Add monitoring and metrics

modules/binary-cache.nix

@@ -3,7 +3,6 @@
 
   sops.secrets = {
     nixCacheKey = {
-      owner = config.services.borgbackup.jobs.data.user;
       key = "nix/cache_secret_key";
     };
   };

modules/grafana.nix

@@ -0,0 +1,162 @@
+{ config, ... }:
+{
+
+  sops.secrets = {
+    grafanaAdminPassword = {
+      owner = config.users.users.grafana.name;
+      key = "grafana/admin_password";
+    };
+  };
+
+
+  services.grafana = {
+    enable = true;
+    domain = "grafana.${config.networking.domain}";
+    security.adminPasswordFile = config.sops.secrets.grafanaAdminPassword.path;
+    dataDir = "/nix/var/data/grafana";
+    provision = {
+      enable = true;
+      datasources = [
+        {
+          name = "Prometheus";
+          type = "prometheus";
+          url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+          isDefault = true;
+        }
+        {
+          name = "Loki";
+          type = "loki";
+          access = "proxy";
+          url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+        }
+      ];
+      dashboards = [
+        {
+          name = "Config";
+          options.path = ./dashboards;
+        }
+      ];
+    };
+  };
+
+  services.nginx = {
+    virtualHosts = {
+      "${config.services.grafana.domain}" = {
+
+        enableACME = true;
+        forceSSL = true;
+
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+          proxyWebsockets = true;
+        };
+      };
+    };
+  };
+
+  services.prometheus = {
+    enable = true;
+
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [{
+          targets = [
+            "10.0.2.3:${toString config.services.prometheus.exporters.node.port}"
+            "10.0.1.1:${toString config.services.prometheus.exporters.node.port}"
+            "10.0.1.11:${toString config.services.prometheus.exporters.node.port}"
+          ];
+        }];
+      }
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+
+    dataDir = "/nix/var/data/loki";
+
+    configuration = {
+      server.http_listen_port = 3100;
+      auth_enabled = false;
+
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 999999;
+        chunk_retain_period = "30s";
+        max_transfer_retries = 0;
+      };
+
+      limits_config = {
+        ingestion_rate_mb = 16;
+      };
+
+      schema_config = {
+        configs = [{
+          from = "2022-09-15";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "${config.services.loki.dataDir}/boltdb-index";
+          cache_location = "${config.services.loki.dataDir}/boltdb-cache";
+          cache_ttl = "24h";
+          shared_store = "filesystem";
+        };
+
+        filesystem = {
+          directory = "${config.services.loki.dataDir}/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      chunk_store_config = {
+        max_look_back_period = "0s";
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = "${config.services.loki.dataDir}";
+        shared_store = "filesystem";
+        compactor_ring = {
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+      };
+
+      analytics = {
+        reporting_enabled = false;
+      };
+    };
+  };
+}
+
+
+

modules/monitoring-exporters.nix

@@ -0,0 +1,61 @@
+{ config, lib, ... }:
+{
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" "processes" ];
+      };
+    };
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3101;
+        grpc_listen_port = 0;
+      };
+      clients = [{
+        url = "http://10.0.2.3:3100/loki/api/v1/push";
+      }];
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+              host = "${config.networking.hostName}";
+            };
+          };
+          relabel_configs = [{
+            source_labels = [ "__journal__systemd_unit" ];
+            target_label = "unit";
+          }];
+        }
+        (lib.mkIf config.services.nginx.enable {
+          job_name = "nginx";
+          static_configs = [
+            {
+              targets = [ "localhost" ];
+              labels = {
+                job = "nginx";
+                host = "${config.networking.hostName}";
+                __path__ = "/var/log/nginx/*.log";
+              };
+            }
+          ];
+        })
+      ];
+    };
+  };
+
+  systemd.services.promtail.serviceConfig = {
+    ReadOnlyPaths = lib.mkIf config.services.nginx.enable "/var/log/nginx";
+    SupplementaryGroups = lib.mkIf config.services.nginx.enable [ "nginx" ];
+  };
+}
+
+
+