Add monitoring and metrics

dns.tf

@@ -56,6 +56,14 @@ resource "hetznerdns_record" "storage1_a" {
   ttl     = 600
 }
 
+resource "hetznerdns_record" "grafana_a" {
+  zone_id = data.hetznerdns_zone.banditlair_zone.id
+  name    = "grafana"
+  value   = local.storage1_ip
+  type    = "A"
+  ttl     = 600
+}
+
 resource "hetznerdns_record" "cache_a" {
   zone_id = data.hetznerdns_zone.banditlair_zone.id
   name    = "cache"

modules/binary-cache.nix

@@ -3,7 +3,6 @@
 
   sops.secrets = {
     nixCacheKey = {
-      owner = config.services.borgbackup.jobs.data.user;
       key = "nix/cache_secret_key";
     };
   };

modules/grafana.nix

@@ -0,0 +1,162 @@
+{ config, ... }:
+{
+
+  sops.secrets = {
+    grafanaAdminPassword = {
+      owner = config.users.users.grafana.name;
+      key = "grafana/admin_password";
+    };
+  };
+
+
+  services.grafana = {
+    enable = true;
+    domain = "grafana.${config.networking.domain}";
+    security.adminPasswordFile = config.sops.secrets.grafanaAdminPassword.path;
+    dataDir = "/nix/var/data/grafana";
+    provision = {
+      enable = true;
+      datasources = [
+        {
+          name = "Prometheus";
+          type = "prometheus";
+          url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+          isDefault = true;
+        }
+        {
+          name = "Loki";
+          type = "loki";
+          access = "proxy";
+          url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+        }
+      ];
+      dashboards = [
+        {
+          name = "Config";
+          options.path = ./dashboards;
+        }
+      ];
+    };
+  };
+
+  services.nginx = {
+    virtualHosts = {
+      "${config.services.grafana.domain}" = {
+
+        enableACME = true;
+        forceSSL = true;
+
+        locations."/" = {
+          proxyPass = "http://127.0.0.1:${toString config.services.grafana.port}";
+          proxyWebsockets = true;
+        };
+      };
+    };
+  };
+
+  services.prometheus = {
+    enable = true;
+
+    scrapeConfigs = [
+      {
+        job_name = "node";
+        static_configs = [{
+          targets = [
+            "10.0.2.3:${toString config.services.prometheus.exporters.node.port}"
+            "10.0.1.1:${toString config.services.prometheus.exporters.node.port}"
+            "10.0.1.11:${toString config.services.prometheus.exporters.node.port}"
+          ];
+        }];
+      }
+    ];
+  };
+
+  services.loki = {
+    enable = true;
+
+    dataDir = "/nix/var/data/loki";
+
+    configuration = {
+      server.http_listen_port = 3100;
+      auth_enabled = false;
+
+      ingester = {
+        lifecycler = {
+          address = "127.0.0.1";
+          ring = {
+            kvstore = {
+              store = "inmemory";
+            };
+            replication_factor = 1;
+          };
+        };
+        chunk_idle_period = "1h";
+        max_chunk_age = "1h";
+        chunk_target_size = 999999;
+        chunk_retain_period = "30s";
+        max_transfer_retries = 0;
+      };
+
+      limits_config = {
+        ingestion_rate_mb = 16;
+      };
+
+      schema_config = {
+        configs = [{
+          from = "2022-09-15";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+
+      storage_config = {
+        boltdb_shipper = {
+          active_index_directory = "${config.services.loki.dataDir}/boltdb-index";
+          cache_location = "${config.services.loki.dataDir}/boltdb-cache";
+          cache_ttl = "24h";
+          shared_store = "filesystem";
+        };
+
+        filesystem = {
+          directory = "${config.services.loki.dataDir}/chunks";
+        };
+      };
+
+      limits_config = {
+        reject_old_samples = true;
+        reject_old_samples_max_age = "168h";
+      };
+
+      chunk_store_config = {
+        max_look_back_period = "0s";
+      };
+
+      table_manager = {
+        retention_deletes_enabled = false;
+        retention_period = "0s";
+      };
+
+      compactor = {
+        working_directory = "${config.services.loki.dataDir}";
+        shared_store = "filesystem";
+        compactor_ring = {
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+      };
+
+      analytics = {
+        reporting_enabled = false;
+      };
+    };
+  };
+}
+
+
+

modules/monitoring-exporters.nix

@@ -0,0 +1,61 @@
+{ config, lib, ... }:
+{
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" "processes" ];
+      };
+    };
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 3101;
+        grpc_listen_port = 0;
+      };
+      clients = [{
+        url = "http://10.0.2.3:3100/loki/api/v1/push";
+      }];
+      scrape_configs = [
+        {
+          job_name = "journal";
+          journal = {
+            max_age = "12h";
+            labels = {
+              job = "systemd-journal";
+              host = "${config.networking.hostName}";
+            };
+          };
+          relabel_configs = [{
+            source_labels = [ "__journal__systemd_unit" ];
+            target_label = "unit";
+          }];
+        }
+        (lib.mkIf config.services.nginx.enable {
+          job_name = "nginx";
+          static_configs = [
+            {
+              targets = [ "localhost" ];
+              labels = {
+                job = "nginx";
+                host = "${config.networking.hostName}";
+                __path__ = "/var/log/nginx/*.log";
+              };
+            }
+          ];
+        })
+      ];
+    };
+  };
+
+  systemd.services.promtail.serviceConfig = {
+    ReadOnlyPaths = lib.mkIf config.services.nginx.enable "/var/log/nginx";
+    SupplementaryGroups = lib.mkIf config.services.nginx.enable [ "nginx" ];
+  };
+}
+
+
+

profiles/backend.nix

@@ -13,6 +13,7 @@
     ../modules/dokuwiki.nix
     ../modules/website-marie.nix
     ../modules/roundcube.nix
+    ../modules/monitoring-exporters.nix
   ];
 
   sops.secrets = {
@@ -67,5 +68,6 @@
 
   networking.firewall.allowedTCPPorts = [ 80 443 64738 ];
   networking.firewall.allowedUDPPorts = [ 64738 ];
+  networking.firewall.interfaces."enp7s0".allowedTCPPorts = [ config.services.prometheus.exporters.node.port ];
 
 }

profiles/db.nix

@@ -7,9 +7,10 @@
     ../modules/postgresql.nix
     ../modules/custom-backup-job.nix
     ../modules/custom-monit.nix
+    ../modules/monitoring-exporters.nix
   ];
 
-  networking.firewall.interfaces."enp7s0".allowedTCPPorts = [ 5432 ];
+  networking.firewall.interfaces."enp7s0".allowedTCPPorts = [ config.services.prometheus.exporters.node.port config.services.postgresql.port ];
 
   sops.secrets = {
     borgSshKey = {
@@ -30,5 +31,4 @@
     sshKey = config.sops.secrets.borgSshKey.path;
   };
 
-  networking.firewall.interfaces."ens10".allowedTCPPorts = [ 80 ];
 }

profiles/storage.nix

@@ -15,6 +15,8 @@
     ../modules/jitsi.nix
     ../modules/gitlab-runner.nix
     ../modules/binary-cache.nix
+    ../modules/grafana.nix
+    ../modules/monitoring-exporters.nix
   ];
 
   sops.secrets = {
@@ -25,6 +27,7 @@
   };
 
   networking.firewall.allowedTCPPorts = [ 80 443 18080 ];
+  networking.firewall.interfaces.vlan4001.allowedTCPPorts = [ config.services.loki.configuration.server.http_listen_port ];
 
   networking.nat.enable = true;
   networking.nat.internalInterfaces = [ "ve-+" ];

secrets.enc.yml

@@ -1,3 +1,5 @@
+grafana:
+    admin_password: ENC[AES256_GCM,data:seXajvIHrEU7XR/XVD6uG/dmZ5I2oiL5IxsM+sMlV9awLwnYpDI0u0gJbYqSYvMRhXS/ZhXuXaTJhgXD,iv:oavt6HtbCCLznPgpSSLKHcHPuJSP+7hPPLepu5orqm0=,tag:Gubg8LEYUMInZpXE1SDYtQ==,type:str]
 nix:
     cache_secret_key: ENC[AES256_GCM,data:Q2mRU+EuTyqjYNvbuyGLqoDSqa/7EPlzNuCJU7QUBRSozf1D4dDzAPNU47xZ2rKcjz6Eg4OhAZLlGeFw9le8SzHOSJ65UYHoMMc6Rpvv/fPhgg2s2UMArrqyO3ultj1pVe3eIIRzBQcdoFqVDg==,iv:jhMTWEO6ahcZl+Dq6mA+mWIie8T0Dq1ZYe/HHYAD5ss=,tag:2GRmd2z96+TGI7MdvOBEdA==,type:str]
 gitlab:
@@ -52,8 +54,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2022-09-14T22:46:49Z"
+    lastmodified: "2022-09-15T05:50:09Z"
-    mac: ENC[AES256_GCM,data:KfG7/Hp3xxa3ykVkbPGWfzufc22TxvfGykNLxN8CX1BrEjdjZhKDYkTbdrANRxuMh1KlCQ1n9zOptPYT7lylEhEAQN4MpyQ0Mz2aQjZgNqhm2qO+YFlvbNsilK1fIbE3exLELfPTCBuJHYj6zMVgOZd1kXNcbL4VRN8uzct4ZzA=,iv:FvD2nvdsLxr5Yd+TKdP/wYHfr9Av5chPYxbwbltnpNI=,tag:hOlapLfrbW+hJlAHp5jX/w==,type:str]
+    mac: ENC[AES256_GCM,data:mmKdFdYWID4oTFCsRrq3idCr+2m/VA22sPOB8V1IVTQISrAQ8j9zwO5JymgXq3+X/1ghNoaFsqmFamzN+uZQ4bd7K2lG2LXzLlzDV4NanPRJGq0szHQ3/DF/hPJij85GREs9OKoPu5zrHVub3B/kymtotc+xUs9x/MdnR+IA9qY=,iv:LdBCUee6YNSMjNtDktsV8LVQIbQVv0ABQgoOSYyu0mg=,tag:WtvpYnxwNRveA+pYd0IGcA==,type:str]
     pgp:
         - created_at: "2021-11-29T00:57:34Z"
           enc: |