Configure Monit on all hosts

modules/custom-backup-job.nix

@@ -20,6 +20,11 @@ in
       default = "";
     };
 
+    postHook = mkOption {
+      type = types.lines;
+      default = "";
+    };
+
     startAt = mkOption {
       type = with types; either str (listOf str);
       default = "03:30";
@@ -48,6 +53,7 @@ in
       };
       readWritePaths = cfg.readWritePaths;
       preHook = cfg.preHook;
+      postHook = cfg.postHook;
       environment = { BORG_RSH = "ssh -i ${cfg.sshKey}"; };
       compression = "lz4";
       startAt = cfg.startAt;

modules/custom-monit.nix

@@ -0,0 +1,58 @@
+{ config, lib, pkgs, ... }:
+with lib;
+let
+  cfg = config.services.custom-monit;
+in
+{
+  options.services.custom-monit = {
+    additionalConfig = mkOption {
+      type = types.lines;
+      default = "";
+    };
+  };
+
+  config = {
+
+    sops.secrets = {
+      monitMailserverConfig = {
+        owner = config.services.borgbackup.jobs.data.user;
+        key = "monit/mailserver_config";
+      };
+    };
+
+    services.monit = {
+      enable = true;
+      config = ''
+        set daemon 30
+          with start delay 90
+
+        set httpd
+          port 2812
+          use address 127.0.0.1
+          allow localhost
+
+        set ssl {
+          verify     : enable,
+        }
+
+        include ${config.sops.secrets.monitMailserverConfig.path}
+
+        set mail-format { from: monit@banditlair.com }
+        set alert alerts@banditlair.com
+
+        check system $HOST
+          if cpu usage > 95% for 10 cycles then alert
+          if memory usage > 75% then alert
+          if swap usage > 25% then alert
+
+        check filesystem root with path /
+          if SPACE usage > 90% then alert
+
+        check file daily-backup-done with path /nix/var/data/backup/backup-ok
+          if changed timestamp then alert
+        
+        ${cfg.additionalConfig}
+      '';
+    };
+  };
+}

profiles/backend.nix

@@ -9,6 +9,7 @@
     ../modules/synapse.nix
     ../modules/nextcloud.nix
     ../modules/custom-backup-job.nix
+    ../modules/custom-monit.nix
     ../modules/dokuwiki.nix
     ../modules/website-marie.nix
   ];
@@ -22,12 +23,35 @@
 
   services.custom-backup-job = {
     additionalPaths = [ "/var/lib/nextcloud/config" ];
-    readWritePaths = [ "/nix/var/data/murmur" ];
-    preHook = "cp /var/lib/murmur/murmur.sqlite /nix/var/data/murmur/murmur.sqlite";
+    readWritePaths = [ "/nix/var/data/murmur" "/nix/var/data/backup/" ];
+    preHook = ''
+      cp /var/lib/murmur/murmur.sqlite /nix/var/data/murmur/murmur.sqlite
+    '';
+    postHook = ''
+      touch /nix/var/data/backup/backup-ok
+    '';
     startAt = "03:30";
     sshKey = config.sops.secrets.borgSshKey.path;
   };
 
+  services.custom-monit.additionalConfig = ''
+    check file nextcloud-data-mounted with path /var/lib/nextcloud/data/index.html
+      start = "${pkgs.systemd}/bin/systemctl start nextcloud-data-sshfs.service"
+
+    check host jellyfin with address jellyfin.banditlair.com
+      if failed port 443 protocol https with timeout 20 seconds then alert
+    check host stb with address www.societe-de-tir-bertrix.com
+      if failed port 443 protocol https with timeout 20 seconds then alert
+
+    check host transmission with address transmission.banditlair.com
+      if failed
+          port 443
+          protocol https
+          status = 401
+          with timeout 20 seconds
+      then alert
+  '';
+
   networking.interfaces.enp1s0 = {
     useDHCP = true;
     ipv4 = {
@@ -43,19 +67,4 @@
   networking.firewall.allowedTCPPorts = [ 80 443 64738 ];
   networking.firewall.allowedUDPPorts = [ 64738 ];
 
-  services.monit = {
-    enable = true;
-    config = ''
-      set daemon 30
-        with start delay 90
-
-      set httpd
-        port 2812
-        use address 127.0.0.1
-        allow localhost
-
-      check file nextcloud-data-mounted with path /var/lib/nextcloud/data/index.html
-        start = "${pkgs.systemd}/bin/systemctl start nextcloud-data-sshfs.service"
-    '';
-  };
 }

profiles/db.nix

@@ -6,6 +6,7 @@
     ../modules/openssh.nix
     ../modules/postgresql.nix
     ../modules/custom-backup-job.nix
+    ../modules/custom-monit.nix
   ];
 
   networking.firewall.interfaces."enp7s0".allowedTCPPorts = [ 5432 ];
@@ -18,11 +19,12 @@
   };
 
   services.custom-backup-job = {
-    readWritePaths = [ "/nix/var/data/postgresql" ];
+    readWritePaths = [ "/nix/var/data/postgresql" "/nix/var/data/backup/" ];
     preHook = ''
       ${pkgs.postgresql_12}/bin/pg_dump -U synapse synapse > /nix/var/data/postgresql/synapse.dmp
       ${pkgs.postgresql_12}/bin/pg_dump -U nextcloud nextcloud > /nix/var/data/postgresql/nextcloud.dmp
     '';
+    postHook = "touch /nix/var/data/backup/backup-ok";
     startAt = "03:00";
     sshKey = config.sops.secrets.borgSshKey.path;
   };

profiles/storage.nix

@@ -11,6 +11,7 @@
     ../modules/monero.nix
     ../modules/torrents.nix
     ../modules/custom-backup-job.nix
+    ../modules/custom-monit.nix
   ];
 
   sops.secrets = {
@@ -45,7 +46,19 @@
   services.custom-backup-job = {
     readWritePaths = [ "/nix/var/data/backup" ];
     preHook = "${pkgs.docker}/bin/docker exec stb-mariadb sh -c 'mysqldump -u stb -pstb stb' > /nix/var/data/backup/stb_mariadb.sql";
+    postHook = "touch /nix/var/data/backup/backup-ok";
     startAt = "04:00";
     sshKey = config.sops.secrets.borgSshKey.path;
   };
+
+  services.custom-monit.additionalConfig = ''
+    check host nextcloud with address cloud.banditlair.com
+      if failed port 443 protocol https with timeout 20 seconds then alert
+    check host anderia-wiki with address anderia.banditlair.com
+      if failed port 443 protocol https with timeout 20 seconds then alert
+    check host arkadia-wiki with address arkadia.banditlair.com
+      if failed port 443 protocol https with timeout 20 seconds then alert
+    check host website-marie with address osteopathie.froidmont.org
+      if failed port 443 protocol https with timeout 20 seconds then alert
+  '';
 }

secrets.enc.yml

@@ -14,6 +14,8 @@ email:
         marie: ENC[AES256_GCM,data:XM1Gt2fY0GqOq+J3+CQflnWPLMmILqTWviWxzkrluovweQ+iMWmfGAS9o2K/GAS1Rr0G3P4NFmhPe6YL,iv:g9Y3WClUzvE4bkXaV82q2/cFME20KvsIV1T/q0ysBIo=,tag:Gc5rE/WubuD66uz+8OOclQ==,type:str]
         alice: ENC[AES256_GCM,data:wLnrPro2FIsT+i5rpcmen63waTE6RBF/aw5yUz6BmsMRXCMmJyoLxrGgB4faIaBEnRNT68iozP8dSCIG,iv:2Tjvz/5JMBby+OBAYShIAz7Tl3gSQAYmUepJcHM9my0=,tag:ulrfLiTBExN5D9hjg3rgSA==,type:str]
         monit: ENC[AES256_GCM,data:p/Vtc9MM8BeNF2V3l0VL82oOk0JUeKY/hAqPtW45Sdm8hiZbCNdF68jurvoI2oBu8b0d2Fer0n4ybAQJ,iv:R7PhqwaWaxx7g1gyYnh0UdoQILYHKuFG84AGghiOJ9g=,tag:S/IpeyVHLzHyqPDHIxAT8w==,type:str]
+monit:
+    mailserver_config: ENC[AES256_GCM,data:LLYTJ0sg5WoJxJg2/9LgD8xAs6ZKuXv8F3ZaEyhW+9spsLbyNibFh9lBxkW009/zyfnI4AU6Fq2WbIWTQKXvCgUWUNr2VCs1BJZkIgijzqJ7edovIcokvm47+eDjeYO2VXBvovtlsryo1BIl+RLulbWFn9Y6fDZbZm4esSL1Vcq9SsSGcMpw8cvUHdWjn7c=,iv:ZfJWtUBPwraKNf5HnVxUCkfvp/xYQwSOK4+4GmWWwn4=,tag:HFboSw/TVczjfGnL1/K29g==,type:str]
 wiki:
     anderia:
         users_file: ENC[AES256_GCM,data:Zx5QTmtqqrRwbHUMiVFfvMnvzaLSlKiouOg57H+4RYS/5Zavl4y3Awswuiz9y7iRDGZhsxba6Ki3jEg/sSwlmB/hICQikQlRfsnx1ibAKeTv9A==,iv:R7vQBU/4thmBVcydHPNiwUOavkhl6OGEVL9WdexJzAw=,tag:FQ/9LjQ6c+ErAhH3erzOBQ==,type:str]
@@ -42,8 +44,8 @@ sops:
     azure_kv: []
     hc_vault: []
     age: []
-    lastmodified: "2021-12-26T22:05:40Z"
-    mac: ENC[AES256_GCM,data:1oCGxIdzNyUjbTxSk788ZvOag5nXSp6RmdJlU1GtJl7eRb6D4rAwUovaKlZy+Wd7jc21U/bA+Hp2W877YMgCasUjBUeSNb5YjyqjCpJCMAqnO2QX40PgcO+v+pNZQLN70e5LZDPG3c9v3DnMDzZycrM2+5LXifYeqyVPyBbwmIg=,iv:UcNQjG2D1yTIFBc/7gBcnptBNcxS621vshTDSDadDbM=,tag:bQfabSuzvBZp+CgDHtrNqQ==,type:str]
+    lastmodified: "2021-12-27T03:39:30Z"
+    mac: ENC[AES256_GCM,data:5M1iio7bFvUVlP0ECwlRMUbboVtTaXGiM0pQkiv738W9hzsCPEKSqOGFNv02r0I6CKf1f88Yvet5VJVK1vbneCAvlp9Io9uSAH+dHPQ4OHABi0LHj58b2lUB0o8DZAL87GlpL7/riTvonEoO1GuO+e4ELoZAtkt2zQXm5576ZWo=,iv:ZfVPvAFGrLCvT8dHql9BWXnBJssvIE3fFih9lEOZJ+U=,tag:CHePtD5xW8Guni6xYc11FQ==,type:str]
     pgp:
         - created_at: "2021-11-29T00:57:34Z"
           enc: |