Configure Monit on all hosts

modules/custom-backup-job.nix

@@ -20,6 +20,11 @@ in
       default = "";
     };
 
+    postHook = mkOption {
+      type = types.lines;
+      default = "";
+    };
+
     startAt = mkOption {
       type = with types; either str (listOf str);
       default = "03:30";
@@ -48,6 +53,7 @@ in
       };
       readWritePaths = cfg.readWritePaths;
       preHook = cfg.preHook;
+      postHook = cfg.postHook;
       environment = { BORG_RSH = "ssh -i ${cfg.sshKey}"; };
       compression = "lz4";
       startAt = cfg.startAt;

modules/custom-monit.nix

@@ -0,0 +1,58 @@
+{ config, lib, pkgs, ... }:
+with lib;
+let
+  cfg = config.services.custom-monit;
+in
+{
+  options.services.custom-monit = {
+    additionalConfig = mkOption {
+      type = types.lines;
+      default = "";
+    };
+  };
+
+  config = {
+
+    sops.secrets = {
+      monitMailserverConfig = {
+        owner = config.services.borgbackup.jobs.data.user;
+        key = "monit/mailserver_config";
+      };
+    };
+
+    services.monit = {
+      enable = true;
+      config = ''
+        set daemon 30
+          with start delay 90
+
+        set httpd
+          port 2812
+          use address 127.0.0.1
+          allow localhost
+
+        set ssl {
+          verify     : enable,
+        }
+
+        include ${config.sops.secrets.monitMailserverConfig.path}
+
+        set mail-format { from: monit@banditlair.com }
+        set alert alerts@banditlair.com
+
+        check system $HOST
+          if cpu usage > 95% for 10 cycles then alert
+          if memory usage > 75% then alert
+          if swap usage > 25% then alert
+
+        check filesystem root with path /
+          if SPACE usage > 90% then alert
+
+        check file daily-backup-done with path /nix/var/data/backup/backup-ok
+          if changed timestamp then alert
+        
+        ${cfg.additionalConfig}
+      '';
+    };
+  };
+}