diff --git a/modules/custom-backup-job.nix b/modules/custom-backup-job.nix index 9fe0683..e18d961 100644 --- a/modules/custom-backup-job.nix +++ b/modules/custom-backup-job.nix @@ -20,6 +20,11 @@ in default = ""; }; + postHook = mkOption { + type = types.lines; + default = ""; + }; + startAt = mkOption { type = with types; either str (listOf str); default = "03:30"; @@ -48,6 +53,7 @@ in }; readWritePaths = cfg.readWritePaths; preHook = cfg.preHook; + postHook = cfg.postHook; environment = { BORG_RSH = "ssh -i ${cfg.sshKey}"; }; compression = "lz4"; startAt = cfg.startAt; diff --git a/modules/custom-monit.nix b/modules/custom-monit.nix new file mode 100644 index 0000000..c6c54a0 --- /dev/null +++ b/modules/custom-monit.nix @@ -0,0 +1,58 @@ +{ config, lib, pkgs, ... }: +with lib; +let + cfg = config.services.custom-monit; +in +{ + options.services.custom-monit = { + additionalConfig = mkOption { + type = types.lines; + default = ""; + }; + }; + + config = { + + sops.secrets = { + monitMailserverConfig = { + owner = config.services.borgbackup.jobs.data.user; + key = "monit/mailserver_config"; + }; + }; + + services.monit = { + enable = true; + config = '' + set daemon 30 + with start delay 90 + + set httpd + port 2812 + use address 127.0.0.1 + allow localhost + + set ssl { + verify : enable, + } + + include ${config.sops.secrets.monitMailserverConfig.path} + + set mail-format { from: monit@banditlair.com } + set alert alerts@banditlair.com + + check system $HOST + if cpu usage > 95% for 10 cycles then alert + if memory usage > 75% then alert + if swap usage > 25% then alert + + check filesystem root with path / + if SPACE usage > 90% then alert + + check file daily-backup-done with path /nix/var/data/backup/backup-ok + if changed timestamp then alert + + ${cfg.additionalConfig} + ''; + }; + }; +} diff --git a/profiles/backend.nix b/profiles/backend.nix index c7c486b..9cbfd18 100644 --- a/profiles/backend.nix +++ b/profiles/backend.nix @@ -9,6 +9,7 @@ ../modules/synapse.nix ../modules/nextcloud.nix ../modules/custom-backup-job.nix + ../modules/custom-monit.nix ../modules/dokuwiki.nix ../modules/website-marie.nix ]; @@ -22,12 +23,35 @@ services.custom-backup-job = { additionalPaths = [ "/var/lib/nextcloud/config" ]; - readWritePaths = [ "/nix/var/data/murmur" ]; - preHook = "cp /var/lib/murmur/murmur.sqlite /nix/var/data/murmur/murmur.sqlite"; + readWritePaths = [ "/nix/var/data/murmur" "/nix/var/data/backup/" ]; + preHook = '' + cp /var/lib/murmur/murmur.sqlite /nix/var/data/murmur/murmur.sqlite + ''; + postHook = '' + touch /nix/var/data/backup/backup-ok + ''; startAt = "03:30"; sshKey = config.sops.secrets.borgSshKey.path; }; + services.custom-monit.additionalConfig = '' + check file nextcloud-data-mounted with path /var/lib/nextcloud/data/index.html + start = "${pkgs.systemd}/bin/systemctl start nextcloud-data-sshfs.service" + + check host jellyfin with address jellyfin.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + check host stb with address www.societe-de-tir-bertrix.com + if failed port 443 protocol https with timeout 20 seconds then alert + + check host transmission with address transmission.banditlair.com + if failed + port 443 + protocol https + status = 401 + with timeout 20 seconds + then alert + ''; + networking.interfaces.enp1s0 = { useDHCP = true; ipv4 = { @@ -43,19 +67,4 @@ networking.firewall.allowedTCPPorts = [ 80 443 64738 ]; networking.firewall.allowedUDPPorts = [ 64738 ]; - services.monit = { - enable = true; - config = '' - set daemon 30 - with start delay 90 - - set httpd - port 2812 - use address 127.0.0.1 - allow localhost - - check file nextcloud-data-mounted with path /var/lib/nextcloud/data/index.html - start = "${pkgs.systemd}/bin/systemctl start nextcloud-data-sshfs.service" - ''; - }; } diff --git a/profiles/db.nix b/profiles/db.nix index 0ba7477..dc02d15 100644 --- a/profiles/db.nix +++ b/profiles/db.nix @@ -6,6 +6,7 @@ ../modules/openssh.nix ../modules/postgresql.nix ../modules/custom-backup-job.nix + ../modules/custom-monit.nix ]; networking.firewall.interfaces."enp7s0".allowedTCPPorts = [ 5432 ]; @@ -18,11 +19,12 @@ }; services.custom-backup-job = { - readWritePaths = [ "/nix/var/data/postgresql" ]; + readWritePaths = [ "/nix/var/data/postgresql" "/nix/var/data/backup/" ]; preHook = '' ${pkgs.postgresql_12}/bin/pg_dump -U synapse synapse > /nix/var/data/postgresql/synapse.dmp ${pkgs.postgresql_12}/bin/pg_dump -U nextcloud nextcloud > /nix/var/data/postgresql/nextcloud.dmp ''; + postHook = "touch /nix/var/data/backup/backup-ok"; startAt = "03:00"; sshKey = config.sops.secrets.borgSshKey.path; }; diff --git a/profiles/storage.nix b/profiles/storage.nix index 8d8e26a..f5b4275 100644 --- a/profiles/storage.nix +++ b/profiles/storage.nix @@ -11,6 +11,7 @@ ../modules/monero.nix ../modules/torrents.nix ../modules/custom-backup-job.nix + ../modules/custom-monit.nix ]; sops.secrets = { @@ -45,7 +46,19 @@ services.custom-backup-job = { readWritePaths = [ "/nix/var/data/backup" ]; preHook = "${pkgs.docker}/bin/docker exec stb-mariadb sh -c 'mysqldump -u stb -pstb stb' > /nix/var/data/backup/stb_mariadb.sql"; + postHook = "touch /nix/var/data/backup/backup-ok"; startAt = "04:00"; sshKey = config.sops.secrets.borgSshKey.path; }; + + services.custom-monit.additionalConfig = '' + check host nextcloud with address cloud.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + check host anderia-wiki with address anderia.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + check host arkadia-wiki with address arkadia.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + check host website-marie with address osteopathie.froidmont.org + if failed port 443 protocol https with timeout 20 seconds then alert + ''; } diff --git a/secrets.enc.yml b/secrets.enc.yml index 9181712..87d6f92 100644 --- a/secrets.enc.yml +++ b/secrets.enc.yml @@ -14,6 +14,8 @@ email: marie: ENC[AES256_GCM,data:XM1Gt2fY0GqOq+J3+CQflnWPLMmILqTWviWxzkrluovweQ+iMWmfGAS9o2K/GAS1Rr0G3P4NFmhPe6YL,iv:g9Y3WClUzvE4bkXaV82q2/cFME20KvsIV1T/q0ysBIo=,tag:Gc5rE/WubuD66uz+8OOclQ==,type:str] alice: ENC[AES256_GCM,data:wLnrPro2FIsT+i5rpcmen63waTE6RBF/aw5yUz6BmsMRXCMmJyoLxrGgB4faIaBEnRNT68iozP8dSCIG,iv:2Tjvz/5JMBby+OBAYShIAz7Tl3gSQAYmUepJcHM9my0=,tag:ulrfLiTBExN5D9hjg3rgSA==,type:str] monit: ENC[AES256_GCM,data:p/Vtc9MM8BeNF2V3l0VL82oOk0JUeKY/hAqPtW45Sdm8hiZbCNdF68jurvoI2oBu8b0d2Fer0n4ybAQJ,iv:R7PhqwaWaxx7g1gyYnh0UdoQILYHKuFG84AGghiOJ9g=,tag:S/IpeyVHLzHyqPDHIxAT8w==,type:str] +monit: + mailserver_config: ENC[AES256_GCM,data:LLYTJ0sg5WoJxJg2/9LgD8xAs6ZKuXv8F3ZaEyhW+9spsLbyNibFh9lBxkW009/zyfnI4AU6Fq2WbIWTQKXvCgUWUNr2VCs1BJZkIgijzqJ7edovIcokvm47+eDjeYO2VXBvovtlsryo1BIl+RLulbWFn9Y6fDZbZm4esSL1Vcq9SsSGcMpw8cvUHdWjn7c=,iv:ZfJWtUBPwraKNf5HnVxUCkfvp/xYQwSOK4+4GmWWwn4=,tag:HFboSw/TVczjfGnL1/K29g==,type:str] wiki: anderia: users_file: ENC[AES256_GCM,data:Zx5QTmtqqrRwbHUMiVFfvMnvzaLSlKiouOg57H+4RYS/5Zavl4y3Awswuiz9y7iRDGZhsxba6Ki3jEg/sSwlmB/hICQikQlRfsnx1ibAKeTv9A==,iv:R7vQBU/4thmBVcydHPNiwUOavkhl6OGEVL9WdexJzAw=,tag:FQ/9LjQ6c+ErAhH3erzOBQ==,type:str] @@ -42,8 +44,8 @@ sops: azure_kv: [] hc_vault: [] age: [] - lastmodified: "2021-12-26T22:05:40Z" - mac: ENC[AES256_GCM,data:1oCGxIdzNyUjbTxSk788ZvOag5nXSp6RmdJlU1GtJl7eRb6D4rAwUovaKlZy+Wd7jc21U/bA+Hp2W877YMgCasUjBUeSNb5YjyqjCpJCMAqnO2QX40PgcO+v+pNZQLN70e5LZDPG3c9v3DnMDzZycrM2+5LXifYeqyVPyBbwmIg=,iv:UcNQjG2D1yTIFBc/7gBcnptBNcxS621vshTDSDadDbM=,tag:bQfabSuzvBZp+CgDHtrNqQ==,type:str] + lastmodified: "2021-12-27T03:39:30Z" + mac: ENC[AES256_GCM,data:5M1iio7bFvUVlP0ECwlRMUbboVtTaXGiM0pQkiv738W9hzsCPEKSqOGFNv02r0I6CKf1f88Yvet5VJVK1vbneCAvlp9Io9uSAH+dHPQ4OHABi0LHj58b2lUB0o8DZAL87GlpL7/riTvonEoO1GuO+e4ELoZAtkt2zQXm5576ZWo=,iv:ZfVPvAFGrLCvT8dHql9BWXnBJssvIE3fFih9lEOZJ+U=,tag:CHePtD5xW8Guni6xYc11FQ==,type:str] pgp: - created_at: "2021-11-29T00:57:34Z" enc: |