From 830f6a56092b405956e6eac7a7471a638863552c Mon Sep 17 00:00:00 2001 From: Paul-Henri Froidmont Date: Tue, 2 Jan 2018 05:03:19 +0100 Subject: [PATCH] Add monit and a few utility scripts --- playbook.yml | 10 +- roles/gitlab-docker/tasks/main.yml | 4 +- .../templates/gitlab/config/gitlab.rb | 4 +- roles/matrix-docker/tasks/main.yml | 15 +- roles/monit/handlers/main.yml | 3 + roles/monit/tasks/main.yml | 14 + roles/monit/templates/monitrc | 357 ++++++++++++++++++ roles/scripts/tasks/main.yml | 21 ++ roles/scripts/templates/dockerComposeAll.sh | 13 + roles/scripts/templates/fullBackup.sh | 55 +++ roles/scripts/templates/syncData.sh | 39 ++ roles/scripts/templates/updateAll.sh | 16 + 12 files changed, 537 insertions(+), 14 deletions(-) create mode 100644 roles/monit/handlers/main.yml create mode 100644 roles/monit/tasks/main.yml create mode 100755 roles/monit/templates/monitrc create mode 100644 roles/scripts/tasks/main.yml create mode 100755 roles/scripts/templates/dockerComposeAll.sh create mode 100755 roles/scripts/templates/fullBackup.sh create mode 100644 roles/scripts/templates/syncData.sh create mode 100755 roles/scripts/templates/updateAll.sh diff --git a/playbook.yml b/playbook.yml index c69d82e..3f358b6 100644 --- a/playbook.yml +++ b/playbook.yml @@ -8,6 +8,7 @@ sub_domains: - rpg roles: + - role: scripts - role: murmur-docker - role: searx-docker - role: wiki-docker @@ -18,9 +19,10 @@ - role: matrix-docker - role: plex-docker - role: deluge-docker - vars_prompt: - - name: "ansible_sudo_pass" - prompt: "Sudo password" - private: yes + - role: monit +# vars_prompt: +# - name: "ansible_sudo_pass" +# prompt: "Sudo password" +# private: yes vars_files: - "passwords.yml" \ No newline at end of file diff --git a/roles/gitlab-docker/tasks/main.yml b/roles/gitlab-docker/tasks/main.yml index 20287f5..54c6d10 100644 --- a/roles/gitlab-docker/tasks/main.yml +++ b/roles/gitlab-docker/tasks/main.yml @@ -1,6 +1,8 @@ --- -- name: Copy gitlab config +- name: Copy docker-compose.yml copy: src=gitlab dest={{docker_compose_files_folder}} +- name: Create gitlab config folder + file: dest={{docker_compose_files_folder}}/gitlab/config state=directory - name: Create gitlab config template: src=gitlab/config/gitlab.rb dest={{docker_compose_files_folder}}/gitlab/config/gitlab.rb - name: Start gitlab docker project diff --git a/roles/gitlab-docker/templates/gitlab/config/gitlab.rb b/roles/gitlab-docker/templates/gitlab/config/gitlab.rb index 9ab6326..34923fa 100644 --- a/roles/gitlab-docker/templates/gitlab/config/gitlab.rb +++ b/roles/gitlab-docker/templates/gitlab/config/gitlab.rb @@ -1227,8 +1227,8 @@ nginx['proxy_set_headers'] = { ##! Docs: https://docs.gitlab.com/ce/administration/monitoring/prometheus/ ################################################################################ -# prometheus['enable'] = true -# prometheus['monitor_kubernetes'] = true +prometheus['enable'] = false +prometheus['monitor_kubernetes'] = false # prometheus['username'] = 'gitlab-prometheus' # prometheus['uid'] = nil # prometheus['gid'] = nil diff --git a/roles/matrix-docker/tasks/main.yml b/roles/matrix-docker/tasks/main.yml index 06d3a7a..f3a2c39 100644 --- a/roles/matrix-docker/tasks/main.yml +++ b/roles/matrix-docker/tasks/main.yml @@ -10,15 +10,16 @@ docker_service: project_src: "{{docker_compose_files_folder}}/matrix" state: present -- name: Check if database tables exist - command: docker-compose exec -T db psql -U synapse synapse -c "\dt" +- name: Wait for database to start and count matrix users + shell: docker-compose exec -T db psql -U synapse synapse -c "select count(*) from users;" -t args: chdir: "{{docker_compose_files_folder}}/matrix/" - register: db_tables_exist - ignore_errors: false - changed_when: '"No relations found." in db_tables_exist.stdout_lines' -- name: Restore Matrix database + register: matrix_users_count + until: matrix_users_count.rc == 0 + retries: 10 + changed_when: false +- name: Restore Matrix database if needed command: docker-compose exec -T db sh -c "psql -U synapse synapse < /backups/database.dmp" args: chdir: "{{docker_compose_files_folder}}/matrix/" - when: '"No relations found." in db_tables_exist.stdout_lines' \ No newline at end of file + when: matrix_users_count.stdout|int == 0 \ No newline at end of file diff --git a/roles/monit/handlers/main.yml b/roles/monit/handlers/main.yml new file mode 100644 index 0000000..5ad2dd6 --- /dev/null +++ b/roles/monit/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: reload monit + command: monit reload diff --git a/roles/monit/tasks/main.yml b/roles/monit/tasks/main.yml new file mode 100644 index 0000000..765333a --- /dev/null +++ b/roles/monit/tasks/main.yml @@ -0,0 +1,14 @@ +--- +- name: Install monit + package: name=monit state=present update_cache=yes +- name: Enable and start monit service + systemd: + name: monit.service + state: started + enabled: True +- name: Copy monit config + template: + src: monitrc + dest: /etc/monitrc + notify: + - reload monit diff --git a/roles/monit/templates/monitrc b/roles/monit/templates/monitrc new file mode 100755 index 0000000..d94552c --- /dev/null +++ b/roles/monit/templates/monitrc @@ -0,0 +1,357 @@ +############################################################################### +## Monit control file +############################################################################### +## +## Comments begin with a '#' and extend through the end of the line. Keywords +## are case insensitive. All path's MUST BE FULLY QUALIFIED, starting with '/'. +## +## Below you will find examples of some frequently used statements. For +## information about the control file and a complete list of statements and +## options, please have a look in the Monit manual. +## +## +############################################################################### +## Global section +############################################################################### +## +## Start Monit in the background (run as a daemon): +# +set daemon 30 # check services at 30 seconds intervals + with start delay 300 # optional: delay the first check by 4-minutes (by +# # default Monit check immediately after Monit start) +# +# +## Set syslog logging. If you want to log to a standalone log file instead, +## specify the full path to the log file +# +set log syslog + +# +# +## Set the location of the Monit lock file which stores the process id of the +## running Monit instance. By default this file is stored in $HOME/.monit.pid +# +# set pidfile /var/run/monit.pid +# +## Set the location of the Monit id file which stores the unique id for the +## Monit instance. The id is generated and stored on first Monit start. By +## default the file is placed in $HOME/.monit.id. +# +# set idfile /var/.monit.id +# +## Set the location of the Monit state file which saves monitoring states +## on each cycle. By default the file is placed in $HOME/.monit.state. If +## the state file is stored on a persistent filesystem, Monit will recover +## the monitoring state across reboots. If it is on temporary filesystem, the +## state will be lost on reboot which may be convenient in some situations. +# +# set statefile /var/.monit.state +# +# + +## Set limits for various tests. The following example shows the default values: +## +# set limits { +# programOutput: 512 B, # check program's output truncate limit +# sendExpectBuffer: 256 B, # limit for send/expect protocol test +# fileContentBuffer: 512 B, # limit for file content test +# httpContentBuffer: 1 MB, # limit for HTTP content test +# networkTimeout: 5 seconds # timeout for network I/O +# programTimeout: 300 seconds # timeout for check program +# stopTimeout: 30 seconds # timeout for service stop +# startTimeout: 30 seconds # timeout for service start +# restartTimeout: 30 seconds # timeout for service restart +# } + +## Set global SSL options (just most common options showed, see manual for +## full list). +# +# set ssl { +# verify : enable, # verify SSL certificates (disabled by default but STRONGLY RECOMMENDED) +# selfsigned : allow # allow self signed SSL certificates (reject by default) +# } +# +# +## Set the list of mail servers for alert delivery. Multiple servers may be +## specified using a comma separator. If the first mail server fails, Monit +# will use the second mail server in the list and so on. By default Monit uses +# port 25 - it is possible to override this with the PORT option. +# +# set mailserver mail.bar.baz, # primary mailserver +# backup.bar.baz port 10025, # backup mailserver on port 10025 +# localhost # fallback relay +set mailserver mail.banditlair.com PORT 465 + USERNAME noreply@banditlair.com PASSWORD {{email_password}} + using SSL +# +# +## By default Monit will drop alert events if no mail servers are available. +## If you want to keep the alerts for later delivery retry, you can use the +## EVENTQUEUE statement. The base directory where undelivered alerts will be +## stored is specified by the BASEDIR option. You can limit the queue size +## by using the SLOTS option (if omitted, the queue is limited by space +## available in the back end filesystem). +# +set eventqueue + basedir /var/monit # set the base directory where events will be stored + slots 100 # optionally limit the queue size +# +# +## Send status and events to M/Monit (for more informations about M/Monit +## see https://mmonit.com/). By default Monit registers credentials with +## M/Monit so M/Monit can smoothly communicate back to Monit and you don't +## have to register Monit credentials manually in M/Monit. It is possible to +## disable credential registration using the commented out option below. +## Though, if safety is a concern we recommend instead using https when +## communicating with M/Monit and send credentials encrypted. The password +## should be URL encoded if it contains URL-significant characters like +## ":", "?", "@". Default timeout is 5 seconds, you can customize it by +## adding the timeout option. +# +# set mmonit http://monit:monit@192.168.1.10:8080/collector +# # with timeout 30 seconds # Default timeout is 5 seconds +# # and register without credentials # Don't register credentials +# +# +## Monit by default uses the following format for alerts if the mail-format +## statement is missing:: +## --8<-- +## set mail-format { +## from: Monit +## subject: monit alert -- $EVENT $SERVICE +## message: $EVENT Service $SERVICE +## Date: $DATE +## Action: $ACTION +## Host: $HOST +## Description: $DESCRIPTION +## +## Your faithful employee, +## Monit +## } +## --8<-- +## +## You can override this message format or parts of it, such as subject +## or sender using the MAIL-FORMAT statement. Macros such as $DATE, etc. +## are expanded at runtime. For example, to override the sender, use: +# +# set mail-format { from: monit@foo.bar } +# +set mail-format { from: monit@banditlair.com } +# +## You can set alert recipients whom will receive alerts if/when a +## service defined in this file has errors. Alerts may be restricted on +## events by using a filter as in the second example below. +# +# set alert sysadm@foo.bar # receive all alerts +# +set alert self.alert@banditlair.com +set alert pascal.falbo@hotmail.fr +## Do not alert when Monit starts, stops or performs a user initiated action. +## This filter is recommended to avoid getting alerts for trivial cases. +# +# set alert your-name@your.domain not on { instance, action } +# +# +## Monit has an embedded HTTP interface which can be used to view status of +## services monitored and manage services from a web interface. The HTTP +## interface is also required if you want to issue Monit commands from the +## command line, such as 'monit status' or 'monit restart service' The reason +## for this is that the Monit client uses the HTTP interface to send these +## commands to a running Monit daemon. See the Monit Wiki if you want to +## enable SSL for the HTTP interface. +# +set httpd port 2812 and + use address localhost # only accept connection from localhost + allow localhost # allow localhost to connect to the server and + allow admin:monit # require user 'admin' with password 'monit' + #with ssl { # enable SSL/TLS and set path to server certificate + # pemfile: /etc/ssl/certs/monit.pem + #} + +############################################################################### +## Services +############################################################################## +## +## Check general system resources such as load average, cpu and memory +## usage. Each test specifies a resource, conditions and the action to be +## performed should a test fail. +# +# check system $HOST +# if loadavg (1min) > 4 then alert +# if loadavg (5min) > 2 then alert +# if cpu usage > 95% for 10 cycles then alert +# if memory usage > 75% then alert +# if swap usage > 25% then alert +check system $HOST +# if loadavg (1min) > 4 then alert +# if loadavg (5min) > 2 then alert + if cpu usage > 95% for 10 cycles then alert + if memory usage > 75% then alert + if swap usage > 25% then alert + + +# +# +## Check if a file exists, checksum, permissions, uid and gid. In addition +## to alert recipients in the global section, customized alert can be sent to +## additional recipients by specifying a local alert handler. The service may +## be grouped using the GROUP option. More than one group can be specified by +## repeating the 'group name' statement. +# +# check file apache_bin with path /usr/local/apache/bin/httpd +# if failed checksum and +# expect the sum 8f7f419955cefa0b33a2ba316cba3659 then unmonitor +# if failed permission 755 then unmonitor +# if failed uid "root" then unmonitor +# if failed gid "root" then unmonitor +# alert security@foo.bar on { +# checksum, permission, uid, gid, unmonitor +# } with the mail-format { subject: Alarm! } +# group server +# +# +## Check that a process is running, in this case Apache, and that it respond +## to HTTP and HTTPS requests. Check its resource usage such as cpu and memory, +## and number of children. If the process is not running, Monit will restart +## it by default. In case the service is restarted very often and the +## problem remains, it is possible to disable monitoring using the TIMEOUT +## statement. This service depends on another service (apache_bin) which +## is defined above. +# +# check process apache with pidfile /usr/local/apache/logs/httpd.pid +# start program = "/etc/init.d/httpd start" with timeout 60 seconds +# stop program = "/etc/init.d/httpd stop" +# if cpu > 60% for 2 cycles then alert +# if cpu > 80% for 5 cycles then restart +# if totalmem > 200.0 MB for 5 cycles then restart +# if children > 250 then restart +# if loadavg(5min) greater than 10 for 8 cycles then stop +# if disk read > 500 kb/s for 10 cycles then alert +# if disk write > 500 kb/s for 10 cycles then alert +# if failed host www.tildeslash.com port 80 protocol http and request "/somefile.html" then restart +# if failed port 443 protocol https with timeout 15 seconds then restart +# if 3 restarts within 5 cycles then unmonitor +# depends on apache_bin +# group server +# +# +## Check filesystem permissions, uid, gid, space usage, inode usage and disk I/O. +## Other services, such as databases, may depend on this resource and an automatically +## graceful stop may be cascaded to them before the filesystem will become full and data +## lost. +# +# check filesystem datafs with path /dev/sdb1 +# start program = "/bin/mount /data" +# stop program = "/bin/umount /data" +# if failed permission 660 then unmonitor +# if failed uid "root" then unmonitor +# if failed gid "disk" then unmonitor +# if space usage > 80% for 5 times within 15 cycles then alert +# if space usage > 99% then stop +# if inode usage > 30000 then alert +# if inode usage > 99% then stop +# if read rate > 1 MB/s for 5 cycles then alert +# if read rate > 500 operations/s for 5 cycles then alert +# if write rate > 1 MB/s for 5 cycles then alert +# if write rate > 500 operations/s for 5 cycles then alert +# if service time > 10 milliseconds for 3 times within 5 cycles then alert +# group server +# +# +## Check a file's timestamp. In this example, we test if a file is older +## than 15 minutes and assume something is wrong if its not updated. Also, +## if the file size exceed a given limit, execute a script +# +# check file database with path /data/mydatabase.db +# if failed permission 700 then alert +# if failed uid "data" then alert +# if failed gid "data" then alert +# if timestamp > 15 minutes then alert +# if size > 100 MB then exec "/my/cleanup/script" as uid dba and gid dba +# +# +## Check directory permission, uid and gid. An event is triggered if the +## directory does not belong to the user with uid 0 and gid 0. In addition, +## the permissions have to match the octal description of 755 (see chmod(1)). +# +# check directory bin with path /bin +# if failed permission 755 then unmonitor +# if failed uid 0 then unmonitor +# if failed gid 0 then unmonitor +# +# +## Check a remote host availability by issuing a ping test and check the +## content of a response from a web server. Up to three pings are sent and +## connection to a port and an application level network check is performed. +# +# check host myserver with address 192.168.1.1 +# if failed ping then alert +# if failed port 3306 protocol mysql with timeout 15 seconds then alert +# if failed port 80 protocol http +# and request /some/path with content = "a string" +# then alert +# +# +## Check a network link status (up/down), link capacity changes, saturation +## and bandwidth usage. +# +# check network public with interface eth0 +# if failed link then alert +# if changed link then alert +# if saturation > 90% then alert +# if download > 10 MB/s then alert +# if total uploaded > 1 GB in last hour then alert +# +# +## Check custom program status output. +# +# check program myscript with path /usr/local/bin/myscript.sh +# if status != 0 then alert +# +# + +check file daily-backup with path /backups/backup-ok + if timestamp > 25 hours then alert + if changed timestamp then alert + +check host home-ssh with address phf.ddns.net + if failed port 2222 protocol ssh with timeout 20 seconds then alert + +check host searX with address banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + +check host NextCloud with address cloud.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + +check host Gitlab-ssh with address gitlab.banditlair.com + if failed port 2224 protocol ssh with timeout 20 seconds then alert + +check host Gitlab-ui with address gitlab.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + +check host mail-admin with address mail.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + +check host Grafana with address grafana.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + +#check host Plex with address plex.banditlair.com +# if failed port 443 protocol https with timeout 20 seconds then alert + +#check host sonar with address sonar.banditlair.com +# if failed port 443 protocol https with timeout 20 seconds then alert + +check host deluge-ui with address deluge.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert + +check host rpg-wiki with address rpg.banditlair.com + if failed port 443 protocol https with timeout 20 seconds then alert +############################################################################### +## Includes +############################################################################### +## +## It is possible to include additional configuration parts from other files or +## directories. +# +# include /etc/monit.d/* +# diff --git a/roles/scripts/tasks/main.yml b/roles/scripts/tasks/main.yml new file mode 100644 index 0000000..4329c57 --- /dev/null +++ b/roles/scripts/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Create fullBackup.sh + template: + src: fullBackup.sh + dest: /root/fullBackup.sh + mode: 0700 +- name: Create dockerComposeAll.sh + template: + src: dockerComposeAll.sh + dest: /root/dockerComposeAll.sh + mode: 0700 +- name: Create syncData.sh + template: + src: syncData.sh + dest: /root/syncData.sh + mode: 0700 +- name: Create updateAll.sh + template: + src: updateAll.sh + dest: /root/updateAll.sh + mode: 0700 \ No newline at end of file diff --git a/roles/scripts/templates/dockerComposeAll.sh b/roles/scripts/templates/dockerComposeAll.sh new file mode 100755 index 0000000..4547b36 --- /dev/null +++ b/roles/scripts/templates/dockerComposeAll.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +for dir in {{docker_compose_files_folder}}/* +do + if [ -d ${dir} ] + then + echo "docker-compose $1 ${dir}" + cd "${dir}" + docker-compose $1 + echo -------------------------------------------------------------- + fi +done; + diff --git a/roles/scripts/templates/fullBackup.sh b/roles/scripts/templates/fullBackup.sh new file mode 100755 index 0000000..20e5773 --- /dev/null +++ b/roles/scripts/templates/fullBackup.sh @@ -0,0 +1,55 @@ +#!/bin/sh +set -e + +REPOSITORY=ssh://backup@phf.ddns.net:2222/./backup + +export BORG_PASSPHRASE='{{backup_borg_passphrase}}' + +echo 'Dumping NextCloud database' +docker exec nextcloud_db_1 sh -c "mysqldump -u nextcloud -p{{nextcloud_mysql_password}} nextcloud > /backups/database.dmp" + +echo 'Dumping matrix database' +docker exec matrix_db_1 sh -c "pg_dump -U synapse synapse > /backups/database.dmp" + +echo 'Copying murmur database' +docker stop murmur_murmur_1 +cp /var/lib/murmur/murmur.sqlite /backups/murmur/murmur.sqlite +docker start murmur_murmur_1 + +echo 'Creating GitLab backup' +docker exec gitlab_gitlab_1 gitlab-rake gitlab:backup:create + +echo 'Starting Borg backup' +borg create -v --stats --compression lz4 \ + ${REPOSITORY}::'{hostname}-{now:%Y-%m-%d}' \ + /root \ + /etc \ + /var \ + /backups \ + --exclude '/var/lib/nextcloud/db' \ + --exclude '/var/lib/plex/transcode' \ + --exclude '/var/lib/prometheus' \ + --exclude '/var/lib/gitlab/data' + +# Route the normal process logging to journalctl +2>&1 + +# If there is an error backing up, reset password envvar and exit +if [ "$?" = "1" ] ; then + export BORG_PASSPHRASE="" + exit 1 +fi + +# Use the `prune` subcommand to maintain 14 daily, 8 weekly and 12 monthly +# archives of THIS machine. The '{hostname}-' prefix is very important to +# limit prune's operation to this machine's archives and not apply to +# other machine's archives also. +borg prune -v --list ${REPOSITORY} --prefix '{hostname}-' \ + --keep-daily=14 --keep-weekly=8 --keep-monthly=12 + +# Unset the password +export BORG_PASSPHRASE="" + +touch /backups/backup-ok + +exit 0 diff --git a/roles/scripts/templates/syncData.sh b/roles/scripts/templates/syncData.sh new file mode 100644 index 0000000..59ba183 --- /dev/null +++ b/roles/scripts/templates/syncData.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +SOURCE_HOST=62.210.202.162 + +#Sync Media +rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/plex/data/ /media --delete + +#Sync Backups +rsync -aAvh --progress root@${SOURCE_HOST}:/backups/ /backups --delete + +#Sync Deluge +mkdir -p {{docker_compose_files_folder}}/deluge +rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/torrent/config/ {{docker_compose_files_folder}}/deluge/config --delete +rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/deluge/ /var/lib/deluge --delete + +#Sync emby +mkdir -p {{docker_compose_files_folder}}/emby +rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/emby/config/ /etc/images/emby/config --delete + +#Sync Mailu +rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/mailu/ /var/lib/mailu --delete + +#Sync matrix +mkdir -p {{docker_compose_files_folder}}/matrix +mkdir -p /var/lib/matrix +rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/matrix/synapse/ {{docker_compose_files_folder}}/matrix/synapse --delete +rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/matrix/media_store/ /var/lib/matrix/media_store --delete + +#Sync nextcloud +rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/nextcloud/ /var/lib/nextcloud --exclude "db" --delete + +#Sync Wiki +rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/wiki/ /var/lib/wiki --delete + +#Sync certificates +mkdir -p {{docker_compose_files_folder}}/proxy/nginx +rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/proxy/nginx/certs/ {{docker_compose_files_folder}}/proxy/nginx/certs --delete \ No newline at end of file diff --git a/roles/scripts/templates/updateAll.sh b/roles/scripts/templates/updateAll.sh new file mode 100755 index 0000000..02af5cc --- /dev/null +++ b/roles/scripts/templates/updateAll.sh @@ -0,0 +1,16 @@ +#!/bin/bash + + +for dir in {{docker_compose_files_folder}}/* +do + if [ -d ${dir} ] + then + echo "Updating ${dir}" + cd "${dir}" + docker-compose pull + [ ${dir} = 'nextcloud' ] && docker-compose build --pull + docker-compose up -d + echo -------------------------------------------------------------- + fi +done; +