Adapt for Ubuntu and prepare for migration to Hetzner

playbook.yml

@@ -1,16 +1,18 @@
 ---
 - hosts: all
   become: true
-  gather_facts: no
   vars:
-    docker_compose_files_folder: /etc/images
+    docker_compose_files_folder_previous_server: /etc/images
+    docker_compose_files_folder: /etc/compose
     domain_name: banditlair.com
+    docker_version: 18.06.*
     sub_domains:
       - rpg
   roles:
     - { role: base, tags: ['base'] }
     - { role: scripts, tags: [ 'scripts' ] }
     - { role: daily-backup, tags: [ 'backup' ] }
+    - { role: docker, tags: [ 'docker' ] }
     - { role: murmur-docker, tags: [ 'murmur', 'docker' ] }
     - { role: searx-docker, tags: [ 'searx', 'docker' ] }
     - { role: wiki-docker, tags: [ 'wiki', 'docker' ] }
@@ -21,7 +23,6 @@
     - { role: matrix-docker, tags: [ 'matrix', 'docker' ] }
     - { role: torrent-docker, tags: [ 'torrent', 'docker' ] }
     - { role: monit, tags: [ 'monit' ] }
-    - { role: arch-mirror-docker, tags: [ 'mirror', 'docker' ] }
     - { role: stb-wordpress-docker, tags: [ 'stb', 'docker' ] }
     - { role: traefik-proxy-docker, tags: [ 'traefik', 'docker' ] }
 

roles/base-docker/README.md

@@ -1,12 +0,0 @@
-base-docker
-===========
-Installs and configures docker
-
-Role Variables
---------------
-- `docker_compose_files_folder` The path where all the compose projects folders will be stored
-- `docker_compose_persistence_folder` The path where all persistent data will be stored, defaults to `/var/lib`
-
-Dependencies
-------------
-- base

roles/base-docker/defaults/main.yml

@@ -1,3 +0,0 @@
----
-docker_compose_files_folder: /etc/compose
-docker_compose_persistence_folder: /var/lib

roles/base-docker/tasks/main.yml

@@ -1,13 +0,0 @@
----
-- name: Install docker packages
-  package: name={{item}} state=present update_cache=yes
-  with_items:
-    - docker
-    - docker-compose
-- name: Enable and start docker service
-  systemd:
-    name: docker.service
-    state: started
-    enabled: True
-- name: Create images config folder
-  file: dest={{docker_compose_files_folder}} state=directory

roles/base/files/ansible_prerequisites.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-set -e
-#pacman -Syu --noconfirm   #Skip this step because reboot is needed to start docker in case of kernel update
-pacman -S python --noconfirm
-touch /root/.ansible_prerequisites_installed

roles/base/tasks/main.yml

@@ -1,22 +1,10 @@
 ---
-- name: Install ansible prerequisites
-  script: ansible_prerequisites.sh creates=/root/.ansible_prerequisites_installed
 - name: Install base packages
   package: name={{item}} state=present update_cache=yes
   with_items:
     - htop
     - git
     - nload
-    - rsync
     - ufw
-    - vim
+    - borgbackup
-    - wget
-    - borg
-    - openbsd-netcat
-    - cronie
-- name: Enable and start cronie
-  service:
-    name: cronie
-    enabled: true
-    state: started
 

roles/daily-backup/templates/fullBackup.sh

@@ -29,7 +29,7 @@ borg create -v --stats --compression lz4           \
     ${REPOSITORY}::'{hostname}-{now:%Y-%m-%d}'     \
     /root                                          \
     /home                                          \
-    /media                                         \
+    /data                                          \
     /etc                                           \
     /var/lib/deluge                                \
     /var/lib/mailu                                 \

roles/docker/tasks/main.yml

@@ -88,6 +88,19 @@
     register: docker_version
     changed_when: no
 
+  - name: Install python3-pip
+    apt:
+      name: python3-pip
+      state: latest
+      cache_valid_time: 3600
+    register: result
+    retries: 3
+    until: result is success
+
+  - name: Install docker-compose
+    pip:
+      name: docker-compose
+
   - name: Printing Docker version
     debug: var=docker_version
 

roles/emby-docker/files/emby/docker-compose.yml

@@ -10,7 +10,7 @@ services:
     image: emby/embyserver:latest
     volumes:
         - ./config:/config
-        - /media:/media:ro
+        - /data:/media:ro
         - /etc/localtime:/etc/localtime:ro
     environment:
       - UID=33

roles/etcd/tasks/main.yml

@@ -1,19 +1,6 @@
 ---
 - name: etcd replicated and outiside of kubeadm when multimasters
   block:
-    - name: Install python3-pip
-      apt:
-        name: python3-pip
-        state: latest
-        cache_valid_time: 3600
-      register: result
-      retries: 3
-      until: result is success
-
-    - name: Install docker-compose
-      pip:
-        name: docker-compose
-
     - name: Running etcd container on masters nodes
       docker_container:
         name: etcd

roles/gitlab-docker/tasks/main.yml

@@ -20,9 +20,14 @@
   wait_for:
     path: /var/lib/gitlab/postgres-exporter/
     state: present
+    timeout: 600
   when: gitlab_users_repos.matched|int == 0
 - name: Restore backup if no users are found
   script: restore-backup.sh {{gitlab_git_uid.stdout}}
+  register: gitlab_backup_restore
   args:
     chdir: "{{docker_compose_files_folder}}/gitlab/"
+  retries: 5
+  delay: 30
+  until: gitlab_backup_restore.rc == 0
   when: gitlab_users_repos.matched|int == 0

roles/kubernetes/tasks/docker-images.yml

@@ -1,17 +1,4 @@
 ---
-- name: Install python3-pip
-  apt:
-    name: python3-pip
-    state: latest
-    cache_valid_time: 3600
-  register: result
-  retries: 3
-  until: result is success
-
-- name: Install docker-compose
-  pip:
-    name: docker-compose
-
 - name: Pull docker images
   docker_image: name="{{ item }}"
   with_items:

roles/monit/tasks/main.yml

@@ -14,6 +14,7 @@
 - name: Copy monit config
   template:
     src: monitrc
-    dest: /etc/monitrc
+    dest: /etc/monit/monitrc
+    mode: 0600
   notify:
       - reload monit

roles/monit/templates/monitrc

@@ -299,7 +299,7 @@ check filesystem root with path /
 ## Check a network link status (up/down), link capacity changes, saturation
 ## and bandwidth usage.
 #
-check network public with interface eno1
+check network public with interface enp4s0
     if failed link then alert
 #    if changed link then alert
 #    if saturation > 90% then alert

roles/murmur-docker/README.md

@@ -1,7 +0,0 @@
-murmur-docker
-=============
-Installs murmur
-
-Dependencies
-------------
-- base-docker

roles/murmur-docker/meta/main.yml

@@ -1,2 +0,0 @@
-dependencies:
-  - base-docker

roles/murmur-docker/tasks/main.yml

@@ -2,7 +2,7 @@
 - name: Copy murmur config
   copy: src=murmur dest={{docker_compose_files_folder}}
 - name: Create murmur data folder
-  file: dest={{docker_compose_persistence_folder}}/murmur state=directory
+  file: dest=/var/lib/murmur state=directory
 - name: Copy murmur database
   copy: src=/backups/murmur/murmur.sqlite dest=/var/lib/murmur/ force=no remote_src=yes
 - name: Start murmur docker project

roles/nextcloud-docker/files/nextcloud/docker-compose.yml

@@ -31,12 +31,10 @@ services:
     volumes:
       - /var/lib/nextcloud:/var/www/html
       - ./config:/var/www/html/config
-      - /media:/media
+      - /data:/media
       - /etc/localtime:/etc/localtime:ro
     environment:
       - MYSQL_HOST=db
-    env_file:
-      - db.env
     depends_on:
       - db
       - redis
@@ -54,8 +52,6 @@ services:
       - MYSQL_DATABASE=${MYSQL_DATABASE}
       - MYSQL_USER=${MYSQL_USER}
       - MYSQL_PASSWORD=${MYSQL_PASSWORD}
-    env_file:
-      - db.env
     restart: always
 
   redis:

roles/nextcloud-docker/tasks/main.yml

@@ -3,10 +3,10 @@
   copy:
     src: nextcloud
     dest: "{{docker_compose_files_folder}}"
-- name: Create db.env
+- name: Create .env
   template:
-    src: nextcloud/db.env
+    src: nextcloud/.env
-    dest: "{{docker_compose_files_folder}}/nextcloud/db.env"
+    dest: "{{docker_compose_files_folder}}/nextcloud/.env"
 - name: Create nextcloud config
   template:
     src: nextcloud/config/{{item}}
@@ -18,8 +18,8 @@
 - name: Change config folder owner to http
   file:
     path: "{{docker_compose_files_folder}}/nextcloud/config"
-    owner: http
+    owner: 33
-    group: http
+    group: 33
     recurse: yes
 - name: Build and start nextcloud docker project
   docker_service:
@@ -31,8 +31,10 @@
   args:
       chdir: "{{docker_compose_files_folder}}/nextcloud/"
   register: db_tables_exist
-  ignore_errors: true
+  retries: 15
-  changed_when: db_tables_exist.stdout_lines|length == 0
+  delay: 10
+  until: db_tables_exist.rc == 0
+  changed_when: no
 - name: Restore Nextcloud database
   command: docker-compose exec -T db sh -c "mysql -u nextcloud -p{{nextcloud_mysql_password}} nextcloud < /backups/database.dmp"
   args:

roles/nextcloud-docker/templates/nextcloud/db.env

@@ -1,4 +0,0 @@
-MYSQL_PASSWORD={{nextcloud_mysql_password}}
-MYSQL_DATABASE=nextcloud
-MYSQL_USER=nextcloud
-

roles/scripts/files/proxyFirewall.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Clear config
+iptables -t nat -F
+iptables -t mangle -F
+iptables -F
+iptables -X
+
+
+echo 1 > /proc/sys/net/ipv4/ip_forward
+
+PORTS_TO_FORWARD_TCP="25 80 110 143 443 465 587 993 995 2224 3478 8008 8448 27015 64738"
+PORTS_TO_FORWARD_UDP="34197 64738"
+DESTINATION_IP="212.83.165.111"
+#DESTINATION_IP="5.9.66.49"
+
+for port in `echo $PORTS_TO_FORWARD_TCP`
+do
+	iptables -t nat -A PREROUTING -p tcp -m tcp --dport ${port} -j DNAT --to-destination ${DESTINATION_IP}
+	iptables -A FORWARD -d ${DESTINATION_IP}/32 -p tcp -m tcp --dport ${port} -j ACCEPT
+done
+
+for port in `echo $PORTS_TO_FORWARD_UDP`
+do
+	iptables -t nat -A PREROUTING -p udp -m udp --dport ${port} -j DNAT --to-destination ${DESTINATION_IP}
+	iptables -A FORWARD -d ${DESTINATION_IP}/32 -p tcp -m tcp --dport ${port} -j ACCEPT
+done
+iptables -t nat -A POSTROUTING -j MASQUERADE

roles/scripts/templates/syncData.sh

@@ -10,14 +10,14 @@ rsync -aAvh --progress root@${SOURCE_HOST}:/media/ /data --delete
 #Sync Backups
 rsync -aAvh --progress root@${SOURCE_HOST}:/backups/ /backups --delete
 
-#Sync Deluge
+#Sync Torrents
-mkdir -p {{docker_compose_files_folder}}/deluge
+mkdir -p {{docker_compose_files_folder}}/torrent
-rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/torrent/config/ {{docker_compose_files_folder}}/deluge/config --delete
+rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder_previous_server}}/torrent/config/ {{docker_compose_files_folder}}/torrent/config --delete
 rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/deluge/ /var/lib/deluge --delete
 
 #Sync emby
 mkdir -p {{docker_compose_files_folder}}/emby
-rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/emby/config/ {{docker_compose_files_folder}}/emby/config --delete
+rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder_previous_server}}/emby/config/ {{docker_compose_files_folder}}/emby/config --exclude "transcoding-temp"  --delete
 
 #Sync Mailu
 rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/mailu/ /var/lib/mailu --delete
@@ -25,16 +25,25 @@ rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/mailu/ /var/lib/mailu --dele
 #Sync matrix
 mkdir -p {{docker_compose_files_folder}}/matrix
 mkdir -p /var/lib/matrix
-rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/matrix/synapse/ {{docker_compose_files_folder}}/matrix/synapse --delete
+rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder_previous_server}}/matrix/synapse/ {{docker_compose_files_folder}}/matrix/synapse --delete
 rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/matrix/media_store/ /var/lib/matrix/media_store --delete
 
 #Sync nextcloud
-rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/nextcloud/ /var/lib/nextcloud --exclude "db" --delete
+mkdir -p {{docker_compose_files_folder}}/nextcloud/config
+rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder_previous_server}}/nextcloud/config/ {{docker_compose_files_folder}}/nextcloud/config --delete
+rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/nextcloud/ /var/lib/nextcloud --delete
 
 #Sync Wiki
 rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/wiki/ /var/lib/wiki --delete
 
 #Sync certificates
 mkdir -p {{docker_compose_files_folder}}/traefik/certs/
-rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder}}/traefik/certs/ {{docker_compose_files_folder}}/traefik/certs --delete
+rsync -aAvh --progress root@${SOURCE_HOST}:{{docker_compose_files_folder_previous_server}}/traefik/certs/ {{docker_compose_files_folder}}/traefik/certs --delete
 
+#Sync factorio
+mkdir -p /opt/factorio
+rsync -aAvh --progress root@${SOURCE_HOST}:/opt/factorio/ /opt/factorio --delete
+
+#Sync STB wordpress
+mkdir -p /var/lib/stb
+rsync -aAvh --progress root@${SOURCE_HOST}:/var/lib/stb/ /var/lib/stb --delete

roles/stb-wordpress-docker/files/docker-compose.yml

@@ -17,8 +17,6 @@ services:
       - MYSQL_DATABASE=${MYSQL_DATABASE}
       - MYSQL_USER=${MYSQL_USER}
       - MYSQL_PASSWORD=${MYSQL_PASSWORD}
-    env_file:
-      - db.env
     restart: always
   wordpress:
     image: wordpress:4.9.4-php7.1-apache

roles/stb-wordpress-docker/tasks/main.yml

@@ -7,10 +7,10 @@
   copy:
     src: docker-compose.yml
     dest: "{{docker_compose_files_folder}}/stb/"
-- name: Create db.env
+- name: Create .env
   template:
-    src: db.env
+    src: .env
-    dest: "{{docker_compose_files_folder}}/stb/db.env"
+    dest: "{{docker_compose_files_folder}}/stb/.env"
 - name: Pull and start docker project
   docker_service:
     project_src: "{{docker_compose_files_folder}}/stb"
@@ -20,8 +20,10 @@
   args:
       chdir: "{{docker_compose_files_folder}}/stb/"
   register: db_tables_exist
-  ignore_errors: true
+  retries: 15
-  changed_when: db_tables_exist.stdout_lines|length == 0
+  delay: 10
+  until: db_tables_exist.rc == 0
+  changed_when: no
 - name: Restore STB database
   command: docker-compose exec -T db sh -c "mysql -u stb -p{{stb_mysql_password}} stb < /backups/database.dmp"
   args:

roles/torrent-docker/files/torrent/docker-compose.yml

@@ -16,7 +16,7 @@ services:
     volumes:
       - /var/lib/deluge:/data
       - ./config/deluge:/config
-      - /media:/media
+      - /data:/media
       - /etc/localtime:/etc/localtime:ro
     environment:
       - VPN_ENABLED=yes
@@ -60,7 +60,7 @@ services:
       - /var/lib/deluge/completed:/downloads
       - /var/lib/nzbget/downloads:/nzbget
       - ./config/sonarr:/config
-      - /media/TV:/tv
+      - /data/TV:/tv
       - /etc/localtime:/etc/localtime:ro
     restart: always
     networks:
@@ -84,7 +84,7 @@ services:
       - /var/lib/deluge/completed:/downloads
       - /var/lib/nzbget/downloads:/nzbget
       - ./config/radarr:/config
-      - /media/Movies:/movies
+      - /data/Movies:/movies
       - /etc/localtime:/etc/localtime:ro
     restart: always
     networks:
@@ -107,7 +107,7 @@ services:
     volumes:
       - /var/lib/deluge/completed:/downloads
       - ./config/headphones:/config
-      - /media/Music:/music
+      - /data/Music:/music
       - /etc/localtime:/etc/localtime:ro
     restart: always
     networks: