From c311cd4f7e9e587b7aa80b16a201504f864bc3d5 Mon Sep 17 00:00:00 2001 From: Paul-Henri Froidmont Date: Thu, 22 Aug 2019 05:11:27 +0200 Subject: [PATCH] Migrate to Hetzner cloud --- .gitignore | 1 + ansible-playbook.sh | 9 ++ ansible.cfg | 4 +- ansible.sh | 8 -- get_hcloud_token.sh | 14 ++ group_vars/all/vault | 133 ++++++++---------- group_vars/k8s-cluster.yml | 13 +- group_vars/kube-master.yml | 2 +- group_vars/kube-node.yml | 2 +- inventories/test/group_vars/k8s-cluster.yml | 4 +- inventories/test/groups | 11 +- inventories/test/masters_hcloud.yml | 6 + inventories/test/nodes_hcloud.yml | 6 + inventories/test/scaleway_inventory.yml | 12 -- k8s.yml | 33 +---- kubespray | 2 +- roles/keepalived-hcloud/files/check_nginx.sh | 3 + .../files/hcloud_failover.py | 60 ++++++++ roles/keepalived-hcloud/handlers/main.yml | 4 + roles/keepalived-hcloud/tasks/main.yml | 34 +++++ .../templates/config.json.j2 | 13 ++ .../templates/keepalived.conf.j2 | 41 ++++++ set_environment.sh | 12 ++ setup_virtualenv.sh | 5 + terraform.sh | 12 +- terraform/config.tf | 8 +- terraform/instances.tf | 58 +++++--- terraform/lb.tf | 22 --- terraform/network.tf | 22 +++ terraform/outputs.tf | 9 +- terraform/scripts/create_lb.sh | 16 --- terraform/scripts/delete_lb.sh | 17 --- terraform/scripts/update_lb_rules.sh | 48 ------- terraform/security.tf | 39 +++++ terraform/ssh_keys/desktop.pub | 1 + terraform/variables.tf | 27 ++-- terraform/versions.tf | 4 - 37 files changed, 416 insertions(+), 299 deletions(-) create mode 100755 ansible-playbook.sh delete mode 100755 ansible.sh create mode 100755 get_hcloud_token.sh create mode 100644 inventories/test/masters_hcloud.yml create mode 100644 inventories/test/nodes_hcloud.yml delete mode 100644 inventories/test/scaleway_inventory.yml create mode 100644 roles/keepalived-hcloud/files/check_nginx.sh create mode 100644 roles/keepalived-hcloud/files/hcloud_failover.py create mode 100644 roles/keepalived-hcloud/handlers/main.yml create mode 100644 roles/keepalived-hcloud/tasks/main.yml create mode 100644 roles/keepalived-hcloud/templates/config.json.j2 create mode 100644 roles/keepalived-hcloud/templates/keepalived.conf.j2 create mode 100755 set_environment.sh create mode 100755 setup_virtualenv.sh delete mode 100644 terraform/lb.tf create mode 100644 terraform/network.tf delete mode 100755 terraform/scripts/create_lb.sh delete mode 100755 terraform/scripts/delete_lb.sh delete mode 100755 terraform/scripts/update_lb_rules.sh create mode 100644 terraform/security.tf create mode 100644 terraform/ssh_keys/desktop.pub delete mode 100644 terraform/versions.tf diff --git a/.gitignore b/.gitignore index 86d417c..33db01a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ .terraform* terraform.tfstate terraform.tfstate.backup +.environment \ No newline at end of file diff --git a/ansible-playbook.sh b/ansible-playbook.sh new file mode 100755 index 0000000..f651fbf --- /dev/null +++ b/ansible-playbook.sh @@ -0,0 +1,9 @@ +#! /bin/bash + +set -e + +export HCLOUD_TOKEN=$(./get_hcloud_token.sh) +ENVIRONMENT=$(cat .environment) +source .virtualenv/bin/activate + +ansible-playbook -i inventories/$ENVIRONMENT --vault-id=~/.ssh/vault-pass "$@" diff --git a/ansible.cfg b/ansible.cfg index ec2aed7..c3bcdf8 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -4,6 +4,8 @@ deprecation_warnings = True display_skipped_hosts = False host_key_checking = False nocows = 1 +stdout_callback=skippy +callback_whitelist=profile_tasks remote_user = root retry_files_enabled = False library = kubespray/library/ @@ -16,4 +18,4 @@ pipelining = True ssh_args = -C -o ControlMaster=auto -o ControlPersist=5m -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null [inventory] -enable_plugins = host_list, scaleway, ini, script, yaml +enable_plugins = hcloud, ini, script, yaml diff --git a/ansible.sh b/ansible.sh deleted file mode 100755 index afd7102..0000000 --- a/ansible.sh +++ /dev/null @@ -1,8 +0,0 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p ansible jq - -set -e - -export SCW_TOKEN=`jq '.token' -r ~/.scwrc` - -ansible-playbook "$@" diff --git a/get_hcloud_token.sh b/get_hcloud_token.sh new file mode 100755 index 0000000..9ab6054 --- /dev/null +++ b/get_hcloud_token.sh @@ -0,0 +1,14 @@ +#! /bin/bash + +set -e + +ENVIRONMENT=$(cat .environment) +HCLOUD_TOKEN=$(cat ~/.ssh/hcloud-$ENVIRONMENT-token) + +if [ -z "$HCLOUD_TOKEN" ] +then + echo "Couldn't find your hetzner cloud token in '~/.ssh/hcloud-$ENVIRONMENT-token'" + exit 1 +fi + +echo $HCLOUD_TOKEN \ No newline at end of file diff --git a/group_vars/all/vault b/group_vars/all/vault index c366250..1f201af 100644 --- a/group_vars/all/vault +++ b/group_vars/all/vault @@ -1,74 +1,61 @@ $ANSIBLE_VAULT;1.1;AES256 -33636335313832373461323863353761386564383734663232383265663165613238636361316437 -3761356337366536636330383365633865373233663531330a623935333463343532363234323536 -62383461343065666237366233386631616630393561383432393330316266353631313833633861 -3233623337343536370a313536643734323862396235666236393738666430373461313737653538 -39366165653131623836343834656137306234663230313665323161363932316663636135643361 -34373866613535306333306562343536376262643436303234383031323236633733333765396130 -36316666373936326663643763386435366235623664626463663838346133336134313363663738 -36623462303065653734653937313032323532626434333136366339623962616664356330646564 -35626339633637353863623737373063636535653964323139626561363036653535333338303836 -65373837306436346234366334343336343733343735383266656131343331623464306430623333 -36356163363133363030663837656133323134663739646236656166356630363061623162376137 -37353231346463636337393539386664343763633035356266336638366564386139306338353564 -66353436663439653061306338343437626332653562643137636165396461616133616164346537 -64303665646435613562376234623036363434636631663864626335646161316430366233623932 -38373931393932313337343264306361666230643636643864636331313562353337663066326532 -30356137653961316635633165623339613861306634323162383333306233343965363164323634 -38376266333633656461663430616261353133346538356532646535616438333233373061626263 -37346137623539343239643261616634363661333130383738636531313135626566393830643163 -61666238653334626661376462313534393834393363613434333863326134393039666432313064 -61333031353364646633373538643366396566303932633666656331333239643438653637656430 -39316430393435366638396662313032643939646533373036636434656438363963613162656236 -38616637323931303437353532366131653237326131663234663961376565356564643561636631 -35363334643835363538306138323430303434613562366164336234313236616162373738303731 -32663732306232373462636638306565383536383136373831376661373964306262646534306431 -31663965616436313464636437353632323763646438393831333937393965303032633465616137 -32313739623464343263386565623666333366626363656538323266653630383431653035366663 -37623663616661306538363261633230393733306261656236356266373933393265323565303562 -66643363613063653131326562373639616633653537303361396238363033343238653163626537 -64623236303337623934366137353235366439633435343362616338316433326539323766623633 -32666665336163633130653462376338653862633765366237313237616666383837323363363662 -65653962643966366137386135373638623330643434663366386632356636323465383062616138 -30663638326430316565306332383632366363663739663831353738363134303635333762623264 -32646435376663646430373239336661383533383630656466343933343631633634613033353435 -35663164343761396637613364363934663136613336623836373231366165373131373861353339 -36613563666237613335346631636666646165336133636633396233613330346438393338396431 -65366135636137623933326664393338653032353166636165393664393365646366323331303337 -39313438656235626139613061353238396330306432383230326239346664323437326130386636 -35313762303863373736326238623064346232396464633766306666613739303061633664653766 -65323739343930623035386439643835383336333638383265323465656339356666643461363233 -62346161396435343266613536316666353838303237356534323330303162343761633139376630 -39386261386238323565633563386332313735613932353262643262623834313066306433326633 -30383063303461366338396161396264656136623932613131333239303938346263363738626538 -66303365333034373838326462343436616430643831303533303631656537336338633937626666 -66323535383638383638663533376139633135323762666632636439393165343037616330333735 -63323461656536353634653863656635393836323565376236666232623461383639336537363032 -65333165393832366262616636306631343566333864656261323330386561396561356134303236 -61303765343437623939303131393639373936383932643062333431383031336666383136316432 -30393539303132396535303631613032666133333638633638353063663964643863343764616339 -65306464666238396663613038363039353131633363323736616134613536383337396561366131 -32336636303134383632343763613131313838656230396266313763333835363730386631363936 -30393635646438643565646637633131383033306638363062636437646137663734616263346132 -31646234383436393562646661363264643964666162393066323965626336366337343462666438 -63383834306432393630396637386237383863303431663139393566636539373339326264653062 -63313831383663356364383163343739343865386163663562623038303433333139346565353838 -36353765373838366131363238383633356431383964643933663433373861336436333134393261 -62383136306538323633323636366632316234303432623830376634646431663634303135316563 -61303836313033633665306634353738326634313834306133313534313161393961623166353761 -37306230653264383131346264633137613731343830633162616134643866363736666333326635 -30373162653463363336663430343635343030613761363530383864356230656532633639336266 -38653063313164613132373138373165623437363763356466303863643136663263376436336130 -37306239353234363739613039346231393831653435373739616266363834343938663233333639 -65633966656162303965643737353934363330326232356431353636323133333337656130303661 -66666335313339353133303135396466636132396466353265386538653534313665356461303539 -32353066323230613931323134346439613064653734343661643866396162653031623662623035 -34333533663866356565373936386535353962653630633838326630623038323837666662393831 -31306530373638333030623337626665653434656534363861666464323739333464336565643339 -30326131336631626537356132646337666236616462393339353033343536623830666333373762 -33386432623537363738393331313539363461393535636237373864346662666534663732303839 -63656532393465356465636230613463373662613065396534323139363164343332376637373539 -39386639663233343866376534383831653733633662613962653330376239323163333761333531 -31643662343562613933353336313637356563393037343936393365376263333062356137386630 -62623135306663666161633734326337616338316130366237336136326363393165383539303863 -35663434333961646134356165343231626532303035393237663764363739646463 +37623833363930356539333739623333646134393137613530646463353538396639623263336534 +3435653535353337376633386161323065383064313436350a363932663165646562653539353836 +31363830643663313035376238663764653836306265333564366436313031393561393164666262 +6535623234353339310a633739633863646363653938343733313032316231333063636630323038 +39336130656534646433316531666563616361383133333938663966373630656165363832663732 +38373533393365643233373636353663623937373062663530646238326462663339393961313633 +35663733353936303962336262356663366461653236333731636233383736376161376463623639 +38373162363437313138313530656538386666633034633434666639393134383630613238613061 +32626162626165313262623732373939313166666435326436316335303937323265343136303834 +62313531353830343766396537366662386561643034316130343738653163386331623838626365 +61393866643463323364633664333461666266396135356130633861333034616561613934666661 +35366161643963646165343031306161653330373933393736316261656437373232396438396565 +30313633303065646231393532386530333231313665306534343234383830313932623162393664 +66643665653466613265306365613133653731333262653039303437323565326265323064326536 +35356239356533323235663232346138356639336165366265353631316263646564346463383831 +61313464396166333762326263363865653436643435653833343536616632386265363266316663 +34643832326564313366613237396138396131393465346439333032373833326339326630656333 +65613538616337386636623965306562323163623265306339353061376466326336653031363437 +35326161323233616363643037316564306164613761343765323566313836303934353436303863 +35356365346139653032376432356133393464626263626339666361333863323635383562323963 +30366532623639353735333762363565373834643335303630303664666663373830326336663933 +61366238353238633437363634333562343039653332333034356463636363346437313833626238 +35623138666464396531323731363832663035336533386332313832663563343863343331653833 +66316134303839346536363431373362316530633739646238633232343334343066396636613537 +61623935616262316164313639643162373035616365646537633631383830376563666136326264 +32313666373635633339393734636266643536653430336364643832633530383966383361663666 +35393135373464643738306463616633363539623834323035626262356634316636616332643964 +33633136383231353238383661363932623537313937633935613865303161663530353561623035 +37323334323339303631356338306164333864633033666336626266386235313431663130633163 +37663261653939633236633165363233353535353939343262316338633631363433656533356436 +36623266393539383732643134383038653636323661653430326132656435373161356639666637 +64376534346662613930333932633737333963336533316135383438353264303136383931393633 +30343037323761633764303461356334313635373937613438386466303934366530356163636163 +39323535626332653736326338303232396532613533376331616264646164633239666663613961 +62343530626431396234613565376531353237663833626532633336616561383438626234386264 +61613333653764663130333132653438633738613566373164653562633066623939653965366231 +65356634613931643234356434366562306337353331643030313336653162646163393933326433 +34383331333630646432383635636430333232633034303362306331376532613363656462316662 +33386130303230613536323832646534336562663733336535643731376663666530663930343430 +39326661653766386365376562316135313766306333313439376562613938363361363936663032 +34393231633536383231376362653336336637393235343039626335323331383935366164376461 +32633762303837343363643736623631376138323933326162356463313665303830363062373562 +61343731633139636638626437313933663736656162663130336563393464316161343866353764 +34383239373264323838663731643737666334383333336561323935353465363135616535336636 +30653139663866376535616433333266303437656430366132666330376263656664616638323236 +64323464656339306339373362656134366335353831316565643539363436343931656239616335 +61626264646133336137333262356630396138626133376433613031386363623161373334666539 +34383234613337306262626639353063366239646235396233373231303134353130343462646365 +66326433346137626232613565376630623361393163636632303734323931623630386439333363 +33326133326263616464313164306236656433643864343661306436636337303962646236656131 +62393734353663613431386263636536336234343932646266663435323063323566353539313562 +39373765373539303133663866643263383266326364333863326263666166353237353734376566 +30393636316632636538656466353237343466346334363864323439303265653833363366633034 +39663764353138633039316263316531623430636634643538363066623737346363646564613632 +66306461336535303365383362633933623332643534343437653234613036363264376466653465 +32343636373737653463653534666433373561656438363038373933633630633966363533663537 +30663061396263303334366363363538643135663731656434326238353165383030653932613432 +38613764336632316563623938333430333262333364363962666337623636346664626332653039 +62663363313062323137386638653762663437346164643932646364303361623961643261643262 +38616563643732613739 diff --git a/group_vars/k8s-cluster.yml b/group_vars/k8s-cluster.yml index f1f11b9..87fbc36 100644 --- a/group_vars/k8s-cluster.yml +++ b/group_vars/k8s-cluster.yml @@ -1,13 +1,16 @@ --- -ip: "{{vpn_ip}}" -kube_network_plugin: flannel +kube_network_plugin: weave bin_dir: /usr/local/bin -kube_config_dir: "/etc/kubernetes" +kube_config_dir: /etc/kubernetes upstream_dns_servers: - - 195.20.55.180 + - 213.133.98.98 + - 213.133.99.99 -#Addons kube_api_anonymous_auth: true cert_manager_enabled: true +ingress_nginx_enabled: true +dashboard_enabled: false +helm_enabled: false metrics_server_enabled: true +enable_nodelocaldns: true diff --git a/group_vars/kube-master.yml b/group_vars/kube-master.yml index 787ffe9..bddfe92 100644 --- a/group_vars/kube-master.yml +++ b/group_vars/kube-master.yml @@ -1,2 +1,2 @@ --- -vpn_ip: 192.168.66.{{ 0 +(inventory_hostname|regex_replace('\D+','')|int) }} +ip: 192.168.1.{{ 0 +(inventory_hostname|regex_replace('\D+','')|int) }} diff --git a/group_vars/kube-node.yml b/group_vars/kube-node.yml index 4e2fc3d..be8c5a6 100644 --- a/group_vars/kube-node.yml +++ b/group_vars/kube-node.yml @@ -1,2 +1,2 @@ --- -vpn_ip: 192.168.66.{{ 100 +( inventory_hostname|regex_replace('\D+','')|int) }} \ No newline at end of file +ip: 192.168.2.{{ 0 +(inventory_hostname|regex_replace('\D+','')|int) }} \ No newline at end of file diff --git a/inventories/test/group_vars/k8s-cluster.yml b/inventories/test/group_vars/k8s-cluster.yml index a6d95f8..d905895 100644 --- a/inventories/test/group_vars/k8s-cluster.yml +++ b/inventories/test/group_vars/k8s-cluster.yml @@ -1,3 +1,5 @@ --- cert_manager_issuer: letsencrypt-staging -banditlair_domain: k8s.banditlair.com \ No newline at end of file +banditlair_domain: k8s.banditlair.com +floating_ip: 116.203.8.164 +floating_ip_id: 91174 diff --git a/inventories/test/groups b/inventories/test/groups index 547aa4c..edd2fbf 100644 --- a/inventories/test/groups +++ b/inventories/test/groups @@ -1,15 +1,14 @@ -[test-master] -[test-etcd] -[test-node] +[master] +[node] [kube-master:children] -test-master +master [etcd:children] -test-etcd +master [kube-node:children] -test-node +node [k8s-cluster:children] kube-master diff --git a/inventories/test/masters_hcloud.yml b/inventories/test/masters_hcloud.yml new file mode 100644 index 0000000..f49f7b7 --- /dev/null +++ b/inventories/test/masters_hcloud.yml @@ -0,0 +1,6 @@ +plugin: hcloud + +label_selector: type=master + +groups: + master: yes diff --git a/inventories/test/nodes_hcloud.yml b/inventories/test/nodes_hcloud.yml new file mode 100644 index 0000000..2062b40 --- /dev/null +++ b/inventories/test/nodes_hcloud.yml @@ -0,0 +1,6 @@ +plugin: hcloud + +label_selector: type=node + +groups: + node: yes diff --git a/inventories/test/scaleway_inventory.yml b/inventories/test/scaleway_inventory.yml deleted file mode 100644 index 57c211f..0000000 --- a/inventories/test/scaleway_inventory.yml +++ /dev/null @@ -1,12 +0,0 @@ -plugin: scaleway -hostnames: - - hostname -regions: - - par1 - - ams1 -tags: - - test-master - - test-etcd - - test-node -variables: - ansible_host: public_ip.address diff --git a/k8s.yml b/k8s.yml index b01ca11..21aebd0 100644 --- a/k8s.yml +++ b/k8s.yml @@ -1,33 +1,8 @@ ---- -- hosts: k8s-cluster +- hosts: kube-node roles: - - role: tinc - tags: tinc + - role: keepalived-hcloud + tags: keepalived - name: Include kubespray tasks import_playbook: kubespray.yml - - -# - hosts: k8s_proxy:k8s_masters:k8s_workers -# roles: -# - role: proxy -# tags: proxy -# - role: docker -# tags: docker -# - hosts: k8s_masters -# gather_facts: false -# roles: -# - role: etcd -# tags: etcd -# - hosts: k8s_proxy:k8s_masters:k8s_workers -# gather_facts: false -# roles: -# - role: kubernetes -# tags: kubernetes -# - hosts: k8s_masters:k8s_proxy -# gather_facts: false -# roles: -# - role: ingress -# tags: ingress -# - role: kubernetes-dashboard -# tags: dashboard + \ No newline at end of file diff --git a/kubespray b/kubespray index 7d8da83..86cc703 160000 --- a/kubespray +++ b/kubespray @@ -1 +1 @@ -Subproject commit 7d8da8348e095a5f0b160c1e05c4c399d201d1f0 +Subproject commit 86cc703c75768207e1943ddf8f6a8082d756cb83 diff --git a/roles/keepalived-hcloud/files/check_nginx.sh b/roles/keepalived-hcloud/files/check_nginx.sh new file mode 100644 index 0000000..756ff33 --- /dev/null +++ b/roles/keepalived-hcloud/files/check_nginx.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +curl 127.0.0.1/healthz -fsS diff --git a/roles/keepalived-hcloud/files/hcloud_failover.py b/roles/keepalived-hcloud/files/hcloud_failover.py new file mode 100644 index 0000000..a3952db --- /dev/null +++ b/roles/keepalived-hcloud/files/hcloud_failover.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +# (c) 2018 Maximilian Siegl + +import sys +import json +import os +import requests +from multiprocessing import Process + +CONFIG_PATH = os.path.join(os.path.abspath( + os.path.dirname(__file__)), "config.json") + + +def del_ip(ip_bin_path, floating_ip, interface): + os.system(ip_bin_path + " addr del " + floating_ip + " dev " + interface) + + +def add_ip(ip_bin_path, floating_ip, interface): + os.system(ip_bin_path + " addr add " + floating_ip + " dev " + interface) + + +def change_request(endstate, url, header, payload, ip_bin_path, floating_ip, interface): + if endstate == "BACKUP": + del_ip(ip_bin_path, floating_ip, interface) + elif endstate == "FAULT": + del_ip(ip_bin_path, floating_ip, interface) + elif endstate == "MASTER": + add_ip(ip_bin_path, floating_ip, interface) + print("Post request to: " + url) + print("Header: " + str(header)) + print("Data: " + str(payload)) + r = requests.post(url, data=payload, headers=header) + print("Response:") + print(r.status_code, r.reason) + print(r.text) + else: + print("Error: Endstate not defined!") + + +def main(arg_type, arg_name, arg_endstate): + with open(CONFIG_PATH, "r") as config_file: + config = json.load(config_file) + + header = { + "Content-Type": "application/json", + "Authorization": "Bearer " + config["api-token"] + } + + payload = '''{"server": ''' + str(config["server-id"]) + "}" + + print("Perform action for transition to " + arg_endstate + " state") + + for ips in config["ips"]: + url = config["url"].format(ips["floating-ip-id"]) + Process(target=change_request, args=(arg_endstate, url, header, payload, + config["ip_bin_path"], ips["floating-ip"], config["interface"])).start() + + +if __name__ == "__main__": + main(arg_type=sys.argv[1], arg_name=sys.argv[2], arg_endstate=sys.argv[3]) diff --git a/roles/keepalived-hcloud/handlers/main.yml b/roles/keepalived-hcloud/handlers/main.yml new file mode 100644 index 0000000..904ae8c --- /dev/null +++ b/roles/keepalived-hcloud/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart keepalived + systemd: + name: keepalived + state: restarted diff --git a/roles/keepalived-hcloud/tasks/main.yml b/roles/keepalived-hcloud/tasks/main.yml new file mode 100644 index 0000000..de40413 --- /dev/null +++ b/roles/keepalived-hcloud/tasks/main.yml @@ -0,0 +1,34 @@ +- name: Install keepalived + package: + name: keepalived + state: present + +- name: Keepalived config + template: + src: keepalived.conf.j2 + dest: /etc/keepalived/keepalived.conf + notify: restart keepalived + +- name: Copy nginx healtcheck script + copy: + src: check_nginx.sh + dest: /etc/keepalived/check_nginx.sh + mode: 0700 + +- name: Copy hcloud failover script + copy: + src: hcloud_failover.py + dest: /etc/keepalived/hcloud_failover.py + mode: 0700 + +- name: Copy hcloud failover script config + template: + src: config.json.j2 + dest: /etc/keepalived/config.json + mode: 0700 + +- name: Start and enable keepalived + systemd: + name: keepalived + enabled: yes + state: started diff --git a/roles/keepalived-hcloud/templates/config.json.j2 b/roles/keepalived-hcloud/templates/config.json.j2 new file mode 100644 index 0000000..9415860 --- /dev/null +++ b/roles/keepalived-hcloud/templates/config.json.j2 @@ -0,0 +1,13 @@ +{ + "url": "https://api.hetzner.cloud/v1/floating_ips/{}/actions/assign", + "api-token": "{{ hcloud_token_vip }}", + "ips": [ + { + "floating-ip-id": "{{ floating_ip_id }}", + "floating-ip": "{{ floating_ip }}" + } + ], + "server-id": {{ hostvars[inventory_hostname]['id'] }}, + "interface": "eth0", + "ip_bin_path": "/bin/ip" +} \ No newline at end of file diff --git a/roles/keepalived-hcloud/templates/keepalived.conf.j2 b/roles/keepalived-hcloud/templates/keepalived.conf.j2 new file mode 100644 index 0000000..e7b2ffe --- /dev/null +++ b/roles/keepalived-hcloud/templates/keepalived.conf.j2 @@ -0,0 +1,41 @@ +vrrp_script check_nginx { + script /etc/keepalived/check_nginx.sh + interval 3 + fall 5 + rise 1 +} + +vrrp_instance VI_1 { +{% if inventory_hostname == groups['kube-node'][0] %} + state MASTER +{% else %} + state BACKUP +{% endif %} + priority 100 + interface eth0 + virtual_router_id 50 + + unicast_src_ip {{ hostvars[inventory_hostname]['ipv4'] }} + unicast_peer { +{% for host in (groups['kube-node']) %} +{% if host != inventory_hostname %} + {{ hostvars[host]['ipv4'] }} +{% endif %} +{% endfor %} + } + + authentication { + auth_type PASS + auth_pass "{{ keepalived_shared_secret }}" + } + + virtual_ipaddress { + {{ floating_ip }} + } + + track_script { + chk_haproxy + } + + notify /etc/keepalived/hcloud_failover.py +} diff --git a/set_environment.sh b/set_environment.sh new file mode 100755 index 0000000..991cafd --- /dev/null +++ b/set_environment.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +if [ -z "$1" ] +then + echo 'You must specify an environment' + exit 1 +fi + +echo "$1" > .environment +./setup_virtualenv.sh diff --git a/setup_virtualenv.sh b/setup_virtualenv.sh new file mode 100755 index 0000000..3684c52 --- /dev/null +++ b/setup_virtualenv.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +virtualenv .virtualenv +. .virtualenv/bin/activate +pip install ansible hcloud netaddr diff --git a/terraform.sh b/terraform.sh index f6dbf03..cc71c08 100755 --- a/terraform.sh +++ b/terraform.sh @@ -1,10 +1,12 @@ -#! /usr/bin/env nix-shell -#! nix-shell -i bash -p terraform jq +#! /bin/bash set -e -export AWS_ACCESS_KEY_ID=`cat ~/.ssh/scw_key_id` -export AWS_SECRET_ACCESS_KEY=`jq '.token' -r ~/.scwrc` +export HCLOUD_TOKEN=$(./get_hcloud_token.sh) +ENVIRONMENT=$(cat .environment) -terraform "$@" terraform +cd terraform + +terraform workspace select $ENVIRONMENT +terraform "$@" diff --git a/terraform/config.tf b/terraform/config.tf index 342042a..d78845d 100644 --- a/terraform/config.tf +++ b/terraform/config.tf @@ -14,6 +14,10 @@ terraform { } } -provider "scaleway" { - region = var.region +provider "hcloud" { +} + +resource "hcloud_ssh_key" "desktop" { + name = "Desktop" + public_key = "${file("ssh_keys/desktop.pub")}" } diff --git a/terraform/instances.tf b/terraform/instances.tf index 531754e..f5042df 100644 --- a/terraform/instances.tf +++ b/terraform/instances.tf @@ -1,27 +1,41 @@ -data "scaleway_image" "ubuntu" { - architecture = var.architecture - name = var.image +resource "hcloud_server" "node" { + count = var.node_server_count + name = "node${count.index + 1}" + image = "ubuntu-18.04" + server_type = var.node_server_type + ssh_keys = [hcloud_ssh_key.desktop.id] + keep_disk = true + + labels = { + environment = local.environment + type = "node" + } } -resource "scaleway_server" "node" { - count = var.node_instance_count - name = "node${count.index + 1}" - image = data.scaleway_image.ubuntu.id - type = var.node_instance_type - state = "running" - dynamic_ip_required = true - tags = ["${local.environment}-node"] +resource "hcloud_server_network" "node_network" { + count = var.node_server_count + server_id = "${hcloud_server.node[count.index].id}" + network_id = "${hcloud_network.private_network.id}" + ip = "192.168.2.${count.index + 1}" } -resource "scaleway_server" "master" { - count = var.master_instance_count - name = "master${count.index + 1}" - image = data.scaleway_image.ubuntu.id - type = var.master_instance_type - state = "running" - dynamic_ip_required = true - tags = [ - "${local.environment}-master", - "${local.environment}-etcd", - ] +resource "hcloud_server" "master" { + count = var.master_server_count + name = "master${count.index + 1}" + image = "ubuntu-18.04" + server_type = var.master_server_type + ssh_keys = [hcloud_ssh_key.desktop.id] + keep_disk = true + + labels = { + environment = local.environment + type = "master" + } +} + +resource "hcloud_server_network" "master_network" { + count = var.master_server_count + server_id = "${hcloud_server.master[count.index].id}" + network_id = "${hcloud_network.private_network.id}" + ip = "192.168.1.${count.index + 1}" } diff --git a/terraform/lb.tf b/terraform/lb.tf deleted file mode 100644 index 86d8438..0000000 --- a/terraform/lb.tf +++ /dev/null @@ -1,22 +0,0 @@ -resource "null_resource" "load_balancer" { - provisioner "local-exec" { - command = "./scripts/create_lb.sh lb-k8s-${local.environment} ${var.lb_ip}" - } - - provisioner "local-exec" { - when = "destroy" - command = "./scripts/delete_lb.sh ${var.lb_ip}" - } -} - -resource "null_resource" "update_load_balancer_rules" { - triggers = { - node_instance_ids = "${join(",", scaleway_server.node.*.private_ip)}" - } - - provisioner "local-exec" { - command = "./scripts/update_lb_rules.sh ${var.lb_ip} '${jsonencode(scaleway_server.node.*.private_ip)}'" - } - - depends_on = [null_resource.load_balancer] -} diff --git a/terraform/network.tf b/terraform/network.tf new file mode 100644 index 0000000..1bfef41 --- /dev/null +++ b/terraform/network.tf @@ -0,0 +1,22 @@ +resource "hcloud_network" "private_network" { + name = "private_network" + ip_range = "192.168.0.0/16" + + labels = { + environment = local.environment + } +} + +resource "hcloud_network_subnet" "master_network" { + network_id = "${hcloud_network.private_network.id}" + type = "server" + network_zone = "eu-central" + ip_range = "192.168.1.0/24" +} + +resource "hcloud_network_subnet" "node_network" { + network_id = "${hcloud_network.private_network.id}" + type = "server" + network_zone = "eu-central" + ip_range = "192.168.2.0/24" +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf index e1c597b..9dea0fa 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -1,11 +1,8 @@ -output "loadbalancer_ip" { - value = var.lb_ip +output "master_public_ips" { + value = [hcloud_server.master.*.ipv4_address] } output "node_public_ips" { - value = [scaleway_server.node.*.public_ip] + value = [hcloud_server.node.*.ipv4_address] } -output "master_public_ips" { - value = [scaleway_server.master.*.public_ip] -} diff --git a/terraform/scripts/create_lb.sh b/terraform/scripts/create_lb.sh deleted file mode 100755 index 3ea0468..0000000 --- a/terraform/scripts/create_lb.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e -set -x - -export TOKEN=`jq '.token' -r ~/.scwrc` -REGION="fr-par" -ORGANIZATION_ID=`jq '.organization' -r ~/.scwrc` - -LB_NAME=$1 -LB_IP=$2 - -IP_ID=$(http GET "https://api.scaleway.com/lb/v1/regions/$REGION/ips" X-Auth-Token:$TOKEN | jq -r ".ips[] | select(.ip_address == \"$LB_IP\") | .id") -echo "IP_ID: $IP_ID" - -http POST "https://api.scaleway.com/lb/v1/regions/$REGION/lbs" X-Auth-Token:$TOKEN name=$LB_NAME organization_id=$ORGANIZATION_ID ip_id=$IP_ID --ignore-stdin | jq -r '.id' diff --git a/terraform/scripts/delete_lb.sh b/terraform/scripts/delete_lb.sh deleted file mode 100755 index 77565ee..0000000 --- a/terraform/scripts/delete_lb.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e -set -x - -export TOKEN=`jq '.token' -r ~/.scwrc` -REGION="fr-par" -ORGANIZATION_ID=`jq '.organization' -r ~/.scwrc` - -LB_IP=$1 - -IP_ID=$(http GET "https://api.scaleway.com/lb/v1/regions/$REGION/ips" X-Auth-Token:$TOKEN | jq -r ".ips[] | select(.ip_address == \"$LB_IP\") | .id") -echo "IP_ID: $IP_ID" - -LB_ID=$(http GET "https://api.scaleway.com/lb/v1/regions/$REGION/lbs" X-Auth-Token:$TOKEN | jq -r ".lbs[] | select(.ip[0].id == \"$IP_ID\") | .id") - -http DELETE "https://api.scaleway.com/lb/v1/regions/$REGION/lbs/$LB_ID" X-Auth-Token:$TOKEN diff --git a/terraform/scripts/update_lb_rules.sh b/terraform/scripts/update_lb_rules.sh deleted file mode 100755 index fe7f02e..0000000 --- a/terraform/scripts/update_lb_rules.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/bash - -set -e -set -x - -export TOKEN=`jq '.token' -r ~/.scwrc` -REGION="fr-par" -ORGANIZATION_ID=`jq '.organization' -r ~/.scwrc` - -LB_IP=$1 -LB_TARGET_IPS=$2 - -function create_rules() { - LB_ID=$1 - declare -A RULES - RULES[http]=80 - RULES[https]=443 - - for PROTOCOL in "${!RULES[@]}"; do - PORT=${RULES[$PROTOCOL]} - BACKEND_ID=$(http POST "https://api.scaleway.com/lb/v1/regions/$REGION/lbs/$LB_ID/backends" X-Auth-Token:$TOKEN name=lbb-$PROTOCOL forward_protocol=tcp forward_port=$PORT forward_port_algorithm=roundrobin sticky_sessions=none health_check:="{\"http_config\":{\"uri\":\"/\",\"method\":\"GET\",\"code\":404},\"check_delay\":1001,\"check_max_retries\":3,\"check_timeout\":3000,\"port\":$PORT}" server_ip:=$LB_TARGET_IPS --ignore-stdin | jq -r '.id') - http POST "https://api.scaleway.com/lb/v1/regions/$REGION/lbs/$LB_ID/frontends" X-Auth-Token:$TOKEN backend_id=$BACKEND_ID inbound_port=$PORT name=lbf-$PROTOCOL --ignore-stdin - done -} - -function update_rules() { - LB_ID=$1 - BACKENDS_IDS$2 - - for BACKEND_ID in $BACKENDS_IDS - do - http PUT "https://api.scaleway.com/lb/v1/regions/$REGION/backends/$BACKEND_ID/servers" X-Auth-Token:$TOKEN server_ip:="$LB_TARGET_IPS" --ignore-stdin - done -} - -IP_ID=$(http GET "https://api.scaleway.com/lb/v1/regions/$REGION/ips" X-Auth-Token:$TOKEN | jq -r ".ips[] | select(.ip_address == \"$LB_IP\") | .id") -echo "IP_ID: $IP_ID" - -LB_ID=$(http GET "https://api.scaleway.com/lb/v1/regions/$REGION/lbs" X-Auth-Token:$TOKEN | jq -r ".lbs[] | select(.ip[0].id == \"$IP_ID\") | .id") - -BACKENDS_IDS=$(http GET "https://api.scaleway.com/lb/v1/regions/$REGION/lbs/$LB_ID/backends" X-Auth-Token:$TOKEN | jq -r ".backends[] | .id") - -if [ -n "$BACKENDS_IDS" ] -then - update_rules $LB_ID $BACKENDS_IDS -else - create_rules $LB_ID -fi diff --git a/terraform/security.tf b/terraform/security.tf new file mode 100644 index 0000000..527b761 --- /dev/null +++ b/terraform/security.tf @@ -0,0 +1,39 @@ +# resource "scaleway_security_group" "k8s-sg" { +# name = "Kubernetes" +# description = "Kubernetes cluster nodes security group" +# enable_default_security = true +# stateful = true +# inbound_default_policy = "drop" +# outbound_default_policy = "accept" +# } + + +# resource "scaleway_security_group_rule" "k8s-sg-rule-ssh" { +# security_group = "${scaleway_security_group.k8s-sg.id}" + +# action = "accept" +# direction = "inbound" +# ip_range = "0.0.0.0/0" +# protocol = "TCP" +# port = 22 +# } + +# resource "scaleway_security_group_rule" "k8s-sg-rule-http" { +# security_group = "${scaleway_security_group.k8s-sg.id}" + +# action = "accept" +# direction = "inbound" +# ip_range = "0.0.0.0/0" +# protocol = "TCP" +# port = 80 +# } + +# resource "scaleway_security_group_rule" "k8s-sg-rule-https" { +# security_group = "${scaleway_security_group.k8s-sg.id}" + +# action = "accept" +# direction = "inbound" +# ip_range = "0.0.0.0/0" +# protocol = "TCP" +# port = 443 +# } diff --git a/terraform/ssh_keys/desktop.pub b/terraform/ssh_keys/desktop.pub new file mode 100644 index 0000000..98d7377 --- /dev/null +++ b/terraform/ssh_keys/desktop.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDMPhCld0dsDzpdkMvPRdiwd6IX8HF8Mb2V6uQzBl8/syeny8FbZxlZR8gk39RGxNYcLaZ+nA50DS6mOIplXCGdtozfw0Vm+FdITN3apMufWIdobG7Igs1vxKBBbkAb5lwxkEFUCUMzPdCLFHd5zabVH0WE42Be8+hYPLd5W/ikPCOgxRaGwryHHroxRMdkD3PcNE8upSEMdGl51pzgXhO6Fcig8UokOYHxV92SiQ0KEsCbc+oe8e9Gkr7g78tz+6YcTYLY2p2ygR7Vrh/WyTaUVnrNNqL8NIqp+Lc2kVtnqGXHFBJ0Wggaly+AeKWygy+dnOMEGSirhQ6/dUcB/Phz phfroidmont@archdesktop-2017-07-31 diff --git a/terraform/variables.tf b/terraform/variables.tf index edde46b..9a43df5 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -1,31 +1,20 @@ variable "region" { - default = "par1" + default = "ams1" } -variable "architecture" { - default = "x86_64" +variable "master_server_type" { + default = "cx11" } -variable "image" { - default = "Ubuntu Bionic" -} - -variable "master_instance_type" { - default = "DEV1-S" -} - -variable "master_instance_count" { +variable "master_server_count" { default = 1 } -variable "node_instance_type" { - default = "DEV1-S" +variable "node_server_type" { + default = "cx21" } -variable "node_instance_count" { - default = 2 +variable "node_server_count" { + default = 3 } -variable "lb_ip" { - default = "51.159.26.139" -} diff --git a/terraform/versions.tf b/terraform/versions.tf deleted file mode 100644 index ac97c6a..0000000 --- a/terraform/versions.tf +++ /dev/null @@ -1,4 +0,0 @@ - -terraform { - required_version = ">= 0.12" -}