From 88ec9b3eafa699041c877b6eecb155c8d808a69e Mon Sep 17 00:00:00 2001 From: Garth Bushell Date: Wed, 28 Mar 2018 18:24:53 +0100 Subject: [PATCH 1/2] Initial cut of moving all etcd stuff to a module --- bashsource.tf | 55 ------- datasources.tf | 5 - etcd-cluster/datasources.tf | 9 ++ .../cloud_init/#bootstrap.template.sh# | 106 +++++++++++++ .../cloud_init/bootstrap.template.sh | 0 .../instance}/datasources.tf | 0 .../etcd => etcd-cluster/instance}/main.tf | 2 +- .../etcd => etcd-cluster/instance}/outputs.tf | 0 .../instance}/variables.tf | 5 +- .../loadbalancer}/main.tf | 0 .../loadbalancer}/outputs.tf | 0 .../loadbalancer}/variables.tf | 0 etcd-cluster/main.tf | 148 ++++++++++++++++++ etcd-cluster/output.tf | 23 +++ etcd-cluster/variables.tf | 91 +++++++++++ k8s-oci.tf | 124 +++------------ outputs.tf | 17 +- 17 files changed, 400 insertions(+), 185 deletions(-) create mode 100644 etcd-cluster/datasources.tf create mode 100644 etcd-cluster/instance/cloud_init/#bootstrap.template.sh# rename {instances/etcd => etcd-cluster/instance}/cloud_init/bootstrap.template.sh (100%) rename {instances/etcd => etcd-cluster/instance}/datasources.tf (100%) rename {instances/etcd => etcd-cluster/instance}/main.tf (96%) rename {instances/etcd => etcd-cluster/instance}/outputs.tf (100%) rename {instances/etcd => etcd-cluster/instance}/variables.tf (95%) rename {network/loadbalancers/etcd => etcd-cluster/loadbalancer}/main.tf (100%) rename {network/loadbalancers/etcd => etcd-cluster/loadbalancer}/outputs.tf (100%) rename {network/loadbalancers/etcd => etcd-cluster/loadbalancer}/variables.tf (100%) create mode 100644 etcd-cluster/main.tf create mode 100644 etcd-cluster/output.tf create mode 100644 etcd-cluster/variables.tf diff --git a/bashsource.tf b/bashsource.tf index 8a9c4b7..a4fcf64 100644 --- a/bashsource.tf +++ b/bashsource.tf @@ -4,61 +4,6 @@ resource null_resource "build_source" { } } -resource null_resource "etcd-ad1" { - count = "${var.etcdAd1Count}" - depends_on = [ - "module.instances-etcd-ad1", - "null_resource.build_source" - ] - - triggers { - etcd_id = "${element(module.instances-etcd-ad1.ids, count.index)}" - build_source_id = "${null_resource.build_source.id}" - } - - provisioner "local-exec" { - command = "echo 'alias ${var.label_prefix}etcdad1-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-etcd-ad1.instance_public_ips, count.index)}\"' >> source.sh" - } -} - -resource null_resource "etcd-ad2" { - count = "${var.etcdAd2Count}" - depends_on = [ - "module.instances-etcd-ad2", - "null_resource.build_source" - ] - - triggers { - etcd_id = "${element(module.instances-etcd-ad2.ids, count.index)}" - build_source_id = "${null_resource.build_source.id}" - } - - provisioner "local-exec" { - command = "echo 'alias ${var.label_prefix}etcdad2-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-etcd-ad2.instance_public_ips, count.index)}\"' >> source.sh" - } -} - - -resource null_resource "etcd-ad3" { - count = "${var.etcdAd3Count}" - depends_on = [ - "module.instances-etcd-ad3", - ] - - triggers { - etcd_id = "${element(module.instances-etcd-ad3.ids, count.index)}" - build_source_id = "${null_resource.build_source.id}" - } - - provisioner "local-exec" { - command = "echo 'alias ${var.label_prefix}etcad3-${count.index}=\"ssh -i ${path.root}/generated/instances_id_rsa opc@${element(module.instances-etcd-ad3.instance_public_ips, count.index)}\"' >> source.sh" - } -} - - - - - resource null_resource "k8smaster-ad1" { count = "${var.k8sMasterAd1Count}" depends_on = [ diff --git a/datasources.tf b/datasources.tf index 75b1db1..5f949fb 100644 --- a/datasources.tf +++ b/datasources.tf @@ -2,8 +2,3 @@ data "oci_identity_availability_domains" "ADs" { compartment_id = "${var.tenancy_ocid}" } -resource "template_file" "etcd_discovery_url" { - provisioner "local-exec" { - command = "[ -d ${path.root}/generated ] || mkdir -p ${path.root}/generated && curl --retry 3 https://discovery.etcd.io/new?size=${var.etcdAd1Count + var.etcdAd2Count + var.etcdAd3Count} > ${path.root}/generated/discovery${self.id}" - } -} diff --git a/etcd-cluster/datasources.tf b/etcd-cluster/datasources.tf new file mode 100644 index 0000000..75b1db1 --- /dev/null +++ b/etcd-cluster/datasources.tf @@ -0,0 +1,9 @@ +data "oci_identity_availability_domains" "ADs" { + compartment_id = "${var.tenancy_ocid}" +} + +resource "template_file" "etcd_discovery_url" { + provisioner "local-exec" { + command = "[ -d ${path.root}/generated ] || mkdir -p ${path.root}/generated && curl --retry 3 https://discovery.etcd.io/new?size=${var.etcdAd1Count + var.etcdAd2Count + var.etcdAd3Count} > ${path.root}/generated/discovery${self.id}" + } +} diff --git a/etcd-cluster/instance/cloud_init/#bootstrap.template.sh# b/etcd-cluster/instance/cloud_init/#bootstrap.template.sh# new file mode 100644 index 0000000..38ab78f --- /dev/null +++ b/etcd-cluster/instance/cloud_init/#bootstrap.template.sh# @@ -0,0 +1,106 @@ +#!/bin/bash -x + +# Turn off SELinux +sudo sed -i s/SELINUX=enforcing/SELINUX=permissive/ /etc/selinux/config +setenforce 0 + +# Set working dir +cd /home/opc + +# enable ol7 addons +yum-config-manager --disable ol7_UEKR3 +yum-config-manager --enable ol7_addons ol7_latest ol7_UEKR4 ol7_optional ol7_optional_latest + + + +# Install Docker +until yum -y install docker-engine-${docker_ver}; do sleep 1 && echo -n "."; done + +cat < /etc/sysconfig/docker +OPTIONS="--selinux-enabled --log-opt max-size=${docker_max_log_size} --log-opt max-file=${docker_max_log_files}" +DOCKER_CERT_PATH=/etc/docker +GOTRACEBACK=crash +EOF + +# Start Docker +systemctl daemon-reload +systemctl enable docker +systemctl restart docker + +docker info + +################### +# Drop firewall rules +iptables -F + +################### +# etcd + +# Get IP Address of self +IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip addr show | grep -Po 'inet \K[\d.]+') +SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2) + +HOSTNAME=$(hostname) +FQDN_HOSTNAME="$(getent hosts $IP_LOCAL | awk '{print $2}')" + +## Login iSCSI volume mount and create filesystem at /etcd +###################################### +iqn=$(iscsiadm --mode discoverydb --type sendtargets --portal 169.254.2.2:3260 --discover| cut -f2 -d" ") + +if [ -n "$${iqn}" ]; then + echo "iSCSI Login $${iqn}" + iscsiadm -m node -o new -T $${iqn} -p 169.254.2.2:3260 + iscsiadm -m node -o update -T $${iqn} -n node.startup -v automatic + iscsiadm -m node -T $${iqn} -p 169.254.2.2:3260 -l + # Wait for device to apear... + until [[ -e "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" ]]; do sleep 1 && echo -n "."; done + # If the volume has been created and formatted before but it's just a new instance this may fail + # but if so ignore and carry on. + mkfs -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1"; + echo "$$(readlink -f /dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1) /etcd xfs defaults,noatime,_netdev 0 2" >> /etc/fstab + mkdir -p /etcd + mount -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" /etcd +fi + +docker run -d \ + --restart=always \ + -p 2380:2380 -p 2379:2379 \ + -v /etc/ssl/certs/ca-bundle.crt:/etc/ssl/certs/ca-bundle.crt \ + -v /etcd:/$HOSTNAME.etcd \ + --net=host \ + quay.io/coreos/etcd:${etcd_ver} \ + /usr/local/bin/etcd \ + -name $HOSTNAME \ + -advertise-client-urls http://$IP_LOCAL:2379 \ + -listen-client-urls http://$IP_LOCAL:2379,http://127.0.0.1:2379 \ + -listen-peer-urls http://0.0.0.0:2380 \ + -discovery ${etcd_discovery_url} + +# Generate a flannel configuration JSON that we will store into etcd using curl. +cat >/tmp/flannel-network.json < Date: Wed, 28 Mar 2018 18:26:21 +0100 Subject: [PATCH 2/2] Remove emacsy file --- .../cloud_init/#bootstrap.template.sh# | 106 ------------------ 1 file changed, 106 deletions(-) delete mode 100644 etcd-cluster/instance/cloud_init/#bootstrap.template.sh# diff --git a/etcd-cluster/instance/cloud_init/#bootstrap.template.sh# b/etcd-cluster/instance/cloud_init/#bootstrap.template.sh# deleted file mode 100644 index 38ab78f..0000000 --- a/etcd-cluster/instance/cloud_init/#bootstrap.template.sh# +++ /dev/null @@ -1,106 +0,0 @@ -#!/bin/bash -x - -# Turn off SELinux -sudo sed -i s/SELINUX=enforcing/SELINUX=permissive/ /etc/selinux/config -setenforce 0 - -# Set working dir -cd /home/opc - -# enable ol7 addons -yum-config-manager --disable ol7_UEKR3 -yum-config-manager --enable ol7_addons ol7_latest ol7_UEKR4 ol7_optional ol7_optional_latest - - - -# Install Docker -until yum -y install docker-engine-${docker_ver}; do sleep 1 && echo -n "."; done - -cat < /etc/sysconfig/docker -OPTIONS="--selinux-enabled --log-opt max-size=${docker_max_log_size} --log-opt max-file=${docker_max_log_files}" -DOCKER_CERT_PATH=/etc/docker -GOTRACEBACK=crash -EOF - -# Start Docker -systemctl daemon-reload -systemctl enable docker -systemctl restart docker - -docker info - -################### -# Drop firewall rules -iptables -F - -################### -# etcd - -# Get IP Address of self -IP_LOCAL=$(ip route show to 0.0.0.0/0 | awk '{ print $5 }' | xargs ip addr show | grep -Po 'inet \K[\d.]+') -SUBNET=$(getent hosts $IP_LOCAL | awk '{print $2}' | cut -d. -f2) - -HOSTNAME=$(hostname) -FQDN_HOSTNAME="$(getent hosts $IP_LOCAL | awk '{print $2}')" - -## Login iSCSI volume mount and create filesystem at /etcd -###################################### -iqn=$(iscsiadm --mode discoverydb --type sendtargets --portal 169.254.2.2:3260 --discover| cut -f2 -d" ") - -if [ -n "$${iqn}" ]; then - echo "iSCSI Login $${iqn}" - iscsiadm -m node -o new -T $${iqn} -p 169.254.2.2:3260 - iscsiadm -m node -o update -T $${iqn} -n node.startup -v automatic - iscsiadm -m node -T $${iqn} -p 169.254.2.2:3260 -l - # Wait for device to apear... - until [[ -e "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" ]]; do sleep 1 && echo -n "."; done - # If the volume has been created and formatted before but it's just a new instance this may fail - # but if so ignore and carry on. - mkfs -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1"; - echo "$$(readlink -f /dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1) /etcd xfs defaults,noatime,_netdev 0 2" >> /etc/fstab - mkdir -p /etcd - mount -t xfs "/dev/disk/by-path/ip-169.254.2.2:3260-iscsi-$${iqn}-lun-1" /etcd -fi - -docker run -d \ - --restart=always \ - -p 2380:2380 -p 2379:2379 \ - -v /etc/ssl/certs/ca-bundle.crt:/etc/ssl/certs/ca-bundle.crt \ - -v /etcd:/$HOSTNAME.etcd \ - --net=host \ - quay.io/coreos/etcd:${etcd_ver} \ - /usr/local/bin/etcd \ - -name $HOSTNAME \ - -advertise-client-urls http://$IP_LOCAL:2379 \ - -listen-client-urls http://$IP_LOCAL:2379,http://127.0.0.1:2379 \ - -listen-peer-urls http://0.0.0.0:2380 \ - -discovery ${etcd_discovery_url} - -# Generate a flannel configuration JSON that we will store into etcd using curl. -cat >/tmp/flannel-network.json <