From b07deec7863213126a7ba1ea397fa207021b9ee1 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Tue, 9 Nov 2021 17:06:17 -0500 Subject: [PATCH 01/40] Add example configs from libvirt provider. --- cloud_init.cfg | 20 ++++++++++++++++++++ network_config.cfg | 4 ++++ 2 files changed, 24 insertions(+) create mode 100644 cloud_init.cfg create mode 100644 network_config.cfg diff --git a/cloud_init.cfg b/cloud_init.cfg new file mode 100644 index 0000000..10cbe9d --- /dev/null +++ b/cloud_init.cfg @@ -0,0 +1,20 @@ +#cloud-config +# vim: syntax=yaml +# +# *********************** +# ---- for more examples look at: ------ +# ---> https://cloudinit.readthedocs.io/en/latest/topics/examples.html +# ****************************** +# +# This is the configuration syntax that the write_files module +# will know how to understand. encoding can be given b64 or gzip or (gz+b64). +# The content will be decoded accordingly and then written to the path that is +# provided. +# +# Note: Content strings here are truncated for example purposes. +ssh_pwauth: True +chpasswd: + list: | + root:terraform-libvirt-linux + expire: False + diff --git a/network_config.cfg b/network_config.cfg new file mode 100644 index 0000000..5b2cbca --- /dev/null +++ b/network_config.cfg @@ -0,0 +1,4 @@ +version: 2 +ethernets: + ens3: + dhcp4: true From 14166ca8be9bb3e7ee1f417575ab0a8b9c26647f Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 10 Nov 2021 11:24:50 -0500 Subject: [PATCH 02/40] Initial attempt is working. --- .gitignore | 38 +++++++++++++++ README.md | 10 ++++ get-vm-ips.sh | 17 +++++++ main.tf | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++ variables.tf | 26 +++++++++++ 5 files changed, 218 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100755 get-vm-ips.sh create mode 100644 main.tf create mode 100644 variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f8c1f36 --- /dev/null +++ b/.gitignore @@ -0,0 +1,38 @@ +################################################################################ +# Pulled from github/gitignore 2021-11-10 commit 1a84870 +################################################################################ + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc diff --git a/README.md b/README.md new file mode 100644 index 0000000..3324cb0 --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ + +Dependencies +---------------------------------------- + +TODO REM add libvirt provider +libvirt provider depends on mkisofs + +security_driver = none for ubuntu host, link github issue. +https://github.com/dmacvicar/terraform-provider-libvirt/issues/546 + diff --git a/get-vm-ips.sh b/get-vm-ips.sh new file mode 100755 index 0000000..cef5323 --- /dev/null +++ b/get-vm-ips.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +# Use eval $(./get-vm-ips.sh) to set env vars for ips. + +terraform refresh > /dev/null +IPS_JSON="$(terraform show -json | jq '.values.root_module.resources[] | select(.type == "libvirt_domain") | {name: .values.name, ip: .values.network_interface[0].addresses[0]}')" + +echo $IPS_JSON | \ + jq 'select(.name | contains("master")) | .ip' | \ + xargs -I% echo export MASTER=% + +echo $IPS_JSON | \ + jq 'select(.name | contains("worker")) | .ip' | \ + nl -v 0 | \ + awk '{print "export WORKER" $1 "=" $2}' | \ + sed 's/"//g' + diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..a158472 --- /dev/null +++ b/main.tf @@ -0,0 +1,127 @@ +terraform { + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + } + } +} + +provider "libvirt" { + uri = "qemu+ssh://gert@gertie/system" +} + +resource "libvirt_pool" "images" { + name = "k8s-tf-images" + type = "dir" + path = var.disk-image-dir +} + +resource "libvirt_volume" "master-image" { + name = "k8s-tf-master" + pool = libvirt_pool.images.name + source = var.ubuntu-image + format = "qcow2" +} + +resource "libvirt_volume" "worker-volumes" { + name = "k8s-tf-worker-${count.index}" + pool = libvirt_pool.images.name + source = var.ubuntu-image + format = "qcow2" + count = var.worker-nodes +} + +data "template_file" "user_data" { + template = file("${path.module}/cloud_init.cfg") +} + +data "template_file" "network_config" { + template = file("${path.module}/network_config.cfg") +} + +# for more info about paramater check this out +# https://github.com/dmacvicar/terraform-provider-libvirt/blob/master/website/docs/r/cloudinit.html.markdown +# Use CloudInit to add our ssh-key to the instance +# you can add also meta_data field +resource "libvirt_cloudinit_disk" "commoninit" { + name = "commoninit.images" + user_data = data.template_file.user_data.rendered + network_config = data.template_file.network_config.rendered + pool = libvirt_pool.images.name +} + +# Create the machine +resource "libvirt_domain" "master-domain" { + name = "k8s-tf-master" + memory = var.node-memory + vcpu = var.node-vcpus + + cloudinit = libvirt_cloudinit_disk.commoninit.id + + network_interface { + network_name = "default" + } + + # IMPORTANT: this is a known bug on cloud images, since they expect a console + # we need to pass it + # https://bugs.launchpad.net/cloud-images/+bug/1573095 + console { + type = "pty" + target_port = "0" + target_type = "serial" + } + + console { + type = "pty" + target_type = "virtio" + target_port = "1" + } + + disk { + volume_id = libvirt_volume.master-image.id + } + + graphics { + type = "spice" + listen_type = "address" + autoport = true + } +} + +resource "libvirt_domain" "worker-domains" { + count = var.worker-nodes + name = "k8s-tf-worker-${count.index}" + memory = var.node-memory + vcpu = var.node-vcpus + + cloudinit = libvirt_cloudinit_disk.commoninit.id + + network_interface { + network_name = "default" + } + + # IMPORTANT: this is a known bug on cloud images, since they expect a console + # we need to pass it + # https://bugs.launchpad.net/cloud-images/+bug/1573095 + console { + type = "pty" + target_port = "0" + target_type = "serial" + } + + console { + type = "pty" + target_type = "virtio" + target_port = "1" + } + + disk { + volume_id = element(libvirt_volume.worker-volumes.*.id, count.index) + } + + graphics { + type = "spice" + listen_type = "address" + autoport = true + } +} diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..acece76 --- /dev/null +++ b/variables.tf @@ -0,0 +1,26 @@ + +variable "disk-image-dir" { + description = "This is the location on the KVM hypervisor host where all the disk images will be kept." +} + +variable "node-memory" { + default = "2048" + description = "The amount of memory to be used for all the nodes." + type = number +} + +variable "node-vcpus" { + default = "2" + description = "The amount of vcpus to be used for all the nodes." + type = number +} + +variable "worker-nodes" { + default = "2" + description = "The number of worker nodes to create." + type = number +} + +variable "ubuntu-image" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" +} From cdfe21bc07356ed962da6d0dc055078674f787b9 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 10 Nov 2021 13:59:07 -0500 Subject: [PATCH 03/40] Setting keys and password with cloud-init working. --- README.md | 7 +++++++ cloud_init.cfg | 40 ++++++++++++++++++++++++---------------- main.tf | 8 ++++---- 3 files changed, 35 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 3324cb0..b27f538 100644 --- a/README.md +++ b/README.md @@ -8,3 +8,10 @@ libvirt provider depends on mkisofs security_driver = none for ubuntu host, link github issue. https://github.com/dmacvicar/terraform-provider-libvirt/issues/546 +Other +---------------------------------------- + +Create a password hash. +```shell +python3 -c 'import crypt; print(crypt.crypt("test", crypt.mksalt(crypt.METHOD_SHA512)))' +``` diff --git a/cloud_init.cfg b/cloud_init.cfg index 10cbe9d..f3ff57c 100644 --- a/cloud_init.cfg +++ b/cloud_init.cfg @@ -1,20 +1,28 @@ #cloud-config # vim: syntax=yaml -# -# *********************** -# ---- for more examples look at: ------ -# ---> https://cloudinit.readthedocs.io/en/latest/topics/examples.html -# ****************************** -# -# This is the configuration syntax that the write_files module -# will know how to understand. encoding can be given b64 or gzip or (gz+b64). -# The content will be decoded accordingly and then written to the path that is -# provided. -# -# Note: Content strings here are truncated for example purposes. -ssh_pwauth: True + +users: + - name: admin + # If we don't supress the user group then cloud init will fail because there + # is allready an admin group in the ubuntu base image. + no_user_group: true + groups: users, admin, sudo + shell: /usr/bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lxKOiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin + - name: root + ssh_authorized_keys: + - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lxKOiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin + +ssh_pwauth: true +disable_root: false chpasswd: - list: | - root:terraform-libvirt-linux - expire: False + list: + expire: false +# Use this when it's determined that we need a bigger disk image. +# This must be used in conjuction with 'size' in 'libvirt_volume' +# growpart: +# mode: auto +# devices: ['/'] diff --git a/main.tf b/main.tf index a158472..295bcea 100644 --- a/main.tf +++ b/main.tf @@ -1,7 +1,9 @@ terraform { + required_version = ">= 0.13" required_providers { libvirt = { source = "dmacvicar/libvirt" + version = "0.6.11" } } } @@ -16,6 +18,8 @@ resource "libvirt_pool" "images" { path = var.disk-image-dir } +# Add 'size' when we need more space. It must be used in conjuction with +# 'growpart' in cloud-init as well. resource "libvirt_volume" "master-image" { name = "k8s-tf-master" pool = libvirt_pool.images.name @@ -39,10 +43,6 @@ data "template_file" "network_config" { template = file("${path.module}/network_config.cfg") } -# for more info about paramater check this out -# https://github.com/dmacvicar/terraform-provider-libvirt/blob/master/website/docs/r/cloudinit.html.markdown -# Use CloudInit to add our ssh-key to the instance -# you can add also meta_data field resource "libvirt_cloudinit_disk" "commoninit" { name = "commoninit.images" user_data = data.template_file.user_data.rendered From 99d7ba1c43c2b2d67a22fac8cef07c1f7ac8dc6f Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 10 Nov 2021 15:29:22 -0500 Subject: [PATCH 04/40] Setting passwords and keys working via variables. - Setting hostnames is working also. --- cloud_init.cfg | 8 ++++++-- main.tf | 37 +++++++++++++++++++++++++++++++------ variables.tf | 8 ++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/cloud_init.cfg b/cloud_init.cfg index f3ff57c..1023821 100644 --- a/cloud_init.cfg +++ b/cloud_init.cfg @@ -10,17 +10,21 @@ users: shell: /usr/bin/bash sudo: ALL=(ALL) NOPASSWD:ALL ssh_authorized_keys: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lxKOiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin + - ${admin-pub-key} - name: root ssh_authorized_keys: - - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lxKOiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin + - ${admin-pub-key} ssh_pwauth: true disable_root: false chpasswd: list: + - root:${admin-passwd} + - admin:${admin-passwd} expire: false +hostname: ${hostname} + # Use this when it's determined that we need a bigger disk image. # This must be used in conjuction with 'size' in 'libvirt_volume' # growpart: diff --git a/main.tf b/main.tf index 295bcea..3c1f9e5 100644 --- a/main.tf +++ b/main.tf @@ -35,31 +35,55 @@ resource "libvirt_volume" "worker-volumes" { count = var.worker-nodes } -data "template_file" "user_data" { +data "template_file" "master-user-data" { template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "k8s-tf-master" + } +} + +data "template_file" "worker-user-data" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "k8s-tf-worker-${count.index}" + } + count = var.worker-nodes } data "template_file" "network_config" { template = file("${path.module}/network_config.cfg") } -resource "libvirt_cloudinit_disk" "commoninit" { - name = "commoninit.images" - user_data = data.template_file.user_data.rendered +resource "libvirt_cloudinit_disk" "master-init" { + name = "k8s-tf-master-init" + user_data = data.template_file.master-user-data.rendered network_config = data.template_file.network_config.rendered pool = libvirt_pool.images.name } +resource "libvirt_cloudinit_disk" "worker-init" { + name = "k8s-tf-worker-${count.index}-init" + user_data = element(data.template_file.worker-user-data.*.rendered, count.index) + network_config = data.template_file.network_config.rendered + pool = libvirt_pool.images.name + count = var.worker-nodes +} + # Create the machine resource "libvirt_domain" "master-domain" { name = "k8s-tf-master" memory = var.node-memory vcpu = var.node-vcpus - cloudinit = libvirt_cloudinit_disk.commoninit.id + cloudinit = libvirt_cloudinit_disk.master-init.id network_interface { network_name = "default" + hostname = "k8s-tf-master" } # IMPORTANT: this is a known bug on cloud images, since they expect a console @@ -94,10 +118,11 @@ resource "libvirt_domain" "worker-domains" { memory = var.node-memory vcpu = var.node-vcpus - cloudinit = libvirt_cloudinit_disk.commoninit.id + cloudinit = element(libvirt_cloudinit_disk.worker-init.*.id, count.index) network_interface { network_name = "default" + hostname = "k8s-tf-worker-${count.index}" } # IMPORTANT: this is a known bug on cloud images, since they expect a console diff --git a/variables.tf b/variables.tf index acece76..95b161e 100644 --- a/variables.tf +++ b/variables.tf @@ -15,6 +15,14 @@ variable "node-vcpus" { type = number } +variable "root-admin-passwd" { + description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." +} + +variable "root-admin-pub-key" { + description = "The public key to be added to authorized_keys for the root and admin accounts." +} + variable "worker-nodes" { default = "2" description = "The number of worker nodes to create." From f4889f042862c1c1de9508dea68968336e3b96e3 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 10 Nov 2021 22:22:17 -0500 Subject: [PATCH 05/40] `node` module is working. --- main.tf | 157 ++++++-------------------------------- modules/node/main.tf | 83 ++++++++++++++++++++ modules/node/outpus.tf | 0 modules/node/variables.tf | 42 ++++++++++ variables.tf | 22 +++++- 5 files changed, 169 insertions(+), 135 deletions(-) create mode 100644 modules/node/main.tf create mode 100644 modules/node/outpus.tf create mode 100644 modules/node/variables.tf diff --git a/main.tf b/main.tf index 3c1f9e5..80a181c 100644 --- a/main.tf +++ b/main.tf @@ -9,144 +9,33 @@ terraform { } provider "libvirt" { - uri = "qemu+ssh://gert@gertie/system" + uri = var.libvirt-connection-url } -resource "libvirt_pool" "images" { - name = "k8s-tf-images" - type = "dir" - path = var.disk-image-dir -} - -# Add 'size' when we need more space. It must be used in conjuction with -# 'growpart' in cloud-init as well. -resource "libvirt_volume" "master-image" { - name = "k8s-tf-master" - pool = libvirt_pool.images.name - source = var.ubuntu-image - format = "qcow2" -} - -resource "libvirt_volume" "worker-volumes" { - name = "k8s-tf-worker-${count.index}" - pool = libvirt_pool.images.name - source = var.ubuntu-image - format = "qcow2" - count = var.worker-nodes -} - -data "template_file" "master-user-data" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "k8s-tf-master" - } -} - -data "template_file" "worker-user-data" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "k8s-tf-worker-${count.index}" - } - count = var.worker-nodes -} - -data "template_file" "network_config" { - template = file("${path.module}/network_config.cfg") -} - -resource "libvirt_cloudinit_disk" "master-init" { - name = "k8s-tf-master-init" - user_data = data.template_file.master-user-data.rendered - network_config = data.template_file.network_config.rendered - pool = libvirt_pool.images.name +module "master-nodes" { + source = "./modules/node" + pool-name = libvirt_pool.images.name + name-prefix = "${var.vm-name-prefix}-master" + num-nodes = var.master-nodes + base-image = var.ubuntu-image + root-admin-passwd = var.root-admin-passwd + root-admin-pub-key = var.root-admin-pub-key + libvirt-connection-url = var.libvirt-connection-url } -resource "libvirt_cloudinit_disk" "worker-init" { - name = "k8s-tf-worker-${count.index}-init" - user_data = element(data.template_file.worker-user-data.*.rendered, count.index) - network_config = data.template_file.network_config.rendered - pool = libvirt_pool.images.name - count = var.worker-nodes -} - -# Create the machine -resource "libvirt_domain" "master-domain" { - name = "k8s-tf-master" - memory = var.node-memory - vcpu = var.node-vcpus - - cloudinit = libvirt_cloudinit_disk.master-init.id - - network_interface { - network_name = "default" - hostname = "k8s-tf-master" - } - - # IMPORTANT: this is a known bug on cloud images, since they expect a console - # we need to pass it - # https://bugs.launchpad.net/cloud-images/+bug/1573095 - console { - type = "pty" - target_port = "0" - target_type = "serial" - } - - console { - type = "pty" - target_type = "virtio" - target_port = "1" - } - - disk { - volume_id = libvirt_volume.master-image.id - } - - graphics { - type = "spice" - listen_type = "address" - autoport = true - } +module "worker-nodes" { + source = "./modules/node" + pool-name = libvirt_pool.images.name + name-prefix = "${var.vm-name-prefix}-worker" + num-nodes = var.worker-nodes + base-image = var.ubuntu-image + root-admin-passwd = var.root-admin-passwd + root-admin-pub-key = var.root-admin-pub-key + libvirt-connection-url = var.libvirt-connection-url } -resource "libvirt_domain" "worker-domains" { - count = var.worker-nodes - name = "k8s-tf-worker-${count.index}" - memory = var.node-memory - vcpu = var.node-vcpus - - cloudinit = element(libvirt_cloudinit_disk.worker-init.*.id, count.index) - - network_interface { - network_name = "default" - hostname = "k8s-tf-worker-${count.index}" - } - - # IMPORTANT: this is a known bug on cloud images, since they expect a console - # we need to pass it - # https://bugs.launchpad.net/cloud-images/+bug/1573095 - console { - type = "pty" - target_port = "0" - target_type = "serial" - } - - console { - type = "pty" - target_type = "virtio" - target_port = "1" - } - - disk { - volume_id = element(libvirt_volume.worker-volumes.*.id, count.index) - } - - graphics { - type = "spice" - listen_type = "address" - autoport = true - } +resource "libvirt_pool" "images" { + name = var.disk-image-pool-name + type = "dir" + path = var.disk-image-dir } diff --git a/modules/node/main.tf b/modules/node/main.tf new file mode 100644 index 0000000..b4b0835 --- /dev/null +++ b/modules/node/main.tf @@ -0,0 +1,83 @@ +terraform { + required_version = ">= 0.13" + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "0.6.11" + } + } +} + +provider "libvirt" { + uri = var.libvirt-connection-url +} + +resource "libvirt_volume" "node-images" { + name = "${var.name-prefix}-${count.index}" + pool = var.pool-name + source = var.base-image + count = var.num-nodes + format = "qcow2" +} + +data "template_file" "node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.name-prefix}-${count.index}" + } + count = var.num-nodes +} + +data "template_file" "network-config" { + template = file("${path.module}/network_config.cfg") +} + +resource "libvirt_cloudinit_disk" "node-inits" { + name = "${var.name-prefix}-${count.index}-init" + user_data = element(data.template_file.node-user-datas.*.rendered, count.index) + network_config = data.template_file.network-config.rendered + pool = var.pool-name + count = var.num-nodes +} + +resource "libvirt_domain" "nodes" { + count = var.num-nodes + name = "${var.name-prefix}-${count.index}" + memory = var.node-memory + vcpu = var.node-vcpus + + cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) + + network_interface { + network_name = "default" + hostname = "${var.name-prefix}-${count.index}" + } + + # IMPORTANT: this is a known bug on cloud images, since they expect a console + # we need to pass it + # https://bugs.launchpad.net/cloud-images/+bug/1573095 + console { + type = "pty" + target_port = "0" + target_type = "serial" + } + + console { + type = "pty" + target_type = "virtio" + target_port = "1" + } + + disk { + volume_id = element(libvirt_volume.node-images.*.id, count.index) + } + + graphics { + type = "spice" + listen_type = "address" + autoport = true + } +} + diff --git a/modules/node/outpus.tf b/modules/node/outpus.tf new file mode 100644 index 0000000..e69de29 diff --git a/modules/node/variables.tf b/modules/node/variables.tf new file mode 100644 index 0000000..93d7ad0 --- /dev/null +++ b/modules/node/variables.tf @@ -0,0 +1,42 @@ +variable "base-image" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" + description = "The base image to be used for all nodes." +} + +variable "libvirt-connection-url" { + description = "The libvirt connection URI, ie. qemu+ssh://@/system" +} + +variable "name-prefix" { + default = "k8s-node" + description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"." +} + +variable "node-memory" { + default = "2048" + description = "The amount of memory to be used for all the nodes." + type = number +} + +variable "node-vcpus" { + default = "2" + description = "The amount of vcpus to be used for all the nodes." + type = number +} + +variable "num-nodes" { + description = "The number of nodes to create with this config." +} + +variable "pool-name" { + default = "default" + description = "The name of the pool to put all disk images in." +} + +variable "root-admin-passwd" { + description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." +} + +variable "root-admin-pub-key" { + description = "The public key to be added to authorized_keys for the root and admin accounts." +} diff --git a/variables.tf b/variables.tf index 95b161e..30dcc51 100644 --- a/variables.tf +++ b/variables.tf @@ -3,6 +3,15 @@ variable "disk-image-dir" { description = "This is the location on the KVM hypervisor host where all the disk images will be kept." } +variable "disk-image-pool-name" { + default = "k8s-tf-images" + description = "The name of the disk pool where all the images will be kept." +} + +variable "libvirt-connection-url" { + description = "The libvirt connection URI, ie. qemu+ssh://@/system" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." @@ -23,8 +32,14 @@ variable "root-admin-pub-key" { description = "The public key to be added to authorized_keys for the root and admin accounts." } +variable "master-nodes" { + default = 1 + description = "The number of master nodes to create." + type = number +} + variable "worker-nodes" { - default = "2" + default = 2 description = "The number of worker nodes to create." type = number } @@ -32,3 +47,8 @@ variable "worker-nodes" { variable "ubuntu-image" { default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" } + +variable "vm-name-prefix" { + default = "k8s-tf" + description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0." +} From 6f2b0a03b944f421d38374f548b6168884a7c766 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 11 Nov 2021 14:50:45 -0500 Subject: [PATCH 06/40] Added outputs for IPs. - Made sure node-memory ad node-vcpus is passed down to module. - Fixed get-vm-ips.sh after move to module. --- get-vm-ips.sh | 12 +++++++----- main.tf | 12 ++++++++++++ modules/node/main.tf | 5 +++-- modules/node/outpus.tf | 4 ++++ 4 files changed, 26 insertions(+), 7 deletions(-) diff --git a/get-vm-ips.sh b/get-vm-ips.sh index cef5323..25e36fe 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -3,15 +3,17 @@ # Use eval $(./get-vm-ips.sh) to set env vars for ips. terraform refresh > /dev/null -IPS_JSON="$(terraform show -json | jq '.values.root_module.resources[] | select(.type == "libvirt_domain") | {name: .values.name, ip: .values.network_interface[0].addresses[0]}')" + +IPS_JSON="$(terraform show -json | jq '.values.outputs')" echo $IPS_JSON | \ - jq 'select(.name | contains("master")) | .ip' | \ - xargs -I% echo export MASTER=% + jq '."master-ips".value[]' | \ + nl -v 0 | \ + awk '{print "export MASTER" $1 "=" $2}' | \ + sed 's/"//g' echo $IPS_JSON | \ - jq 'select(.name | contains("worker")) | .ip' | \ + jq '."worker-ips".value[]' | \ nl -v 0 | \ awk '{print "export WORKER" $1 "=" $2}' | \ sed 's/"//g' - diff --git a/main.tf b/main.tf index 80a181c..d273d9b 100644 --- a/main.tf +++ b/main.tf @@ -17,6 +17,8 @@ module "master-nodes" { pool-name = libvirt_pool.images.name name-prefix = "${var.vm-name-prefix}-master" num-nodes = var.master-nodes + node-memory = var.node-memory + node-vcpus = var.node-vcpus base-image = var.ubuntu-image root-admin-passwd = var.root-admin-passwd root-admin-pub-key = var.root-admin-pub-key @@ -28,6 +30,8 @@ module "worker-nodes" { pool-name = libvirt_pool.images.name name-prefix = "${var.vm-name-prefix}-worker" num-nodes = var.worker-nodes + node-memory = var.node-memory + node-vcpus = var.node-vcpus base-image = var.ubuntu-image root-admin-passwd = var.root-admin-passwd root-admin-pub-key = var.root-admin-pub-key @@ -39,3 +43,11 @@ resource "libvirt_pool" "images" { type = "dir" path = var.disk-image-dir } + +output "master-ips" { + value = module.master-nodes.ips +} + +output "worker-ips" { + value = module.worker-nodes.ips +} diff --git a/modules/node/main.tf b/modules/node/main.tf index b4b0835..0507076 100644 --- a/modules/node/main.tf +++ b/modules/node/main.tf @@ -51,8 +51,9 @@ resource "libvirt_domain" "nodes" { cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) network_interface { - network_name = "default" - hostname = "${var.name-prefix}-${count.index}" + network_name = "default" + hostname = "${var.name-prefix}-${count.index}" + wait_for_lease = true } # IMPORTANT: this is a known bug on cloud images, since they expect a console diff --git a/modules/node/outpus.tf b/modules/node/outpus.tf index e69de29..4d399a6 100644 --- a/modules/node/outpus.tf +++ b/modules/node/outpus.tf @@ -0,0 +1,4 @@ + +output "ips" { + value = libvirt_domain.nodes.*.network_interface.0.addresses.0 +} From 5d21c16b6904521bf7898d8a3ebf870379792478 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 11 Nov 2021 18:37:17 -0500 Subject: [PATCH 07/40] Tested with centos 7 image. --- .gitignore | 2 ++ example.tfvars | 19 +++++++++++++++++++ main.tf | 4 ++-- variables.tf | 2 +- 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 example.tfvars diff --git a/.gitignore b/.gitignore index f8c1f36..02fd6f3 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ # Crash log files crash.log +# Added to original pulled from github/gitignore +crash*.log # Exclude all .tfvars files, which are likely to contain sentitive data, such as # password, private keys, and other secrets. These should not be part of version diff --git a/example.tfvars b/example.tfvars new file mode 100644 index 0000000..af4fae3 --- /dev/null +++ b/example.tfvars @@ -0,0 +1,19 @@ +disk-image-dir = "/path/to/disk/pool/" +libvirt-connection-url = "qemu+ssh://@/system" + +master-nodes = 1 +worker-nodes = 2 + +node-memory = 2048 +node-vcpus = 2 + +base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" +# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 +# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" + +# Password hash created with: +# python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))' +# where "linux" is the password. +root-admin-passwd = "$6$fiLRWvGQkdK.MnZA$Co9NkA5ruuBUA389JzmKJiC8gKRohmyM09AFnVBOD7ErZnxK4RHMUlKvYg1HSgwaCXTl7H/q1svoeQeUfgc6f0" + +root-admin-pub-key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lx4OiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin" diff --git a/main.tf b/main.tf index d273d9b..9cb11a3 100644 --- a/main.tf +++ b/main.tf @@ -19,7 +19,7 @@ module "master-nodes" { num-nodes = var.master-nodes node-memory = var.node-memory node-vcpus = var.node-vcpus - base-image = var.ubuntu-image + base-image = var.base-image root-admin-passwd = var.root-admin-passwd root-admin-pub-key = var.root-admin-pub-key libvirt-connection-url = var.libvirt-connection-url @@ -32,7 +32,7 @@ module "worker-nodes" { num-nodes = var.worker-nodes node-memory = var.node-memory node-vcpus = var.node-vcpus - base-image = var.ubuntu-image + base-image = var.base-image root-admin-passwd = var.root-admin-passwd root-admin-pub-key = var.root-admin-pub-key libvirt-connection-url = var.libvirt-connection-url diff --git a/variables.tf b/variables.tf index 30dcc51..ee7e655 100644 --- a/variables.tf +++ b/variables.tf @@ -44,7 +44,7 @@ variable "worker-nodes" { type = number } -variable "ubuntu-image" { +variable "base-image" { default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" } From 1f5b81116fe8af81262f38b0583d5f38e7fa49dc Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 12 Nov 2021 14:03:45 -0500 Subject: [PATCH 08/40] Single AWS node is working. --- .terraform.lock.hcl | 57 ++++++ example.tfvars | 17 +- main.tf | 184 +++++++++++++++---- modules/{node => libvirt-nodes}/main.tf | 0 modules/{node => libvirt-nodes}/outpus.tf | 0 modules/{node => libvirt-nodes}/variables.tf | 0 variables.tf | 19 ++ 7 files changed, 238 insertions(+), 39 deletions(-) create mode 100644 .terraform.lock.hcl rename modules/{node => libvirt-nodes}/main.tf (100%) rename modules/{node => libvirt-nodes}/outpus.tf (100%) rename modules/{node => libvirt-nodes}/variables.tf (100%) diff --git a/.terraform.lock.hcl b/.terraform.lock.hcl new file mode 100644 index 0000000..5f76ac4 --- /dev/null +++ b/.terraform.lock.hcl @@ -0,0 +1,57 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/dmacvicar/libvirt" { + version = "0.6.11" + hashes = [ + "h1:6QzHY/7aNdaaDxJZKygotWnM5uHoS2gs/03CzUCJX60=", + "zh:15300a1c3c294eccade4c8a678412d81602ab041dc0a5aab72fee5425d778e89", + "zh:1605806de0d3b86b7e94b5d04a7ad9b6ac695781f9672ab6002c23caef43b98e", + "zh:21efc5937d89f9ec96bc626d2ce3621c0919b3da97ab63b4e520c37d3f5c7357", + "zh:2c143a6909917fd11191447de4c496f084c7da5200beb9f512791a80a1f33e7c", + "zh:3ca369718cc49feefc3a6ffa795a9055e60de33989a9f1c72b6db16048a181fa", + "zh:71db1d1cf2c06984bba408ad5dc9b4e25285684ee5c530a61583b202cff21b96", + "zh:a67adfc988311d34adcc119500c2ef048a45d632b00bb5a15ea6d3ffdc1c3d1c", + "zh:a83448cbcc194e3b52af9b89b9273a116082d83f2c966035bf8a8c5d5606ca9c", + "zh:a9c5a818dd2606460d4d6f33af7cb387f3e984d631fc233aaec0dda4e0756c2b", + "zh:af3263e66cf9138361d6d7408533edd6de8498e67c88cf0084421ae31fe89054", + "zh:b9596cb26c1e391172472de4ada9b3b0a08e4777e41327db8e021454cc6aae20", + "zh:e28124b9ee0b8c18b6f776eb6523d8935f3072c47cd803ea2f1a06206effaa48", + "zh:edd00638d8c088b8a38e7ab8b9e8ab3bd710f7357d0f6b4a38e0028bd49d8460", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.64.2" + hashes = [ + "h1:oFsgTmmt/eJ8KLo59PSmYu/irUHJiPnQEey0VqaOOck=", + "zh:0b029a2282beabfe410eb2969e18ca773d3473415e442be4dc8ce0eb6d1cd8c5", + "zh:3209de3266a1138f1ccb09f094fdd98b6f55afc06e291db0abe092ec5dbe7640", + "zh:40648266551631cbc15f8a76e80faf300510e3b38c2544d43fc25e37e6802727", + "zh:483c8af92ae70146f2790a70c1a810251e7135aa912b66e769c934eddceebe32", + "zh:4d106d8d415d8df342f3f85e58c35418e6c55e3cb7f02897f832cefac4dca68c", + "zh:972626a6ddb31d5216606d12ab5c30fbf8d51ed2bbe0efcdd7cffa68c1141557", + "zh:a230d55ec52b1695148d40296877ee23e0b302e817154f9b838eb117c87b13fa", + "zh:c95fddfbd7f870db949da0601323e866e0f0fb0d4a93e96725ae5b88029e84d5", + "zh:ea0c7f568074f835f22273c8e7e61e87f5277e32004c72122915fd3c8df49ccc", + "zh:f96d25887e6e2d2ae47659e2c586efea2167995b59a479ae65a02b097da86474", + "zh:fe7502d8e52d3b5ccb2b3c178e7ea894344783093aa71ffb20e978914c976182", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} diff --git a/example.tfvars b/example.tfvars index af4fae3..33a78d3 100644 --- a/example.tfvars +++ b/example.tfvars @@ -1,3 +1,7 @@ +# A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0' +# equates to any ip address. +admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] + disk-image-dir = "/path/to/disk/pool/" libvirt-connection-url = "qemu+ssh://@/system" @@ -7,7 +11,18 @@ worker-nodes = 2 node-memory = 2048 node-vcpus = 2 -base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" +# 1 GiB, 1 vcpu, only one that is free. +# This one won't work with k8s because it requires at least 2 vcpus. +aws-ec2-instance-type = "t2.micro" + +# 4 GiB, 2 vcpus +# aws-ec2-instnce-type = "t2.medium" + +# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free +# base-image = "ami-00be885d550dcee43" +# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free +base-image = "ami-0dd0ccab7e2801812" +# base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" # From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 # base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" diff --git a/main.tf b/main.tf index 9cb11a3..a3fc99d 100644 --- a/main.tf +++ b/main.tf @@ -8,46 +8,154 @@ terraform { } } -provider "libvirt" { - uri = var.libvirt-connection-url -} - -module "master-nodes" { - source = "./modules/node" - pool-name = libvirt_pool.images.name - name-prefix = "${var.vm-name-prefix}-master" - num-nodes = var.master-nodes - node-memory = var.node-memory - node-vcpus = var.node-vcpus - base-image = var.base-image - root-admin-passwd = var.root-admin-passwd - root-admin-pub-key = var.root-admin-pub-key - libvirt-connection-url = var.libvirt-connection-url -} - -module "worker-nodes" { - source = "./modules/node" - pool-name = libvirt_pool.images.name - name-prefix = "${var.vm-name-prefix}-worker" - num-nodes = var.worker-nodes - node-memory = var.node-memory - node-vcpus = var.node-vcpus - base-image = var.base-image - root-admin-passwd = var.root-admin-passwd - root-admin-pub-key = var.root-admin-pub-key - libvirt-connection-url = var.libvirt-connection-url -} - -resource "libvirt_pool" "images" { - name = var.disk-image-pool-name - type = "dir" - path = var.disk-image-dir +provider "aws" { + region = "us-east-2" } -output "master-ips" { - value = module.master-nodes.ips +resource "aws_vpc" "vpc" { + cidr_block = var.aws-vpc-cidr-block + tags = { + Name = "${var.vm-name-prefix}-vpc" + } +} + +resource "aws_subnet" "subnet" { + vpc_id = aws_vpc.vpc.id + cidr_block = var.aws-subnet-cidr-block + # availability_zone = var.avail_zone + tags = { + Name = "${var.vm-name-prefix}-subnet" + } +} + +resource "aws_default_security_group" "sg" { + vpc_id = aws_vpc.vpc.id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = var.admin-ips + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + prefix_list_ids = [] + } + + tags = { + Name = "${var.vm-name-prefix}-sg" + } +} + +resource "aws_internet_gateway" "igw" { + vpc_id = aws_vpc.vpc.id + + tags = { + Name = "${var.vm-name-prefix}-igw" + } +} + +resource "aws_default_route_table" "route-table" { + default_route_table_id = aws_vpc.vpc.main_route_table_id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.igw.id + } + + # default route, mapping VPC CIDR block to "local", created implicitly and + # cannot be specified. + + tags = { + Name = "${var.vm-name-prefix}-route-table" + } } -output "worker-ips" { - value = module.worker-nodes.ips +# Associate subnet with Route Table +resource "aws_route_table_association" "a-rtb-subnet" { + subnet_id = aws_subnet.subnet.id + route_table_id = aws_default_route_table.route-table.id +} + +resource "aws_key_pair" "debug1" { + key_name = "debug1" + public_key = var.root-admin-pub-key +} + +data "template_file" "node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-${count.index}" + } + count = var.master-nodes } + +resource "aws_instance" "test-node" { + ami = var.base-image + instance_type = var.aws-ec2-instance-type + key_name = aws_key_pair.debug1.key_name + associate_public_ip_address = true + subnet_id = aws_subnet.subnet.id + vpc_security_group_ids = [aws_default_security_group.sg.id] + # user_data = element(data.template_file.node-user-datas.*.rendered, count.index) + count = var.master-nodes + + tags = { + Name = "${var.vm-name-prefix}-test" + } +} + +output "master-ips" { + value = aws_instance.test-node.*.public_ip +} + +# provider "libvirt" { +# uri = var.libvirt-connection-url +# } +# +# module "master-nodes" { +# source = "./modules/libvirt-nodes" +# pool-name = libvirt_pool.images.name +# name-prefix = "${var.vm-name-prefix}-master" +# num-nodes = var.master-nodes +# node-memory = var.node-memory +# node-vcpus = var.node-vcpus +# base-image = var.base-image +# root-admin-passwd = var.root-admin-passwd +# root-admin-pub-key = var.root-admin-pub-key +# libvirt-connection-url = var.libvirt-connection-url +# } +# +# module "worker-nodes" { +# source = "./modules/libvirt-nodes" +# pool-name = libvirt_pool.images.name +# name-prefix = "${var.vm-name-prefix}-worker" +# num-nodes = var.worker-nodes +# node-memory = var.node-memory +# node-vcpus = var.node-vcpus +# base-image = var.base-image +# root-admin-passwd = var.root-admin-passwd +# root-admin-pub-key = var.root-admin-pub-key +# libvirt-connection-url = var.libvirt-connection-url +# } +# +# resource "libvirt_pool" "images" { +# name = var.disk-image-pool-name +# type = "dir" +# path = var.disk-image-dir +# } +# +# # TODO REM move to other file? +# output "master-ips" { +# value = module.master-nodes.ips +# } +# +# output "worker-ips" { +# value = module.worker-nodes.ips +# } diff --git a/modules/node/main.tf b/modules/libvirt-nodes/main.tf similarity index 100% rename from modules/node/main.tf rename to modules/libvirt-nodes/main.tf diff --git a/modules/node/outpus.tf b/modules/libvirt-nodes/outpus.tf similarity index 100% rename from modules/node/outpus.tf rename to modules/libvirt-nodes/outpus.tf diff --git a/modules/node/variables.tf b/modules/libvirt-nodes/variables.tf similarity index 100% rename from modules/node/variables.tf rename to modules/libvirt-nodes/variables.tf diff --git a/variables.tf b/variables.tf index ee7e655..3455efa 100644 --- a/variables.tf +++ b/variables.tf @@ -1,3 +1,22 @@ +variable "admin-ips" { + description = "A list of ips or cidr blocks that are allowed to connect to the nodes." + type = list(string) +} + +variable "aws-ec2-instance-type" { + default = "t2.micro" + description = "The AWS instance type to use for all nodes." +} + +variable "aws-subnet-cidr-block" { + default = "10.0.1.0/24" + description = "The address space to be used for this subnet." +} + +variable "aws-vpc-cidr-block" { + default = "10.0.0.0/16" + description = "The address space to be used for this VPC." +} variable "disk-image-dir" { description = "This is the location on the KVM hypervisor host where all the disk images will be kept." From c2909d078b2be48f455a96d8fcc898c8417ff7f6 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 12 Nov 2021 15:55:25 -0500 Subject: [PATCH 09/40] Moved AWS network resources into a aws-network module. --- main.tf | 88 ++++++-------------------------- modules/aws-network/main.tf | 67 ++++++++++++++++++++++++ modules/aws-network/outputs.tf | 11 ++++ modules/aws-network/variables.tf | 23 +++++++++ 4 files changed, 117 insertions(+), 72 deletions(-) create mode 100644 modules/aws-network/main.tf create mode 100644 modules/aws-network/outputs.tf create mode 100644 modules/aws-network/variables.tf diff --git a/main.tf b/main.tf index a3fc99d..5bb2c7b 100644 --- a/main.tf +++ b/main.tf @@ -12,80 +12,24 @@ provider "aws" { region = "us-east-2" } -resource "aws_vpc" "vpc" { - cidr_block = var.aws-vpc-cidr-block - tags = { - Name = "${var.vm-name-prefix}-vpc" - } +module "aws-network" { + source = "./modules/aws-network" + name-prefix = var.vm-name-prefix + vpc-cidr-block = var.aws-vpc-cidr-block + subnet-cidr-block = var.aws-subnet-cidr-block + admin-ips = var.admin-ips } -resource "aws_subnet" "subnet" { - vpc_id = aws_vpc.vpc.id - cidr_block = var.aws-subnet-cidr-block - # availability_zone = var.avail_zone - tags = { - Name = "${var.vm-name-prefix}-subnet" - } -} - -resource "aws_default_security_group" "sg" { - vpc_id = aws_vpc.vpc.id - - ingress { - from_port = 22 - to_port = 22 - protocol = "tcp" - cidr_blocks = var.admin-ips - } - - egress { - from_port = 0 - to_port = 0 - protocol = "-1" - cidr_blocks = ["0.0.0.0/0"] - prefix_list_ids = [] - } - +# This key pair is not actually used. Keys are added to the nodes via cloud-init +# instead. We just add this here that this key will show up in the AWS console." +resource "aws_key_pair" "key" { + key_name = "${var.vm-name-prefix}-key}" + public_key = var.root-admin-pub-key tags = { - Name = "${var.vm-name-prefix}-sg" + Name = "${var.vm-name-prefix}-key" } } -resource "aws_internet_gateway" "igw" { - vpc_id = aws_vpc.vpc.id - - tags = { - Name = "${var.vm-name-prefix}-igw" - } -} - -resource "aws_default_route_table" "route-table" { - default_route_table_id = aws_vpc.vpc.main_route_table_id - - route { - cidr_block = "0.0.0.0/0" - gateway_id = aws_internet_gateway.igw.id - } - - # default route, mapping VPC CIDR block to "local", created implicitly and - # cannot be specified. - - tags = { - Name = "${var.vm-name-prefix}-route-table" - } -} - -# Associate subnet with Route Table -resource "aws_route_table_association" "a-rtb-subnet" { - subnet_id = aws_subnet.subnet.id - route_table_id = aws_default_route_table.route-table.id -} - -resource "aws_key_pair" "debug1" { - key_name = "debug1" - public_key = var.root-admin-pub-key -} - data "template_file" "node-user-datas" { template = file("${path.module}/cloud_init.cfg") vars = { @@ -99,11 +43,11 @@ data "template_file" "node-user-datas" { resource "aws_instance" "test-node" { ami = var.base-image instance_type = var.aws-ec2-instance-type - key_name = aws_key_pair.debug1.key_name + # key_name = aws_key_pair.debug1.key_name associate_public_ip_address = true - subnet_id = aws_subnet.subnet.id - vpc_security_group_ids = [aws_default_security_group.sg.id] - # user_data = element(data.template_file.node-user-datas.*.rendered, count.index) + subnet_id = module.aws-network.subnet.id + vpc_security_group_ids = [module.aws-network.default-security-group.id] + user_data = element(data.template_file.node-user-datas.*.rendered, count.index) count = var.master-nodes tags = { diff --git a/modules/aws-network/main.tf b/modules/aws-network/main.tf new file mode 100644 index 0000000..be001ac --- /dev/null +++ b/modules/aws-network/main.tf @@ -0,0 +1,67 @@ +resource "aws_vpc" "vpc" { + cidr_block = var.vpc-cidr-block + tags = { + Name = "${var.name-prefix}-vpc" + } +} + +resource "aws_subnet" "subnet" { + vpc_id = aws_vpc.vpc.id + cidr_block = var.subnet-cidr-block + # availability_zone = var.avail_zone + tags = { + Name = "${var.name-prefix}-subnet" + } +} + +resource "aws_default_security_group" "sg" { + vpc_id = aws_vpc.vpc.id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = var.admin-ips + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + prefix_list_ids = [] + } + + tags = { + Name = "${var.name-prefix}-ssh-from-admins--sg" + } +} + +resource "aws_internet_gateway" "igw" { + vpc_id = aws_vpc.vpc.id + tags = { + Name = "${var.name-prefix}-igw" + } +} + +resource "aws_default_route_table" "route-table" { + default_route_table_id = aws_vpc.vpc.main_route_table_id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.igw.id + } + + # default route, mapping VPC CIDR block to "local", created implicitly and + # cannot be specified. + + tags = { + Name = "${var.name-prefix}-route-table" + } +} + +# Associate subnet with Route Table +resource "aws_route_table_association" "a-rtb-subnet" { + subnet_id = aws_subnet.subnet.id + route_table_id = aws_default_route_table.route-table.id +} diff --git a/modules/aws-network/outputs.tf b/modules/aws-network/outputs.tf new file mode 100644 index 0000000..5d43ec7 --- /dev/null +++ b/modules/aws-network/outputs.tf @@ -0,0 +1,11 @@ +output "vpc" { + value = aws_vpc.vpc +} + +output "subnet" { + value = aws_subnet.subnet +} + +output "default-security-group" { + value = aws_default_security_group.sg +} diff --git a/modules/aws-network/variables.tf b/modules/aws-network/variables.tf new file mode 100644 index 0000000..527523e --- /dev/null +++ b/modules/aws-network/variables.tf @@ -0,0 +1,23 @@ +variable "admin-ips" { + description = "A list of ips or cidr blocks that are allowed to connect to the nodes." + type = list(string) +} + +variable "name-prefix" { + default = "tf" + description = "This prefix will be used in all the names of the resources creates in our AWS network." + type = string +} + +variable "subnet-cidr-block" { + default = "10.0.1.0/24" + description = "The address space to be used for this subnet." + type = string +} + +variable "vpc-cidr-block" { + default = "10.0.0.0/16" + description = "The address space to be used for out networks VPC." + type = string +} + From b9da870158379e35fe5526859192a24ef0f3a364 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 12 Nov 2021 18:31:01 -0500 Subject: [PATCH 10/40] Created aws-node module. --- main.tf | 61 +++++++++++++++++++++++++++++++--------------------- variables.tf | 2 +- 2 files changed, 38 insertions(+), 25 deletions(-) diff --git a/main.tf b/main.tf index 5bb2c7b..c41a45d 100644 --- a/main.tf +++ b/main.tf @@ -40,25 +40,34 @@ data "template_file" "node-user-datas" { count = var.master-nodes } -resource "aws_instance" "test-node" { - ami = var.base-image - instance_type = var.aws-ec2-instance-type - # key_name = aws_key_pair.debug1.key_name - associate_public_ip_address = true - subnet_id = module.aws-network.subnet.id - vpc_security_group_ids = [module.aws-network.default-security-group.id] - user_data = element(data.template_file.node-user-datas.*.rendered, count.index) - count = var.master-nodes - - tags = { - Name = "${var.vm-name-prefix}-test" - } +module "master-nodes" { + source = "./modules/aws-nodes" + ami = var.base-image + ec2-instance-type = var.aws-ec2-instance-type + subnet-id = module.aws-network.subnet.id + security-group-ids = [module.aws-network.default-security-group.id] + user-datas = data.template_file.node-user-datas + num-nodes = var.master-nodes + name-prefix = "${var.vm-name-prefix}-master" } -output "master-ips" { - value = aws_instance.test-node.*.public_ip +module "worker-nodes" { + source = "./modules/aws-nodes" + ami = var.base-image + ec2-instance-type = var.aws-ec2-instance-type + subnet-id = module.aws-network.subnet.id + security-group-ids = [module.aws-network.default-security-group.id] + user-datas = data.template_file.node-user-datas + num-nodes = var.worker-nodes + name-prefix = "${var.vm-name-prefix}-worker" } +################################################################################ +# libvirt +# To use the libvirt module, uncomment the libvirt modules/resources and comment +# out the aws modules/resources. +################################################################################ + # provider "libvirt" { # uri = var.libvirt-connection-url # } @@ -94,12 +103,16 @@ output "master-ips" { # type = "dir" # path = var.disk-image-dir # } -# -# # TODO REM move to other file? -# output "master-ips" { -# value = module.master-nodes.ips -# } -# -# output "worker-ips" { -# value = module.worker-nodes.ips -# } + +################################################################################ +# end libvirt +################################################################################ + +# TODO REM move to other file? +output "master-ips" { + value = module.master-nodes.ips +} + +output "worker-ips" { + value = module.worker-nodes.ips +} diff --git a/variables.tf b/variables.tf index 3455efa..52cf7a2 100644 --- a/variables.tf +++ b/variables.tf @@ -15,7 +15,7 @@ variable "aws-subnet-cidr-block" { variable "aws-vpc-cidr-block" { default = "10.0.0.0/16" - description = "The address space to be used for this VPC." + description = "The address space to be used for the VPC that all the AWS nodes will be in." } variable "disk-image-dir" { From 58f2a14a27cbd74aff2e627c97cc8b17df6beca6 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 12 Nov 2021 20:28:12 -0500 Subject: [PATCH 11/40] Fixed cloud-init for aws. - Organized main.tf - Still need to add cloud-init as an input parameter for the libvirt module. --- main.tf | 48 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/main.tf b/main.tf index c41a45d..cd96132 100644 --- a/main.tf +++ b/main.tf @@ -8,6 +8,36 @@ terraform { } } +################################################################################ +# cloud-init +################################################################################ + +data "template_file" "master-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-master-${count.index}" + } + count = var.master-nodes +} + +data "template_file" "worker-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-worker-${count.index}" + } + count = var.worker-nodes +} + +################################################################################ +# aws +# To use the aws module, uncomment the aws modules/resources and comment out the +# libvirt modules/resources. +################################################################################ + provider "aws" { region = "us-east-2" } @@ -30,23 +60,13 @@ resource "aws_key_pair" "key" { } } -data "template_file" "node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-${count.index}" - } - count = var.master-nodes -} - module "master-nodes" { source = "./modules/aws-nodes" ami = var.base-image ec2-instance-type = var.aws-ec2-instance-type subnet-id = module.aws-network.subnet.id security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.node-user-datas + user-datas = data.template_file.master-node-user-datas num-nodes = var.master-nodes name-prefix = "${var.vm-name-prefix}-master" } @@ -57,11 +77,15 @@ module "worker-nodes" { ec2-instance-type = var.aws-ec2-instance-type subnet-id = module.aws-network.subnet.id security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.node-user-datas + user-datas = data.template_file.worker-node-user-datas num-nodes = var.worker-nodes name-prefix = "${var.vm-name-prefix}-worker" } +################################################################################ +# end aws +################################################################################ + ################################################################################ # libvirt # To use the libvirt module, uncomment the libvirt modules/resources and comment From 097a150b570da1287bd6ab8d425fe880c99aef5a Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 12 Nov 2021 20:54:05 -0500 Subject: [PATCH 12/40] Made cloud-init configs an input to libvirt-nodes. --- main.tf | 38 ++++++++++++++++-------------- modules/libvirt-nodes/main.tf | 16 +++---------- modules/libvirt-nodes/variables.tf | 4 ++++ 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/main.tf b/main.tf index cd96132..20f8bfc 100644 --- a/main.tf +++ b/main.tf @@ -97,29 +97,31 @@ module "worker-nodes" { # } # # module "master-nodes" { -# source = "./modules/libvirt-nodes" -# pool-name = libvirt_pool.images.name -# name-prefix = "${var.vm-name-prefix}-master" -# num-nodes = var.master-nodes -# node-memory = var.node-memory -# node-vcpus = var.node-vcpus -# base-image = var.base-image -# root-admin-passwd = var.root-admin-passwd -# root-admin-pub-key = var.root-admin-pub-key +# source = "./modules/libvirt-nodes" +# pool-name = libvirt_pool.images.name +# name-prefix = "${var.vm-name-prefix}-master" +# num-nodes = var.master-nodes +# node-memory = var.node-memory +# node-vcpus = var.node-vcpus +# base-image = var.base-image +# root-admin-passwd = var.root-admin-passwd +# root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url +# user-datas = data.template_file.master-node-user-datas # } # # module "worker-nodes" { -# source = "./modules/libvirt-nodes" -# pool-name = libvirt_pool.images.name -# name-prefix = "${var.vm-name-prefix}-worker" -# num-nodes = var.worker-nodes -# node-memory = var.node-memory -# node-vcpus = var.node-vcpus -# base-image = var.base-image -# root-admin-passwd = var.root-admin-passwd -# root-admin-pub-key = var.root-admin-pub-key +# source = "./modules/libvirt-nodes" +# pool-name = libvirt_pool.images.name +# name-prefix = "${var.vm-name-prefix}-worker" +# num-nodes = var.worker-nodes +# node-memory = var.node-memory +# node-vcpus = var.node-vcpus +# base-image = var.base-image +# root-admin-passwd = var.root-admin-passwd +# root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url +# user-datas = data.template_file.worker-node-user-datas # } # # resource "libvirt_pool" "images" { diff --git a/modules/libvirt-nodes/main.tf b/modules/libvirt-nodes/main.tf index 0507076..9b792d0 100644 --- a/modules/libvirt-nodes/main.tf +++ b/modules/libvirt-nodes/main.tf @@ -20,31 +20,21 @@ resource "libvirt_volume" "node-images" { format = "qcow2" } -data "template_file" "node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.name-prefix}-${count.index}" - } - count = var.num-nodes -} - data "template_file" "network-config" { template = file("${path.module}/network_config.cfg") } resource "libvirt_cloudinit_disk" "node-inits" { name = "${var.name-prefix}-${count.index}-init" - user_data = element(data.template_file.node-user-datas.*.rendered, count.index) + user_data = element(var.user-datas.*.rendered, count.index) network_config = data.template_file.network-config.rendered pool = var.pool-name count = var.num-nodes } resource "libvirt_domain" "nodes" { - count = var.num-nodes - name = "${var.name-prefix}-${count.index}" + count = var.num-nodes + name = "${var.name-prefix}-${count.index}" memory = var.node-memory vcpu = var.node-vcpus diff --git a/modules/libvirt-nodes/variables.tf b/modules/libvirt-nodes/variables.tf index 93d7ad0..4253b62 100644 --- a/modules/libvirt-nodes/variables.tf +++ b/modules/libvirt-nodes/variables.tf @@ -24,6 +24,10 @@ variable "node-vcpus" { type = number } +variable "user-datas" { + description = "A list of cloud-init configs that get applied to their corresponding node." +} + variable "num-nodes" { description = "The number of nodes to create with this config." } From 4791bf9b6dee32542858bb9d1a4c2531b6bab582 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 12 Nov 2021 21:58:22 -0500 Subject: [PATCH 13/40] Forgot to add aws-nodes files. - Moved network_config.cfg to libvirt-nodes. --- .gitignore | 8 +++++ modules/aws-nodes/main.tf | 14 ++++++++ modules/aws-nodes/outputs.tf | 3 ++ modules/aws-nodes/variables.tf | 36 +++++++++++++++++++ .../libvirt-nodes/network_config.cfg | 0 5 files changed, 61 insertions(+) create mode 100644 modules/aws-nodes/main.tf create mode 100644 modules/aws-nodes/outputs.tf create mode 100644 modules/aws-nodes/variables.tf rename network_config.cfg => modules/libvirt-nodes/network_config.cfg (100%) diff --git a/.gitignore b/.gitignore index 02fd6f3..b47f542 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,11 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc + +################################################################################ +# end Pulled from github/gitignore 2021-11-10 commit 1a84870 +################################################################################ + +k8s-key* +STARTHERE + diff --git a/modules/aws-nodes/main.tf b/modules/aws-nodes/main.tf new file mode 100644 index 0000000..039fb20 --- /dev/null +++ b/modules/aws-nodes/main.tf @@ -0,0 +1,14 @@ +resource "aws_instance" "nodes" { + ami = var.ami + instance_type = var.ec2-instance-type + # key_name = aws_key_pair.debug1.key_name + associate_public_ip_address = true + subnet_id = var.subnet-id + vpc_security_group_ids = var.security-group-ids + user_data = element(var.user-datas.*.rendered, count.index) + count = var.num-nodes + + tags = { + Name = "${var.name-prefix}-${count.index}" + } +} diff --git a/modules/aws-nodes/outputs.tf b/modules/aws-nodes/outputs.tf new file mode 100644 index 0000000..d6faaf0 --- /dev/null +++ b/modules/aws-nodes/outputs.tf @@ -0,0 +1,3 @@ +output "ips" { + value = aws_instance.nodes.*.public_ip +} diff --git a/modules/aws-nodes/variables.tf b/modules/aws-nodes/variables.tf new file mode 100644 index 0000000..4968522 --- /dev/null +++ b/modules/aws-nodes/variables.tf @@ -0,0 +1,36 @@ +variable "ami" { + description = "The AWS AMI to be used for all the nodes" + type = string +} + +variable "ec2-instance-type" { + default = "t2.micro" + description = "The AWS instance type to use for all nodes." + type = string +} + +variable "name-prefix" { + default = "tf-node" + description = "This prefix will be applied to all names created by this module." + type = string +} + +variable "num-nodes" { + default = 1 + description = "The number of nodes to create from the given input parameters." + type = number +} + +variable "user-datas" { + description = "A list of cloud-init configs that get applied to their corresponding node." +} + +variable "subnet-id" { + description = "The ID of the subnet that all the nodes will be added to." + type = string +} + +variable "security-group-ids" { + description = "A list of security group IDs to be applied to all the nodes." + type = list(string) +} diff --git a/network_config.cfg b/modules/libvirt-nodes/network_config.cfg similarity index 100% rename from network_config.cfg rename to modules/libvirt-nodes/network_config.cfg From 925ff307dd8f721001d7fe1fa240b4d9d24b2c5a Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Sun, 14 Nov 2021 11:49:25 -0500 Subject: [PATCH 14/40] Creates 1 amazn, 1 ubuntu to test ansible. --- example.tfvars | 8 ++++- main.tf | 88 +++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 76 insertions(+), 20 deletions(-) diff --git a/example.tfvars b/example.tfvars index 33a78d3..61ac0d0 100644 --- a/example.tfvars +++ b/example.tfvars @@ -1,3 +1,5 @@ +vm-name-prefix = "docker-ansible-test" + # A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0' # equates to any ip address. admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] @@ -21,7 +23,11 @@ aws-ec2-instance-type = "t2.micro" # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free # base-image = "ami-00be885d550dcee43" # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free -base-image = "ami-0dd0ccab7e2801812" +# base-image = "ami-0dd0ccab7e2801812" +# Ubuntu Server 20.04 LTS (HVM), SSD Volume Type +# us-east-2 - (64-bit x86) - 2021.11.12 - free +base-image = "ami-0629230e074c580f2" + # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" # From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 # base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" diff --git a/main.tf b/main.tf index 20f8bfc..f8e8418 100644 --- a/main.tf +++ b/main.tf @@ -12,24 +12,44 @@ terraform { # cloud-init ################################################################################ -data "template_file" "master-node-user-datas" { +# data "template_file" "master-node-user-datas" { +# template = file("${path.module}/cloud_init.cfg") +# vars = { +# admin-passwd = "${var.root-admin-passwd}" +# admin-pub-key = "${var.root-admin-pub-key}" +# hostname = "${var.vm-name-prefix}-master-${count.index}" +# } +# count = var.master-nodes +# } +# +# data "template_file" "worker-node-user-datas" { +# template = file("${path.module}/cloud_init.cfg") +# vars = { +# admin-passwd = "${var.root-admin-passwd}" +# admin-pub-key = "${var.root-admin-pub-key}" +# hostname = "${var.vm-name-prefix}-worker-${count.index}" +# } +# count = var.worker-nodes +# } + +data "template_file" "amzn2-node-user-datas" { template = file("${path.module}/cloud_init.cfg") vars = { admin-passwd = "${var.root-admin-passwd}" admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-master-${count.index}" + hostname = "${var.vm-name-prefix}-amzn2-${count.index}" } - count = var.master-nodes + count = 1 } -data "template_file" "worker-node-user-datas" { +data "template_file" "ubuntu-node-user-datas" { template = file("${path.module}/cloud_init.cfg") vars = { admin-passwd = "${var.root-admin-passwd}" admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-worker-${count.index}" + hostname = "${var.vm-name-prefix}-ubuntu-${count.index}" } - count = var.worker-nodes + count = 1 } ################################################################################ @@ -60,28 +80,50 @@ resource "aws_key_pair" "key" { } } -module "master-nodes" { +module "amzn2-nodes" { source = "./modules/aws-nodes" ami = var.base-image ec2-instance-type = var.aws-ec2-instance-type subnet-id = module.aws-network.subnet.id security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.master-node-user-datas - num-nodes = var.master-nodes - name-prefix = "${var.vm-name-prefix}-master" + user-datas = data.template_file.amzn2-node-user-datas + num-nodes = 1 + name-prefix = "${var.vm-name-prefix}-amzn2" } -module "worker-nodes" { +module "ubuntu-nodes" { source = "./modules/aws-nodes" ami = var.base-image ec2-instance-type = var.aws-ec2-instance-type subnet-id = module.aws-network.subnet.id security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.worker-node-user-datas - num-nodes = var.worker-nodes - name-prefix = "${var.vm-name-prefix}-worker" + user-datas = data.template_file.ubuntu-node-user-datas + num-nodes = 1 + name-prefix = "${var.vm-name-prefix}-ubuntu" } +# module "master-nodes" { +# source = "./modules/aws-nodes" +# ami = var.base-image +# ec2-instance-type = var.aws-ec2-instance-type +# subnet-id = module.aws-network.subnet.id +# security-group-ids = [module.aws-network.default-security-group.id] +# user-datas = data.template_file.master-node-user-datas +# num-nodes = var.master-nodes +# name-prefix = "${var.vm-name-prefix}-master" +# } +# +# module "worker-nodes" { +# source = "./modules/aws-nodes" +# ami = var.base-image +# ec2-instance-type = var.aws-ec2-instance-type +# subnet-id = module.aws-network.subnet.id +# security-group-ids = [module.aws-network.default-security-group.id] +# user-datas = data.template_file.worker-node-user-datas +# num-nodes = var.worker-nodes +# name-prefix = "${var.vm-name-prefix}-worker" +# } + ################################################################################ # end aws ################################################################################ @@ -134,11 +176,19 @@ module "worker-nodes" { # end libvirt ################################################################################ -# TODO REM move to other file? -output "master-ips" { - value = module.master-nodes.ips +output "amzn2-ips" { + value = module.amzn2-nodes.ips } -output "worker-ips" { - value = module.worker-nodes.ips +output "ubuntu-ips" { + value = module.ubuntu-nodes.ips } + +# TODO REM move to other file? +# output "master-ips" { +# value = module.master-nodes.ips +# } +# +# output "worker-ips" { +# value = module.worker-nodes.ips +# } From 678c44d462ce82730e0a6aea7641d5192f4a5342 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Mon, 15 Nov 2021 09:14:33 -0500 Subject: [PATCH 15/40] Added arch AWS image. --- README.md | 12 ++++++++++++ example.tfvars | 19 +++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b27f538..e495e0a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +A Terraform script to create k8s nodes. This script has modules for creating the +nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS. + +The modules allow you create N VMs of a specific type. So you could create 1 +master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS +VMs, or whatever fits your needs. + +Cloud-Init +---------------------------------------- + +Both the libvirt and aws modules use cloud-init for initial configuration of the +VMs. Dependencies ---------------------------------------- diff --git a/example.tfvars b/example.tfvars index 61ac0d0..37c353f 100644 --- a/example.tfvars +++ b/example.tfvars @@ -20,13 +20,28 @@ aws-ec2-instance-type = "t2.micro" # 4 GiB, 2 vcpus # aws-ec2-instnce-type = "t2.medium" +################################################################################ +# AWS images (AMIs) +################################################################################ + +## Amazon Linux 2 # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free # base-image = "ami-00be885d550dcee43" # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free -# base-image = "ami-0dd0ccab7e2801812" +base-image = "ami-0dd0ccab7e2801812" + +## Ubuntu # Ubuntu Server 20.04 LTS (HVM), SSD Volume Type # us-east-2 - (64-bit x86) - 2021.11.12 - free -base-image = "ami-0629230e074c580f2" +# base-image = "ami-0629230e074c580f2" + +## Arch linux +# arch-linux-lts-hvm-2021.06.02.x86_64-ebs - us-east-2 +# base-image = "ami-02653f06de985e3ba" + +################################################################################ +# libvirt images +################################################################################ # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" # From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 From 6aed115478a31990766126d5405a40f9601e5401 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Tue, 16 Nov 2021 14:36:57 -0500 Subject: [PATCH 16/40] Added CentOS 7 & 8 and Arch --- .gitignore | 1 + example.tfvars | 6 ++++ get-vm-ips.sh | 63 +++++++++++++++++++++++++++++++++-------- main.tf | 77 +++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 135 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index b47f542..7103111 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,5 @@ terraform.rc k8s-key* STARTHERE +inventory diff --git a/example.tfvars b/example.tfvars index 37c353f..4b6100d 100644 --- a/example.tfvars +++ b/example.tfvars @@ -30,6 +30,12 @@ aws-ec2-instance-type = "t2.micro" # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free base-image = "ami-0dd0ccab7e2801812" +## CentOS +# CentOS 7.9.2009 x86_64 - us-east-2 - 2021-11-15 +# base-image = "ami-00f8e2c955f7ffa9b" +# CentOS 8.4.2105 x86_64 - us-east-2 - 2021-11015 +# base-image = "ami-057cacbfbbb471bb3" + ## Ubuntu # Ubuntu Server 20.04 LTS (HVM), SSD Volume Type # us-east-2 - (64-bit x86) - 2021.11.12 - free diff --git a/get-vm-ips.sh b/get-vm-ips.sh index 25e36fe..8eceb68 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -1,19 +1,60 @@ #!/bin/sh +# This script will create environment variables for all of the output IPs. It +# will also create a `ANSIBLE_INV` variable that will be a comma separated +# string of all the IPs. A anisble inventory file called "inventory is created +# as well. +# # Use eval $(./get-vm-ips.sh) to set env vars for ips. terraform refresh > /dev/null -IPS_JSON="$(terraform show -json | jq '.values.outputs')" +# All terraform outputs in json format. +OUTPUTS_JSON="$( + terraform show -json | \ + jq '.values.outputs' | \ + sed 's/-/_/g')" +# Just the IP address outputs in json format. Also all '-' characters are +# replaced by '_' becuase '-' causes jq some problems. +IPS_JSON="$( + echo $OUTPUTS_JSON | \ + jq 'to_entries | .[] | select(.key | contains("ips"))')" +# An array of all node "types" +NODE_TYPE_ARRAY="$( + echo $IPS_JSON | \ + jq '.key' | \ + sed 's/"//g' | \ + sed -z 's/\n/ /g;s/ $/\n/g')" -echo $IPS_JSON | \ - jq '."master-ips".value[]' | \ - nl -v 0 | \ - awk '{print "export MASTER" $1 "=" $2}' | \ - sed 's/"//g' +# Loop over all the node types and create an export line for each IP. +VM_IP_EXPORTS="$( + for TYPE in $NODE_TYPE_ARRAY; do -echo $IPS_JSON | \ - jq '."worker-ips".value[]' | \ - nl -v 0 | \ - awk '{print "export WORKER" $1 "=" $2}' | \ - sed 's/"//g' + # Convert type, converts "master-ips" to "MASTER" + TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)" + echo "$OUTPUTS_JSON" | \ + jq '.'"$TYPE"'.value[]' | \ + # Add line numbers starting with 0. + nl -v 0 | \ + # Print an export string with a type placeholder "__TYPE__". + awk '{print "export __TYPE__" $1 "=" $2}' | \ + sed s/__TYPE__/$TYPE_UPPER/g + done)" + +ANSIBLE_INV="$( + echo "$VM_IP_EXPORTS" | \ + sed 's/"//g' | \ + sed 's/^.*=//g' | \ + sed -z 's/\n/,/g;s/,$/\n/g')" + +# Create an inventory file for ansible. +echo "[k8s_nodes]" > inventory +echo $VM_IP_EXPORTS | \ + sed 's/"//g' | \ + sed 's/export //g' | \ + sed 's/ /\n/g' | \ + sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \ + >> inventory + +echo $VM_IP_EXPORTS | sed 's/" /"\n/g' +echo export ANSIBLE_INV=$ANSIBLE_INV diff --git a/main.tf b/main.tf index f8e8418..50982d6 100644 --- a/main.tf +++ b/main.tf @@ -52,6 +52,36 @@ data "template_file" "ubuntu-node-user-datas" { count = 1 } +data "template_file" "arch-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-arch-${count.index}" + } + count = 1 +} + +data "template_file" "centos7-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-centos7-${count.index}" + } + count = 1 +} + +data "template_file" "centos8-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-centos8-${count.index}" + } + count = 1 +} + ################################################################################ # aws # To use the aws module, uncomment the aws modules/resources and comment out the @@ -93,7 +123,7 @@ module "amzn2-nodes" { module "ubuntu-nodes" { source = "./modules/aws-nodes" - ami = var.base-image + ami = "ami-0629230e074c580f2" ec2-instance-type = var.aws-ec2-instance-type subnet-id = module.aws-network.subnet.id security-group-ids = [module.aws-network.default-security-group.id] @@ -102,6 +132,39 @@ module "ubuntu-nodes" { name-prefix = "${var.vm-name-prefix}-ubuntu" } +module "arch-nodes" { + source = "./modules/aws-nodes" + ami = "ami-02653f06de985e3ba" + ec2-instance-type = var.aws-ec2-instance-type + subnet-id = module.aws-network.subnet.id + security-group-ids = [module.aws-network.default-security-group.id] + user-datas = data.template_file.ubuntu-node-user-datas + num-nodes = 1 + name-prefix = "${var.vm-name-prefix}-arch" +} + +module "centos7-nodes" { + source = "./modules/aws-nodes" + ami = "ami-00f8e2c955f7ffa9b" + ec2-instance-type = var.aws-ec2-instance-type + subnet-id = module.aws-network.subnet.id + security-group-ids = [module.aws-network.default-security-group.id] + user-datas = data.template_file.ubuntu-node-user-datas + num-nodes = 1 + name-prefix = "${var.vm-name-prefix}-centos7" +} + +module "centos8-nodes" { + source = "./modules/aws-nodes" + ami = "ami-057cacbfbbb471bb3" + ec2-instance-type = var.aws-ec2-instance-type + subnet-id = module.aws-network.subnet.id + security-group-ids = [module.aws-network.default-security-group.id] + user-datas = data.template_file.ubuntu-node-user-datas + num-nodes = 1 + name-prefix = "${var.vm-name-prefix}-centos8" +} + # module "master-nodes" { # source = "./modules/aws-nodes" # ami = var.base-image @@ -184,6 +247,18 @@ output "ubuntu-ips" { value = module.ubuntu-nodes.ips } +output "arch-ips" { + value = module.arch-nodes.ips +} + +output "centos7-ips" { + value = module.centos7-nodes.ips +} + +output "centos8-ips" { + value = module.centos8-nodes.ips +} + # TODO REM move to other file? # output "master-ips" { # value = module.master-nodes.ips From 6b4c56e5fb5f7e615da30dbfcc4bbb54dee73cce Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Tue, 16 Nov 2021 14:38:06 -0500 Subject: [PATCH 17/40] Create ansible invetory with get-vm-ips.sh --- .gitignore | 1 + example.tfvars | 27 ++++++++++++++++++++++ get-vm-ips.sh | 63 +++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/.gitignore b/.gitignore index b47f542..7103111 100644 --- a/.gitignore +++ b/.gitignore @@ -45,4 +45,5 @@ terraform.rc k8s-key* STARTHERE +inventory diff --git a/example.tfvars b/example.tfvars index 33a78d3..4b6100d 100644 --- a/example.tfvars +++ b/example.tfvars @@ -1,3 +1,5 @@ +vm-name-prefix = "docker-ansible-test" + # A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0' # equates to any ip address. admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] @@ -18,10 +20,35 @@ aws-ec2-instance-type = "t2.micro" # 4 GiB, 2 vcpus # aws-ec2-instnce-type = "t2.medium" +################################################################################ +# AWS images (AMIs) +################################################################################ + +## Amazon Linux 2 # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free # base-image = "ami-00be885d550dcee43" # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free base-image = "ami-0dd0ccab7e2801812" + +## CentOS +# CentOS 7.9.2009 x86_64 - us-east-2 - 2021-11-15 +# base-image = "ami-00f8e2c955f7ffa9b" +# CentOS 8.4.2105 x86_64 - us-east-2 - 2021-11015 +# base-image = "ami-057cacbfbbb471bb3" + +## Ubuntu +# Ubuntu Server 20.04 LTS (HVM), SSD Volume Type +# us-east-2 - (64-bit x86) - 2021.11.12 - free +# base-image = "ami-0629230e074c580f2" + +## Arch linux +# arch-linux-lts-hvm-2021.06.02.x86_64-ebs - us-east-2 +# base-image = "ami-02653f06de985e3ba" + +################################################################################ +# libvirt images +################################################################################ + # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" # From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 # base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" diff --git a/get-vm-ips.sh b/get-vm-ips.sh index 25e36fe..8eceb68 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -1,19 +1,60 @@ #!/bin/sh +# This script will create environment variables for all of the output IPs. It +# will also create a `ANSIBLE_INV` variable that will be a comma separated +# string of all the IPs. A anisble inventory file called "inventory is created +# as well. +# # Use eval $(./get-vm-ips.sh) to set env vars for ips. terraform refresh > /dev/null -IPS_JSON="$(terraform show -json | jq '.values.outputs')" +# All terraform outputs in json format. +OUTPUTS_JSON="$( + terraform show -json | \ + jq '.values.outputs' | \ + sed 's/-/_/g')" +# Just the IP address outputs in json format. Also all '-' characters are +# replaced by '_' becuase '-' causes jq some problems. +IPS_JSON="$( + echo $OUTPUTS_JSON | \ + jq 'to_entries | .[] | select(.key | contains("ips"))')" +# An array of all node "types" +NODE_TYPE_ARRAY="$( + echo $IPS_JSON | \ + jq '.key' | \ + sed 's/"//g' | \ + sed -z 's/\n/ /g;s/ $/\n/g')" -echo $IPS_JSON | \ - jq '."master-ips".value[]' | \ - nl -v 0 | \ - awk '{print "export MASTER" $1 "=" $2}' | \ - sed 's/"//g' +# Loop over all the node types and create an export line for each IP. +VM_IP_EXPORTS="$( + for TYPE in $NODE_TYPE_ARRAY; do -echo $IPS_JSON | \ - jq '."worker-ips".value[]' | \ - nl -v 0 | \ - awk '{print "export WORKER" $1 "=" $2}' | \ - sed 's/"//g' + # Convert type, converts "master-ips" to "MASTER" + TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)" + echo "$OUTPUTS_JSON" | \ + jq '.'"$TYPE"'.value[]' | \ + # Add line numbers starting with 0. + nl -v 0 | \ + # Print an export string with a type placeholder "__TYPE__". + awk '{print "export __TYPE__" $1 "=" $2}' | \ + sed s/__TYPE__/$TYPE_UPPER/g + done)" + +ANSIBLE_INV="$( + echo "$VM_IP_EXPORTS" | \ + sed 's/"//g' | \ + sed 's/^.*=//g' | \ + sed -z 's/\n/,/g;s/,$/\n/g')" + +# Create an inventory file for ansible. +echo "[k8s_nodes]" > inventory +echo $VM_IP_EXPORTS | \ + sed 's/"//g' | \ + sed 's/export //g' | \ + sed 's/ /\n/g' | \ + sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \ + >> inventory + +echo $VM_IP_EXPORTS | sed 's/" /"\n/g' +echo export ANSIBLE_INV=$ANSIBLE_INV From 7c266ec4aa37af12f11cc7b8afa203781d207edc Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Tue, 16 Nov 2021 21:01:34 -0500 Subject: [PATCH 18/40] Added aws-ami module. --- main.tf | 46 +++++++++++++++++------------ modules/aws-amis/main.tf | 58 +++++++++++++++++++++++++++++++++++++ modules/aws-amis/outputs.tf | 3 ++ 3 files changed, 88 insertions(+), 19 deletions(-) create mode 100644 modules/aws-amis/main.tf create mode 100644 modules/aws-amis/outputs.tf diff --git a/main.tf b/main.tf index 50982d6..7113df0 100644 --- a/main.tf +++ b/main.tf @@ -12,25 +12,25 @@ terraform { # cloud-init ################################################################################ -# data "template_file" "master-node-user-datas" { -# template = file("${path.module}/cloud_init.cfg") -# vars = { -# admin-passwd = "${var.root-admin-passwd}" -# admin-pub-key = "${var.root-admin-pub-key}" -# hostname = "${var.vm-name-prefix}-master-${count.index}" -# } -# count = var.master-nodes -# } -# -# data "template_file" "worker-node-user-datas" { -# template = file("${path.module}/cloud_init.cfg") -# vars = { -# admin-passwd = "${var.root-admin-passwd}" -# admin-pub-key = "${var.root-admin-pub-key}" -# hostname = "${var.vm-name-prefix}-worker-${count.index}" -# } -# count = var.worker-nodes -# } +data "template_file" "master-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-master-${count.index}" + } + count = var.master-nodes +} + +data "template_file" "worker-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-worker-${count.index}" + } + count = var.worker-nodes +} data "template_file" "amzn2-node-user-datas" { template = file("${path.module}/cloud_init.cfg") @@ -92,6 +92,10 @@ provider "aws" { region = "us-east-2" } +module "aws-amis" { + source = "./modules/aws-amis" +} + module "aws-network" { source = "./modules/aws-network" name-prefix = var.vm-name-prefix @@ -165,6 +169,10 @@ module "centos8-nodes" { name-prefix = "${var.vm-name-prefix}-centos8" } +output "amis" { + value = module.aws-amis.amis +} + # module "master-nodes" { # source = "./modules/aws-nodes" # ami = var.base-image diff --git a/modules/aws-amis/main.tf b/modules/aws-amis/main.tf new file mode 100644 index 0000000..1949653 --- /dev/null +++ b/modules/aws-amis/main.tf @@ -0,0 +1,58 @@ +locals { + amis = { + amzn2 = { + owner-id = "137112412989" + name = "amzn2-ami-hvm-2*x86_64-gp2" + }, + ubuntu = { + owner-id = "099720109477" + name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*" + }, + centos7 = { + owner-id = "125523088429" + name = "CentOS 7.*x86_64" + }, + centos8 = { + owner-id = "125523088429" + name = "CentOS 8.*x86_64" + }, + arch = { + owner-id = "093273469852" + name = "arch-linux-lts-hvm*x86_64-ebs" + }, + rhel7 = { + owner-id = "309956199498" + name = "RHEL-7.*HVM*x86_64*GP2" + }, + rhel8 = { + owner-id = "309956199498" + name = "RHEL-8.*HVM*x86_64*GP2" + } + } +} + +data "aws_ami" "amis" { + for_each = local.amis + most_recent = true + owners = [each.value.owner-id] + + filter { + name = "name" + values = [each.value.name] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } +} diff --git a/modules/aws-amis/outputs.tf b/modules/aws-amis/outputs.tf new file mode 100644 index 0000000..3174fd9 --- /dev/null +++ b/modules/aws-amis/outputs.tf @@ -0,0 +1,3 @@ +output "amis" { + value = tomap({ for type, ami in data.aws_ami.amis : type => ami.id }) +} From bb80e8f3da194ab409e5041d0cfaf6f255566eae Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Tue, 16 Nov 2021 21:02:41 -0500 Subject: [PATCH 19/40] Recomment user-data from master. --- main.tf | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/main.tf b/main.tf index 7113df0..49485b4 100644 --- a/main.tf +++ b/main.tf @@ -12,25 +12,25 @@ terraform { # cloud-init ################################################################################ -data "template_file" "master-node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-master-${count.index}" - } - count = var.master-nodes -} - -data "template_file" "worker-node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-worker-${count.index}" - } - count = var.worker-nodes -} +# data "template_file" "master-node-user-datas" { +# template = file("${path.module}/cloud_init.cfg") +# vars = { +# admin-passwd = "${var.root-admin-passwd}" +# admin-pub-key = "${var.root-admin-pub-key}" +# hostname = "${var.vm-name-prefix}-master-${count.index}" +# } +# count = var.master-nodes +# } +# +# data "template_file" "worker-node-user-datas" { +# template = file("${path.module}/cloud_init.cfg") +# vars = { +# admin-passwd = "${var.root-admin-passwd}" +# admin-pub-key = "${var.root-admin-pub-key}" +# hostname = "${var.vm-name-prefix}-worker-${count.index}" +# } +# count = var.worker-nodes +# } data "template_file" "amzn2-node-user-datas" { template = file("${path.module}/cloud_init.cfg") From a730a9d94045c2ec34ebd2d35873d7f9847a3357 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 17 Nov 2021 14:18:31 -0500 Subject: [PATCH 20/40] Added a aws_ami module. --- main.tf | 8 +++++ modules/aws-amis/main.tf | 58 +++++++++++++++++++++++++++++++++++++ modules/aws-amis/outputs.tf | 3 ++ 3 files changed, 69 insertions(+) create mode 100644 modules/aws-amis/main.tf create mode 100644 modules/aws-amis/outputs.tf diff --git a/main.tf b/main.tf index 20f8bfc..69c7943 100644 --- a/main.tf +++ b/main.tf @@ -42,6 +42,10 @@ provider "aws" { region = "us-east-2" } +module "aws-amis" { + source = "./modules/aws-amis" +} + module "aws-network" { source = "./modules/aws-network" name-prefix = var.vm-name-prefix @@ -82,6 +86,10 @@ module "worker-nodes" { name-prefix = "${var.vm-name-prefix}-worker" } +output "amis" { + value = module.aws-amis.amis +} + ################################################################################ # end aws ################################################################################ diff --git a/modules/aws-amis/main.tf b/modules/aws-amis/main.tf new file mode 100644 index 0000000..1949653 --- /dev/null +++ b/modules/aws-amis/main.tf @@ -0,0 +1,58 @@ +locals { + amis = { + amzn2 = { + owner-id = "137112412989" + name = "amzn2-ami-hvm-2*x86_64-gp2" + }, + ubuntu = { + owner-id = "099720109477" + name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*" + }, + centos7 = { + owner-id = "125523088429" + name = "CentOS 7.*x86_64" + }, + centos8 = { + owner-id = "125523088429" + name = "CentOS 8.*x86_64" + }, + arch = { + owner-id = "093273469852" + name = "arch-linux-lts-hvm*x86_64-ebs" + }, + rhel7 = { + owner-id = "309956199498" + name = "RHEL-7.*HVM*x86_64*GP2" + }, + rhel8 = { + owner-id = "309956199498" + name = "RHEL-8.*HVM*x86_64*GP2" + } + } +} + +data "aws_ami" "amis" { + for_each = local.amis + most_recent = true + owners = [each.value.owner-id] + + filter { + name = "name" + values = [each.value.name] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } +} diff --git a/modules/aws-amis/outputs.tf b/modules/aws-amis/outputs.tf new file mode 100644 index 0000000..3174fd9 --- /dev/null +++ b/modules/aws-amis/outputs.tf @@ -0,0 +1,3 @@ +output "amis" { + value = tomap({ for type, ami in data.aws_ami.amis : type => ami.id }) +} From df53ae047d0b11dcb1aa378ed107cda191932096 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 17 Nov 2021 21:50:07 -0500 Subject: [PATCH 21/40] AWS nodes and user-data created from config map. --- example.tfvars | 23 ++++++- main.tf | 83 +++++++++----------------- modules/cloud-init-config/main.tf | 9 +++ modules/cloud-init-config/outputs.tf | 3 + modules/cloud-init-config/variables.tf | 22 +++++++ variables.tf | 5 ++ 6 files changed, 90 insertions(+), 55 deletions(-) create mode 100644 modules/cloud-init-config/main.tf create mode 100644 modules/cloud-init-config/outputs.tf create mode 100644 modules/cloud-init-config/variables.tf diff --git a/example.tfvars b/example.tfvars index 4b6100d..4ea7b3c 100644 --- a/example.tfvars +++ b/example.tfvars @@ -1,4 +1,4 @@ -vm-name-prefix = "docker-ansible-test" +vm-name-prefix = "k8s-tf" # A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0' # equates to any ip address. @@ -13,6 +13,10 @@ worker-nodes = 2 node-memory = 2048 node-vcpus = 2 +################################################################################ +# AWS EC2 instance types +################################################################################ + # 1 GiB, 1 vcpu, only one that is free. # This one won't work with k8s because it requires at least 2 vcpus. aws-ec2-instance-type = "t2.micro" @@ -20,6 +24,19 @@ aws-ec2-instance-type = "t2.micro" # 4 GiB, 2 vcpus # aws-ec2-instnce-type = "t2.medium" +################################################################################ + +nodes-config = { + "master" = { + base-image = "ami-0dd0ccab7e2801812" + num = 1 + }, + "worker" = { + base-image = "ami-0dd0ccab7e2801812" + num = 2 + } +} + ################################################################################ # AWS images (AMIs) ################################################################################ @@ -53,6 +70,10 @@ base-image = "ami-0dd0ccab7e2801812" # From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 # base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" +################################################################################ +# Keys/Passwords +################################################################################ + # Password hash created with: # python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))' # where "linux" is the password. diff --git a/main.tf b/main.tf index 69c7943..ce42da9 100644 --- a/main.tf +++ b/main.tf @@ -1,8 +1,8 @@ terraform { - required_version = ">= 0.13" + required_version = ">= 1.0.8" required_providers { libvirt = { - source = "dmacvicar/libvirt" + source = "dmacvicar/libvirt" version = "0.6.11" } } @@ -12,24 +12,14 @@ terraform { # cloud-init ################################################################################ -data "template_file" "master-node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-master-${count.index}" - } - count = var.master-nodes -} - -data "template_file" "worker-node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-worker-${count.index}" - } - count = var.worker-nodes +module "cloud-init-config" { + for_each = var.nodes-config + source = "./modules/cloud-init-config" + cloud-init-template = "${path.module}/cloud_init.cfg" + hostname-prefix = "${var.vm-name-prefix}-${each.key}" + num = each.value.num + root-admin-passwd = var.root-admin-passwd + root-admin-pub-key = var.root-admin-pub-key } ################################################################################ @@ -42,16 +32,20 @@ provider "aws" { region = "us-east-2" } -module "aws-amis" { - source = "./modules/aws-amis" -} +# This module will grab the latest ami for a variety of distros. +# module "aws-amis" { +# source = "./modules/aws-amis" +# } +# output "amis" { +# value = module.aws-amis.amis +# } module "aws-network" { - source = "./modules/aws-network" - name-prefix = var.vm-name-prefix - vpc-cidr-block = var.aws-vpc-cidr-block + source = "./modules/aws-network" + name-prefix = var.vm-name-prefix + vpc-cidr-block = var.aws-vpc-cidr-block subnet-cidr-block = var.aws-subnet-cidr-block - admin-ips = var.admin-ips + admin-ips = var.admin-ips } # This key pair is not actually used. Keys are added to the nodes via cloud-init @@ -64,30 +58,16 @@ resource "aws_key_pair" "key" { } } -module "master-nodes" { +module "nodes" { + for_each = var.nodes-config source = "./modules/aws-nodes" - ami = var.base-image + ami = each.value.base-image ec2-instance-type = var.aws-ec2-instance-type subnet-id = module.aws-network.subnet.id security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.master-node-user-datas - num-nodes = var.master-nodes - name-prefix = "${var.vm-name-prefix}-master" -} - -module "worker-nodes" { - source = "./modules/aws-nodes" - ami = var.base-image - ec2-instance-type = var.aws-ec2-instance-type - subnet-id = module.aws-network.subnet.id - security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.worker-node-user-datas - num-nodes = var.worker-nodes - name-prefix = "${var.vm-name-prefix}-worker" -} - -output "amis" { - value = module.aws-amis.amis + user-datas = lookup(module.cloud-init-config, each.key, null).user-datas + num-nodes = each.value.num + name-prefix = "${var.vm-name-prefix}-${each.key}" } ################################################################################ @@ -142,11 +122,6 @@ output "amis" { # end libvirt ################################################################################ -# TODO REM move to other file? -output "master-ips" { - value = module.master-nodes.ips -} - -output "worker-ips" { - value = module.worker-nodes.ips +output "ips" { + value = { for type, node in module.nodes : type => node.ips } } diff --git a/modules/cloud-init-config/main.tf b/modules/cloud-init-config/main.tf new file mode 100644 index 0000000..e1c75c4 --- /dev/null +++ b/modules/cloud-init-config/main.tf @@ -0,0 +1,9 @@ +data "template_file" "user-datas" { + template = file("${var.cloud-init-template}") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.hostname-prefix}-${count.index}" + } + count = var.num +} diff --git a/modules/cloud-init-config/outputs.tf b/modules/cloud-init-config/outputs.tf new file mode 100644 index 0000000..15415d4 --- /dev/null +++ b/modules/cloud-init-config/outputs.tf @@ -0,0 +1,3 @@ +output "user-datas" { + value = data.template_file.user-datas +} diff --git a/modules/cloud-init-config/variables.tf b/modules/cloud-init-config/variables.tf new file mode 100644 index 0000000..fc4f437 --- /dev/null +++ b/modules/cloud-init-config/variables.tf @@ -0,0 +1,22 @@ +variable "cloud-init-template" { + default = "../../cloud_init.cfg" + description = "The path to the cloud-init config template." + type = string +} + +variable "hostname-prefix" { + description = "This prefix wil be applied as a prefix for the hostnames." +} + +variable "num" { + description = "The number of user-datas to create with these parameters." +} + +variable "root-admin-passwd" { + description = "This value will be substituted for any occurence of 'admin-password' in the cloud-init config template." +} + +variable "root-admin-pub-key" { + description = "This value will be substituted for any occurence of 'admin-pub-key' in the cloud-init config template." +} + diff --git a/variables.tf b/variables.tf index 52cf7a2..e8f3211 100644 --- a/variables.tf +++ b/variables.tf @@ -43,6 +43,11 @@ variable "node-vcpus" { type = number } +variable "nodes-config" { + description = "A config that declares how many nodes of each type you want created." + type = map(object({base-image=string,num=number})) +} + variable "root-admin-passwd" { description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." } From 1e81dc036b9d404f4ff61c37547095b2e1f9766c Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 18 Nov 2021 09:45:16 -0500 Subject: [PATCH 22/40] Make libvirt module accept config map. --- example.tfvars | 3 --- main.tf | 25 ++++++------------------- modules/libvirt-nodes/main.tf | 4 ---- 3 files changed, 6 insertions(+), 26 deletions(-) diff --git a/example.tfvars b/example.tfvars index 4ea7b3c..df3f604 100644 --- a/example.tfvars +++ b/example.tfvars @@ -7,9 +7,6 @@ admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] disk-image-dir = "/path/to/disk/pool/" libvirt-connection-url = "qemu+ssh://@/system" -master-nodes = 1 -worker-nodes = 2 - node-memory = 2048 node-vcpus = 2 diff --git a/main.tf b/main.tf index ce42da9..7510f3e 100644 --- a/main.tf +++ b/main.tf @@ -84,32 +84,19 @@ module "nodes" { # uri = var.libvirt-connection-url # } # -# module "master-nodes" { +# module "nodes" { +# for_each = var.nodes-config # source = "./modules/libvirt-nodes" # pool-name = libvirt_pool.images.name -# name-prefix = "${var.vm-name-prefix}-master" -# num-nodes = var.master-nodes +# name-prefix = "${var.vm-name-prefix}-${each.key}" +# num-nodes = each.value.num # node-memory = var.node-memory # node-vcpus = var.node-vcpus -# base-image = var.base-image +# base-image = each.value.base-image # root-admin-passwd = var.root-admin-passwd # root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url -# user-datas = data.template_file.master-node-user-datas -# } -# -# module "worker-nodes" { -# source = "./modules/libvirt-nodes" -# pool-name = libvirt_pool.images.name -# name-prefix = "${var.vm-name-prefix}-worker" -# num-nodes = var.worker-nodes -# node-memory = var.node-memory -# node-vcpus = var.node-vcpus -# base-image = var.base-image -# root-admin-passwd = var.root-admin-passwd -# root-admin-pub-key = var.root-admin-pub-key -# libvirt-connection-url = var.libvirt-connection-url -# user-datas = data.template_file.worker-node-user-datas +# user-datas = lookup(module.cloud-init-config, each.key, null).user-datas # } # # resource "libvirt_pool" "images" { diff --git a/modules/libvirt-nodes/main.tf b/modules/libvirt-nodes/main.tf index 9b792d0..cbd0dac 100644 --- a/modules/libvirt-nodes/main.tf +++ b/modules/libvirt-nodes/main.tf @@ -8,10 +8,6 @@ terraform { } } -provider "libvirt" { - uri = var.libvirt-connection-url -} - resource "libvirt_volume" "node-images" { name = "${var.name-prefix}-${count.index}" pool = var.pool-name From dab37be485b3848cf7d9b86d5b65282b8583a210 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 18 Nov 2021 10:45:58 -0500 Subject: [PATCH 23/40] Update get-vm-ips.sh to work with new map config. --- get-vm-ips.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/get-vm-ips.sh b/get-vm-ips.sh index 8eceb68..230b5cf 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -22,7 +22,7 @@ IPS_JSON="$( # An array of all node "types" NODE_TYPE_ARRAY="$( echo $IPS_JSON | \ - jq '.key' | \ + jq '.value.value | to_entries | .[] | .key' | \ sed 's/"//g' | \ sed -z 's/\n/ /g;s/ $/\n/g')" @@ -32,12 +32,12 @@ VM_IP_EXPORTS="$( # Convert type, converts "master-ips" to "MASTER" TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)" - echo "$OUTPUTS_JSON" | \ - jq '.'"$TYPE"'.value[]' | \ + echo "$IPS_JSON" | \ + jq '.value.value.'"$TYPE"'[]' | \ # Add line numbers starting with 0. nl -v 0 | \ # Print an export string with a type placeholder "__TYPE__". - awk '{print "export __TYPE__" $1 "=" $2}' | \ + awk '{print "export __TYPE___" $1 "=" $2}' | \ sed s/__TYPE__/$TYPE_UPPER/g done)" From 17d70ea35181f44a4070d62fcf344d2a044dc62c Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 18 Nov 2021 11:19:59 -0500 Subject: [PATCH 24/40] Moved node-config to be local, added AMI vars. --- example.tfvars | 13 ------------- get-vm-ips.sh | 1 + main.tf | 22 ++++++++++++++++++---- variables.tf | 41 ++++++++++++++++++++++++++++++++++++----- 4 files changed, 55 insertions(+), 22 deletions(-) diff --git a/example.tfvars b/example.tfvars index df3f604..41ca648 100644 --- a/example.tfvars +++ b/example.tfvars @@ -21,19 +21,6 @@ aws-ec2-instance-type = "t2.micro" # 4 GiB, 2 vcpus # aws-ec2-instnce-type = "t2.medium" -################################################################################ - -nodes-config = { - "master" = { - base-image = "ami-0dd0ccab7e2801812" - num = 1 - }, - "worker" = { - base-image = "ami-0dd0ccab7e2801812" - num = 2 - } -} - ################################################################################ # AWS images (AMIs) ################################################################################ diff --git a/get-vm-ips.sh b/get-vm-ips.sh index 230b5cf..cbdc820 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -48,6 +48,7 @@ ANSIBLE_INV="$( sed -z 's/\n/,/g;s/,$/\n/g')" # Create an inventory file for ansible. +echo "# Wrote an Ansible inventory file at ./inventory" echo "[k8s_nodes]" > inventory echo $VM_IP_EXPORTS | \ sed 's/"//g' | \ diff --git a/main.tf b/main.tf index 7510f3e..caa213c 100644 --- a/main.tf +++ b/main.tf @@ -8,12 +8,25 @@ terraform { } } +locals { + nodes-config = { + "master" = { + base-image = var.amzn2-ami + num = 1 + }, + "worker" = { + base-image = var.amzn2-ami + num = 2 + } + } +} + ################################################################################ # cloud-init ################################################################################ module "cloud-init-config" { - for_each = var.nodes-config + for_each = local.nodes-config source = "./modules/cloud-init-config" cloud-init-template = "${path.module}/cloud_init.cfg" hostname-prefix = "${var.vm-name-prefix}-${each.key}" @@ -32,7 +45,8 @@ provider "aws" { region = "us-east-2" } -# This module will grab the latest ami for a variety of distros. +# This module will grab the latest ami for a variety of distros. Uncomment to +# get a list of the latest AMIs for our supported distros. # module "aws-amis" { # source = "./modules/aws-amis" # } @@ -59,7 +73,7 @@ resource "aws_key_pair" "key" { } module "nodes" { - for_each = var.nodes-config + for_each = local.nodes-config source = "./modules/aws-nodes" ami = each.value.base-image ec2-instance-type = var.aws-ec2-instance-type @@ -85,7 +99,7 @@ module "nodes" { # } # # module "nodes" { -# for_each = var.nodes-config +# for_each = local.nodes-config # source = "./modules/libvirt-nodes" # pool-name = libvirt_pool.images.name # name-prefix = "${var.vm-name-prefix}-${each.key}" diff --git a/variables.tf b/variables.tf index e8f3211..3719268 100644 --- a/variables.tf +++ b/variables.tf @@ -43,11 +43,6 @@ variable "node-vcpus" { type = number } -variable "nodes-config" { - description = "A config that declares how many nodes of each type you want created." - type = map(object({base-image=string,num=number})) -} - variable "root-admin-passwd" { description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." } @@ -76,3 +71,39 @@ variable "vm-name-prefix" { default = "k8s-tf" description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0." } + +################################################################################ +# AWS AMI vars +# These variables are really mor like constants. Using variables improves +# readability. The defaults are manually updated. Use the aws-amis module to get +# the latest for each distro. +################################################################################ + +variable "amzn2-ami" { + default = "ami-0dd0ccab7e2801812" + description = "The AMI to use for Amazon Linux 2." +} +variable "ubuntu-ami" { + default = "ami-06c7d6c0987eaa46c" + description = "The AMI to use for Ubuntu." +} +variable "centos7-ami" { + default = "ami-00f8e2c955f7ffa9b" + description = "The AMI to use for CentOS 7." +} +variable "centos8-ami" { + default = "ami-057cacbfbbb471bb3" + description = "The AMI to use for CentOS 8." +} +variable "arch-ami" { + default = "ami-02653f06de985e3ba" + description = "The AMI to use for Arch Linux." +} +variable "rhel7-ami" { + default = "ami-0a509b3c2a4d05b3f" + description = "The AMI to use for RHEL 7." +} +variable "rhel8-ami" { + default = "ami-0d871ca8a77af2948" + description = "The AMI to use for RHEL 8." +} From 191c12dcd0b340c446f3810d9410366de405730c Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 18 Nov 2021 11:41:26 -0500 Subject: [PATCH 25/40] Use config map and added RHEL 7 & 8. --- main.tf | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/main.tf b/main.tf index caa213c..0589fa6 100644 --- a/main.tf +++ b/main.tf @@ -10,13 +10,33 @@ terraform { locals { nodes-config = { - "master" = { + "amzn2" = { base-image = var.amzn2-ami num = 1 }, - "worker" = { - base-image = var.amzn2-ami - num = 2 + "ubuntu" = { + base-image = var.ubuntu-ami + num = 1 + }, + "arch" = { + base-image = var.arch-ami + num = 1 + }, + "centos7" = { + base-image = var.centos7-ami + num = 1 + }, + "centos8" = { + base-image = var.centos8-ami + num = 1 + }, + "rhel7" = { + base-image = var.rhel7-ami + num = 1 + }, + "rhel8" = { + base-image = var.rhel8-ami + num = 1 } } } From ca2cbea60464bbd251023336c61a8dfbb91c0767 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 26 Nov 2021 15:01:54 -0500 Subject: [PATCH 26/40] Added a script to poll AWS node status. --- get-aws-node-status.sh | 4 ++++ get-vm-ips.sh | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100755 get-aws-node-status.sh diff --git a/get-aws-node-status.sh b/get-aws-node-status.sh new file mode 100755 index 0000000..aad14b0 --- /dev/null +++ b/get-aws-node-status.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +aws ec2 describe-instance-status | \ + jq '.InstanceStatuses[] | {id: .InstanceId, instance_status: .InstanceStatus.Status, system_status: .SystemStatus.Status}' diff --git a/get-vm-ips.sh b/get-vm-ips.sh index cbdc820..230b5cf 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -48,7 +48,6 @@ ANSIBLE_INV="$( sed -z 's/\n/,/g;s/,$/\n/g')" # Create an inventory file for ansible. -echo "# Wrote an Ansible inventory file at ./inventory" echo "[k8s_nodes]" > inventory echo $VM_IP_EXPORTS | \ sed 's/"//g' | \ From ae2bb1425145a886083c2adf9e3d350afa9e1ebc Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 26 Nov 2021 15:21:32 -0500 Subject: [PATCH 27/40] Rolled back main.tf to have master and worker. --- main.tf | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/main.tf b/main.tf index 0589fa6..caa213c 100644 --- a/main.tf +++ b/main.tf @@ -10,33 +10,13 @@ terraform { locals { nodes-config = { - "amzn2" = { + "master" = { base-image = var.amzn2-ami num = 1 }, - "ubuntu" = { - base-image = var.ubuntu-ami - num = 1 - }, - "arch" = { - base-image = var.arch-ami - num = 1 - }, - "centos7" = { - base-image = var.centos7-ami - num = 1 - }, - "centos8" = { - base-image = var.centos8-ami - num = 1 - }, - "rhel7" = { - base-image = var.rhel7-ami - num = 1 - }, - "rhel8" = { - base-image = var.rhel8-ami - num = 1 + "worker" = { + base-image = var.amzn2-ami + num = 2 } } } From f8cdec1bb87e3113d15733a03c91fb64cdc9f3a5 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Mon, 29 Nov 2021 10:40:33 -0500 Subject: [PATCH 28/40] Added Arch libvirt image. --- example.tfvars | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/example.tfvars b/example.tfvars index 9d7f2b9..10c8f4b 100644 --- a/example.tfvars +++ b/example.tfvars @@ -54,6 +54,11 @@ base-image = "ami-0dd0ccab7e2801812" # From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 # base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" +## Arch +# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output +# on 2021-11-28 +# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" + ################################################################################ # Keys/Passwords ################################################################################ From c9b56a8db50d2482f011efa4fee6d05238201ae2 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 9 Dec 2021 13:46:54 -0500 Subject: [PATCH 29/40] libvirt bridged network interface - Added support for libvirt VMs that use a bridged network interface. --- cloud_init.cfg | 18 +++++-- example.tfvars | 18 ++++++- get-libvirt-bridge-ips.sh | 81 ++++++++++++++++++++++++++++++ main.tf | 2 + modules/libvirt-nodes/main.tf | 17 +++++-- modules/libvirt-nodes/outpus.tf | 8 +-- modules/libvirt-nodes/variables.tf | 10 ++++ variables.tf | 32 ++++++++++++ 8 files changed, 171 insertions(+), 15 deletions(-) create mode 100755 get-libvirt-bridge-ips.sh diff --git a/cloud_init.cfg b/cloud_init.cfg index 1023821..4b4f11c 100644 --- a/cloud_init.cfg +++ b/cloud_init.cfg @@ -24,9 +24,17 @@ chpasswd: expire: false hostname: ${hostname} +fqdn: ${hostname} -# Use this when it's determined that we need a bigger disk image. -# This must be used in conjuction with 'size' in 'libvirt_volume' -# growpart: -# mode: auto -# devices: ['/'] +packages: + # TODO These are only necessary for libvirt. + - qemu-guest-agent +runcmd: + # TODO At some point revisit this, this was added because it seemed like + # apparmor was causing dhclient to not get an IP address for ubuntu. This + # should be double checked. + - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient + # TODO These are only necessary for libvirt. + - systemctl enable qemu-guest-agent + - systemctl start qemu-guest-agent + - systemctl status qemu-guest-agent diff --git a/example.tfvars b/example.tfvars index 9d7f2b9..5754315 100644 --- a/example.tfvars +++ b/example.tfvars @@ -10,6 +10,15 @@ libvirt-connection-url = "qemu+ssh://@/system" node-memory = 2048 node-vcpus = 2 +## libvirt disk size +# 1 GiB = 1073741824 +# 4 GiB +# libvirt-node-disk-size = "${4 * 1073741824}" +# 8 GiB +# libvirt-node-disk-size = "${8 * 1073741824}" +# 12 GiB +# libvirt-node-disk-size = "${12 * 1073741824}" + ################################################################################ # AWS EC2 instance types ################################################################################ @@ -51,8 +60,13 @@ base-image = "ami-0dd0ccab7e2801812" ################################################################################ # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" -# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 -# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" +# From https://cloud.centos.org/centos/7/images/ from 2020-12-06 +# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + +## Arch +# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output +# on 2021-11-28 +# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" ################################################################################ # Keys/Passwords diff --git a/get-libvirt-bridge-ips.sh b/get-libvirt-bridge-ips.sh new file mode 100755 index 0000000..ef82c8f --- /dev/null +++ b/get-libvirt-bridge-ips.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# This script will grab the IPs for libvirt VMs. This script is only needed when +# using a bridge as the network for the VMs. This should only be needed while +# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is +# unresolved. + +# These are the network interfaces that this script will attempt to get the IP +# address for. +# Ubuntu 20.04 ens3 +# Centos 7 & 8 eth0 +NET_INTERFACES="eth0 ens3" + +LIBVIRT_CONNECTION_URL="libvirt-connection-url" +VM_NAME_PREFIX="vm-name-prefix" + +INV_GROUPS="$( \ +cat terraform.tfstate | \ + jq '.resources[] | select(.type=="libvirt_domain") | .module' | \ + sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )" + +# Grab the connection URL and the vm name prefix. We do this by greping all +# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the +# last grep result, this way we make sure any value in terraform.tfvars will +# take priority. +CONN_URLS="$( \ + find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \ + grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)" + +CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +NAME_PREFIXES="$( \ + find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \ + grep "$VM_NAME_PREFIX" terraform.tfvars)" + +NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +# These can be used for debugging. +# echo "Using connection URL: $CONN_URL" +# echo "Using prefix: $NAME_PREFIX" + +# Get the names of our VMs from libvirt. +VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')" + +# Convert the lines of VM names to an array. +OLD_IFS=$IFS +IFS=$'\n' +VMS=($VMS) +IFS=$OLD_IFS + +# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the +# result to VM_IP_PAIRS as :. +VM_IP_PAIRS="" +for VM in "${VMS[@]}"; do + for INTERFACE in $NET_INTERFACES; do + IP="$( \ + virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \ + jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \ + sed 's/"//g')" + # Add the VM:IP pair if IP is not empty. + if [ ! -z "$IP" ]; then + VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP" + fi + done +done + +# Write inventory +cat /dev/null > inventory +for GROUP in $INV_GROUPS; do + echo "[$GROUP]" >> inventory + echo "$VM_IP_PAIRS" | \ + grep $GROUP | \ + sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory +done + +# Print vars +echo "$VM_IP_PAIRS" | \ + sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \ + sed s/$NAME_PREFIX-//g | \ + sed 's/-/_/g' | \ + awk '{print toupper($1)}' diff --git a/main.tf b/main.tf index 0589fa6..36222c5 100644 --- a/main.tf +++ b/main.tf @@ -126,7 +126,9 @@ module "nodes" { # num-nodes = each.value.num # node-memory = var.node-memory # node-vcpus = var.node-vcpus +# node-disk-size = var.libvirt-node-disk-size # base-image = each.value.base-image +# network-name = var.libvirt-network-name # root-admin-passwd = var.root-admin-passwd # root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url diff --git a/modules/libvirt-nodes/main.tf b/modules/libvirt-nodes/main.tf index cbd0dac..d258c46 100644 --- a/modules/libvirt-nodes/main.tf +++ b/modules/libvirt-nodes/main.tf @@ -9,13 +9,20 @@ terraform { } resource "libvirt_volume" "node-images" { - name = "${var.name-prefix}-${count.index}" + name = "${var.name-prefix}-base" pool = var.pool-name source = var.base-image - count = var.num-nodes format = "qcow2" } +resource "libvirt_volume" "node-images-resized" { + name = "${var.name-prefix}-${count.index}-resized" + pool = var.pool-name + base_volume_id = libvirt_volume.node-images.id + count = var.num-nodes + size = var.node-disk-size +} + data "template_file" "network-config" { template = file("${path.module}/network_config.cfg") } @@ -37,9 +44,9 @@ resource "libvirt_domain" "nodes" { cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) network_interface { - network_name = "default" + network_name = var.network-name hostname = "${var.name-prefix}-${count.index}" - wait_for_lease = true + # wait_for_lease = true } # IMPORTANT: this is a known bug on cloud images, since they expect a console @@ -58,7 +65,7 @@ resource "libvirt_domain" "nodes" { } disk { - volume_id = element(libvirt_volume.node-images.*.id, count.index) + volume_id = element(libvirt_volume.node-images-resized.*.id, count.index) } graphics { diff --git a/modules/libvirt-nodes/outpus.tf b/modules/libvirt-nodes/outpus.tf index 4d399a6..62af47b 100644 --- a/modules/libvirt-nodes/outpus.tf +++ b/modules/libvirt-nodes/outpus.tf @@ -1,4 +1,6 @@ -output "ips" { - value = libvirt_domain.nodes.*.network_interface.0.addresses.0 -} +# This only works on the default network. They will not work using the bridged +# network. +# output "ips" { +# value = libvirt_domain.nodes.*.network_interface.0.addresses.0 +# } diff --git a/modules/libvirt-nodes/variables.tf b/modules/libvirt-nodes/variables.tf index 4253b62..101321d 100644 --- a/modules/libvirt-nodes/variables.tf +++ b/modules/libvirt-nodes/variables.tf @@ -12,6 +12,16 @@ variable "name-prefix" { description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"." } +variable "network-name" { + default = "default" + description = "The name of a pre-existing virtual-network." +} + +variable "node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." diff --git a/variables.tf b/variables.tf index 3719268..cdba553 100644 --- a/variables.tf +++ b/variables.tf @@ -31,6 +31,16 @@ variable "libvirt-connection-url" { description = "The libvirt connection URI, ie. qemu+ssh://@/system" } +variable "libvirt-network-name" { + default = "default" + description = "The name of a pre-existing libvirt virtual-network." +} + +variable "libvirt-node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." @@ -107,3 +117,25 @@ variable "rhel8-ami" { default = "ami-0d871ca8a77af2948" description = "The AMI to use for RHEL 8." } + +################################################################################ +# Libvirt Images +# These variables are really mor like constants. Using variables improves +# readability. The defaults are manually updated. +################################################################################ + +variable "ubuntu-img" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" + description = "The libvirt image tp use for Ubuntu." +} + +variable "centos7-img" { + # Latest as of 2021-12-06. + default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + description = "The libvirt image tp use for CentOS 7." +} + +variable "centos8-img" { + default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2" + description = "The libvirt image tp use for CentOS 8." +} From 96374f2c1a8e6ecc0e423549ecc553ac5cff65b7 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Thu, 9 Dec 2021 23:13:00 -0500 Subject: [PATCH 30/40] Updated get-vim-ips script to use hostnames. - Made the qemu-agent content in the cloud-init config to be controlled by a 'install-qemu-agent' flag. --- cloud_init.cfg | 6 +- get-vm-ips.sh | 138 +++++++++++++++---------- main.tf | 12 ++- modules/aws-nodes/outputs.tf | 4 + modules/cloud-init-config/main.tf | 7 +- modules/cloud-init-config/variables.tf | 6 ++ 6 files changed, 111 insertions(+), 62 deletions(-) diff --git a/cloud_init.cfg b/cloud_init.cfg index 4b4f11c..5266096 100644 --- a/cloud_init.cfg +++ b/cloud_init.cfg @@ -26,15 +26,17 @@ chpasswd: hostname: ${hostname} fqdn: ${hostname} +%{ if install-qemu-agent } packages: - # TODO These are only necessary for libvirt. + # This are only necessary for libvirt. - qemu-guest-agent runcmd: # TODO At some point revisit this, this was added because it seemed like # apparmor was causing dhclient to not get an IP address for ubuntu. This # should be double checked. - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient - # TODO These are only necessary for libvirt. + # These are only necessary for libvirt. - systemctl enable qemu-guest-agent - systemctl start qemu-guest-agent - systemctl status qemu-guest-agent +%{ endif } diff --git a/get-vm-ips.sh b/get-vm-ips.sh index 230b5cf..b8ab1df 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -1,60 +1,92 @@ -#!/bin/sh +#!/bin/bash -# This script will create environment variables for all of the output IPs. It -# will also create a `ANSIBLE_INV` variable that will be a comma separated -# string of all the IPs. A anisble inventory file called "inventory is created -# as well. +# This script will create environment variables for all of the output IPs. An +# anisble inventory file is created as well. # # Use eval $(./get-vm-ips.sh) to set env vars for ips. terraform refresh > /dev/null -# All terraform outputs in json format. -OUTPUTS_JSON="$( - terraform show -json | \ - jq '.values.outputs' | \ - sed 's/-/_/g')" -# Just the IP address outputs in json format. Also all '-' characters are -# replaced by '_' becuase '-' causes jq some problems. -IPS_JSON="$( - echo $OUTPUTS_JSON | \ - jq 'to_entries | .[] | select(.key | contains("ips"))')" -# An array of all node "types" -NODE_TYPE_ARRAY="$( - echo $IPS_JSON | \ - jq '.value.value | to_entries | .[] | .key' | \ - sed 's/"//g' | \ - sed -z 's/\n/ /g;s/ $/\n/g')" - -# Loop over all the node types and create an export line for each IP. -VM_IP_EXPORTS="$( - for TYPE in $NODE_TYPE_ARRAY; do - - # Convert type, converts "master-ips" to "MASTER" - TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)" - echo "$IPS_JSON" | \ - jq '.value.value.'"$TYPE"'[]' | \ - # Add line numbers starting with 0. - nl -v 0 | \ - # Print an export string with a type placeholder "__TYPE__". - awk '{print "export __TYPE___" $1 "=" $2}' | \ - sed s/__TYPE__/$TYPE_UPPER/g - done)" - -ANSIBLE_INV="$( - echo "$VM_IP_EXPORTS" | \ +# The file to write the inventory to. This file will be completely overridden. +INVENTORY_FILE="inventory" + +# Grab the the vm name prefix. We do this by greping all *.tfvars files making +# sure to cat terraform.tfvars last. Then we just grab the last grep result, +# this way we make sure any value in terraform.tfvars will take priority. +VM_NAME_PREFIX_VAR="vm-name-prefix" +VM_NAME_PREFIXES="$( \ + find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \ + grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)" +VM_NAME_PREFIX="$( + echo "$VM_NAME_PREFIXES" | \ + tail -n 1 | \ + sed 's/^.*=\s*"\(.*\)"/\1/g')" + +# This command stores the output data in the format below. +# [ +# { +# "group": "master", +# "vms": [ +# { +# "hostname": "ansible-test-master-0", +# "ip": "52.14.114.48" +# } +# ] +# }, +# { +# "group": "worker", +# "vms": [ +# { +# "hostname": "ansible-test-worker-0", +# "ip": "3.145.121.159" +# }, +# { +# "hostname": "ansible-test-worker-1", +# "ip": "18.217.112.176" +# } +# ] +# } +# ] +DATA="$(terraform show -json | \ + jq '.values.outputs.groups_hostnames_ips.value | to_entries | + map({group: .key, vms:.value | to_entries | + map({hostname:.key,ip:.value})})')" + +# Pull out the groups from $DATA. The format is a single string with the groups +# separated by spaces, ie. "group1 group2 group3". +ANS_GROUPS="$( + echo $DATA | \ + jq '.[] | .group' | \ sed 's/"//g' | \ - sed 's/^.*=//g' | \ - sed -z 's/\n/,/g;s/,$/\n/g')" - -# Create an inventory file for ansible. -echo "[k8s_nodes]" > inventory -echo $VM_IP_EXPORTS | \ - sed 's/"//g' | \ - sed 's/export //g' | \ - sed 's/ /\n/g' | \ - sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \ - >> inventory - -echo $VM_IP_EXPORTS | sed 's/" /"\n/g' -echo export ANSIBLE_INV=$ANSIBLE_INV + tr '\n' ' ' + )" + +# Clear the inventory file. +cat /dev/null > $INVENTORY_FILE + +# For each group, write the VM info to $INVENTORY_FILE and also print a variable +# expression to stdout. +for GROUP in $ANS_GROUPS; do + + # Write the inventory file to $INVENTORY_FILE. + echo "[$GROUP]" >> $INVENTORY_FILE + echo $DATA | \ + jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | + "\(.hostname) ansible_host=\(.ip)"' | \ + sed 's/"//g' \ + >> $INVENTORY_FILE + + # For this group, collect expressions into VARS. The format is: + # HOSTNAME1=0.0.0.0 + # HOSTNAME2=0.0.0.0 + VARS="$( + echo $DATA | \ + jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | + "\(.hostname)=\(.ip)"' | \ + sed 's/"//g' | \ + sed "s/$VM_NAME_PREFIX-//g" | \ + sed 's/-/_/g' + )" + # Print the contents of $VARS converted to uppercase. + echo "${VARS^^}" +done diff --git a/main.tf b/main.tf index 10cb32b..c20f4e6 100644 --- a/main.tf +++ b/main.tf @@ -11,14 +11,15 @@ terraform { locals { nodes-config = { "master" = { - base-image = var.amzn2-ami + base-image = var.centos8-ami num = 1 }, "worker" = { - base-image = var.amzn2-ami + base-image = var.centos8-ami num = 2 } } + install-qemu-agent = false } ################################################################################ @@ -33,6 +34,7 @@ module "cloud-init-config" { num = each.value.num root-admin-passwd = var.root-admin-passwd root-admin-pub-key = var.root-admin-pub-key + install-qemu-agent = local.install-qemu-agent } ################################################################################ @@ -125,6 +127,8 @@ module "nodes" { # end libvirt ################################################################################ -output "ips" { - value = { for type, node in module.nodes : type => node.ips } +# This will outpus a map of group => [{hostname, ip}]. +# TODO A 'names' output needs to be added to libvirt. +output "groups_hostnames_ips" { + value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) } } diff --git a/modules/aws-nodes/outputs.tf b/modules/aws-nodes/outputs.tf index d6faaf0..0b4fe7c 100644 --- a/modules/aws-nodes/outputs.tf +++ b/modules/aws-nodes/outputs.tf @@ -1,3 +1,7 @@ output "ips" { value = aws_instance.nodes.*.public_ip } + +output "names" { + value = aws_instance.nodes.*.tags.Name +} diff --git a/modules/cloud-init-config/main.tf b/modules/cloud-init-config/main.tf index e1c75c4..6b5beed 100644 --- a/modules/cloud-init-config/main.tf +++ b/modules/cloud-init-config/main.tf @@ -1,9 +1,10 @@ data "template_file" "user-datas" { template = file("${var.cloud-init-template}") vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.hostname-prefix}-${count.index}" + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.hostname-prefix}-${count.index}" + install-qemu-agent = var.install-qemu-agent } count = var.num } diff --git a/modules/cloud-init-config/variables.tf b/modules/cloud-init-config/variables.tf index fc4f437..a1b0f2d 100644 --- a/modules/cloud-init-config/variables.tf +++ b/modules/cloud-init-config/variables.tf @@ -8,6 +8,12 @@ variable "hostname-prefix" { description = "This prefix wil be applied as a prefix for the hostnames." } +variable "install-qemu-agent" { + default = false + description = "This flag determines whether or not qemu-agent is installed." + type = bool +} + variable "num" { description = "The number of user-datas to create with these parameters." } From 710f09b9045f68a662b907539c3b97096b71137b Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 10 Dec 2021 09:56:06 -0500 Subject: [PATCH 31/40] Updated get-vim-ips script to use hostnames. - Made the qemu-agent content in the cloud-init config to be controlled by a 'install-qemu-agent' flag. --- modules/aws-network/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/aws-network/main.tf b/modules/aws-network/main.tf index be001ac..6056fc0 100644 --- a/modules/aws-network/main.tf +++ b/modules/aws-network/main.tf @@ -33,7 +33,7 @@ resource "aws_default_security_group" "sg" { } tags = { - Name = "${var.name-prefix}-ssh-from-admins--sg" + Name = "${var.name-prefix}-ssh-from-admins-sg" } } From c17f4a7bda4e66476005b73c04a54f713d37718c Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Mon, 15 Nov 2021 14:01:22 -0500 Subject: [PATCH 32/40] k8s-nodes initial work --- k8s-nodes/README.md | 17 +++ k8s-nodes/cloud_init.cfg | 32 ++++ k8s-nodes/example.tfvars | 34 +++++ k8s-nodes/get-vm-ips.sh | 19 +++ k8s-nodes/main.tf | 144 ++++++++++++++++++ k8s-nodes/modules/aws-network/main.tf | 67 ++++++++ k8s-nodes/modules/aws-network/outputs.tf | 11 ++ k8s-nodes/modules/aws-network/variables.tf | 23 +++ k8s-nodes/modules/aws-nodes/main.tf | 14 ++ k8s-nodes/modules/aws-nodes/outputs.tf | 3 + k8s-nodes/modules/aws-nodes/variables.tf | 36 +++++ k8s-nodes/modules/libvirt-nodes/main.tf | 74 +++++++++ .../modules/libvirt-nodes/network_config.cfg | 4 + k8s-nodes/modules/libvirt-nodes/outpus.tf | 4 + k8s-nodes/modules/libvirt-nodes/variables.tf | 46 ++++++ k8s-nodes/variables.tf | 73 +++++++++ 16 files changed, 601 insertions(+) create mode 100644 k8s-nodes/README.md create mode 100644 k8s-nodes/cloud_init.cfg create mode 100644 k8s-nodes/example.tfvars create mode 100755 k8s-nodes/get-vm-ips.sh create mode 100644 k8s-nodes/main.tf create mode 100644 k8s-nodes/modules/aws-network/main.tf create mode 100644 k8s-nodes/modules/aws-network/outputs.tf create mode 100644 k8s-nodes/modules/aws-network/variables.tf create mode 100644 k8s-nodes/modules/aws-nodes/main.tf create mode 100644 k8s-nodes/modules/aws-nodes/outputs.tf create mode 100644 k8s-nodes/modules/aws-nodes/variables.tf create mode 100644 k8s-nodes/modules/libvirt-nodes/main.tf create mode 100644 k8s-nodes/modules/libvirt-nodes/network_config.cfg create mode 100644 k8s-nodes/modules/libvirt-nodes/outpus.tf create mode 100644 k8s-nodes/modules/libvirt-nodes/variables.tf create mode 100644 k8s-nodes/variables.tf diff --git a/k8s-nodes/README.md b/k8s-nodes/README.md new file mode 100644 index 0000000..b27f538 --- /dev/null +++ b/k8s-nodes/README.md @@ -0,0 +1,17 @@ + +Dependencies +---------------------------------------- + +TODO REM add libvirt provider +libvirt provider depends on mkisofs + +security_driver = none for ubuntu host, link github issue. +https://github.com/dmacvicar/terraform-provider-libvirt/issues/546 + +Other +---------------------------------------- + +Create a password hash. +```shell +python3 -c 'import crypt; print(crypt.crypt("test", crypt.mksalt(crypt.METHOD_SHA512)))' +``` diff --git a/k8s-nodes/cloud_init.cfg b/k8s-nodes/cloud_init.cfg new file mode 100644 index 0000000..1023821 --- /dev/null +++ b/k8s-nodes/cloud_init.cfg @@ -0,0 +1,32 @@ +#cloud-config +# vim: syntax=yaml + +users: + - name: admin + # If we don't supress the user group then cloud init will fail because there + # is allready an admin group in the ubuntu base image. + no_user_group: true + groups: users, admin, sudo + shell: /usr/bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ${admin-pub-key} + - name: root + ssh_authorized_keys: + - ${admin-pub-key} + +ssh_pwauth: true +disable_root: false +chpasswd: + list: + - root:${admin-passwd} + - admin:${admin-passwd} + expire: false + +hostname: ${hostname} + +# Use this when it's determined that we need a bigger disk image. +# This must be used in conjuction with 'size' in 'libvirt_volume' +# growpart: +# mode: auto +# devices: ['/'] diff --git a/k8s-nodes/example.tfvars b/k8s-nodes/example.tfvars new file mode 100644 index 0000000..33a78d3 --- /dev/null +++ b/k8s-nodes/example.tfvars @@ -0,0 +1,34 @@ +# A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0' +# equates to any ip address. +admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] + +disk-image-dir = "/path/to/disk/pool/" +libvirt-connection-url = "qemu+ssh://@/system" + +master-nodes = 1 +worker-nodes = 2 + +node-memory = 2048 +node-vcpus = 2 + +# 1 GiB, 1 vcpu, only one that is free. +# This one won't work with k8s because it requires at least 2 vcpus. +aws-ec2-instance-type = "t2.micro" + +# 4 GiB, 2 vcpus +# aws-ec2-instnce-type = "t2.medium" + +# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free +# base-image = "ami-00be885d550dcee43" +# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free +base-image = "ami-0dd0ccab7e2801812" +# base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" +# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 +# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" + +# Password hash created with: +# python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))' +# where "linux" is the password. +root-admin-passwd = "$6$fiLRWvGQkdK.MnZA$Co9NkA5ruuBUA389JzmKJiC8gKRohmyM09AFnVBOD7ErZnxK4RHMUlKvYg1HSgwaCXTl7H/q1svoeQeUfgc6f0" + +root-admin-pub-key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lx4OiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin" diff --git a/k8s-nodes/get-vm-ips.sh b/k8s-nodes/get-vm-ips.sh new file mode 100755 index 0000000..25e36fe --- /dev/null +++ b/k8s-nodes/get-vm-ips.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# Use eval $(./get-vm-ips.sh) to set env vars for ips. + +terraform refresh > /dev/null + +IPS_JSON="$(terraform show -json | jq '.values.outputs')" + +echo $IPS_JSON | \ + jq '."master-ips".value[]' | \ + nl -v 0 | \ + awk '{print "export MASTER" $1 "=" $2}' | \ + sed 's/"//g' + +echo $IPS_JSON | \ + jq '."worker-ips".value[]' | \ + nl -v 0 | \ + awk '{print "export WORKER" $1 "=" $2}' | \ + sed 's/"//g' diff --git a/k8s-nodes/main.tf b/k8s-nodes/main.tf new file mode 100644 index 0000000..20f8bfc --- /dev/null +++ b/k8s-nodes/main.tf @@ -0,0 +1,144 @@ +terraform { + required_version = ">= 0.13" + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "0.6.11" + } + } +} + +################################################################################ +# cloud-init +################################################################################ + +data "template_file" "master-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-master-${count.index}" + } + count = var.master-nodes +} + +data "template_file" "worker-node-user-datas" { + template = file("${path.module}/cloud_init.cfg") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.vm-name-prefix}-worker-${count.index}" + } + count = var.worker-nodes +} + +################################################################################ +# aws +# To use the aws module, uncomment the aws modules/resources and comment out the +# libvirt modules/resources. +################################################################################ + +provider "aws" { + region = "us-east-2" +} + +module "aws-network" { + source = "./modules/aws-network" + name-prefix = var.vm-name-prefix + vpc-cidr-block = var.aws-vpc-cidr-block + subnet-cidr-block = var.aws-subnet-cidr-block + admin-ips = var.admin-ips +} + +# This key pair is not actually used. Keys are added to the nodes via cloud-init +# instead. We just add this here that this key will show up in the AWS console." +resource "aws_key_pair" "key" { + key_name = "${var.vm-name-prefix}-key}" + public_key = var.root-admin-pub-key + tags = { + Name = "${var.vm-name-prefix}-key" + } +} + +module "master-nodes" { + source = "./modules/aws-nodes" + ami = var.base-image + ec2-instance-type = var.aws-ec2-instance-type + subnet-id = module.aws-network.subnet.id + security-group-ids = [module.aws-network.default-security-group.id] + user-datas = data.template_file.master-node-user-datas + num-nodes = var.master-nodes + name-prefix = "${var.vm-name-prefix}-master" +} + +module "worker-nodes" { + source = "./modules/aws-nodes" + ami = var.base-image + ec2-instance-type = var.aws-ec2-instance-type + subnet-id = module.aws-network.subnet.id + security-group-ids = [module.aws-network.default-security-group.id] + user-datas = data.template_file.worker-node-user-datas + num-nodes = var.worker-nodes + name-prefix = "${var.vm-name-prefix}-worker" +} + +################################################################################ +# end aws +################################################################################ + +################################################################################ +# libvirt +# To use the libvirt module, uncomment the libvirt modules/resources and comment +# out the aws modules/resources. +################################################################################ + +# provider "libvirt" { +# uri = var.libvirt-connection-url +# } +# +# module "master-nodes" { +# source = "./modules/libvirt-nodes" +# pool-name = libvirt_pool.images.name +# name-prefix = "${var.vm-name-prefix}-master" +# num-nodes = var.master-nodes +# node-memory = var.node-memory +# node-vcpus = var.node-vcpus +# base-image = var.base-image +# root-admin-passwd = var.root-admin-passwd +# root-admin-pub-key = var.root-admin-pub-key +# libvirt-connection-url = var.libvirt-connection-url +# user-datas = data.template_file.master-node-user-datas +# } +# +# module "worker-nodes" { +# source = "./modules/libvirt-nodes" +# pool-name = libvirt_pool.images.name +# name-prefix = "${var.vm-name-prefix}-worker" +# num-nodes = var.worker-nodes +# node-memory = var.node-memory +# node-vcpus = var.node-vcpus +# base-image = var.base-image +# root-admin-passwd = var.root-admin-passwd +# root-admin-pub-key = var.root-admin-pub-key +# libvirt-connection-url = var.libvirt-connection-url +# user-datas = data.template_file.worker-node-user-datas +# } +# +# resource "libvirt_pool" "images" { +# name = var.disk-image-pool-name +# type = "dir" +# path = var.disk-image-dir +# } + +################################################################################ +# end libvirt +################################################################################ + +# TODO REM move to other file? +output "master-ips" { + value = module.master-nodes.ips +} + +output "worker-ips" { + value = module.worker-nodes.ips +} diff --git a/k8s-nodes/modules/aws-network/main.tf b/k8s-nodes/modules/aws-network/main.tf new file mode 100644 index 0000000..be001ac --- /dev/null +++ b/k8s-nodes/modules/aws-network/main.tf @@ -0,0 +1,67 @@ +resource "aws_vpc" "vpc" { + cidr_block = var.vpc-cidr-block + tags = { + Name = "${var.name-prefix}-vpc" + } +} + +resource "aws_subnet" "subnet" { + vpc_id = aws_vpc.vpc.id + cidr_block = var.subnet-cidr-block + # availability_zone = var.avail_zone + tags = { + Name = "${var.name-prefix}-subnet" + } +} + +resource "aws_default_security_group" "sg" { + vpc_id = aws_vpc.vpc.id + + ingress { + from_port = 22 + to_port = 22 + protocol = "tcp" + cidr_blocks = var.admin-ips + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + prefix_list_ids = [] + } + + tags = { + Name = "${var.name-prefix}-ssh-from-admins--sg" + } +} + +resource "aws_internet_gateway" "igw" { + vpc_id = aws_vpc.vpc.id + tags = { + Name = "${var.name-prefix}-igw" + } +} + +resource "aws_default_route_table" "route-table" { + default_route_table_id = aws_vpc.vpc.main_route_table_id + + route { + cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.igw.id + } + + # default route, mapping VPC CIDR block to "local", created implicitly and + # cannot be specified. + + tags = { + Name = "${var.name-prefix}-route-table" + } +} + +# Associate subnet with Route Table +resource "aws_route_table_association" "a-rtb-subnet" { + subnet_id = aws_subnet.subnet.id + route_table_id = aws_default_route_table.route-table.id +} diff --git a/k8s-nodes/modules/aws-network/outputs.tf b/k8s-nodes/modules/aws-network/outputs.tf new file mode 100644 index 0000000..5d43ec7 --- /dev/null +++ b/k8s-nodes/modules/aws-network/outputs.tf @@ -0,0 +1,11 @@ +output "vpc" { + value = aws_vpc.vpc +} + +output "subnet" { + value = aws_subnet.subnet +} + +output "default-security-group" { + value = aws_default_security_group.sg +} diff --git a/k8s-nodes/modules/aws-network/variables.tf b/k8s-nodes/modules/aws-network/variables.tf new file mode 100644 index 0000000..527523e --- /dev/null +++ b/k8s-nodes/modules/aws-network/variables.tf @@ -0,0 +1,23 @@ +variable "admin-ips" { + description = "A list of ips or cidr blocks that are allowed to connect to the nodes." + type = list(string) +} + +variable "name-prefix" { + default = "tf" + description = "This prefix will be used in all the names of the resources creates in our AWS network." + type = string +} + +variable "subnet-cidr-block" { + default = "10.0.1.0/24" + description = "The address space to be used for this subnet." + type = string +} + +variable "vpc-cidr-block" { + default = "10.0.0.0/16" + description = "The address space to be used for out networks VPC." + type = string +} + diff --git a/k8s-nodes/modules/aws-nodes/main.tf b/k8s-nodes/modules/aws-nodes/main.tf new file mode 100644 index 0000000..039fb20 --- /dev/null +++ b/k8s-nodes/modules/aws-nodes/main.tf @@ -0,0 +1,14 @@ +resource "aws_instance" "nodes" { + ami = var.ami + instance_type = var.ec2-instance-type + # key_name = aws_key_pair.debug1.key_name + associate_public_ip_address = true + subnet_id = var.subnet-id + vpc_security_group_ids = var.security-group-ids + user_data = element(var.user-datas.*.rendered, count.index) + count = var.num-nodes + + tags = { + Name = "${var.name-prefix}-${count.index}" + } +} diff --git a/k8s-nodes/modules/aws-nodes/outputs.tf b/k8s-nodes/modules/aws-nodes/outputs.tf new file mode 100644 index 0000000..d6faaf0 --- /dev/null +++ b/k8s-nodes/modules/aws-nodes/outputs.tf @@ -0,0 +1,3 @@ +output "ips" { + value = aws_instance.nodes.*.public_ip +} diff --git a/k8s-nodes/modules/aws-nodes/variables.tf b/k8s-nodes/modules/aws-nodes/variables.tf new file mode 100644 index 0000000..4968522 --- /dev/null +++ b/k8s-nodes/modules/aws-nodes/variables.tf @@ -0,0 +1,36 @@ +variable "ami" { + description = "The AWS AMI to be used for all the nodes" + type = string +} + +variable "ec2-instance-type" { + default = "t2.micro" + description = "The AWS instance type to use for all nodes." + type = string +} + +variable "name-prefix" { + default = "tf-node" + description = "This prefix will be applied to all names created by this module." + type = string +} + +variable "num-nodes" { + default = 1 + description = "The number of nodes to create from the given input parameters." + type = number +} + +variable "user-datas" { + description = "A list of cloud-init configs that get applied to their corresponding node." +} + +variable "subnet-id" { + description = "The ID of the subnet that all the nodes will be added to." + type = string +} + +variable "security-group-ids" { + description = "A list of security group IDs to be applied to all the nodes." + type = list(string) +} diff --git a/k8s-nodes/modules/libvirt-nodes/main.tf b/k8s-nodes/modules/libvirt-nodes/main.tf new file mode 100644 index 0000000..9b792d0 --- /dev/null +++ b/k8s-nodes/modules/libvirt-nodes/main.tf @@ -0,0 +1,74 @@ +terraform { + required_version = ">= 0.13" + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "0.6.11" + } + } +} + +provider "libvirt" { + uri = var.libvirt-connection-url +} + +resource "libvirt_volume" "node-images" { + name = "${var.name-prefix}-${count.index}" + pool = var.pool-name + source = var.base-image + count = var.num-nodes + format = "qcow2" +} + +data "template_file" "network-config" { + template = file("${path.module}/network_config.cfg") +} + +resource "libvirt_cloudinit_disk" "node-inits" { + name = "${var.name-prefix}-${count.index}-init" + user_data = element(var.user-datas.*.rendered, count.index) + network_config = data.template_file.network-config.rendered + pool = var.pool-name + count = var.num-nodes +} + +resource "libvirt_domain" "nodes" { + count = var.num-nodes + name = "${var.name-prefix}-${count.index}" + memory = var.node-memory + vcpu = var.node-vcpus + + cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) + + network_interface { + network_name = "default" + hostname = "${var.name-prefix}-${count.index}" + wait_for_lease = true + } + + # IMPORTANT: this is a known bug on cloud images, since they expect a console + # we need to pass it + # https://bugs.launchpad.net/cloud-images/+bug/1573095 + console { + type = "pty" + target_port = "0" + target_type = "serial" + } + + console { + type = "pty" + target_type = "virtio" + target_port = "1" + } + + disk { + volume_id = element(libvirt_volume.node-images.*.id, count.index) + } + + graphics { + type = "spice" + listen_type = "address" + autoport = true + } +} + diff --git a/k8s-nodes/modules/libvirt-nodes/network_config.cfg b/k8s-nodes/modules/libvirt-nodes/network_config.cfg new file mode 100644 index 0000000..5b2cbca --- /dev/null +++ b/k8s-nodes/modules/libvirt-nodes/network_config.cfg @@ -0,0 +1,4 @@ +version: 2 +ethernets: + ens3: + dhcp4: true diff --git a/k8s-nodes/modules/libvirt-nodes/outpus.tf b/k8s-nodes/modules/libvirt-nodes/outpus.tf new file mode 100644 index 0000000..4d399a6 --- /dev/null +++ b/k8s-nodes/modules/libvirt-nodes/outpus.tf @@ -0,0 +1,4 @@ + +output "ips" { + value = libvirt_domain.nodes.*.network_interface.0.addresses.0 +} diff --git a/k8s-nodes/modules/libvirt-nodes/variables.tf b/k8s-nodes/modules/libvirt-nodes/variables.tf new file mode 100644 index 0000000..4253b62 --- /dev/null +++ b/k8s-nodes/modules/libvirt-nodes/variables.tf @@ -0,0 +1,46 @@ +variable "base-image" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" + description = "The base image to be used for all nodes." +} + +variable "libvirt-connection-url" { + description = "The libvirt connection URI, ie. qemu+ssh://@/system" +} + +variable "name-prefix" { + default = "k8s-node" + description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"." +} + +variable "node-memory" { + default = "2048" + description = "The amount of memory to be used for all the nodes." + type = number +} + +variable "node-vcpus" { + default = "2" + description = "The amount of vcpus to be used for all the nodes." + type = number +} + +variable "user-datas" { + description = "A list of cloud-init configs that get applied to their corresponding node." +} + +variable "num-nodes" { + description = "The number of nodes to create with this config." +} + +variable "pool-name" { + default = "default" + description = "The name of the pool to put all disk images in." +} + +variable "root-admin-passwd" { + description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." +} + +variable "root-admin-pub-key" { + description = "The public key to be added to authorized_keys for the root and admin accounts." +} diff --git a/k8s-nodes/variables.tf b/k8s-nodes/variables.tf new file mode 100644 index 0000000..52cf7a2 --- /dev/null +++ b/k8s-nodes/variables.tf @@ -0,0 +1,73 @@ +variable "admin-ips" { + description = "A list of ips or cidr blocks that are allowed to connect to the nodes." + type = list(string) +} + +variable "aws-ec2-instance-type" { + default = "t2.micro" + description = "The AWS instance type to use for all nodes." +} + +variable "aws-subnet-cidr-block" { + default = "10.0.1.0/24" + description = "The address space to be used for this subnet." +} + +variable "aws-vpc-cidr-block" { + default = "10.0.0.0/16" + description = "The address space to be used for the VPC that all the AWS nodes will be in." +} + +variable "disk-image-dir" { + description = "This is the location on the KVM hypervisor host where all the disk images will be kept." +} + +variable "disk-image-pool-name" { + default = "k8s-tf-images" + description = "The name of the disk pool where all the images will be kept." +} + +variable "libvirt-connection-url" { + description = "The libvirt connection URI, ie. qemu+ssh://@/system" +} + +variable "node-memory" { + default = "2048" + description = "The amount of memory to be used for all the nodes." + type = number +} + +variable "node-vcpus" { + default = "2" + description = "The amount of vcpus to be used for all the nodes." + type = number +} + +variable "root-admin-passwd" { + description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." +} + +variable "root-admin-pub-key" { + description = "The public key to be added to authorized_keys for the root and admin accounts." +} + +variable "master-nodes" { + default = 1 + description = "The number of master nodes to create." + type = number +} + +variable "worker-nodes" { + default = 2 + description = "The number of worker nodes to create." + type = number +} + +variable "base-image" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" +} + +variable "vm-name-prefix" { + default = "k8s-tf" + description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0." +} From f2f02aeab41a28ae33c013adb2c8da02d961bb0c Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 26 Nov 2021 16:06:19 -0500 Subject: [PATCH 33/40] Simplified main. Added AMI module. --- k8s-nodes/.gitignore | 49 ++++++++ k8s-nodes/.terraform.lock.hcl | 57 +++++++++ k8s-nodes/README.md | 12 ++ k8s-nodes/example.tfvars | 38 +++++- k8s-nodes/get-aws-node-status.sh | 4 + k8s-nodes/get-vm-ips.sh | 63 ++++++++-- k8s-nodes/main.tf | 116 ++++++++---------- k8s-nodes/modules/aws-amis/main.tf | 58 +++++++++ k8s-nodes/modules/aws-amis/outputs.tf | 3 + k8s-nodes/modules/cloud-init-config/main.tf | 9 ++ .../modules/cloud-init-config/outputs.tf | 3 + .../modules/cloud-init-config/variables.tf | 22 ++++ k8s-nodes/modules/libvirt-nodes/main.tf | 4 - k8s-nodes/variables.tf | 36 ++++++ 14 files changed, 390 insertions(+), 84 deletions(-) create mode 100644 k8s-nodes/.gitignore create mode 100644 k8s-nodes/.terraform.lock.hcl create mode 100755 k8s-nodes/get-aws-node-status.sh create mode 100644 k8s-nodes/modules/aws-amis/main.tf create mode 100644 k8s-nodes/modules/aws-amis/outputs.tf create mode 100644 k8s-nodes/modules/cloud-init-config/main.tf create mode 100644 k8s-nodes/modules/cloud-init-config/outputs.tf create mode 100644 k8s-nodes/modules/cloud-init-config/variables.tf diff --git a/k8s-nodes/.gitignore b/k8s-nodes/.gitignore new file mode 100644 index 0000000..7103111 --- /dev/null +++ b/k8s-nodes/.gitignore @@ -0,0 +1,49 @@ +################################################################################ +# Pulled from github/gitignore 2021-11-10 commit 1a84870 +################################################################################ + +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log +# Added to original pulled from github/gitignore +crash*.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +################################################################################ +# end Pulled from github/gitignore 2021-11-10 commit 1a84870 +################################################################################ + +k8s-key* +STARTHERE +inventory + diff --git a/k8s-nodes/.terraform.lock.hcl b/k8s-nodes/.terraform.lock.hcl new file mode 100644 index 0000000..5f76ac4 --- /dev/null +++ b/k8s-nodes/.terraform.lock.hcl @@ -0,0 +1,57 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/dmacvicar/libvirt" { + version = "0.6.11" + hashes = [ + "h1:6QzHY/7aNdaaDxJZKygotWnM5uHoS2gs/03CzUCJX60=", + "zh:15300a1c3c294eccade4c8a678412d81602ab041dc0a5aab72fee5425d778e89", + "zh:1605806de0d3b86b7e94b5d04a7ad9b6ac695781f9672ab6002c23caef43b98e", + "zh:21efc5937d89f9ec96bc626d2ce3621c0919b3da97ab63b4e520c37d3f5c7357", + "zh:2c143a6909917fd11191447de4c496f084c7da5200beb9f512791a80a1f33e7c", + "zh:3ca369718cc49feefc3a6ffa795a9055e60de33989a9f1c72b6db16048a181fa", + "zh:71db1d1cf2c06984bba408ad5dc9b4e25285684ee5c530a61583b202cff21b96", + "zh:a67adfc988311d34adcc119500c2ef048a45d632b00bb5a15ea6d3ffdc1c3d1c", + "zh:a83448cbcc194e3b52af9b89b9273a116082d83f2c966035bf8a8c5d5606ca9c", + "zh:a9c5a818dd2606460d4d6f33af7cb387f3e984d631fc233aaec0dda4e0756c2b", + "zh:af3263e66cf9138361d6d7408533edd6de8498e67c88cf0084421ae31fe89054", + "zh:b9596cb26c1e391172472de4ada9b3b0a08e4777e41327db8e021454cc6aae20", + "zh:e28124b9ee0b8c18b6f776eb6523d8935f3072c47cd803ea2f1a06206effaa48", + "zh:edd00638d8c088b8a38e7ab8b9e8ab3bd710f7357d0f6b4a38e0028bd49d8460", + ] +} + +provider "registry.terraform.io/hashicorp/aws" { + version = "3.64.2" + hashes = [ + "h1:oFsgTmmt/eJ8KLo59PSmYu/irUHJiPnQEey0VqaOOck=", + "zh:0b029a2282beabfe410eb2969e18ca773d3473415e442be4dc8ce0eb6d1cd8c5", + "zh:3209de3266a1138f1ccb09f094fdd98b6f55afc06e291db0abe092ec5dbe7640", + "zh:40648266551631cbc15f8a76e80faf300510e3b38c2544d43fc25e37e6802727", + "zh:483c8af92ae70146f2790a70c1a810251e7135aa912b66e769c934eddceebe32", + "zh:4d106d8d415d8df342f3f85e58c35418e6c55e3cb7f02897f832cefac4dca68c", + "zh:972626a6ddb31d5216606d12ab5c30fbf8d51ed2bbe0efcdd7cffa68c1141557", + "zh:a230d55ec52b1695148d40296877ee23e0b302e817154f9b838eb117c87b13fa", + "zh:c95fddfbd7f870db949da0601323e866e0f0fb0d4a93e96725ae5b88029e84d5", + "zh:ea0c7f568074f835f22273c8e7e61e87f5277e32004c72122915fd3c8df49ccc", + "zh:f96d25887e6e2d2ae47659e2c586efea2167995b59a479ae65a02b097da86474", + "zh:fe7502d8e52d3b5ccb2b3c178e7ea894344783093aa71ffb20e978914c976182", + ] +} + +provider "registry.terraform.io/hashicorp/template" { + version = "2.2.0" + hashes = [ + "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=", + "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386", + "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53", + "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603", + "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16", + "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776", + "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451", + "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae", + "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde", + "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d", + "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2", + ] +} diff --git a/k8s-nodes/README.md b/k8s-nodes/README.md index b27f538..e495e0a 100644 --- a/k8s-nodes/README.md +++ b/k8s-nodes/README.md @@ -1,3 +1,15 @@ +A Terraform script to create k8s nodes. This script has modules for creating the +nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS. + +The modules allow you create N VMs of a specific type. So you could create 1 +master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS +VMs, or whatever fits your needs. + +Cloud-Init +---------------------------------------- + +Both the libvirt and aws modules use cloud-init for initial configuration of the +VMs. Dependencies ---------------------------------------- diff --git a/k8s-nodes/example.tfvars b/k8s-nodes/example.tfvars index 33a78d3..9d7f2b9 100644 --- a/k8s-nodes/example.tfvars +++ b/k8s-nodes/example.tfvars @@ -1,3 +1,5 @@ +vm-name-prefix = "ansible-test" + # A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0' # equates to any ip address. admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] @@ -5,12 +7,13 @@ admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] disk-image-dir = "/path/to/disk/pool/" libvirt-connection-url = "qemu+ssh://@/system" -master-nodes = 1 -worker-nodes = 2 - node-memory = 2048 node-vcpus = 2 +################################################################################ +# AWS EC2 instance types +################################################################################ + # 1 GiB, 1 vcpu, only one that is free. # This one won't work with k8s because it requires at least 2 vcpus. aws-ec2-instance-type = "t2.micro" @@ -18,14 +21,43 @@ aws-ec2-instance-type = "t2.micro" # 4 GiB, 2 vcpus # aws-ec2-instnce-type = "t2.medium" +################################################################################ +# AWS images (AMIs) +################################################################################ + +## Amazon Linux 2 # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free # base-image = "ami-00be885d550dcee43" # AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free base-image = "ami-0dd0ccab7e2801812" + +## CentOS +# CentOS 7.9.2009 x86_64 - us-east-2 - 2021-11-15 +# base-image = "ami-00f8e2c955f7ffa9b" +# CentOS 8.4.2105 x86_64 - us-east-2 - 2021-11015 +# base-image = "ami-057cacbfbbb471bb3" + +## Ubuntu +# Ubuntu Server 20.04 LTS (HVM), SSD Volume Type +# us-east-2 - (64-bit x86) - 2021.11.12 - free +# base-image = "ami-0629230e074c580f2" + +## Arch linux +# arch-linux-lts-hvm-2021.06.02.x86_64-ebs - us-east-2 +# base-image = "ami-02653f06de985e3ba" + +################################################################################ +# libvirt images +################################################################################ + # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" # From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 # base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" +################################################################################ +# Keys/Passwords +################################################################################ + # Password hash created with: # python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))' # where "linux" is the password. diff --git a/k8s-nodes/get-aws-node-status.sh b/k8s-nodes/get-aws-node-status.sh new file mode 100755 index 0000000..aad14b0 --- /dev/null +++ b/k8s-nodes/get-aws-node-status.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +aws ec2 describe-instance-status | \ + jq '.InstanceStatuses[] | {id: .InstanceId, instance_status: .InstanceStatus.Status, system_status: .SystemStatus.Status}' diff --git a/k8s-nodes/get-vm-ips.sh b/k8s-nodes/get-vm-ips.sh index 25e36fe..230b5cf 100755 --- a/k8s-nodes/get-vm-ips.sh +++ b/k8s-nodes/get-vm-ips.sh @@ -1,19 +1,60 @@ #!/bin/sh +# This script will create environment variables for all of the output IPs. It +# will also create a `ANSIBLE_INV` variable that will be a comma separated +# string of all the IPs. A anisble inventory file called "inventory is created +# as well. +# # Use eval $(./get-vm-ips.sh) to set env vars for ips. terraform refresh > /dev/null -IPS_JSON="$(terraform show -json | jq '.values.outputs')" +# All terraform outputs in json format. +OUTPUTS_JSON="$( + terraform show -json | \ + jq '.values.outputs' | \ + sed 's/-/_/g')" +# Just the IP address outputs in json format. Also all '-' characters are +# replaced by '_' becuase '-' causes jq some problems. +IPS_JSON="$( + echo $OUTPUTS_JSON | \ + jq 'to_entries | .[] | select(.key | contains("ips"))')" +# An array of all node "types" +NODE_TYPE_ARRAY="$( + echo $IPS_JSON | \ + jq '.value.value | to_entries | .[] | .key' | \ + sed 's/"//g' | \ + sed -z 's/\n/ /g;s/ $/\n/g')" -echo $IPS_JSON | \ - jq '."master-ips".value[]' | \ - nl -v 0 | \ - awk '{print "export MASTER" $1 "=" $2}' | \ - sed 's/"//g' +# Loop over all the node types and create an export line for each IP. +VM_IP_EXPORTS="$( + for TYPE in $NODE_TYPE_ARRAY; do -echo $IPS_JSON | \ - jq '."worker-ips".value[]' | \ - nl -v 0 | \ - awk '{print "export WORKER" $1 "=" $2}' | \ - sed 's/"//g' + # Convert type, converts "master-ips" to "MASTER" + TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)" + echo "$IPS_JSON" | \ + jq '.value.value.'"$TYPE"'[]' | \ + # Add line numbers starting with 0. + nl -v 0 | \ + # Print an export string with a type placeholder "__TYPE__". + awk '{print "export __TYPE___" $1 "=" $2}' | \ + sed s/__TYPE__/$TYPE_UPPER/g + done)" + +ANSIBLE_INV="$( + echo "$VM_IP_EXPORTS" | \ + sed 's/"//g' | \ + sed 's/^.*=//g' | \ + sed -z 's/\n/,/g;s/,$/\n/g')" + +# Create an inventory file for ansible. +echo "[k8s_nodes]" > inventory +echo $VM_IP_EXPORTS | \ + sed 's/"//g' | \ + sed 's/export //g' | \ + sed 's/ /\n/g' | \ + sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \ + >> inventory + +echo $VM_IP_EXPORTS | sed 's/" /"\n/g' +echo export ANSIBLE_INV=$ANSIBLE_INV diff --git a/k8s-nodes/main.tf b/k8s-nodes/main.tf index 20f8bfc..caa213c 100644 --- a/k8s-nodes/main.tf +++ b/k8s-nodes/main.tf @@ -1,35 +1,38 @@ terraform { - required_version = ">= 0.13" + required_version = ">= 1.0.8" required_providers { libvirt = { - source = "dmacvicar/libvirt" + source = "dmacvicar/libvirt" version = "0.6.11" } } } +locals { + nodes-config = { + "master" = { + base-image = var.amzn2-ami + num = 1 + }, + "worker" = { + base-image = var.amzn2-ami + num = 2 + } + } +} + ################################################################################ # cloud-init ################################################################################ -data "template_file" "master-node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-master-${count.index}" - } - count = var.master-nodes -} - -data "template_file" "worker-node-user-datas" { - template = file("${path.module}/cloud_init.cfg") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.vm-name-prefix}-worker-${count.index}" - } - count = var.worker-nodes +module "cloud-init-config" { + for_each = local.nodes-config + source = "./modules/cloud-init-config" + cloud-init-template = "${path.module}/cloud_init.cfg" + hostname-prefix = "${var.vm-name-prefix}-${each.key}" + num = each.value.num + root-admin-passwd = var.root-admin-passwd + root-admin-pub-key = var.root-admin-pub-key } ################################################################################ @@ -42,12 +45,21 @@ provider "aws" { region = "us-east-2" } +# This module will grab the latest ami for a variety of distros. Uncomment to +# get a list of the latest AMIs for our supported distros. +# module "aws-amis" { +# source = "./modules/aws-amis" +# } +# output "amis" { +# value = module.aws-amis.amis +# } + module "aws-network" { - source = "./modules/aws-network" - name-prefix = var.vm-name-prefix - vpc-cidr-block = var.aws-vpc-cidr-block + source = "./modules/aws-network" + name-prefix = var.vm-name-prefix + vpc-cidr-block = var.aws-vpc-cidr-block subnet-cidr-block = var.aws-subnet-cidr-block - admin-ips = var.admin-ips + admin-ips = var.admin-ips } # This key pair is not actually used. Keys are added to the nodes via cloud-init @@ -60,26 +72,16 @@ resource "aws_key_pair" "key" { } } -module "master-nodes" { - source = "./modules/aws-nodes" - ami = var.base-image - ec2-instance-type = var.aws-ec2-instance-type - subnet-id = module.aws-network.subnet.id - security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.master-node-user-datas - num-nodes = var.master-nodes - name-prefix = "${var.vm-name-prefix}-master" -} - -module "worker-nodes" { +module "nodes" { + for_each = local.nodes-config source = "./modules/aws-nodes" - ami = var.base-image + ami = each.value.base-image ec2-instance-type = var.aws-ec2-instance-type subnet-id = module.aws-network.subnet.id security-group-ids = [module.aws-network.default-security-group.id] - user-datas = data.template_file.worker-node-user-datas - num-nodes = var.worker-nodes - name-prefix = "${var.vm-name-prefix}-worker" + user-datas = lookup(module.cloud-init-config, each.key, null).user-datas + num-nodes = each.value.num + name-prefix = "${var.vm-name-prefix}-${each.key}" } ################################################################################ @@ -96,32 +98,19 @@ module "worker-nodes" { # uri = var.libvirt-connection-url # } # -# module "master-nodes" { -# source = "./modules/libvirt-nodes" -# pool-name = libvirt_pool.images.name -# name-prefix = "${var.vm-name-prefix}-master" -# num-nodes = var.master-nodes -# node-memory = var.node-memory -# node-vcpus = var.node-vcpus -# base-image = var.base-image -# root-admin-passwd = var.root-admin-passwd -# root-admin-pub-key = var.root-admin-pub-key -# libvirt-connection-url = var.libvirt-connection-url -# user-datas = data.template_file.master-node-user-datas -# } -# -# module "worker-nodes" { +# module "nodes" { +# for_each = local.nodes-config # source = "./modules/libvirt-nodes" # pool-name = libvirt_pool.images.name -# name-prefix = "${var.vm-name-prefix}-worker" -# num-nodes = var.worker-nodes +# name-prefix = "${var.vm-name-prefix}-${each.key}" +# num-nodes = each.value.num # node-memory = var.node-memory # node-vcpus = var.node-vcpus -# base-image = var.base-image +# base-image = each.value.base-image # root-admin-passwd = var.root-admin-passwd # root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url -# user-datas = data.template_file.worker-node-user-datas +# user-datas = lookup(module.cloud-init-config, each.key, null).user-datas # } # # resource "libvirt_pool" "images" { @@ -134,11 +123,6 @@ module "worker-nodes" { # end libvirt ################################################################################ -# TODO REM move to other file? -output "master-ips" { - value = module.master-nodes.ips -} - -output "worker-ips" { - value = module.worker-nodes.ips +output "ips" { + value = { for type, node in module.nodes : type => node.ips } } diff --git a/k8s-nodes/modules/aws-amis/main.tf b/k8s-nodes/modules/aws-amis/main.tf new file mode 100644 index 0000000..1949653 --- /dev/null +++ b/k8s-nodes/modules/aws-amis/main.tf @@ -0,0 +1,58 @@ +locals { + amis = { + amzn2 = { + owner-id = "137112412989" + name = "amzn2-ami-hvm-2*x86_64-gp2" + }, + ubuntu = { + owner-id = "099720109477" + name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*" + }, + centos7 = { + owner-id = "125523088429" + name = "CentOS 7.*x86_64" + }, + centos8 = { + owner-id = "125523088429" + name = "CentOS 8.*x86_64" + }, + arch = { + owner-id = "093273469852" + name = "arch-linux-lts-hvm*x86_64-ebs" + }, + rhel7 = { + owner-id = "309956199498" + name = "RHEL-7.*HVM*x86_64*GP2" + }, + rhel8 = { + owner-id = "309956199498" + name = "RHEL-8.*HVM*x86_64*GP2" + } + } +} + +data "aws_ami" "amis" { + for_each = local.amis + most_recent = true + owners = [each.value.owner-id] + + filter { + name = "name" + values = [each.value.name] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } +} diff --git a/k8s-nodes/modules/aws-amis/outputs.tf b/k8s-nodes/modules/aws-amis/outputs.tf new file mode 100644 index 0000000..3174fd9 --- /dev/null +++ b/k8s-nodes/modules/aws-amis/outputs.tf @@ -0,0 +1,3 @@ +output "amis" { + value = tomap({ for type, ami in data.aws_ami.amis : type => ami.id }) +} diff --git a/k8s-nodes/modules/cloud-init-config/main.tf b/k8s-nodes/modules/cloud-init-config/main.tf new file mode 100644 index 0000000..e1c75c4 --- /dev/null +++ b/k8s-nodes/modules/cloud-init-config/main.tf @@ -0,0 +1,9 @@ +data "template_file" "user-datas" { + template = file("${var.cloud-init-template}") + vars = { + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.hostname-prefix}-${count.index}" + } + count = var.num +} diff --git a/k8s-nodes/modules/cloud-init-config/outputs.tf b/k8s-nodes/modules/cloud-init-config/outputs.tf new file mode 100644 index 0000000..15415d4 --- /dev/null +++ b/k8s-nodes/modules/cloud-init-config/outputs.tf @@ -0,0 +1,3 @@ +output "user-datas" { + value = data.template_file.user-datas +} diff --git a/k8s-nodes/modules/cloud-init-config/variables.tf b/k8s-nodes/modules/cloud-init-config/variables.tf new file mode 100644 index 0000000..fc4f437 --- /dev/null +++ b/k8s-nodes/modules/cloud-init-config/variables.tf @@ -0,0 +1,22 @@ +variable "cloud-init-template" { + default = "../../cloud_init.cfg" + description = "The path to the cloud-init config template." + type = string +} + +variable "hostname-prefix" { + description = "This prefix wil be applied as a prefix for the hostnames." +} + +variable "num" { + description = "The number of user-datas to create with these parameters." +} + +variable "root-admin-passwd" { + description = "This value will be substituted for any occurence of 'admin-password' in the cloud-init config template." +} + +variable "root-admin-pub-key" { + description = "This value will be substituted for any occurence of 'admin-pub-key' in the cloud-init config template." +} + diff --git a/k8s-nodes/modules/libvirt-nodes/main.tf b/k8s-nodes/modules/libvirt-nodes/main.tf index 9b792d0..cbd0dac 100644 --- a/k8s-nodes/modules/libvirt-nodes/main.tf +++ b/k8s-nodes/modules/libvirt-nodes/main.tf @@ -8,10 +8,6 @@ terraform { } } -provider "libvirt" { - uri = var.libvirt-connection-url -} - resource "libvirt_volume" "node-images" { name = "${var.name-prefix}-${count.index}" pool = var.pool-name diff --git a/k8s-nodes/variables.tf b/k8s-nodes/variables.tf index 52cf7a2..3719268 100644 --- a/k8s-nodes/variables.tf +++ b/k8s-nodes/variables.tf @@ -71,3 +71,39 @@ variable "vm-name-prefix" { default = "k8s-tf" description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0." } + +################################################################################ +# AWS AMI vars +# These variables are really mor like constants. Using variables improves +# readability. The defaults are manually updated. Use the aws-amis module to get +# the latest for each distro. +################################################################################ + +variable "amzn2-ami" { + default = "ami-0dd0ccab7e2801812" + description = "The AMI to use for Amazon Linux 2." +} +variable "ubuntu-ami" { + default = "ami-06c7d6c0987eaa46c" + description = "The AMI to use for Ubuntu." +} +variable "centos7-ami" { + default = "ami-00f8e2c955f7ffa9b" + description = "The AMI to use for CentOS 7." +} +variable "centos8-ami" { + default = "ami-057cacbfbbb471bb3" + description = "The AMI to use for CentOS 8." +} +variable "arch-ami" { + default = "ami-02653f06de985e3ba" + description = "The AMI to use for Arch Linux." +} +variable "rhel7-ami" { + default = "ami-0a509b3c2a4d05b3f" + description = "The AMI to use for RHEL 7." +} +variable "rhel8-ami" { + default = "ami-0d871ca8a77af2948" + description = "The AMI to use for RHEL 8." +} From 01a5e56d91f3b9e8736a7253ff2d5b98ae08bffc Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 10 Dec 2021 15:32:10 -0500 Subject: [PATCH 34/40] changed region to west and added code for backend state management --- k8s-nodes/main.tf | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/k8s-nodes/main.tf b/k8s-nodes/main.tf index caa213c..81e0da1 100644 --- a/k8s-nodes/main.tf +++ b/k8s-nodes/main.tf @@ -1,5 +1,16 @@ terraform { required_version = ">= 1.0.8" + + backend "s3" { + + bucket = "mss-terraform-state" + key = "global/s3/terraform.tfstate" + region = "us-gov-west-1" + + dynamodb_table = "mss-terraform-state-lock" + encrypt = true + + } required_providers { libvirt = { source = "dmacvicar/libvirt" @@ -42,7 +53,7 @@ module "cloud-init-config" { ################################################################################ provider "aws" { - region = "us-east-2" + region = "us-gov-west-1" } # This module will grab the latest ami for a variety of distros. Uncomment to From 3bb1cb4a951d2b37f7e538238aa495991e01ae89 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 10 Dec 2021 11:00:45 -0500 Subject: [PATCH 35/40] Added support for bridge network interfaces for libvirt. --- k8s-nodes/cloud_init.cfg | 20 ++- k8s-nodes/example.tfvars | 23 ++- k8s-nodes/get-libvirt-bridge-ips.sh | 81 ++++++++++ k8s-nodes/get-vm-ips.sh | 138 +++++++++++------- k8s-nodes/main.tf | 14 +- k8s-nodes/modules/aws-network/main.tf | 2 +- k8s-nodes/modules/aws-nodes/outputs.tf | 4 + k8s-nodes/modules/cloud-init-config/main.tf | 7 +- .../modules/cloud-init-config/variables.tf | 6 + k8s-nodes/modules/libvirt-nodes/main.tf | 17 ++- k8s-nodes/modules/libvirt-nodes/outpus.tf | 8 +- k8s-nodes/modules/libvirt-nodes/variables.tf | 10 ++ k8s-nodes/variables.tf | 32 ++++ 13 files changed, 286 insertions(+), 76 deletions(-) create mode 100755 k8s-nodes/get-libvirt-bridge-ips.sh diff --git a/k8s-nodes/cloud_init.cfg b/k8s-nodes/cloud_init.cfg index 1023821..5266096 100644 --- a/k8s-nodes/cloud_init.cfg +++ b/k8s-nodes/cloud_init.cfg @@ -24,9 +24,19 @@ chpasswd: expire: false hostname: ${hostname} +fqdn: ${hostname} -# Use this when it's determined that we need a bigger disk image. -# This must be used in conjuction with 'size' in 'libvirt_volume' -# growpart: -# mode: auto -# devices: ['/'] +%{ if install-qemu-agent } +packages: + # This are only necessary for libvirt. + - qemu-guest-agent +runcmd: + # TODO At some point revisit this, this was added because it seemed like + # apparmor was causing dhclient to not get an IP address for ubuntu. This + # should be double checked. + - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient + # These are only necessary for libvirt. + - systemctl enable qemu-guest-agent + - systemctl start qemu-guest-agent + - systemctl status qemu-guest-agent +%{ endif } diff --git a/k8s-nodes/example.tfvars b/k8s-nodes/example.tfvars index 9d7f2b9..2131e2e 100644 --- a/k8s-nodes/example.tfvars +++ b/k8s-nodes/example.tfvars @@ -10,6 +10,15 @@ libvirt-connection-url = "qemu+ssh://@/system" node-memory = 2048 node-vcpus = 2 +## libvirt disk size +# 1 GiB = 1073741824 +# 4 GiB +# libvirt-node-disk-size = "${4 * 1073741824}" +# 8 GiB +# libvirt-node-disk-size = "${8 * 1073741824}" +# 12 GiB +# libvirt-node-disk-size = "${12 * 1073741824}" + ################################################################################ # AWS EC2 instance types ################################################################################ @@ -51,8 +60,18 @@ base-image = "ami-0dd0ccab7e2801812" ################################################################################ # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" -# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 -# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" +# From https://cloud.centos.org/centos/7/images/ from 2020-12-06 +# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + +## Arch +# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output +# on 2021-11-28 +# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" + +## Arch +# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output +# on 2021-11-28 +# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" ################################################################################ # Keys/Passwords diff --git a/k8s-nodes/get-libvirt-bridge-ips.sh b/k8s-nodes/get-libvirt-bridge-ips.sh new file mode 100755 index 0000000..ef82c8f --- /dev/null +++ b/k8s-nodes/get-libvirt-bridge-ips.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# This script will grab the IPs for libvirt VMs. This script is only needed when +# using a bridge as the network for the VMs. This should only be needed while +# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is +# unresolved. + +# These are the network interfaces that this script will attempt to get the IP +# address for. +# Ubuntu 20.04 ens3 +# Centos 7 & 8 eth0 +NET_INTERFACES="eth0 ens3" + +LIBVIRT_CONNECTION_URL="libvirt-connection-url" +VM_NAME_PREFIX="vm-name-prefix" + +INV_GROUPS="$( \ +cat terraform.tfstate | \ + jq '.resources[] | select(.type=="libvirt_domain") | .module' | \ + sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )" + +# Grab the connection URL and the vm name prefix. We do this by greping all +# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the +# last grep result, this way we make sure any value in terraform.tfvars will +# take priority. +CONN_URLS="$( \ + find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \ + grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)" + +CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +NAME_PREFIXES="$( \ + find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \ + grep "$VM_NAME_PREFIX" terraform.tfvars)" + +NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +# These can be used for debugging. +# echo "Using connection URL: $CONN_URL" +# echo "Using prefix: $NAME_PREFIX" + +# Get the names of our VMs from libvirt. +VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')" + +# Convert the lines of VM names to an array. +OLD_IFS=$IFS +IFS=$'\n' +VMS=($VMS) +IFS=$OLD_IFS + +# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the +# result to VM_IP_PAIRS as :. +VM_IP_PAIRS="" +for VM in "${VMS[@]}"; do + for INTERFACE in $NET_INTERFACES; do + IP="$( \ + virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \ + jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \ + sed 's/"//g')" + # Add the VM:IP pair if IP is not empty. + if [ ! -z "$IP" ]; then + VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP" + fi + done +done + +# Write inventory +cat /dev/null > inventory +for GROUP in $INV_GROUPS; do + echo "[$GROUP]" >> inventory + echo "$VM_IP_PAIRS" | \ + grep $GROUP | \ + sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory +done + +# Print vars +echo "$VM_IP_PAIRS" | \ + sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \ + sed s/$NAME_PREFIX-//g | \ + sed 's/-/_/g' | \ + awk '{print toupper($1)}' diff --git a/k8s-nodes/get-vm-ips.sh b/k8s-nodes/get-vm-ips.sh index 230b5cf..b8ab1df 100755 --- a/k8s-nodes/get-vm-ips.sh +++ b/k8s-nodes/get-vm-ips.sh @@ -1,60 +1,92 @@ -#!/bin/sh +#!/bin/bash -# This script will create environment variables for all of the output IPs. It -# will also create a `ANSIBLE_INV` variable that will be a comma separated -# string of all the IPs. A anisble inventory file called "inventory is created -# as well. +# This script will create environment variables for all of the output IPs. An +# anisble inventory file is created as well. # # Use eval $(./get-vm-ips.sh) to set env vars for ips. terraform refresh > /dev/null -# All terraform outputs in json format. -OUTPUTS_JSON="$( - terraform show -json | \ - jq '.values.outputs' | \ - sed 's/-/_/g')" -# Just the IP address outputs in json format. Also all '-' characters are -# replaced by '_' becuase '-' causes jq some problems. -IPS_JSON="$( - echo $OUTPUTS_JSON | \ - jq 'to_entries | .[] | select(.key | contains("ips"))')" -# An array of all node "types" -NODE_TYPE_ARRAY="$( - echo $IPS_JSON | \ - jq '.value.value | to_entries | .[] | .key' | \ - sed 's/"//g' | \ - sed -z 's/\n/ /g;s/ $/\n/g')" - -# Loop over all the node types and create an export line for each IP. -VM_IP_EXPORTS="$( - for TYPE in $NODE_TYPE_ARRAY; do - - # Convert type, converts "master-ips" to "MASTER" - TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)" - echo "$IPS_JSON" | \ - jq '.value.value.'"$TYPE"'[]' | \ - # Add line numbers starting with 0. - nl -v 0 | \ - # Print an export string with a type placeholder "__TYPE__". - awk '{print "export __TYPE___" $1 "=" $2}' | \ - sed s/__TYPE__/$TYPE_UPPER/g - done)" - -ANSIBLE_INV="$( - echo "$VM_IP_EXPORTS" | \ +# The file to write the inventory to. This file will be completely overridden. +INVENTORY_FILE="inventory" + +# Grab the the vm name prefix. We do this by greping all *.tfvars files making +# sure to cat terraform.tfvars last. Then we just grab the last grep result, +# this way we make sure any value in terraform.tfvars will take priority. +VM_NAME_PREFIX_VAR="vm-name-prefix" +VM_NAME_PREFIXES="$( \ + find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \ + grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)" +VM_NAME_PREFIX="$( + echo "$VM_NAME_PREFIXES" | \ + tail -n 1 | \ + sed 's/^.*=\s*"\(.*\)"/\1/g')" + +# This command stores the output data in the format below. +# [ +# { +# "group": "master", +# "vms": [ +# { +# "hostname": "ansible-test-master-0", +# "ip": "52.14.114.48" +# } +# ] +# }, +# { +# "group": "worker", +# "vms": [ +# { +# "hostname": "ansible-test-worker-0", +# "ip": "3.145.121.159" +# }, +# { +# "hostname": "ansible-test-worker-1", +# "ip": "18.217.112.176" +# } +# ] +# } +# ] +DATA="$(terraform show -json | \ + jq '.values.outputs.groups_hostnames_ips.value | to_entries | + map({group: .key, vms:.value | to_entries | + map({hostname:.key,ip:.value})})')" + +# Pull out the groups from $DATA. The format is a single string with the groups +# separated by spaces, ie. "group1 group2 group3". +ANS_GROUPS="$( + echo $DATA | \ + jq '.[] | .group' | \ sed 's/"//g' | \ - sed 's/^.*=//g' | \ - sed -z 's/\n/,/g;s/,$/\n/g')" - -# Create an inventory file for ansible. -echo "[k8s_nodes]" > inventory -echo $VM_IP_EXPORTS | \ - sed 's/"//g' | \ - sed 's/export //g' | \ - sed 's/ /\n/g' | \ - sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \ - >> inventory - -echo $VM_IP_EXPORTS | sed 's/" /"\n/g' -echo export ANSIBLE_INV=$ANSIBLE_INV + tr '\n' ' ' + )" + +# Clear the inventory file. +cat /dev/null > $INVENTORY_FILE + +# For each group, write the VM info to $INVENTORY_FILE and also print a variable +# expression to stdout. +for GROUP in $ANS_GROUPS; do + + # Write the inventory file to $INVENTORY_FILE. + echo "[$GROUP]" >> $INVENTORY_FILE + echo $DATA | \ + jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | + "\(.hostname) ansible_host=\(.ip)"' | \ + sed 's/"//g' \ + >> $INVENTORY_FILE + + # For this group, collect expressions into VARS. The format is: + # HOSTNAME1=0.0.0.0 + # HOSTNAME2=0.0.0.0 + VARS="$( + echo $DATA | \ + jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | + "\(.hostname)=\(.ip)"' | \ + sed 's/"//g' | \ + sed "s/$VM_NAME_PREFIX-//g" | \ + sed 's/-/_/g' + )" + # Print the contents of $VARS converted to uppercase. + echo "${VARS^^}" +done diff --git a/k8s-nodes/main.tf b/k8s-nodes/main.tf index 81e0da1..64849b6 100644 --- a/k8s-nodes/main.tf +++ b/k8s-nodes/main.tf @@ -22,14 +22,15 @@ terraform { locals { nodes-config = { "master" = { - base-image = var.amzn2-ami + base-image = var.centos8-ami num = 1 }, "worker" = { - base-image = var.amzn2-ami + base-image = var.centos8-ami num = 2 } } + install-qemu-agent = false } ################################################################################ @@ -44,6 +45,7 @@ module "cloud-init-config" { num = each.value.num root-admin-passwd = var.root-admin-passwd root-admin-pub-key = var.root-admin-pub-key + install-qemu-agent = local.install-qemu-agent } ################################################################################ @@ -117,7 +119,9 @@ module "nodes" { # num-nodes = each.value.num # node-memory = var.node-memory # node-vcpus = var.node-vcpus +# node-disk-size = var.libvirt-node-disk-size # base-image = each.value.base-image +# network-name = var.libvirt-network-name # root-admin-passwd = var.root-admin-passwd # root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url @@ -134,6 +138,8 @@ module "nodes" { # end libvirt ################################################################################ -output "ips" { - value = { for type, node in module.nodes : type => node.ips } +# This will outpus a map of group => [{hostname, ip}]. +# TODO A 'names' output needs to be added to libvirt. +output "groups_hostnames_ips" { + value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) } } diff --git a/k8s-nodes/modules/aws-network/main.tf b/k8s-nodes/modules/aws-network/main.tf index be001ac..6056fc0 100644 --- a/k8s-nodes/modules/aws-network/main.tf +++ b/k8s-nodes/modules/aws-network/main.tf @@ -33,7 +33,7 @@ resource "aws_default_security_group" "sg" { } tags = { - Name = "${var.name-prefix}-ssh-from-admins--sg" + Name = "${var.name-prefix}-ssh-from-admins-sg" } } diff --git a/k8s-nodes/modules/aws-nodes/outputs.tf b/k8s-nodes/modules/aws-nodes/outputs.tf index d6faaf0..0b4fe7c 100644 --- a/k8s-nodes/modules/aws-nodes/outputs.tf +++ b/k8s-nodes/modules/aws-nodes/outputs.tf @@ -1,3 +1,7 @@ output "ips" { value = aws_instance.nodes.*.public_ip } + +output "names" { + value = aws_instance.nodes.*.tags.Name +} diff --git a/k8s-nodes/modules/cloud-init-config/main.tf b/k8s-nodes/modules/cloud-init-config/main.tf index e1c75c4..6b5beed 100644 --- a/k8s-nodes/modules/cloud-init-config/main.tf +++ b/k8s-nodes/modules/cloud-init-config/main.tf @@ -1,9 +1,10 @@ data "template_file" "user-datas" { template = file("${var.cloud-init-template}") vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.hostname-prefix}-${count.index}" + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.hostname-prefix}-${count.index}" + install-qemu-agent = var.install-qemu-agent } count = var.num } diff --git a/k8s-nodes/modules/cloud-init-config/variables.tf b/k8s-nodes/modules/cloud-init-config/variables.tf index fc4f437..a1b0f2d 100644 --- a/k8s-nodes/modules/cloud-init-config/variables.tf +++ b/k8s-nodes/modules/cloud-init-config/variables.tf @@ -8,6 +8,12 @@ variable "hostname-prefix" { description = "This prefix wil be applied as a prefix for the hostnames." } +variable "install-qemu-agent" { + default = false + description = "This flag determines whether or not qemu-agent is installed." + type = bool +} + variable "num" { description = "The number of user-datas to create with these parameters." } diff --git a/k8s-nodes/modules/libvirt-nodes/main.tf b/k8s-nodes/modules/libvirt-nodes/main.tf index cbd0dac..d258c46 100644 --- a/k8s-nodes/modules/libvirt-nodes/main.tf +++ b/k8s-nodes/modules/libvirt-nodes/main.tf @@ -9,13 +9,20 @@ terraform { } resource "libvirt_volume" "node-images" { - name = "${var.name-prefix}-${count.index}" + name = "${var.name-prefix}-base" pool = var.pool-name source = var.base-image - count = var.num-nodes format = "qcow2" } +resource "libvirt_volume" "node-images-resized" { + name = "${var.name-prefix}-${count.index}-resized" + pool = var.pool-name + base_volume_id = libvirt_volume.node-images.id + count = var.num-nodes + size = var.node-disk-size +} + data "template_file" "network-config" { template = file("${path.module}/network_config.cfg") } @@ -37,9 +44,9 @@ resource "libvirt_domain" "nodes" { cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) network_interface { - network_name = "default" + network_name = var.network-name hostname = "${var.name-prefix}-${count.index}" - wait_for_lease = true + # wait_for_lease = true } # IMPORTANT: this is a known bug on cloud images, since they expect a console @@ -58,7 +65,7 @@ resource "libvirt_domain" "nodes" { } disk { - volume_id = element(libvirt_volume.node-images.*.id, count.index) + volume_id = element(libvirt_volume.node-images-resized.*.id, count.index) } graphics { diff --git a/k8s-nodes/modules/libvirt-nodes/outpus.tf b/k8s-nodes/modules/libvirt-nodes/outpus.tf index 4d399a6..62af47b 100644 --- a/k8s-nodes/modules/libvirt-nodes/outpus.tf +++ b/k8s-nodes/modules/libvirt-nodes/outpus.tf @@ -1,4 +1,6 @@ -output "ips" { - value = libvirt_domain.nodes.*.network_interface.0.addresses.0 -} +# This only works on the default network. They will not work using the bridged +# network. +# output "ips" { +# value = libvirt_domain.nodes.*.network_interface.0.addresses.0 +# } diff --git a/k8s-nodes/modules/libvirt-nodes/variables.tf b/k8s-nodes/modules/libvirt-nodes/variables.tf index 4253b62..101321d 100644 --- a/k8s-nodes/modules/libvirt-nodes/variables.tf +++ b/k8s-nodes/modules/libvirt-nodes/variables.tf @@ -12,6 +12,16 @@ variable "name-prefix" { description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"." } +variable "network-name" { + default = "default" + description = "The name of a pre-existing virtual-network." +} + +variable "node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." diff --git a/k8s-nodes/variables.tf b/k8s-nodes/variables.tf index 3719268..cdba553 100644 --- a/k8s-nodes/variables.tf +++ b/k8s-nodes/variables.tf @@ -31,6 +31,16 @@ variable "libvirt-connection-url" { description = "The libvirt connection URI, ie. qemu+ssh://@/system" } +variable "libvirt-network-name" { + default = "default" + description = "The name of a pre-existing libvirt virtual-network." +} + +variable "libvirt-node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." @@ -107,3 +117,25 @@ variable "rhel8-ami" { default = "ami-0d871ca8a77af2948" description = "The AMI to use for RHEL 8." } + +################################################################################ +# Libvirt Images +# These variables are really mor like constants. Using variables improves +# readability. The defaults are manually updated. +################################################################################ + +variable "ubuntu-img" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" + description = "The libvirt image tp use for Ubuntu." +} + +variable "centos7-img" { + # Latest as of 2021-12-06. + default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + description = "The libvirt image tp use for CentOS 7." +} + +variable "centos8-img" { + default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2" + description = "The libvirt image tp use for CentOS 8." +} From c9f16a587d6f2b148e4e1c0246a6c2d854f20b20 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Mon, 13 Dec 2021 16:08:19 -0500 Subject: [PATCH 36/40] Branches k8s-nodes-updated and add_backend consolidted. --- k8s-nodes/modules/aws-nodes/main.tf | 1 + k8s-nodes/providers.tf | 76 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 k8s-nodes/providers.tf diff --git a/k8s-nodes/modules/aws-nodes/main.tf b/k8s-nodes/modules/aws-nodes/main.tf index 039fb20..282a488 100644 --- a/k8s-nodes/modules/aws-nodes/main.tf +++ b/k8s-nodes/modules/aws-nodes/main.tf @@ -1,6 +1,7 @@ resource "aws_instance" "nodes" { ami = var.ami instance_type = var.ec2-instance-type + # TODO REM double check this key. # key_name = aws_key_pair.debug1.key_name associate_public_ip_address = true subnet_id = var.subnet-id diff --git a/k8s-nodes/providers.tf b/k8s-nodes/providers.tf new file mode 100644 index 0000000..68b5fd1 --- /dev/null +++ b/k8s-nodes/providers.tf @@ -0,0 +1,76 @@ + +provider "aws" { + region = "us-gov-west-1" +} + + +terraform { + required_version = ">= 1.0.8" + + backend "s3" { + + bucket = "mss-terraform-state" + key = "global/s3/terraform.tfstate" + region = "us-gov-west-1" + + dynamodb_table = "mss-terraform-state-lock" + encrypt = true + + } + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "0.6.11" + } + } +} + + + + + +## +#S3 bucket create to hold our TFState file so we can all share env settings +resource "aws_s3_bucket" "terraform_state" { + bucket = "mss-terraform-state" + + # enable versioning for the state files + versioning { + enabled = true + } + + #enable server-side encryption + server_side_encryption_configuration { + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } + } +} + +## +# no sql database used so that we can lock the TFstate file in the S3 bucket to ensure two people +# are not running a terraform command at the same time +resource "aws_dynamodb_table" "terraform_locks" { + name = "mss-terraform-state-lock" + billing_mode = "PAY_PER_REQUEST" + hash_key = "LockID" + + attribute { + name = "LockID" + type = "S" + } + + +} + + +## +# output variable to give details on the s3 bucket created +#TODO: move to output.tf +output "s3_bucket_arn" { + value = aws_s3_bucket.terraform_state.arn + description = "The ARN of the S3 bucket" +} \ No newline at end of file From 2e96171329b214435e06ce7eb7a87103eab52776 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Mon, 13 Dec 2021 19:42:41 -0500 Subject: [PATCH 37/40] Shared state is working. Qeurying for default VPC. --- k8s-nodes/main.tf | 49 +++++++------------ .../modules/aws-network-existing/main.tf | 23 +++++++++ .../modules/aws-network-existing/outputs.tf | 18 +++++++ .../modules/aws-network-existing/variables.tf | 4 ++ .../main.tf | 0 .../outputs.tf | 0 .../variables.tf | 0 k8s-nodes/variables.tf | 5 ++ 8 files changed, 69 insertions(+), 30 deletions(-) create mode 100644 k8s-nodes/modules/aws-network-existing/main.tf create mode 100644 k8s-nodes/modules/aws-network-existing/outputs.tf create mode 100644 k8s-nodes/modules/aws-network-existing/variables.tf rename k8s-nodes/modules/{aws-network => aws-network-from-scratch}/main.tf (100%) rename k8s-nodes/modules/{aws-network => aws-network-from-scratch}/outputs.tf (100%) rename k8s-nodes/modules/{aws-network => aws-network-from-scratch}/variables.tf (100%) diff --git a/k8s-nodes/main.tf b/k8s-nodes/main.tf index 64849b6..0b7b0ab 100644 --- a/k8s-nodes/main.tf +++ b/k8s-nodes/main.tf @@ -1,23 +1,3 @@ -terraform { - required_version = ">= 1.0.8" - - backend "s3" { - - bucket = "mss-terraform-state" - key = "global/s3/terraform.tfstate" - region = "us-gov-west-1" - - dynamodb_table = "mss-terraform-state-lock" - encrypt = true - - } - required_providers { - libvirt = { - source = "dmacvicar/libvirt" - version = "0.6.11" - } - } -} locals { nodes-config = { @@ -54,10 +34,6 @@ module "cloud-init-config" { # libvirt modules/resources. ################################################################################ -provider "aws" { - region = "us-gov-west-1" -} - # This module will grab the latest ami for a variety of distros. Uncomment to # get a list of the latest AMIs for our supported distros. # module "aws-amis" { @@ -67,14 +43,27 @@ provider "aws" { # value = module.aws-amis.amis # } -module "aws-network" { - source = "./modules/aws-network" - name-prefix = var.vm-name-prefix - vpc-cidr-block = var.aws-vpc-cidr-block - subnet-cidr-block = var.aws-subnet-cidr-block - admin-ips = var.admin-ips +################################################################################ +# AWS Networking +# Use of the 2 modules below to create resources for the AWS network. +# aws-network-from-scratch will build the AWS network from scratch. +# aws-network-existing will query AWS for an existing VPC. +################################################################################ + +# module "aws-network-from-scratch" { +# source = "./modules/aws-network-from-scratch" +# name-prefix = var.vm-name-prefix +# vpc-cidr-block = var.aws-vpc-cidr-block +# subnet-cidr-block = var.aws-subnet-cidr-block +# admin-ips = var.admin-ips +# } + +module "aws-network-existing" { + source = "./modules/aws-network-existing" } +################################################################################ + # This key pair is not actually used. Keys are added to the nodes via cloud-init # instead. We just add this here that this key will show up in the AWS console." resource "aws_key_pair" "key" { diff --git a/k8s-nodes/modules/aws-network-existing/main.tf b/k8s-nodes/modules/aws-network-existing/main.tf new file mode 100644 index 0000000..81d25c3 --- /dev/null +++ b/k8s-nodes/modules/aws-network-existing/main.tf @@ -0,0 +1,23 @@ +locals { + az-to-subnets = { + for s in data.aws_subnet.subnets : s.availability_zone => s.id... + } +} + +data "aws_vpc" "default" { + tags = { + Name = var.default-vpc-name + } +} + +data "aws_subnets" "subnet-ids" { + filter { + name = "vpc-id" + values = [data.aws_vpc.default.id] + } +} + +data "aws_subnet" "subnets" { + for_each = toset(data.aws_subnets.subnet-ids.ids) + id = each.key +} diff --git a/k8s-nodes/modules/aws-network-existing/outputs.tf b/k8s-nodes/modules/aws-network-existing/outputs.tf new file mode 100644 index 0000000..f16e968 --- /dev/null +++ b/k8s-nodes/modules/aws-network-existing/outputs.tf @@ -0,0 +1,18 @@ +output "default-vpc" { + value = data.aws_vpc.default +} + +output "subnets" { + description = "An array of all subnets in default-vpc." + value = data.aws_subnet.subnets +} + +output "k8s-subnets" { + description = "An array of subnets to be used for k8s VMs. These subnets were chosen by selecting a single subnet from each availability_zone." + value = [for k,v in local.az-to-subnets : v[0]] +} + +output "az-to-subnets" { + description = "A map of availability zone to array of subnets that are in thet availability zone." + value = local.az-to-subnets +} diff --git a/k8s-nodes/modules/aws-network-existing/variables.tf b/k8s-nodes/modules/aws-network-existing/variables.tf new file mode 100644 index 0000000..b03ebd1 --- /dev/null +++ b/k8s-nodes/modules/aws-network-existing/variables.tf @@ -0,0 +1,4 @@ +variable "default-vpc-name" { + description = "The name of the existing default VPC. This module will query AWS for a VPC with this name," + default = "Managed VPC" +} diff --git a/k8s-nodes/modules/aws-network/main.tf b/k8s-nodes/modules/aws-network-from-scratch/main.tf similarity index 100% rename from k8s-nodes/modules/aws-network/main.tf rename to k8s-nodes/modules/aws-network-from-scratch/main.tf diff --git a/k8s-nodes/modules/aws-network/outputs.tf b/k8s-nodes/modules/aws-network-from-scratch/outputs.tf similarity index 100% rename from k8s-nodes/modules/aws-network/outputs.tf rename to k8s-nodes/modules/aws-network-from-scratch/outputs.tf diff --git a/k8s-nodes/modules/aws-network/variables.tf b/k8s-nodes/modules/aws-network-from-scratch/variables.tf similarity index 100% rename from k8s-nodes/modules/aws-network/variables.tf rename to k8s-nodes/modules/aws-network-from-scratch/variables.tf diff --git a/k8s-nodes/variables.tf b/k8s-nodes/variables.tf index cdba553..aa13de2 100644 --- a/k8s-nodes/variables.tf +++ b/k8s-nodes/variables.tf @@ -8,6 +8,11 @@ variable "aws-ec2-instance-type" { description = "The AWS instance type to use for all nodes." } +variable "aws-region" { + default = "us-east-1" + description = "The AWS region to use." +} + variable "aws-subnet-cidr-block" { default = "10.0.1.0/24" description = "The address space to be used for this subnet." From d4922cbcb9884466944297f6faa42ecaa61c6a2c Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Mon, 13 Dec 2021 19:59:30 -0500 Subject: [PATCH 38/40] Moved everything from k8s-nodes to the top-level. --- k8s-nodes/README.md | 29 ---- k8s-nodes/cloud_init.cfg | 42 ----- k8s-nodes/example.tfvars | 85 ---------- k8s-nodes/get-aws-node-status.sh | 4 - k8s-nodes/get-libvirt-bridge-ips.sh | 81 ---------- k8s-nodes/get-vm-ips.sh | 92 ----------- k8s-nodes/main.tf | 134 ---------------- k8s-nodes/modules/aws-amis/main.tf | 58 ------- k8s-nodes/modules/aws-amis/outputs.tf | 3 - k8s-nodes/modules/aws-nodes/main.tf | 15 -- k8s-nodes/modules/aws-nodes/outputs.tf | 7 - k8s-nodes/modules/aws-nodes/variables.tf | 36 ----- k8s-nodes/modules/cloud-init-config/main.tf | 10 -- .../modules/cloud-init-config/outputs.tf | 3 - .../modules/cloud-init-config/variables.tf | 28 ---- k8s-nodes/modules/libvirt-nodes/main.tf | 77 --------- .../modules/libvirt-nodes/network_config.cfg | 4 - k8s-nodes/modules/libvirt-nodes/outpus.tf | 6 - k8s-nodes/modules/libvirt-nodes/variables.tf | 56 ------- k8s-nodes/variables.tf | 146 ------------------ main.tf | 38 ++--- .../aws-network-existing/main.tf | 0 .../aws-network-existing/outputs.tf | 0 .../aws-network-existing/variables.tf | 0 .../aws-network-from-scratch/main.tf | 0 .../aws-network-from-scratch/outputs.tf | 0 .../aws-network-from-scratch/variables.tf | 0 modules/aws-nodes/main.tf | 1 + k8s-nodes/providers.tf => providers.tf | 0 variables.tf | 5 + 30 files changed, 25 insertions(+), 935 deletions(-) delete mode 100644 k8s-nodes/README.md delete mode 100644 k8s-nodes/cloud_init.cfg delete mode 100644 k8s-nodes/example.tfvars delete mode 100755 k8s-nodes/get-aws-node-status.sh delete mode 100755 k8s-nodes/get-libvirt-bridge-ips.sh delete mode 100755 k8s-nodes/get-vm-ips.sh delete mode 100644 k8s-nodes/main.tf delete mode 100644 k8s-nodes/modules/aws-amis/main.tf delete mode 100644 k8s-nodes/modules/aws-amis/outputs.tf delete mode 100644 k8s-nodes/modules/aws-nodes/main.tf delete mode 100644 k8s-nodes/modules/aws-nodes/outputs.tf delete mode 100644 k8s-nodes/modules/aws-nodes/variables.tf delete mode 100644 k8s-nodes/modules/cloud-init-config/main.tf delete mode 100644 k8s-nodes/modules/cloud-init-config/outputs.tf delete mode 100644 k8s-nodes/modules/cloud-init-config/variables.tf delete mode 100644 k8s-nodes/modules/libvirt-nodes/main.tf delete mode 100644 k8s-nodes/modules/libvirt-nodes/network_config.cfg delete mode 100644 k8s-nodes/modules/libvirt-nodes/outpus.tf delete mode 100644 k8s-nodes/modules/libvirt-nodes/variables.tf delete mode 100644 k8s-nodes/variables.tf rename {k8s-nodes/modules => modules}/aws-network-existing/main.tf (100%) rename {k8s-nodes/modules => modules}/aws-network-existing/outputs.tf (100%) rename {k8s-nodes/modules => modules}/aws-network-existing/variables.tf (100%) rename {k8s-nodes/modules => modules}/aws-network-from-scratch/main.tf (100%) rename {k8s-nodes/modules => modules}/aws-network-from-scratch/outputs.tf (100%) rename {k8s-nodes/modules => modules}/aws-network-from-scratch/variables.tf (100%) rename k8s-nodes/providers.tf => providers.tf (100%) diff --git a/k8s-nodes/README.md b/k8s-nodes/README.md deleted file mode 100644 index e495e0a..0000000 --- a/k8s-nodes/README.md +++ /dev/null @@ -1,29 +0,0 @@ -A Terraform script to create k8s nodes. This script has modules for creating the -nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS. - -The modules allow you create N VMs of a specific type. So you could create 1 -master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS -VMs, or whatever fits your needs. - -Cloud-Init ----------------------------------------- - -Both the libvirt and aws modules use cloud-init for initial configuration of the -VMs. - -Dependencies ----------------------------------------- - -TODO REM add libvirt provider -libvirt provider depends on mkisofs - -security_driver = none for ubuntu host, link github issue. -https://github.com/dmacvicar/terraform-provider-libvirt/issues/546 - -Other ----------------------------------------- - -Create a password hash. -```shell -python3 -c 'import crypt; print(crypt.crypt("test", crypt.mksalt(crypt.METHOD_SHA512)))' -``` diff --git a/k8s-nodes/cloud_init.cfg b/k8s-nodes/cloud_init.cfg deleted file mode 100644 index 5266096..0000000 --- a/k8s-nodes/cloud_init.cfg +++ /dev/null @@ -1,42 +0,0 @@ -#cloud-config -# vim: syntax=yaml - -users: - - name: admin - # If we don't supress the user group then cloud init will fail because there - # is allready an admin group in the ubuntu base image. - no_user_group: true - groups: users, admin, sudo - shell: /usr/bin/bash - sudo: ALL=(ALL) NOPASSWD:ALL - ssh_authorized_keys: - - ${admin-pub-key} - - name: root - ssh_authorized_keys: - - ${admin-pub-key} - -ssh_pwauth: true -disable_root: false -chpasswd: - list: - - root:${admin-passwd} - - admin:${admin-passwd} - expire: false - -hostname: ${hostname} -fqdn: ${hostname} - -%{ if install-qemu-agent } -packages: - # This are only necessary for libvirt. - - qemu-guest-agent -runcmd: - # TODO At some point revisit this, this was added because it seemed like - # apparmor was causing dhclient to not get an IP address for ubuntu. This - # should be double checked. - - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient - # These are only necessary for libvirt. - - systemctl enable qemu-guest-agent - - systemctl start qemu-guest-agent - - systemctl status qemu-guest-agent -%{ endif } diff --git a/k8s-nodes/example.tfvars b/k8s-nodes/example.tfvars deleted file mode 100644 index 2131e2e..0000000 --- a/k8s-nodes/example.tfvars +++ /dev/null @@ -1,85 +0,0 @@ -vm-name-prefix = "ansible-test" - -# A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0' -# equates to any ip address. -admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ] - -disk-image-dir = "/path/to/disk/pool/" -libvirt-connection-url = "qemu+ssh://@/system" - -node-memory = 2048 -node-vcpus = 2 - -## libvirt disk size -# 1 GiB = 1073741824 -# 4 GiB -# libvirt-node-disk-size = "${4 * 1073741824}" -# 8 GiB -# libvirt-node-disk-size = "${8 * 1073741824}" -# 12 GiB -# libvirt-node-disk-size = "${12 * 1073741824}" - -################################################################################ -# AWS EC2 instance types -################################################################################ - -# 1 GiB, 1 vcpu, only one that is free. -# This one won't work with k8s because it requires at least 2 vcpus. -aws-ec2-instance-type = "t2.micro" - -# 4 GiB, 2 vcpus -# aws-ec2-instnce-type = "t2.medium" - -################################################################################ -# AWS images (AMIs) -################################################################################ - -## Amazon Linux 2 -# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free -# base-image = "ami-00be885d550dcee43" -# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free -base-image = "ami-0dd0ccab7e2801812" - -## CentOS -# CentOS 7.9.2009 x86_64 - us-east-2 - 2021-11-15 -# base-image = "ami-00f8e2c955f7ffa9b" -# CentOS 8.4.2105 x86_64 - us-east-2 - 2021-11015 -# base-image = "ami-057cacbfbbb471bb3" - -## Ubuntu -# Ubuntu Server 20.04 LTS (HVM), SSD Volume Type -# us-east-2 - (64-bit x86) - 2021.11.12 - free -# base-image = "ami-0629230e074c580f2" - -## Arch linux -# arch-linux-lts-hvm-2021.06.02.x86_64-ebs - us-east-2 -# base-image = "ami-02653f06de985e3ba" - -################################################################################ -# libvirt images -################################################################################ - -# base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" -# From https://cloud.centos.org/centos/7/images/ from 2020-12-06 -# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" - -## Arch -# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output -# on 2021-11-28 -# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" - -## Arch -# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output -# on 2021-11-28 -# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" - -################################################################################ -# Keys/Passwords -################################################################################ - -# Password hash created with: -# python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))' -# where "linux" is the password. -root-admin-passwd = "$6$fiLRWvGQkdK.MnZA$Co9NkA5ruuBUA389JzmKJiC8gKRohmyM09AFnVBOD7ErZnxK4RHMUlKvYg1HSgwaCXTl7H/q1svoeQeUfgc6f0" - -root-admin-pub-key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lx4OiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin" diff --git a/k8s-nodes/get-aws-node-status.sh b/k8s-nodes/get-aws-node-status.sh deleted file mode 100755 index aad14b0..0000000 --- a/k8s-nodes/get-aws-node-status.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh - -aws ec2 describe-instance-status | \ - jq '.InstanceStatuses[] | {id: .InstanceId, instance_status: .InstanceStatus.Status, system_status: .SystemStatus.Status}' diff --git a/k8s-nodes/get-libvirt-bridge-ips.sh b/k8s-nodes/get-libvirt-bridge-ips.sh deleted file mode 100755 index ef82c8f..0000000 --- a/k8s-nodes/get-libvirt-bridge-ips.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/bash - -# This script will grab the IPs for libvirt VMs. This script is only needed when -# using a bridge as the network for the VMs. This should only be needed while -# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is -# unresolved. - -# These are the network interfaces that this script will attempt to get the IP -# address for. -# Ubuntu 20.04 ens3 -# Centos 7 & 8 eth0 -NET_INTERFACES="eth0 ens3" - -LIBVIRT_CONNECTION_URL="libvirt-connection-url" -VM_NAME_PREFIX="vm-name-prefix" - -INV_GROUPS="$( \ -cat terraform.tfstate | \ - jq '.resources[] | select(.type=="libvirt_domain") | .module' | \ - sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )" - -# Grab the connection URL and the vm name prefix. We do this by greping all -# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the -# last grep result, this way we make sure any value in terraform.tfvars will -# take priority. -CONN_URLS="$( \ - find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \ - grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)" - -CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" - -NAME_PREFIXES="$( \ - find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \ - grep "$VM_NAME_PREFIX" terraform.tfvars)" - -NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" - -# These can be used for debugging. -# echo "Using connection URL: $CONN_URL" -# echo "Using prefix: $NAME_PREFIX" - -# Get the names of our VMs from libvirt. -VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')" - -# Convert the lines of VM names to an array. -OLD_IFS=$IFS -IFS=$'\n' -VMS=($VMS) -IFS=$OLD_IFS - -# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the -# result to VM_IP_PAIRS as :. -VM_IP_PAIRS="" -for VM in "${VMS[@]}"; do - for INTERFACE in $NET_INTERFACES; do - IP="$( \ - virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \ - jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \ - sed 's/"//g')" - # Add the VM:IP pair if IP is not empty. - if [ ! -z "$IP" ]; then - VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP" - fi - done -done - -# Write inventory -cat /dev/null > inventory -for GROUP in $INV_GROUPS; do - echo "[$GROUP]" >> inventory - echo "$VM_IP_PAIRS" | \ - grep $GROUP | \ - sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory -done - -# Print vars -echo "$VM_IP_PAIRS" | \ - sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \ - sed s/$NAME_PREFIX-//g | \ - sed 's/-/_/g' | \ - awk '{print toupper($1)}' diff --git a/k8s-nodes/get-vm-ips.sh b/k8s-nodes/get-vm-ips.sh deleted file mode 100755 index b8ab1df..0000000 --- a/k8s-nodes/get-vm-ips.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/bin/bash - -# This script will create environment variables for all of the output IPs. An -# anisble inventory file is created as well. -# -# Use eval $(./get-vm-ips.sh) to set env vars for ips. - -terraform refresh > /dev/null - -# The file to write the inventory to. This file will be completely overridden. -INVENTORY_FILE="inventory" - -# Grab the the vm name prefix. We do this by greping all *.tfvars files making -# sure to cat terraform.tfvars last. Then we just grab the last grep result, -# this way we make sure any value in terraform.tfvars will take priority. -VM_NAME_PREFIX_VAR="vm-name-prefix" -VM_NAME_PREFIXES="$( \ - find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \ - grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)" -VM_NAME_PREFIX="$( - echo "$VM_NAME_PREFIXES" | \ - tail -n 1 | \ - sed 's/^.*=\s*"\(.*\)"/\1/g')" - -# This command stores the output data in the format below. -# [ -# { -# "group": "master", -# "vms": [ -# { -# "hostname": "ansible-test-master-0", -# "ip": "52.14.114.48" -# } -# ] -# }, -# { -# "group": "worker", -# "vms": [ -# { -# "hostname": "ansible-test-worker-0", -# "ip": "3.145.121.159" -# }, -# { -# "hostname": "ansible-test-worker-1", -# "ip": "18.217.112.176" -# } -# ] -# } -# ] -DATA="$(terraform show -json | \ - jq '.values.outputs.groups_hostnames_ips.value | to_entries | - map({group: .key, vms:.value | to_entries | - map({hostname:.key,ip:.value})})')" - -# Pull out the groups from $DATA. The format is a single string with the groups -# separated by spaces, ie. "group1 group2 group3". -ANS_GROUPS="$( - echo $DATA | \ - jq '.[] | .group' | \ - sed 's/"//g' | \ - tr '\n' ' ' - )" - -# Clear the inventory file. -cat /dev/null > $INVENTORY_FILE - -# For each group, write the VM info to $INVENTORY_FILE and also print a variable -# expression to stdout. -for GROUP in $ANS_GROUPS; do - - # Write the inventory file to $INVENTORY_FILE. - echo "[$GROUP]" >> $INVENTORY_FILE - echo $DATA | \ - jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | - "\(.hostname) ansible_host=\(.ip)"' | \ - sed 's/"//g' \ - >> $INVENTORY_FILE - - # For this group, collect expressions into VARS. The format is: - # HOSTNAME1=0.0.0.0 - # HOSTNAME2=0.0.0.0 - VARS="$( - echo $DATA | \ - jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | - "\(.hostname)=\(.ip)"' | \ - sed 's/"//g' | \ - sed "s/$VM_NAME_PREFIX-//g" | \ - sed 's/-/_/g' - )" - # Print the contents of $VARS converted to uppercase. - echo "${VARS^^}" -done diff --git a/k8s-nodes/main.tf b/k8s-nodes/main.tf deleted file mode 100644 index 0b7b0ab..0000000 --- a/k8s-nodes/main.tf +++ /dev/null @@ -1,134 +0,0 @@ - -locals { - nodes-config = { - "master" = { - base-image = var.centos8-ami - num = 1 - }, - "worker" = { - base-image = var.centos8-ami - num = 2 - } - } - install-qemu-agent = false -} - -################################################################################ -# cloud-init -################################################################################ - -module "cloud-init-config" { - for_each = local.nodes-config - source = "./modules/cloud-init-config" - cloud-init-template = "${path.module}/cloud_init.cfg" - hostname-prefix = "${var.vm-name-prefix}-${each.key}" - num = each.value.num - root-admin-passwd = var.root-admin-passwd - root-admin-pub-key = var.root-admin-pub-key - install-qemu-agent = local.install-qemu-agent -} - -################################################################################ -# aws -# To use the aws module, uncomment the aws modules/resources and comment out the -# libvirt modules/resources. -################################################################################ - -# This module will grab the latest ami for a variety of distros. Uncomment to -# get a list of the latest AMIs for our supported distros. -# module "aws-amis" { -# source = "./modules/aws-amis" -# } -# output "amis" { -# value = module.aws-amis.amis -# } - -################################################################################ -# AWS Networking -# Use of the 2 modules below to create resources for the AWS network. -# aws-network-from-scratch will build the AWS network from scratch. -# aws-network-existing will query AWS for an existing VPC. -################################################################################ - -# module "aws-network-from-scratch" { -# source = "./modules/aws-network-from-scratch" -# name-prefix = var.vm-name-prefix -# vpc-cidr-block = var.aws-vpc-cidr-block -# subnet-cidr-block = var.aws-subnet-cidr-block -# admin-ips = var.admin-ips -# } - -module "aws-network-existing" { - source = "./modules/aws-network-existing" -} - -################################################################################ - -# This key pair is not actually used. Keys are added to the nodes via cloud-init -# instead. We just add this here that this key will show up in the AWS console." -resource "aws_key_pair" "key" { - key_name = "${var.vm-name-prefix}-key}" - public_key = var.root-admin-pub-key - tags = { - Name = "${var.vm-name-prefix}-key" - } -} - -module "nodes" { - for_each = local.nodes-config - source = "./modules/aws-nodes" - ami = each.value.base-image - ec2-instance-type = var.aws-ec2-instance-type - subnet-id = module.aws-network.subnet.id - security-group-ids = [module.aws-network.default-security-group.id] - user-datas = lookup(module.cloud-init-config, each.key, null).user-datas - num-nodes = each.value.num - name-prefix = "${var.vm-name-prefix}-${each.key}" -} - -################################################################################ -# end aws -################################################################################ - -################################################################################ -# libvirt -# To use the libvirt module, uncomment the libvirt modules/resources and comment -# out the aws modules/resources. -################################################################################ - -# provider "libvirt" { -# uri = var.libvirt-connection-url -# } -# -# module "nodes" { -# for_each = local.nodes-config -# source = "./modules/libvirt-nodes" -# pool-name = libvirt_pool.images.name -# name-prefix = "${var.vm-name-prefix}-${each.key}" -# num-nodes = each.value.num -# node-memory = var.node-memory -# node-vcpus = var.node-vcpus -# node-disk-size = var.libvirt-node-disk-size -# base-image = each.value.base-image -# network-name = var.libvirt-network-name -# root-admin-passwd = var.root-admin-passwd -# root-admin-pub-key = var.root-admin-pub-key -# libvirt-connection-url = var.libvirt-connection-url -# user-datas = lookup(module.cloud-init-config, each.key, null).user-datas -# } -# -# resource "libvirt_pool" "images" { -# name = var.disk-image-pool-name -# type = "dir" -# path = var.disk-image-dir -# } - -################################################################################ -# end libvirt -################################################################################ - -# This will outpus a map of group => [{hostname, ip}]. -# TODO A 'names' output needs to be added to libvirt. -output "groups_hostnames_ips" { - value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) } -} diff --git a/k8s-nodes/modules/aws-amis/main.tf b/k8s-nodes/modules/aws-amis/main.tf deleted file mode 100644 index 1949653..0000000 --- a/k8s-nodes/modules/aws-amis/main.tf +++ /dev/null @@ -1,58 +0,0 @@ -locals { - amis = { - amzn2 = { - owner-id = "137112412989" - name = "amzn2-ami-hvm-2*x86_64-gp2" - }, - ubuntu = { - owner-id = "099720109477" - name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*" - }, - centos7 = { - owner-id = "125523088429" - name = "CentOS 7.*x86_64" - }, - centos8 = { - owner-id = "125523088429" - name = "CentOS 8.*x86_64" - }, - arch = { - owner-id = "093273469852" - name = "arch-linux-lts-hvm*x86_64-ebs" - }, - rhel7 = { - owner-id = "309956199498" - name = "RHEL-7.*HVM*x86_64*GP2" - }, - rhel8 = { - owner-id = "309956199498" - name = "RHEL-8.*HVM*x86_64*GP2" - } - } -} - -data "aws_ami" "amis" { - for_each = local.amis - most_recent = true - owners = [each.value.owner-id] - - filter { - name = "name" - values = [each.value.name] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "root-device-type" - values = ["ebs"] - } -} diff --git a/k8s-nodes/modules/aws-amis/outputs.tf b/k8s-nodes/modules/aws-amis/outputs.tf deleted file mode 100644 index 3174fd9..0000000 --- a/k8s-nodes/modules/aws-amis/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "amis" { - value = tomap({ for type, ami in data.aws_ami.amis : type => ami.id }) -} diff --git a/k8s-nodes/modules/aws-nodes/main.tf b/k8s-nodes/modules/aws-nodes/main.tf deleted file mode 100644 index 282a488..0000000 --- a/k8s-nodes/modules/aws-nodes/main.tf +++ /dev/null @@ -1,15 +0,0 @@ -resource "aws_instance" "nodes" { - ami = var.ami - instance_type = var.ec2-instance-type - # TODO REM double check this key. - # key_name = aws_key_pair.debug1.key_name - associate_public_ip_address = true - subnet_id = var.subnet-id - vpc_security_group_ids = var.security-group-ids - user_data = element(var.user-datas.*.rendered, count.index) - count = var.num-nodes - - tags = { - Name = "${var.name-prefix}-${count.index}" - } -} diff --git a/k8s-nodes/modules/aws-nodes/outputs.tf b/k8s-nodes/modules/aws-nodes/outputs.tf deleted file mode 100644 index 0b4fe7c..0000000 --- a/k8s-nodes/modules/aws-nodes/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "ips" { - value = aws_instance.nodes.*.public_ip -} - -output "names" { - value = aws_instance.nodes.*.tags.Name -} diff --git a/k8s-nodes/modules/aws-nodes/variables.tf b/k8s-nodes/modules/aws-nodes/variables.tf deleted file mode 100644 index 4968522..0000000 --- a/k8s-nodes/modules/aws-nodes/variables.tf +++ /dev/null @@ -1,36 +0,0 @@ -variable "ami" { - description = "The AWS AMI to be used for all the nodes" - type = string -} - -variable "ec2-instance-type" { - default = "t2.micro" - description = "The AWS instance type to use for all nodes." - type = string -} - -variable "name-prefix" { - default = "tf-node" - description = "This prefix will be applied to all names created by this module." - type = string -} - -variable "num-nodes" { - default = 1 - description = "The number of nodes to create from the given input parameters." - type = number -} - -variable "user-datas" { - description = "A list of cloud-init configs that get applied to their corresponding node." -} - -variable "subnet-id" { - description = "The ID of the subnet that all the nodes will be added to." - type = string -} - -variable "security-group-ids" { - description = "A list of security group IDs to be applied to all the nodes." - type = list(string) -} diff --git a/k8s-nodes/modules/cloud-init-config/main.tf b/k8s-nodes/modules/cloud-init-config/main.tf deleted file mode 100644 index 6b5beed..0000000 --- a/k8s-nodes/modules/cloud-init-config/main.tf +++ /dev/null @@ -1,10 +0,0 @@ -data "template_file" "user-datas" { - template = file("${var.cloud-init-template}") - vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.hostname-prefix}-${count.index}" - install-qemu-agent = var.install-qemu-agent - } - count = var.num -} diff --git a/k8s-nodes/modules/cloud-init-config/outputs.tf b/k8s-nodes/modules/cloud-init-config/outputs.tf deleted file mode 100644 index 15415d4..0000000 --- a/k8s-nodes/modules/cloud-init-config/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "user-datas" { - value = data.template_file.user-datas -} diff --git a/k8s-nodes/modules/cloud-init-config/variables.tf b/k8s-nodes/modules/cloud-init-config/variables.tf deleted file mode 100644 index a1b0f2d..0000000 --- a/k8s-nodes/modules/cloud-init-config/variables.tf +++ /dev/null @@ -1,28 +0,0 @@ -variable "cloud-init-template" { - default = "../../cloud_init.cfg" - description = "The path to the cloud-init config template." - type = string -} - -variable "hostname-prefix" { - description = "This prefix wil be applied as a prefix for the hostnames." -} - -variable "install-qemu-agent" { - default = false - description = "This flag determines whether or not qemu-agent is installed." - type = bool -} - -variable "num" { - description = "The number of user-datas to create with these parameters." -} - -variable "root-admin-passwd" { - description = "This value will be substituted for any occurence of 'admin-password' in the cloud-init config template." -} - -variable "root-admin-pub-key" { - description = "This value will be substituted for any occurence of 'admin-pub-key' in the cloud-init config template." -} - diff --git a/k8s-nodes/modules/libvirt-nodes/main.tf b/k8s-nodes/modules/libvirt-nodes/main.tf deleted file mode 100644 index d258c46..0000000 --- a/k8s-nodes/modules/libvirt-nodes/main.tf +++ /dev/null @@ -1,77 +0,0 @@ -terraform { - required_version = ">= 0.13" - required_providers { - libvirt = { - source = "dmacvicar/libvirt" - version = "0.6.11" - } - } -} - -resource "libvirt_volume" "node-images" { - name = "${var.name-prefix}-base" - pool = var.pool-name - source = var.base-image - format = "qcow2" -} - -resource "libvirt_volume" "node-images-resized" { - name = "${var.name-prefix}-${count.index}-resized" - pool = var.pool-name - base_volume_id = libvirt_volume.node-images.id - count = var.num-nodes - size = var.node-disk-size -} - -data "template_file" "network-config" { - template = file("${path.module}/network_config.cfg") -} - -resource "libvirt_cloudinit_disk" "node-inits" { - name = "${var.name-prefix}-${count.index}-init" - user_data = element(var.user-datas.*.rendered, count.index) - network_config = data.template_file.network-config.rendered - pool = var.pool-name - count = var.num-nodes -} - -resource "libvirt_domain" "nodes" { - count = var.num-nodes - name = "${var.name-prefix}-${count.index}" - memory = var.node-memory - vcpu = var.node-vcpus - - cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) - - network_interface { - network_name = var.network-name - hostname = "${var.name-prefix}-${count.index}" - # wait_for_lease = true - } - - # IMPORTANT: this is a known bug on cloud images, since they expect a console - # we need to pass it - # https://bugs.launchpad.net/cloud-images/+bug/1573095 - console { - type = "pty" - target_port = "0" - target_type = "serial" - } - - console { - type = "pty" - target_type = "virtio" - target_port = "1" - } - - disk { - volume_id = element(libvirt_volume.node-images-resized.*.id, count.index) - } - - graphics { - type = "spice" - listen_type = "address" - autoport = true - } -} - diff --git a/k8s-nodes/modules/libvirt-nodes/network_config.cfg b/k8s-nodes/modules/libvirt-nodes/network_config.cfg deleted file mode 100644 index 5b2cbca..0000000 --- a/k8s-nodes/modules/libvirt-nodes/network_config.cfg +++ /dev/null @@ -1,4 +0,0 @@ -version: 2 -ethernets: - ens3: - dhcp4: true diff --git a/k8s-nodes/modules/libvirt-nodes/outpus.tf b/k8s-nodes/modules/libvirt-nodes/outpus.tf deleted file mode 100644 index 62af47b..0000000 --- a/k8s-nodes/modules/libvirt-nodes/outpus.tf +++ /dev/null @@ -1,6 +0,0 @@ - -# This only works on the default network. They will not work using the bridged -# network. -# output "ips" { -# value = libvirt_domain.nodes.*.network_interface.0.addresses.0 -# } diff --git a/k8s-nodes/modules/libvirt-nodes/variables.tf b/k8s-nodes/modules/libvirt-nodes/variables.tf deleted file mode 100644 index 101321d..0000000 --- a/k8s-nodes/modules/libvirt-nodes/variables.tf +++ /dev/null @@ -1,56 +0,0 @@ -variable "base-image" { - default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" - description = "The base image to be used for all nodes." -} - -variable "libvirt-connection-url" { - description = "The libvirt connection URI, ie. qemu+ssh://@/system" -} - -variable "name-prefix" { - default = "k8s-node" - description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"." -} - -variable "network-name" { - default = "default" - description = "The name of a pre-existing virtual-network." -} - -variable "node-disk-size" { - default = 4294967296 - description = "The size of the disk to be used for libvirt nodes. (in bytes)" -} - -variable "node-memory" { - default = "2048" - description = "The amount of memory to be used for all the nodes." - type = number -} - -variable "node-vcpus" { - default = "2" - description = "The amount of vcpus to be used for all the nodes." - type = number -} - -variable "user-datas" { - description = "A list of cloud-init configs that get applied to their corresponding node." -} - -variable "num-nodes" { - description = "The number of nodes to create with this config." -} - -variable "pool-name" { - default = "default" - description = "The name of the pool to put all disk images in." -} - -variable "root-admin-passwd" { - description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." -} - -variable "root-admin-pub-key" { - description = "The public key to be added to authorized_keys for the root and admin accounts." -} diff --git a/k8s-nodes/variables.tf b/k8s-nodes/variables.tf deleted file mode 100644 index aa13de2..0000000 --- a/k8s-nodes/variables.tf +++ /dev/null @@ -1,146 +0,0 @@ -variable "admin-ips" { - description = "A list of ips or cidr blocks that are allowed to connect to the nodes." - type = list(string) -} - -variable "aws-ec2-instance-type" { - default = "t2.micro" - description = "The AWS instance type to use for all nodes." -} - -variable "aws-region" { - default = "us-east-1" - description = "The AWS region to use." -} - -variable "aws-subnet-cidr-block" { - default = "10.0.1.0/24" - description = "The address space to be used for this subnet." -} - -variable "aws-vpc-cidr-block" { - default = "10.0.0.0/16" - description = "The address space to be used for the VPC that all the AWS nodes will be in." -} - -variable "disk-image-dir" { - description = "This is the location on the KVM hypervisor host where all the disk images will be kept." -} - -variable "disk-image-pool-name" { - default = "k8s-tf-images" - description = "The name of the disk pool where all the images will be kept." -} - -variable "libvirt-connection-url" { - description = "The libvirt connection URI, ie. qemu+ssh://@/system" -} - -variable "libvirt-network-name" { - default = "default" - description = "The name of a pre-existing libvirt virtual-network." -} - -variable "libvirt-node-disk-size" { - default = 4294967296 - description = "The size of the disk to be used for libvirt nodes. (in bytes)" -} - -variable "node-memory" { - default = "2048" - description = "The amount of memory to be used for all the nodes." - type = number -} - -variable "node-vcpus" { - default = "2" - description = "The amount of vcpus to be used for all the nodes." - type = number -} - -variable "root-admin-passwd" { - description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module." -} - -variable "root-admin-pub-key" { - description = "The public key to be added to authorized_keys for the root and admin accounts." -} - -variable "master-nodes" { - default = 1 - description = "The number of master nodes to create." - type = number -} - -variable "worker-nodes" { - default = 2 - description = "The number of worker nodes to create." - type = number -} - -variable "base-image" { - default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" -} - -variable "vm-name-prefix" { - default = "k8s-tf" - description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0." -} - -################################################################################ -# AWS AMI vars -# These variables are really mor like constants. Using variables improves -# readability. The defaults are manually updated. Use the aws-amis module to get -# the latest for each distro. -################################################################################ - -variable "amzn2-ami" { - default = "ami-0dd0ccab7e2801812" - description = "The AMI to use for Amazon Linux 2." -} -variable "ubuntu-ami" { - default = "ami-06c7d6c0987eaa46c" - description = "The AMI to use for Ubuntu." -} -variable "centos7-ami" { - default = "ami-00f8e2c955f7ffa9b" - description = "The AMI to use for CentOS 7." -} -variable "centos8-ami" { - default = "ami-057cacbfbbb471bb3" - description = "The AMI to use for CentOS 8." -} -variable "arch-ami" { - default = "ami-02653f06de985e3ba" - description = "The AMI to use for Arch Linux." -} -variable "rhel7-ami" { - default = "ami-0a509b3c2a4d05b3f" - description = "The AMI to use for RHEL 7." -} -variable "rhel8-ami" { - default = "ami-0d871ca8a77af2948" - description = "The AMI to use for RHEL 8." -} - -################################################################################ -# Libvirt Images -# These variables are really mor like constants. Using variables improves -# readability. The defaults are manually updated. -################################################################################ - -variable "ubuntu-img" { - default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" - description = "The libvirt image tp use for Ubuntu." -} - -variable "centos7-img" { - # Latest as of 2021-12-06. - default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" - description = "The libvirt image tp use for CentOS 7." -} - -variable "centos8-img" { - default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2" - description = "The libvirt image tp use for CentOS 8." -} diff --git a/main.tf b/main.tf index c20f4e6..0b7b0ab 100644 --- a/main.tf +++ b/main.tf @@ -1,12 +1,3 @@ -terraform { - required_version = ">= 1.0.8" - required_providers { - libvirt = { - source = "dmacvicar/libvirt" - version = "0.6.11" - } - } -} locals { nodes-config = { @@ -43,10 +34,6 @@ module "cloud-init-config" { # libvirt modules/resources. ################################################################################ -provider "aws" { - region = "us-east-2" -} - # This module will grab the latest ami for a variety of distros. Uncomment to # get a list of the latest AMIs for our supported distros. # module "aws-amis" { @@ -56,14 +43,27 @@ provider "aws" { # value = module.aws-amis.amis # } -module "aws-network" { - source = "./modules/aws-network" - name-prefix = var.vm-name-prefix - vpc-cidr-block = var.aws-vpc-cidr-block - subnet-cidr-block = var.aws-subnet-cidr-block - admin-ips = var.admin-ips +################################################################################ +# AWS Networking +# Use of the 2 modules below to create resources for the AWS network. +# aws-network-from-scratch will build the AWS network from scratch. +# aws-network-existing will query AWS for an existing VPC. +################################################################################ + +# module "aws-network-from-scratch" { +# source = "./modules/aws-network-from-scratch" +# name-prefix = var.vm-name-prefix +# vpc-cidr-block = var.aws-vpc-cidr-block +# subnet-cidr-block = var.aws-subnet-cidr-block +# admin-ips = var.admin-ips +# } + +module "aws-network-existing" { + source = "./modules/aws-network-existing" } +################################################################################ + # This key pair is not actually used. Keys are added to the nodes via cloud-init # instead. We just add this here that this key will show up in the AWS console." resource "aws_key_pair" "key" { diff --git a/k8s-nodes/modules/aws-network-existing/main.tf b/modules/aws-network-existing/main.tf similarity index 100% rename from k8s-nodes/modules/aws-network-existing/main.tf rename to modules/aws-network-existing/main.tf diff --git a/k8s-nodes/modules/aws-network-existing/outputs.tf b/modules/aws-network-existing/outputs.tf similarity index 100% rename from k8s-nodes/modules/aws-network-existing/outputs.tf rename to modules/aws-network-existing/outputs.tf diff --git a/k8s-nodes/modules/aws-network-existing/variables.tf b/modules/aws-network-existing/variables.tf similarity index 100% rename from k8s-nodes/modules/aws-network-existing/variables.tf rename to modules/aws-network-existing/variables.tf diff --git a/k8s-nodes/modules/aws-network-from-scratch/main.tf b/modules/aws-network-from-scratch/main.tf similarity index 100% rename from k8s-nodes/modules/aws-network-from-scratch/main.tf rename to modules/aws-network-from-scratch/main.tf diff --git a/k8s-nodes/modules/aws-network-from-scratch/outputs.tf b/modules/aws-network-from-scratch/outputs.tf similarity index 100% rename from k8s-nodes/modules/aws-network-from-scratch/outputs.tf rename to modules/aws-network-from-scratch/outputs.tf diff --git a/k8s-nodes/modules/aws-network-from-scratch/variables.tf b/modules/aws-network-from-scratch/variables.tf similarity index 100% rename from k8s-nodes/modules/aws-network-from-scratch/variables.tf rename to modules/aws-network-from-scratch/variables.tf diff --git a/modules/aws-nodes/main.tf b/modules/aws-nodes/main.tf index 039fb20..282a488 100644 --- a/modules/aws-nodes/main.tf +++ b/modules/aws-nodes/main.tf @@ -1,6 +1,7 @@ resource "aws_instance" "nodes" { ami = var.ami instance_type = var.ec2-instance-type + # TODO REM double check this key. # key_name = aws_key_pair.debug1.key_name associate_public_ip_address = true subnet_id = var.subnet-id diff --git a/k8s-nodes/providers.tf b/providers.tf similarity index 100% rename from k8s-nodes/providers.tf rename to providers.tf diff --git a/variables.tf b/variables.tf index cdba553..aa13de2 100644 --- a/variables.tf +++ b/variables.tf @@ -8,6 +8,11 @@ variable "aws-ec2-instance-type" { description = "The AWS instance type to use for all nodes." } +variable "aws-region" { + default = "us-east-1" + description = "The AWS region to use." +} + variable "aws-subnet-cidr-block" { default = "10.0.1.0/24" description = "The address space to be used for this subnet." From afdd92e5ae60403e4ce7c981e59db8eb1ef507c5 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 15 Dec 2021 08:53:37 -0500 Subject: [PATCH 39/40] k8s cluster deployed to AWS. --- .gitignore | 3 +- get-vm-ips.sh | 6 +++- main.tf | 14 +++++++--- modules/aws-amis/main.tf | 53 +++++++++++++++++++++++++----------- modules/aws-nodes/main.tf | 1 + modules/aws-nodes/outputs.tf | 4 +++ variables.tf | 25 ++++++++++++++--- 7 files changed, 80 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 7103111..71e6cdb 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ terraform.rc ################################################################################ k8s-key* -STARTHERE +*admin-key* +TARTHERE inventory diff --git a/get-vm-ips.sh b/get-vm-ips.sh index b8ab1df..7fc5178 100755 --- a/get-vm-ips.sh +++ b/get-vm-ips.sh @@ -22,6 +22,10 @@ VM_NAME_PREFIX="$( tail -n 1 | \ sed 's/^.*=\s*"\(.*\)"/\1/g')" +PUBLIC_IP_OUTPUT="groups_hostnames_ips" +PRIVATE_IP_OUTPUT="groups_hostnames_private_ips" +IP_TYPE="$PRIVATE_IP_OUTPUT" + # This command stores the output data in the format below. # [ # { @@ -48,7 +52,7 @@ VM_NAME_PREFIX="$( # } # ] DATA="$(terraform show -json | \ - jq '.values.outputs.groups_hostnames_ips.value | to_entries | + jq '.values.outputs.'"$IP_TYPE"'.value | to_entries | map({group: .key, vms:.value | to_entries | map({hostname:.key,ip:.value})})')" diff --git a/main.tf b/main.tf index 0b7b0ab..fdcc3a2 100644 --- a/main.tf +++ b/main.tf @@ -2,11 +2,11 @@ locals { nodes-config = { "master" = { - base-image = var.centos8-ami + base-image = var.ubuntu-ami num = 1 }, "worker" = { - base-image = var.centos8-ami + base-image = var.ubuntu-ami num = 2 } } @@ -79,8 +79,8 @@ module "nodes" { source = "./modules/aws-nodes" ami = each.value.base-image ec2-instance-type = var.aws-ec2-instance-type - subnet-id = module.aws-network.subnet.id - security-group-ids = [module.aws-network.default-security-group.id] + subnet-id = module.aws-network-existing.k8s-subnets[0] + security-group-ids = [data.aws_security_group.default.id] user-datas = lookup(module.cloud-init-config, each.key, null).user-datas num-nodes = each.value.num name-prefix = "${var.vm-name-prefix}-${each.key}" @@ -132,3 +132,9 @@ module "nodes" { output "groups_hostnames_ips" { value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) } } + +# This will outpus a map of group => [{hostname, private_ip}]. +# TODO Figure out how what to do about private_ips for libvirt. +output "groups_hostnames_private_ips" { + value = { for type, node in module.nodes : type => zipmap(node.names, node.private_ips) } +} diff --git a/modules/aws-amis/main.tf b/modules/aws-amis/main.tf index 1949653..8cb82d8 100644 --- a/modules/aws-amis/main.tf +++ b/modules/aws-amis/main.tf @@ -1,31 +1,52 @@ locals { amis = { amzn2 = { - owner-id = "137112412989" + # us-east-2 + # owner-id = "137112412989" + # us-gov-west-1 + owner-id = "045324592363" name = "amzn2-ami-hvm-2*x86_64-gp2" }, ubuntu = { - owner-id = "099720109477" + # us-east-2 + # owner-id = "099720109477" + # us-gov-west-1 + owner-id = "513442679011" name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*" }, - centos7 = { - owner-id = "125523088429" - name = "CentOS 7.*x86_64" - }, - centos8 = { - owner-id = "125523088429" - name = "CentOS 8.*x86_64" - }, - arch = { - owner-id = "093273469852" - name = "arch-linux-lts-hvm*x86_64-ebs" - }, + # centos7 = { + # # us-east-2 + # # owner-id = "125523088429" + # # us-gov-west-1 + # # owner-id = THERE IS NO CENTOS7 IMAGE in us-gov-west-1!! + # name = "CentOS 7.*x86_64" + # }, + # centos8 = { + # # us-east-2 + # # owner-id = "125523088429" + # # us-gov-west-1 + # # owner-id = THERE IS NO CENTOS8 IMAGE in us-gov-west-1!! + # name = "CentOS 8.*x86_64" + # }, + # arch = { + # # us-east-2 + # # owner-id = "093273469852" + # # us-gov-west-1 + # # owner-id = THERE IS NO ARCH IMAGE in us-gov-west-1!! + # name = "arch-linux-lts-hvm*x86_64-ebs" + # }, rhel7 = { - owner-id = "309956199498" + # us-east-2 + # owner-id = "309956199498" + # us-gov-west-1 + owner-id = "219670896067" name = "RHEL-7.*HVM*x86_64*GP2" }, rhel8 = { - owner-id = "309956199498" + # us-east-2 + # owner-id = "309956199498" + # us-gov-west-1 + owner-id = "219670896067" name = "RHEL-8.*HVM*x86_64*GP2" } } diff --git a/modules/aws-nodes/main.tf b/modules/aws-nodes/main.tf index 282a488..8a0f793 100644 --- a/modules/aws-nodes/main.tf +++ b/modules/aws-nodes/main.tf @@ -3,6 +3,7 @@ resource "aws_instance" "nodes" { instance_type = var.ec2-instance-type # TODO REM double check this key. # key_name = aws_key_pair.debug1.key_name + # TODO Make this a variable. associate_public_ip_address = true subnet_id = var.subnet-id vpc_security_group_ids = var.security-group-ids diff --git a/modules/aws-nodes/outputs.tf b/modules/aws-nodes/outputs.tf index 0b4fe7c..ac1ae29 100644 --- a/modules/aws-nodes/outputs.tf +++ b/modules/aws-nodes/outputs.tf @@ -2,6 +2,10 @@ output "ips" { value = aws_instance.nodes.*.public_ip } +output "private_ips" { + value = aws_instance.nodes.*.private_ip +} + output "names" { value = aws_instance.nodes.*.tags.Name } diff --git a/variables.tf b/variables.tf index aa13de2..94c8bd6 100644 --- a/variables.tf +++ b/variables.tf @@ -1,4 +1,5 @@ variable "admin-ips" { + default = ["0.0.0.0/0"] description = "A list of ips or cidr blocks that are allowed to connect to the nodes." type = list(string) } @@ -33,6 +34,7 @@ variable "disk-image-pool-name" { } variable "libvirt-connection-url" { + default = "nobody@localhost" description = "The libvirt connection URI, ie. qemu+ssh://@/system" } @@ -95,31 +97,46 @@ variable "vm-name-prefix" { ################################################################################ variable "amzn2-ami" { - default = "ami-0dd0ccab7e2801812" + # us-east-2 + # default = "ami-0dd0ccab7e2801812" + # us-gov-west-1 + default = "ami-098bf51d9a35299f0" description = "The AMI to use for Amazon Linux 2." } variable "ubuntu-ami" { - default = "ami-06c7d6c0987eaa46c" + # us-east-2 + # default = "ami-06c7d6c0987eaa46c" + # us-gov-west-1 + default = "ami-087ee83c8de303181" description = "The AMI to use for Ubuntu." } variable "centos7-ami" { + # us-east-2 default = "ami-00f8e2c955f7ffa9b" description = "The AMI to use for CentOS 7." } variable "centos8-ami" { + # us-east-2 default = "ami-057cacbfbbb471bb3" description = "The AMI to use for CentOS 8." } variable "arch-ami" { + # us-east-2 default = "ami-02653f06de985e3ba" description = "The AMI to use for Arch Linux." } variable "rhel7-ami" { - default = "ami-0a509b3c2a4d05b3f" + # us-east-2 + # default = "ami-0a509b3c2a4d05b3f" + # us-gov-west-1 + default = "ami-04ccdf5793086ea95" description = "The AMI to use for RHEL 7." } variable "rhel8-ami" { - default = "ami-0d871ca8a77af2948" + # us-east-2 + # default = "ami-0d871ca8a77af2948" + # us-gov-west-1 + default = "ami-0b1f10cd1cd107dd2" description = "The AMI to use for RHEL 8." } From 2f6577b82fe8a31a5790e2b3d3bdd138bb815f3d Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Wed, 15 Dec 2021 09:38:11 -0500 Subject: [PATCH 40/40] Cleaned up some TODOs. --- main.tf | 8 +++++--- modules/aws-network-existing/main.tf | 4 ++++ modules/aws-network-existing/outputs.tf | 6 +++++- modules/aws-network-existing/variables.tf | 5 ++++- modules/aws-nodes/main.tf | 2 -- variables.tf | 10 ++++++++++ 6 files changed, 28 insertions(+), 7 deletions(-) diff --git a/main.tf b/main.tf index fdcc3a2..ba9a35f 100644 --- a/main.tf +++ b/main.tf @@ -59,7 +59,9 @@ module "cloud-init-config" { # } module "aws-network-existing" { - source = "./modules/aws-network-existing" + source = "./modules/aws-network-existing" + default-vpc-name = var.aws-existing-vpc-name + default-security-group-name = var.aws-existing-sg-name } ################################################################################ @@ -79,8 +81,8 @@ module "nodes" { source = "./modules/aws-nodes" ami = each.value.base-image ec2-instance-type = var.aws-ec2-instance-type - subnet-id = module.aws-network-existing.k8s-subnets[0] - security-group-ids = [data.aws_security_group.default.id] + subnet-id = module.aws-network-existing.k8s-subnets-ids[0] + security-group-ids = [module.aws-network-existing.default-sg.id] user-datas = lookup(module.cloud-init-config, each.key, null).user-datas num-nodes = each.value.num name-prefix = "${var.vm-name-prefix}-${each.key}" diff --git a/modules/aws-network-existing/main.tf b/modules/aws-network-existing/main.tf index 81d25c3..19fff56 100644 --- a/modules/aws-network-existing/main.tf +++ b/modules/aws-network-existing/main.tf @@ -21,3 +21,7 @@ data "aws_subnet" "subnets" { for_each = toset(data.aws_subnets.subnet-ids.ids) id = each.key } + +data "aws_security_group" "default" { + name = var.default-security-group-name +} diff --git a/modules/aws-network-existing/outputs.tf b/modules/aws-network-existing/outputs.tf index f16e968..affe10d 100644 --- a/modules/aws-network-existing/outputs.tf +++ b/modules/aws-network-existing/outputs.tf @@ -2,12 +2,16 @@ output "default-vpc" { value = data.aws_vpc.default } +output "default-sg" { + value = data.aws_security_group.default +} + output "subnets" { description = "An array of all subnets in default-vpc." value = data.aws_subnet.subnets } -output "k8s-subnets" { +output "k8s-subnets-ids" { description = "An array of subnets to be used for k8s VMs. These subnets were chosen by selecting a single subnet from each availability_zone." value = [for k,v in local.az-to-subnets : v[0]] } diff --git a/modules/aws-network-existing/variables.tf b/modules/aws-network-existing/variables.tf index b03ebd1..9add409 100644 --- a/modules/aws-network-existing/variables.tf +++ b/modules/aws-network-existing/variables.tf @@ -1,4 +1,7 @@ +variable "default-security-group-name" { + description = "The name of the existing default security group. This module will query AWS for a security group with this name," +} + variable "default-vpc-name" { description = "The name of the existing default VPC. This module will query AWS for a VPC with this name," - default = "Managed VPC" } diff --git a/modules/aws-nodes/main.tf b/modules/aws-nodes/main.tf index 8a0f793..a9f0c1e 100644 --- a/modules/aws-nodes/main.tf +++ b/modules/aws-nodes/main.tf @@ -1,8 +1,6 @@ resource "aws_instance" "nodes" { ami = var.ami instance_type = var.ec2-instance-type - # TODO REM double check this key. - # key_name = aws_key_pair.debug1.key_name # TODO Make this a variable. associate_public_ip_address = true subnet_id = var.subnet-id diff --git a/variables.tf b/variables.tf index 94c8bd6..fed0de5 100644 --- a/variables.tf +++ b/variables.tf @@ -9,6 +9,16 @@ variable "aws-ec2-instance-type" { description = "The AWS instance type to use for all nodes." } +variable "aws-existing-sg-name" { + default = "change-me-if-using-aws-network-existing" + description = "The name of the existing security group when using aws-network-existing." +} + +variable "aws-existing-vpc-name" { + default = "change-me-if-using-aws-network-existing" + description = "The name of the existing VPC when using aws-network-existing." +} + variable "aws-region" { default = "us-east-1" description = "The AWS region to use."