Merge branch 'master' into ansible-test

- Made the qemu-agent content in the cloud-init config to be controlled
  by a 'install-qemu-agent' flag.

README.md

@@ -1,3 +1,15 @@
+A Terraform script to create k8s nodes. This script has modules for creating the
+nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS.
+
+The modules allow you create N VMs of a specific type. So you could create 1
+master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS
+VMs, or whatever fits your needs.
+
+Cloud-Init
+----------------------------------------
+
+Both the libvirt and aws modules use cloud-init for initial configuration of the
+VMs.
 
 Dependencies
 ----------------------------------------

cloud_init.cfg

@@ -24,9 +24,19 @@ chpasswd:
   expire: false
 
 hostname: ${hostname}
+fqdn: ${hostname}
 
-# Use this when it's determined that we need a bigger disk image.
+%{ if install-qemu-agent }
-# This must be used in conjuction with 'size' in 'libvirt_volume'
+packages:
-# growpart:
+  # This are only necessary for libvirt.
-#   mode: auto
+  - qemu-guest-agent
-#   devices: ['/']
+runcmd:
+  # TODO At some point revisit this, this was added because it seemed like
+  # apparmor was causing dhclient to not get an IP address for ubuntu. This
+  # should be double checked.
+  - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient
+  # These are only necessary for libvirt.
+  - systemctl enable qemu-guest-agent
+  - systemctl start qemu-guest-agent
+  - systemctl status qemu-guest-agent
+%{ endif }

example.tfvars

@@ -1,4 +1,4 @@
-vm-name-prefix = "k8s-tf"
+vm-name-prefix = "ansible-test"
 
 # A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0'
 # equates to any ip address.
@@ -10,6 +10,15 @@ libvirt-connection-url = "qemu+ssh://<user>@<host>/system"
 node-memory = 2048
 node-vcpus = 2
 
+## libvirt disk size
+# 1 GiB = 1073741824
+# 4 GiB
+# libvirt-node-disk-size = "${4 * 1073741824}"
+# 8 GiB
+# libvirt-node-disk-size = "${8 * 1073741824}"
+# 12 GiB
+# libvirt-node-disk-size = "${12 * 1073741824}"
+
 ################################################################################
 # AWS EC2 instance types
 ################################################################################
@@ -21,19 +30,6 @@ aws-ec2-instance-type = "t2.micro"
 # 4 GiB, 2 vcpus
 # aws-ec2-instnce-type = "t2.medium"
 
-################################################################################
-
-nodes-config = {
-  "master" = {
-    base-image = "ami-0dd0ccab7e2801812"
-    num = 1
-  },
-  "worker" = {
-    base-image = "ami-0dd0ccab7e2801812"
-    num = 2
-  }
-}
-
 ################################################################################
 # AWS images (AMIs)
 ################################################################################
@@ -64,8 +60,18 @@ base-image = "ami-0dd0ccab7e2801812"
 ################################################################################
 
 # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
-# From https://cloud.centos.org/centos/7/images/ from	2020-11-12 06:52
+# From https://cloud.centos.org/centos/7/images/ from	2020-12-06
-# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2"
+# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+
+## Arch
+# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
+# on 2021-11-28
+# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
+
+## Arch
+# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
+# on 2021-11-28
+# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
 
 ################################################################################
 # Keys/Passwords

get-aws-node-status.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+aws ec2 describe-instance-status | \
+    jq '.InstanceStatuses[] | {id: .InstanceId, instance_status: .InstanceStatus.Status, system_status: .SystemStatus.Status}'

get-libvirt-bridge-ips.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# This script will grab the IPs for libvirt VMs. This script is only needed when
+# using a bridge as the network for the VMs. This should only be needed while
+# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is
+# unresolved.
+
+# These are the network interfaces that this script will attempt to get the IP
+# address for.
+# Ubuntu 20.04    ens3
+# Centos 7 & 8    eth0
+NET_INTERFACES="eth0 ens3"
+
+LIBVIRT_CONNECTION_URL="libvirt-connection-url"
+VM_NAME_PREFIX="vm-name-prefix"
+
+INV_GROUPS="$( \
+cat terraform.tfstate | \
+    jq '.resources[] | select(.type=="libvirt_domain") | .module' | \
+    sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )"
+
+# Grab the connection URL and the vm name prefix. We do this by greping all
+# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the
+# last grep result, this way we make sure any value in terraform.tfvars will
+# take priority.
+CONN_URLS="$( \
+    find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \
+    grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)"
+
+CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
+
+NAME_PREFIXES="$( \
+    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \
+    grep "$VM_NAME_PREFIX" terraform.tfvars)"
+
+NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
+
+# These can be used for debugging.
+# echo "Using connection URL: $CONN_URL"
+# echo "Using prefix: $NAME_PREFIX"
+
+# Get the names of our VMs from libvirt.
+VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')"
+
+# Convert the lines of VM names to an array.
+OLD_IFS=$IFS
+IFS=$'\n'
+VMS=($VMS)
+IFS=$OLD_IFS
+
+# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the
+# result to VM_IP_PAIRS as <vm-name>:<ipv4-address>.
+VM_IP_PAIRS=""
+for VM in "${VMS[@]}"; do
+    for INTERFACE in $NET_INTERFACES; do
+        IP="$( \
+            virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \
+            jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \
+            sed 's/"//g')"
+        # Add the VM:IP pair if IP is not empty.
+        if [ ! -z "$IP" ]; then
+            VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP"
+        fi
+    done
+done
+
+# Write inventory
+cat /dev/null > inventory
+for GROUP in $INV_GROUPS; do
+    echo "[$GROUP]" >> inventory
+    echo "$VM_IP_PAIRS" | \
+        grep $GROUP | \
+        sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory
+done
+
+# Print vars
+echo "$VM_IP_PAIRS" | \
+    sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \
+    sed s/$NAME_PREFIX-//g | \
+    sed 's/-/_/g' | \
+    awk '{print toupper($1)}'

get-vm-ips.sh

@@ -1,60 +1,92 @@
-#!/bin/sh
+#!/bin/bash
 
-# This script will create environment variables for all of the output IPs. It
+# This script will create environment variables for all of the output IPs. An
-# will also create a `ANSIBLE_INV` variable that will be a comma separated
+# anisble inventory file is created as well.
-# string of all the IPs. A anisble inventory file called "inventory is created
-# as well.
 #
 # Use eval $(./get-vm-ips.sh) to set env vars for ips.
 
 terraform refresh > /dev/null
 
-# All terraform outputs in json format.
+# The file to write the inventory to. This file will be completely overridden.
-OUTPUTS_JSON="$(
+INVENTORY_FILE="inventory"
-    terraform show -json | \
+
-        jq '.values.outputs' | \
+# Grab the the vm name prefix. We do this by greping all *.tfvars files making
-        sed 's/-/_/g')"
+# sure to cat terraform.tfvars last. Then we just grab the last grep result,
-# Just the IP address outputs in json format. Also all '-' characters are
+# this way we make sure any value in terraform.tfvars will take priority.
-# replaced by '_' becuase '-' causes jq some problems.
+VM_NAME_PREFIX_VAR="vm-name-prefix"
-IPS_JSON="$(
+VM_NAME_PREFIXES="$( \
-    echo $OUTPUTS_JSON | \
+    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \
-        jq 'to_entries | .[] | select(.key | contains("ips"))')"
+    grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)"
-# An array of all node "types"
+VM_NAME_PREFIX="$(
-NODE_TYPE_ARRAY="$(
+    echo "$VM_NAME_PREFIXES" | \
-    echo $IPS_JSON | \
+        tail -n 1 | \
-        jq '.value.value | to_entries | .[] | .key' | \
+        sed 's/^.*=\s*"\(.*\)"/\1/g')"
+
+# This command stores the output data in the format below.
+# [
+#   {
+#     "group": "master",
+#     "vms": [
+#       {
+#         "hostname": "ansible-test-master-0",
+#         "ip": "52.14.114.48"
+#       }
+#     ]
+#   },
+#   {
+#     "group": "worker",
+#     "vms": [
+#       {
+#         "hostname": "ansible-test-worker-0",
+#         "ip": "3.145.121.159"
+#       },
+#       {
+#         "hostname": "ansible-test-worker-1",
+#         "ip": "18.217.112.176"
+#       }
+#     ]
+#   }
+# ]
+DATA="$(terraform show -json | \
+    jq '.values.outputs.groups_hostnames_ips.value | to_entries |
+        map({group: .key, vms:.value | to_entries |
+        map({hostname:.key,ip:.value})})')"
+
+# Pull out the groups from $DATA. The format is a single string with the groups
+# separated by spaces, ie. "group1 group2 group3".
+ANS_GROUPS="$(
+    echo $DATA | \
+        jq '.[] | .group' | \
         sed 's/"//g' | \
-        sed -z 's/\n/ /g;s/ $/\n/g')"
+        tr '\n' ' '
+    )"
 
-# Loop over all the node types and create an export line for each IP.
+# Clear the inventory file.
-VM_IP_EXPORTS="$(
+cat /dev/null > $INVENTORY_FILE
-    for TYPE in $NODE_TYPE_ARRAY; do
 
-        # Convert type, converts "master-ips" to "MASTER"
+# For each group, write the VM info to $INVENTORY_FILE and also print a variable
-        TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)"
+# expression to stdout.
-        echo "$IPS_JSON" | \
+for GROUP in $ANS_GROUPS; do
-            jq '.value.value.'"$TYPE"'[]' | \
-            # Add line numbers starting with 0.
-            nl -v 0 | \
-            # Print an export string with a type placeholder "__TYPE__".
-            awk '{print "export __TYPE___" $1 "=" $2}' | \
-            sed s/__TYPE__/$TYPE_UPPER/g
-    done)"
 
-ANSIBLE_INV="$(
+    # Write the inventory file to $INVENTORY_FILE.
-    echo "$VM_IP_EXPORTS" | \
+    echo "[$GROUP]" >> $INVENTORY_FILE
+    echo $DATA | \
+        jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
+            "\(.hostname) ansible_host=\(.ip)"' | \
+        sed 's/"//g' \
+            >> $INVENTORY_FILE
+
+    # For this group, collect expressions into VARS. The format is:
+    # HOSTNAME1=0.0.0.0
+    # HOSTNAME2=0.0.0.0
+    VARS="$(
+        echo $DATA | \
+            jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
+                "\(.hostname)=\(.ip)"' | \
             sed 's/"//g' | \
-        sed 's/^.*=//g' | \
+            sed "s/$VM_NAME_PREFIX-//g" | \
-        sed -z 's/\n/,/g;s/,$/\n/g')"
+            sed 's/-/_/g'
-
+        )"
-# Create an inventory file for ansible.
+    # Print the contents of $VARS converted to uppercase.
-echo "[k8s_nodes]" > inventory
+    echo "${VARS^^}"
-echo $VM_IP_EXPORTS | \
+done
-    sed 's/"//g' | \
-    sed 's/export //g' | \
-    sed 's/ /\n/g' | \
-    sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \
-        >> inventory
-
-echo $VM_IP_EXPORTS | sed 's/" /"\n/g'
-echo export ANSIBLE_INV=$ANSIBLE_INV

main.tf

@@ -8,18 +8,33 @@ terraform {
   }
 }
 
+locals {
+  nodes-config = {
+    "master" = {
+      base-image = var.centos8-ami
+      num = 1
+    },
+    "worker" = {
+      base-image = var.centos8-ami
+      num = 2
+    }
+  }
+  install-qemu-agent = false
+}
+
 ################################################################################
 # cloud-init
 ################################################################################
 
 module "cloud-init-config" {
-  for_each            = var.nodes-config
+  for_each            = local.nodes-config
   source              = "./modules/cloud-init-config"
   cloud-init-template = "${path.module}/cloud_init.cfg"
   hostname-prefix     = "${var.vm-name-prefix}-${each.key}"
   num                 = each.value.num
   root-admin-passwd   = var.root-admin-passwd
   root-admin-pub-key  = var.root-admin-pub-key
+  install-qemu-agent  = local.install-qemu-agent
 }
 
 ################################################################################
@@ -32,7 +47,8 @@ provider "aws" {
   region = "us-east-2"
 }
 
-# This module will grab the latest ami for a variety of distros.
+# This module will grab the latest ami for a variety of distros. Uncomment to
+# get a list of the latest AMIs for our supported distros.
 # module "aws-amis" {
 #   source = "./modules/aws-amis"
 # }
@@ -59,7 +75,7 @@ resource "aws_key_pair" "key" {
 }
 
 module "nodes" {
-  for_each           = var.nodes-config
+  for_each           = local.nodes-config
   source             = "./modules/aws-nodes"
   ami                = each.value.base-image
   ec2-instance-type  = var.aws-ec2-instance-type
@@ -85,14 +101,16 @@ module "nodes" {
 # }
 # 
 # module "nodes" {
-#   for_each               = var.nodes-config
+#   for_each               = local.nodes-config
 #   source                 = "./modules/libvirt-nodes"
 #   pool-name              = libvirt_pool.images.name
 #   name-prefix            = "${var.vm-name-prefix}-${each.key}"
 #   num-nodes              = each.value.num
 #   node-memory            = var.node-memory
 #   node-vcpus             = var.node-vcpus
+#   node-disk-size         = var.libvirt-node-disk-size
 #   base-image             = each.value.base-image
+#   network-name           = var.libvirt-network-name
 #   root-admin-passwd      = var.root-admin-passwd
 #   root-admin-pub-key     = var.root-admin-pub-key
 #   libvirt-connection-url = var.libvirt-connection-url
@@ -109,6 +127,8 @@ module "nodes" {
 # end libvirt
 ################################################################################
 
-output "ips" {
+# This will outpus a map of group => [{hostname, ip}].
-  value = { for type, node in module.nodes : type => node.ips }
+# TODO A 'names' output needs to be added to libvirt.
+output "groups_hostnames_ips" {
+  value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) }
 }

modules/aws-network/main.tf

@@ -33,7 +33,7 @@ resource "aws_default_security_group" "sg" {
   }
 
   tags = {
-    Name = "${var.name-prefix}-ssh-from-admins--sg"
+    Name = "${var.name-prefix}-ssh-from-admins-sg"
   }
 }
 

modules/aws-nodes/outputs.tf

@@ -1,3 +1,7 @@
 output "ips" {
   value = aws_instance.nodes.*.public_ip
 }
+
+output "names" {
+  value = aws_instance.nodes.*.tags.Name
+}

modules/cloud-init-config/main.tf

@@ -4,6 +4,7 @@ data "template_file" "user-datas" {
     admin-passwd       = "${var.root-admin-passwd}"
     admin-pub-key      = "${var.root-admin-pub-key}"
     hostname           = "${var.hostname-prefix}-${count.index}"
+    install-qemu-agent = var.install-qemu-agent
   }
   count = var.num
 }

modules/cloud-init-config/variables.tf

@@ -8,6 +8,12 @@ variable "hostname-prefix" {
   description = "This prefix wil be applied as a prefix for the hostnames."
 }
 
+variable "install-qemu-agent" {
+  default     = false
+  description = "This flag determines whether or not qemu-agent is installed."
+  type        = bool
+}
+
 variable "num" {
   description = "The number of user-datas to create with these parameters."
 }

modules/libvirt-nodes/main.tf

@@ -9,13 +9,20 @@ terraform {
 }
 
 resource "libvirt_volume" "node-images" {
-  name   = "${var.name-prefix}-${count.index}"
+  name   = "${var.name-prefix}-base"
   pool   = var.pool-name
   source = var.base-image
-  count  = var.num-nodes
   format = "qcow2"
 }
 
+resource "libvirt_volume" "node-images-resized" {
+  name           = "${var.name-prefix}-${count.index}-resized"
+  pool           = var.pool-name
+  base_volume_id = libvirt_volume.node-images.id
+  count          = var.num-nodes
+  size           = var.node-disk-size
+}
+
 data "template_file" "network-config" {
   template = file("${path.module}/network_config.cfg")
 }
@@ -37,9 +44,9 @@ resource "libvirt_domain" "nodes" {
   cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index)
 
   network_interface {
-    network_name   = "default"
+    network_name   = var.network-name
     hostname       = "${var.name-prefix}-${count.index}"
-    wait_for_lease = true
+    # wait_for_lease = true
   }
 
   # IMPORTANT: this is a known bug on cloud images, since they expect a console
@@ -58,7 +65,7 @@ resource "libvirt_domain" "nodes" {
   }
 
   disk {
-    volume_id = element(libvirt_volume.node-images.*.id, count.index)
+    volume_id = element(libvirt_volume.node-images-resized.*.id, count.index)
   }
 
   graphics {

modules/libvirt-nodes/outpus.tf

@@ -1,4 +1,6 @@
 
-output "ips" {
+# This only works on the default network. They will not work using the bridged
-  value = libvirt_domain.nodes.*.network_interface.0.addresses.0
+# network.
-}
+# output "ips" {
+#   value = libvirt_domain.nodes.*.network_interface.0.addresses.0
+# }

modules/libvirt-nodes/variables.tf

@@ -12,6 +12,16 @@ variable "name-prefix" {
   description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"."
 }
 
+variable "network-name" {
+  default = "default"
+  description = "The name of a pre-existing virtual-network."
+}
+
+variable "node-disk-size" {
+  default = 4294967296
+  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
+}
+
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."

variables.tf

@@ -31,6 +31,16 @@ variable "libvirt-connection-url" {
   description = "The libvirt connection URI, ie. qemu+ssh://<user>@<host>/system"
 }
 
+variable "libvirt-network-name" {
+  default = "default"
+  description = "The name of a pre-existing libvirt virtual-network."
+}
+
+variable "libvirt-node-disk-size" {
+  default = 4294967296
+  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
+}
+
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."
@@ -43,11 +53,6 @@ variable "node-vcpus" {
   type        = number
 }
 
-variable "nodes-config" {
-  description = "A config that declares how many nodes of each type you want created."
-  type = map(object({base-image=string,num=number}))
-}
-
 variable "root-admin-passwd" {
   description = "This will be the password for the root and admin user. The format of this can by any format accepted by cloud-init's chpasswd module."
 }
@@ -76,3 +81,61 @@ variable "vm-name-prefix" {
   default = "k8s-tf"
   description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0."
 }
+
+################################################################################
+# AWS AMI vars
+# These variables are really mor like constants. Using variables improves
+# readability. The defaults are manually updated. Use the aws-amis module to get
+# the latest for each distro.
+################################################################################
+
+variable "amzn2-ami" {
+  default     = "ami-0dd0ccab7e2801812"
+  description = "The AMI to use for Amazon Linux 2."
+}
+variable "ubuntu-ami" {
+  default = "ami-06c7d6c0987eaa46c"
+  description = "The AMI to use for Ubuntu."
+}
+variable "centos7-ami" {
+  default = "ami-00f8e2c955f7ffa9b"
+  description = "The AMI to use for CentOS 7."
+}
+variable "centos8-ami" {
+  default = "ami-057cacbfbbb471bb3"
+  description = "The AMI to use for CentOS 8."
+}
+variable "arch-ami" {
+  default = "ami-02653f06de985e3ba"
+  description = "The AMI to use for Arch Linux."
+}
+variable "rhel7-ami" {
+  default = "ami-0a509b3c2a4d05b3f"
+  description = "The AMI to use for RHEL 7."
+}
+variable "rhel8-ami" {
+  default = "ami-0d871ca8a77af2948"
+  description = "The AMI to use for RHEL 8."
+}
+
+################################################################################
+# Libvirt Images
+# These variables are really mor like constants. Using variables improves
+# readability. The defaults are manually updated.
+################################################################################
+
+variable "ubuntu-img" {
+  default     = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
+  description = "The libvirt image tp use for Ubuntu."
+}
+
+variable "centos7-img" {
+  # Latest as of 2021-12-06.
+  default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+  description = "The libvirt image tp use for CentOS 7."
+}
+
+variable "centos8-img" {
+  default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2"
+  description = "The libvirt image tp use for CentOS 8."
+}