Moved everything from k8s-nodes to the top-level.

README.md

@@ -1,3 +1,15 @@
+A Terraform script to create k8s nodes. This script has modules for creating the
+nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS.
+
+The modules allow you create N VMs of a specific type. So you could create 1
+master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS
+VMs, or whatever fits your needs.
+
+Cloud-Init
+----------------------------------------
+
+Both the libvirt and aws modules use cloud-init for initial configuration of the
+VMs.
 
 Dependencies
 ----------------------------------------

cloud_init.cfg

@@ -24,9 +24,19 @@ chpasswd:
   expire: false
 
 hostname: ${hostname}
+fqdn: ${hostname}
 
-# Use this when it's determined that we need a bigger disk image.
+%{ if install-qemu-agent }
-# This must be used in conjuction with 'size' in 'libvirt_volume'
+packages:
-# growpart:
+  # This are only necessary for libvirt.
-#   mode: auto
+  - qemu-guest-agent
-#   devices: ['/']
+runcmd:
+  # TODO At some point revisit this, this was added because it seemed like
+  # apparmor was causing dhclient to not get an IP address for ubuntu. This
+  # should be double checked.
+  - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient
+  # These are only necessary for libvirt.
+  - systemctl enable qemu-guest-agent
+  - systemctl start qemu-guest-agent
+  - systemctl status qemu-guest-agent
+%{ endif }

example.tfvars

@@ -0,0 +1,85 @@
+vm-name-prefix = "ansible-test"
+
+# A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0'
+# equates to any ip address.
+admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ]
+
+disk-image-dir = "/path/to/disk/pool/"
+libvirt-connection-url = "qemu+ssh://<user>@<host>/system"
+
+node-memory = 2048
+node-vcpus = 2
+
+## libvirt disk size
+# 1 GiB = 1073741824
+# 4 GiB
+# libvirt-node-disk-size = "${4 * 1073741824}"
+# 8 GiB
+# libvirt-node-disk-size = "${8 * 1073741824}"
+# 12 GiB
+# libvirt-node-disk-size = "${12 * 1073741824}"
+
+################################################################################
+# AWS EC2 instance types
+################################################################################
+
+# 1 GiB, 1 vcpu, only one that is free.
+# This one won't work with k8s because it requires at least 2 vcpus.
+aws-ec2-instance-type = "t2.micro"
+
+# 4 GiB, 2 vcpus
+# aws-ec2-instnce-type = "t2.medium"
+
+################################################################################
+# AWS images (AMIs)
+################################################################################
+
+## Amazon Linux 2
+# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free
+# base-image = "ami-00be885d550dcee43"
+# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free
+base-image = "ami-0dd0ccab7e2801812"
+
+## CentOS
+# CentOS 7.9.2009 x86_64 - us-east-2 - 2021-11-15
+# base-image = "ami-00f8e2c955f7ffa9b"
+# CentOS 8.4.2105 x86_64 - us-east-2 - 2021-11015
+# base-image = "ami-057cacbfbbb471bb3"
+
+## Ubuntu
+# Ubuntu Server 20.04 LTS (HVM), SSD Volume Type
+# us-east-2 - (64-bit x86) - 2021.11.12 - free
+# base-image = "ami-0629230e074c580f2"
+
+## Arch linux
+# arch-linux-lts-hvm-2021.06.02.x86_64-ebs - us-east-2
+# base-image = "ami-02653f06de985e3ba"
+
+################################################################################
+# libvirt images
+################################################################################
+
+# base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
+# From https://cloud.centos.org/centos/7/images/ from	2020-12-06
+# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+
+## Arch
+# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
+# on 2021-11-28
+# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
+
+## Arch
+# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
+# on 2021-11-28
+# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
+
+################################################################################
+# Keys/Passwords
+################################################################################
+
+# Password hash created with:
+# python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))'
+# where "linux" is the password.
+root-admin-passwd = "$6$fiLRWvGQkdK.MnZA$Co9NkA5ruuBUA389JzmKJiC8gKRohmyM09AFnVBOD7ErZnxK4RHMUlKvYg1HSgwaCXTl7H/q1svoeQeUfgc6f0"
+
+root-admin-pub-key = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDfDcjMFmWd6qy9KIlnIHNbEfeNLHC885UUH3jGwESmMTpFfPUn01t9hq5GGaFDrBR55VgdKebAv2JSVl209+r3tE5XxUX5/s2Pu3o2283PiZhA+D18skL7fzaolygOY8mxi9CZSDFia//lLbqT/OE45VGahVBRtda4gmjrade0XRKqjJUCkIo6huG9Ub6yP4gFtFU/C1rRvQo0hqT/imsMYU0Q5XzrKVWv3CpzA7EIQq8llU0fRGMuXWYYOXznPeqqf5BTbWhMWUXVS0o7Cz+zvbxwq1dOR1qHbJ8Vrkt30Cz5QEd159dIM3LHCtOHnveeOpkFo0RqkhQdpZM+2cKzESvivGNGP9h+PrSjcveADxVwDHcxguumUyM012M3yR8cK9KY+GqW5jPdAs13yXGTG4OWiQKeKEgX910l/FndhQi0tSpSEhIlfcEpa3k3P8RrhKJbwiRgR7Qvus4R/KU+lx4OiOr4RKyPQJobC0i0/bvqkw+UHWp4U0Hqivjsb6k= admin"

get-aws-node-status.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+aws ec2 describe-instance-status | \
+    jq '.InstanceStatuses[] | {id: .InstanceId, instance_status: .InstanceStatus.Status, system_status: .SystemStatus.Status}'

get-libvirt-bridge-ips.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# This script will grab the IPs for libvirt VMs. This script is only needed when
+# using a bridge as the network for the VMs. This should only be needed while
+# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is
+# unresolved.
+
+# These are the network interfaces that this script will attempt to get the IP
+# address for.
+# Ubuntu 20.04    ens3
+# Centos 7 & 8    eth0
+NET_INTERFACES="eth0 ens3"
+
+LIBVIRT_CONNECTION_URL="libvirt-connection-url"
+VM_NAME_PREFIX="vm-name-prefix"
+
+INV_GROUPS="$( \
+cat terraform.tfstate | \
+    jq '.resources[] | select(.type=="libvirt_domain") | .module' | \
+    sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )"
+
+# Grab the connection URL and the vm name prefix. We do this by greping all
+# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the
+# last grep result, this way we make sure any value in terraform.tfvars will
+# take priority.
+CONN_URLS="$( \
+    find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \
+    grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)"
+
+CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
+
+NAME_PREFIXES="$( \
+    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \
+    grep "$VM_NAME_PREFIX" terraform.tfvars)"
+
+NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
+
+# These can be used for debugging.
+# echo "Using connection URL: $CONN_URL"
+# echo "Using prefix: $NAME_PREFIX"
+
+# Get the names of our VMs from libvirt.
+VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')"
+
+# Convert the lines of VM names to an array.
+OLD_IFS=$IFS
+IFS=$'\n'
+VMS=($VMS)
+IFS=$OLD_IFS
+
+# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the
+# result to VM_IP_PAIRS as <vm-name>:<ipv4-address>.
+VM_IP_PAIRS=""
+for VM in "${VMS[@]}"; do
+    for INTERFACE in $NET_INTERFACES; do
+        IP="$( \
+            virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \
+            jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \
+            sed 's/"//g')"
+        # Add the VM:IP pair if IP is not empty.
+        if [ ! -z "$IP" ]; then
+            VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP"
+        fi
+    done
+done
+
+# Write inventory
+cat /dev/null > inventory
+for GROUP in $INV_GROUPS; do
+    echo "[$GROUP]" >> inventory
+    echo "$VM_IP_PAIRS" | \
+        grep $GROUP | \
+        sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory
+done
+
+# Print vars
+echo "$VM_IP_PAIRS" | \
+    sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \
+    sed s/$NAME_PREFIX-//g | \
+    sed 's/-/_/g' | \
+    awk '{print toupper($1)}'

get-vm-ips.sh

@@ -1,19 +1,92 @@
-#!/bin/sh
+#!/bin/bash
 
+# This script will create environment variables for all of the output IPs. An
+# anisble inventory file is created as well.
+#
 # Use eval $(./get-vm-ips.sh) to set env vars for ips.
 
 terraform refresh > /dev/null
 
-IPS_JSON="$(terraform show -json | jq '.values.outputs')"
+# The file to write the inventory to. This file will be completely overridden.
+INVENTORY_FILE="inventory"
 
-echo $IPS_JSON | \
+# Grab the the vm name prefix. We do this by greping all *.tfvars files making
-    jq '."master-ips".value[]' | \
+# sure to cat terraform.tfvars last. Then we just grab the last grep result,
-    nl -v 0 | \
+# this way we make sure any value in terraform.tfvars will take priority.
-    awk '{print "export MASTER" $1 "=" $2}' | \
+VM_NAME_PREFIX_VAR="vm-name-prefix"
-    sed 's/"//g'
+VM_NAME_PREFIXES="$( \
+    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \
+    grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)"
+VM_NAME_PREFIX="$(
+    echo "$VM_NAME_PREFIXES" | \
+        tail -n 1 | \
+        sed 's/^.*=\s*"\(.*\)"/\1/g')"
 
-echo $IPS_JSON | \
+# This command stores the output data in the format below.
-    jq '."worker-ips".value[]' | \
+# [
-    nl -v 0 | \
+#   {
-    awk '{print "export WORKER" $1 "=" $2}' | \
+#     "group": "master",
-    sed 's/"//g'
+#     "vms": [
+#       {
+#         "hostname": "ansible-test-master-0",
+#         "ip": "52.14.114.48"
+#       }
+#     ]
+#   },
+#   {
+#     "group": "worker",
+#     "vms": [
+#       {
+#         "hostname": "ansible-test-worker-0",
+#         "ip": "3.145.121.159"
+#       },
+#       {
+#         "hostname": "ansible-test-worker-1",
+#         "ip": "18.217.112.176"
+#       }
+#     ]
+#   }
+# ]
+DATA="$(terraform show -json | \
+    jq '.values.outputs.groups_hostnames_ips.value | to_entries |
+        map({group: .key, vms:.value | to_entries |
+        map({hostname:.key,ip:.value})})')"
+
+# Pull out the groups from $DATA. The format is a single string with the groups
+# separated by spaces, ie. "group1 group2 group3".
+ANS_GROUPS="$(
+    echo $DATA | \
+        jq '.[] | .group' | \
+        sed 's/"//g' | \
+        tr '\n' ' '
+    )"
+
+# Clear the inventory file.
+cat /dev/null > $INVENTORY_FILE
+
+# For each group, write the VM info to $INVENTORY_FILE and also print a variable
+# expression to stdout.
+for GROUP in $ANS_GROUPS; do
+
+    # Write the inventory file to $INVENTORY_FILE.
+    echo "[$GROUP]" >> $INVENTORY_FILE
+    echo $DATA | \
+        jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
+            "\(.hostname) ansible_host=\(.ip)"' | \
+        sed 's/"//g' \
+            >> $INVENTORY_FILE
+
+    # For this group, collect expressions into VARS. The format is:
+    # HOSTNAME1=0.0.0.0
+    # HOSTNAME2=0.0.0.0
+    VARS="$(
+        echo $DATA | \
+            jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
+                "\(.hostname)=\(.ip)"' | \
+            sed 's/"//g' | \
+            sed "s/$VM_NAME_PREFIX-//g" | \
+            sed 's/-/_/g'
+        )"
+    # Print the contents of $VARS converted to uppercase.
+    echo "${VARS^^}"
+done

k8s-nodes/.gitignore

@@ -11,6 +11,8 @@
 
 # Crash log files
 crash.log
+# Added to original pulled from github/gitignore
+crash*.log
 
 # Exclude all .tfvars files, which are likely to contain sentitive data, such as
 # password, private keys, and other secrets. These should not be part of version
@@ -36,3 +38,12 @@ override.tf.json
 # Ignore CLI configuration files
 .terraformrc
 terraform.rc
+
+################################################################################
+# end Pulled from github/gitignore 2021-11-10 commit 1a84870
+################################################################################
+
+k8s-key*
+STARTHERE
+inventory
+

k8s-nodes/.terraform.lock.hcl

@@ -0,0 +1,57 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/dmacvicar/libvirt" {
+  version = "0.6.11"
+  hashes = [
+    "h1:6QzHY/7aNdaaDxJZKygotWnM5uHoS2gs/03CzUCJX60=",
+    "zh:15300a1c3c294eccade4c8a678412d81602ab041dc0a5aab72fee5425d778e89",
+    "zh:1605806de0d3b86b7e94b5d04a7ad9b6ac695781f9672ab6002c23caef43b98e",
+    "zh:21efc5937d89f9ec96bc626d2ce3621c0919b3da97ab63b4e520c37d3f5c7357",
+    "zh:2c143a6909917fd11191447de4c496f084c7da5200beb9f512791a80a1f33e7c",
+    "zh:3ca369718cc49feefc3a6ffa795a9055e60de33989a9f1c72b6db16048a181fa",
+    "zh:71db1d1cf2c06984bba408ad5dc9b4e25285684ee5c530a61583b202cff21b96",
+    "zh:a67adfc988311d34adcc119500c2ef048a45d632b00bb5a15ea6d3ffdc1c3d1c",
+    "zh:a83448cbcc194e3b52af9b89b9273a116082d83f2c966035bf8a8c5d5606ca9c",
+    "zh:a9c5a818dd2606460d4d6f33af7cb387f3e984d631fc233aaec0dda4e0756c2b",
+    "zh:af3263e66cf9138361d6d7408533edd6de8498e67c88cf0084421ae31fe89054",
+    "zh:b9596cb26c1e391172472de4ada9b3b0a08e4777e41327db8e021454cc6aae20",
+    "zh:e28124b9ee0b8c18b6f776eb6523d8935f3072c47cd803ea2f1a06206effaa48",
+    "zh:edd00638d8c088b8a38e7ab8b9e8ab3bd710f7357d0f6b4a38e0028bd49d8460",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/aws" {
+  version = "3.64.2"
+  hashes = [
+    "h1:oFsgTmmt/eJ8KLo59PSmYu/irUHJiPnQEey0VqaOOck=",
+    "zh:0b029a2282beabfe410eb2969e18ca773d3473415e442be4dc8ce0eb6d1cd8c5",
+    "zh:3209de3266a1138f1ccb09f094fdd98b6f55afc06e291db0abe092ec5dbe7640",
+    "zh:40648266551631cbc15f8a76e80faf300510e3b38c2544d43fc25e37e6802727",
+    "zh:483c8af92ae70146f2790a70c1a810251e7135aa912b66e769c934eddceebe32",
+    "zh:4d106d8d415d8df342f3f85e58c35418e6c55e3cb7f02897f832cefac4dca68c",
+    "zh:972626a6ddb31d5216606d12ab5c30fbf8d51ed2bbe0efcdd7cffa68c1141557",
+    "zh:a230d55ec52b1695148d40296877ee23e0b302e817154f9b838eb117c87b13fa",
+    "zh:c95fddfbd7f870db949da0601323e866e0f0fb0d4a93e96725ae5b88029e84d5",
+    "zh:ea0c7f568074f835f22273c8e7e61e87f5277e32004c72122915fd3c8df49ccc",
+    "zh:f96d25887e6e2d2ae47659e2c586efea2167995b59a479ae65a02b097da86474",
+    "zh:fe7502d8e52d3b5ccb2b3c178e7ea894344783093aa71ffb20e978914c976182",
+  ]
+}
+
+provider "registry.terraform.io/hashicorp/template" {
+  version = "2.2.0"
+  hashes = [
+    "h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=",
+    "zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386",
+    "zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53",
+    "zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603",
+    "zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16",
+    "zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776",
+    "zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451",
+    "zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae",
+    "zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde",
+    "zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d",
+    "zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2",
+  ]
+}

main.tf

@@ -1,53 +1,134 @@
-terraform {
+
-  required_version = ">= 0.13"
+locals {
-  required_providers {
+  nodes-config = {
-    libvirt = {
+    "master" = {
-      source = "dmacvicar/libvirt"
+      base-image = var.centos8-ami
-      version = "0.6.11"
+      num = 1
+    },
+    "worker" = {
+      base-image = var.centos8-ami
+      num = 2
     }
   }
+  install-qemu-agent = false
 }
 
-provider "libvirt" {
+################################################################################
-  uri = var.libvirt-connection-url
+# cloud-init
-}
+################################################################################
 
-module "master-nodes" {
+module "cloud-init-config" {
-  source = "./modules/node"
+  for_each            = local.nodes-config
-  pool-name = libvirt_pool.images.name
+  source              = "./modules/cloud-init-config"
-  name-prefix = "${var.vm-name-prefix}-master"
+  cloud-init-template = "${path.module}/cloud_init.cfg"
-  num-nodes = var.master-nodes
+  hostname-prefix     = "${var.vm-name-prefix}-${each.key}"
-  node-memory = var.node-memory
+  num                 = each.value.num
-  node-vcpus = var.node-vcpus
-  base-image = var.ubuntu-image
   root-admin-passwd   = var.root-admin-passwd
   root-admin-pub-key  = var.root-admin-pub-key
-  libvirt-connection-url = var.libvirt-connection-url
+  install-qemu-agent  = local.install-qemu-agent
 }
 
-module "worker-nodes" {
+################################################################################
-  source = "./modules/node"
+# aws
-  pool-name = libvirt_pool.images.name
+# To use the aws module, uncomment the aws modules/resources and comment out the
-  name-prefix = "${var.vm-name-prefix}-worker"
+# libvirt modules/resources.
-  num-nodes = var.worker-nodes
+################################################################################
-  node-memory = var.node-memory
+
-  node-vcpus = var.node-vcpus
+# This module will grab the latest ami for a variety of distros. Uncomment to
-  base-image = var.ubuntu-image
+# get a list of the latest AMIs for our supported distros.
-  root-admin-passwd = var.root-admin-passwd
+# module "aws-amis" {
-  root-admin-pub-key = var.root-admin-pub-key
+#   source = "./modules/aws-amis"
-  libvirt-connection-url = var.libvirt-connection-url
+# }
+# output "amis" {
+#   value = module.aws-amis.amis
+# }
+
+################################################################################
+# AWS Networking
+# Use of the 2 modules below to create resources for the AWS network.
+# aws-network-from-scratch will build the AWS network from scratch.
+# aws-network-existing will query AWS for an existing VPC.
+################################################################################
+
+# module "aws-network-from-scratch" {
+#   source            = "./modules/aws-network-from-scratch"
+#   name-prefix       = var.vm-name-prefix
+#   vpc-cidr-block    = var.aws-vpc-cidr-block
+#   subnet-cidr-block = var.aws-subnet-cidr-block
+#   admin-ips         = var.admin-ips
+# }
+
+module "aws-network-existing" {
+  source = "./modules/aws-network-existing"
 }
 
-resource "libvirt_pool" "images" {
+################################################################################
-  name = var.disk-image-pool-name
+
-  type = "dir"
+# This key pair is not actually used. Keys are added to the nodes via cloud-init
-  path = var.disk-image-dir
+# instead. We just add this here that this key will show up in the AWS console."
+resource "aws_key_pair" "key" {
+  key_name   = "${var.vm-name-prefix}-key}"
+  public_key = var.root-admin-pub-key
+  tags = {
+    Name = "${var.vm-name-prefix}-key"
+  }
 }
 
-output "master-ips" {
+module "nodes" {
-  value = module.master-nodes.ips
+  for_each           = local.nodes-config
+  source             = "./modules/aws-nodes"
+  ami                = each.value.base-image
+  ec2-instance-type  = var.aws-ec2-instance-type
+  subnet-id          = module.aws-network.subnet.id
+  security-group-ids = [module.aws-network.default-security-group.id]
+  user-datas         = lookup(module.cloud-init-config, each.key, null).user-datas
+  num-nodes          = each.value.num
+  name-prefix        = "${var.vm-name-prefix}-${each.key}"
 }
 
-output "worker-ips" {
+################################################################################
-  value = module.worker-nodes.ips
+# end aws
+################################################################################
+
+################################################################################
+# libvirt
+# To use the libvirt module, uncomment the libvirt modules/resources and comment
+# out the aws modules/resources.
+################################################################################
+
+# provider "libvirt" {
+#   uri = var.libvirt-connection-url
+# }
+# 
+# module "nodes" {
+#   for_each               = local.nodes-config
+#   source                 = "./modules/libvirt-nodes"
+#   pool-name              = libvirt_pool.images.name
+#   name-prefix            = "${var.vm-name-prefix}-${each.key}"
+#   num-nodes              = each.value.num
+#   node-memory            = var.node-memory
+#   node-vcpus             = var.node-vcpus
+#   node-disk-size         = var.libvirt-node-disk-size
+#   base-image             = each.value.base-image
+#   network-name           = var.libvirt-network-name
+#   root-admin-passwd      = var.root-admin-passwd
+#   root-admin-pub-key     = var.root-admin-pub-key
+#   libvirt-connection-url = var.libvirt-connection-url
+#   user-datas             = lookup(module.cloud-init-config, each.key, null).user-datas
+# }
+# 
+# resource "libvirt_pool" "images" {
+#   name = var.disk-image-pool-name
+#   type = "dir"
+#   path = var.disk-image-dir
+# }
+
+################################################################################
+# end libvirt
+################################################################################
+
+# This will output a map of group => [{hostname, ip}].
+# TODO A 'names' output needs to be added to libvirt.
+output "groups_hostnames_ips" {
+  value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) }
 }

modules/aws-amis/main.tf

@@ -0,0 +1,58 @@
+locals {
+  amis = {
+    amzn2 = {
+      owner-id = "137112412989"
+      name = "amzn2-ami-hvm-2*x86_64-gp2"
+    },
+    ubuntu = {
+      owner-id = "099720109477"
+      name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
+    },
+    centos7 = {
+      owner-id = "125523088429"
+      name = "CentOS 7.*x86_64"
+    },
+    centos8 = {
+      owner-id = "125523088429"
+      name = "CentOS 8.*x86_64"
+    },
+    arch = {
+      owner-id = "093273469852"
+      name = "arch-linux-lts-hvm*x86_64-ebs"
+    },
+    rhel7 = {
+      owner-id = "309956199498"
+      name = "RHEL-7.*HVM*x86_64*GP2"
+    },
+    rhel8 = {
+      owner-id = "309956199498"
+      name = "RHEL-8.*HVM*x86_64*GP2"
+    }
+  }
+}
+
+data "aws_ami" "amis" {
+  for_each = local.amis
+  most_recent = true
+  owners = [each.value.owner-id]
+
+  filter {
+    name = "name"
+    values = [each.value.name]
+  }
+
+  filter {
+    name = "virtualization-type"
+    values = ["hvm"]
+  }
+
+  filter {
+    name = "architecture"
+    values = ["x86_64"]
+  }
+
+  filter {
+    name = "root-device-type"
+    values = ["ebs"]
+  }
+}

modules/aws-amis/outputs.tf

@@ -0,0 +1,3 @@
+output "amis" {
+  value = tomap({ for type, ami in data.aws_ami.amis : type => ami.id })
+}

modules/aws-network-existing/main.tf

@@ -0,0 +1,23 @@
+locals {
+  az-to-subnets = {
+    for s in data.aws_subnet.subnets : s.availability_zone => s.id...
+  }
+}
+
+data "aws_vpc" "default" {
+  tags = {
+    Name = var.default-vpc-name
+  }
+}
+
+data "aws_subnets" "subnet-ids" {
+  filter {
+    name = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
+}
+
+data "aws_subnet" "subnets" {
+  for_each = toset(data.aws_subnets.subnet-ids.ids)
+  id = each.key
+}

modules/aws-network-existing/outputs.tf

@@ -0,0 +1,18 @@
+output "default-vpc" {
+  value = data.aws_vpc.default
+}
+
+output "subnets" {
+  description = "An array of all subnets in default-vpc."
+  value = data.aws_subnet.subnets
+}
+
+output "k8s-subnets" {
+  description = "An array of subnets to be used for k8s VMs. These subnets were chosen by selecting a single subnet from each availability_zone."
+  value = [for k,v in local.az-to-subnets : v[0]]
+}
+
+output "az-to-subnets" {
+  description = "A map of availability zone to array of subnets that are in thet availability zone."
+  value = local.az-to-subnets
+}

modules/aws-network-existing/variables.tf

@@ -0,0 +1,4 @@
+variable "default-vpc-name" {
+  description = "The name of the existing default VPC. This module will query AWS for a VPC with this name,"
+  default = "Managed VPC"
+}

modules/aws-network-from-scratch/main.tf

@@ -0,0 +1,67 @@
+resource "aws_vpc" "vpc" {
+  cidr_block = var.vpc-cidr-block
+  tags = {
+      Name = "${var.name-prefix}-vpc"
+  }
+}
+
+resource "aws_subnet" "subnet" {
+  vpc_id = aws_vpc.vpc.id
+  cidr_block = var.subnet-cidr-block
+  # availability_zone = var.avail_zone
+  tags = {
+      Name = "${var.name-prefix}-subnet"
+  }
+}
+
+resource "aws_default_security_group" "sg" {
+  vpc_id = aws_vpc.vpc.id
+
+  ingress {
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = var.admin-ips
+  }
+
+  egress {
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+    cidr_blocks     = ["0.0.0.0/0"]
+    prefix_list_ids = []
+  }
+
+  tags = {
+    Name = "${var.name-prefix}-ssh-from-admins-sg"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+	vpc_id = aws_vpc.vpc.id
+  tags = {
+    Name = "${var.name-prefix}-igw"
+  }
+}
+
+resource "aws_default_route_table" "route-table" {
+   default_route_table_id = aws_vpc.vpc.main_route_table_id
+
+   route {
+     cidr_block = "0.0.0.0/0"
+     gateway_id = aws_internet_gateway.igw.id
+   }
+
+   # default route, mapping VPC CIDR block to "local", created implicitly and
+   # cannot be specified.
+
+   tags = {
+     Name = "${var.name-prefix}-route-table"
+   }
+}
+
+# Associate subnet with Route Table
+resource "aws_route_table_association" "a-rtb-subnet" {
+  subnet_id      = aws_subnet.subnet.id
+  route_table_id = aws_default_route_table.route-table.id
+}

modules/aws-network-from-scratch/outputs.tf

@@ -0,0 +1,11 @@
+output "vpc" {
+  value = aws_vpc.vpc
+}
+
+output "subnet" {
+  value = aws_subnet.subnet
+}
+
+output "default-security-group" {
+  value = aws_default_security_group.sg
+}

modules/aws-network-from-scratch/variables.tf

@@ -0,0 +1,23 @@
+variable "admin-ips" {
+  description = "A list of ips or cidr blocks that are allowed to connect to the nodes."
+  type = list(string)
+}
+
+variable "name-prefix" {
+  default = "tf"
+  description = "This prefix will be used in all the names of the resources creates in our AWS network."
+  type = string
+}
+
+variable "subnet-cidr-block" {
+  default = "10.0.1.0/24"
+  description = "The address space to be used for this subnet."
+  type = string
+}
+
+variable "vpc-cidr-block" {
+  default = "10.0.0.0/16"
+  description = "The address space to be used for out networks VPC."
+  type = string
+}
+

modules/aws-nodes/main.tf

@@ -0,0 +1,15 @@
+resource "aws_instance" "nodes" {
+  ami                         = var.ami
+  instance_type               = var.ec2-instance-type
+  # TODO REM double check this key.
+  # key_name                    = aws_key_pair.debug1.key_name
+  associate_public_ip_address = true
+  subnet_id                   = var.subnet-id
+  vpc_security_group_ids = var.security-group-ids
+  user_data = element(var.user-datas.*.rendered, count.index)
+  count                       = var.num-nodes
+
+  tags = {
+    Name = "${var.name-prefix}-${count.index}"
+  }
+}

modules/aws-nodes/outputs.tf

@@ -0,0 +1,7 @@
+output "ips" {
+  value = aws_instance.nodes.*.public_ip
+}
+
+output "names" {
+  value = aws_instance.nodes.*.tags.Name
+}

modules/aws-nodes/variables.tf

@@ -0,0 +1,36 @@
+variable "ami" {
+  description = "The AWS AMI to be used for all the nodes"
+  type = string
+}
+
+variable "ec2-instance-type" {
+  default = "t2.micro"
+  description = "The AWS instance type to use for all nodes."
+  type = string
+}
+
+variable "name-prefix" {
+  default = "tf-node"
+  description = "This prefix will be applied to all names created by this module."
+  type = string
+}
+
+variable "num-nodes" {
+  default = 1
+  description = "The number of nodes to create from the given input parameters."
+  type = number
+}
+
+variable "user-datas" {
+  description = "A list of cloud-init configs that get applied to their corresponding node."
+}
+
+variable "subnet-id" {
+  description = "The ID of the subnet that all the nodes will be added to."
+  type = string
+}
+
+variable "security-group-ids" {
+  description = "A list of security group IDs to be applied to all the nodes."
+  type = list(string)
+}

modules/cloud-init-config/main.tf

@@ -0,0 +1,10 @@
+data "template_file" "user-datas" {
+  template = file("${var.cloud-init-template}")
+  vars = {
+    admin-passwd       = "${var.root-admin-passwd}"
+    admin-pub-key      = "${var.root-admin-pub-key}"
+    hostname           = "${var.hostname-prefix}-${count.index}"
+    install-qemu-agent = var.install-qemu-agent
+  }
+  count = var.num
+}

modules/cloud-init-config/outputs.tf

@@ -0,0 +1,3 @@
+output "user-datas" {
+  value = data.template_file.user-datas
+}

modules/cloud-init-config/variables.tf

@@ -0,0 +1,28 @@
+variable "cloud-init-template" {
+  default     = "../../cloud_init.cfg"
+  description = "The path to the cloud-init config template."
+  type        = string
+}
+
+variable "hostname-prefix" {
+  description = "This prefix wil be applied as a prefix for the hostnames."
+}
+
+variable "install-qemu-agent" {
+  default     = false
+  description = "This flag determines whether or not qemu-agent is installed."
+  type        = bool
+}
+
+variable "num" {
+  description = "The number of user-datas to create with these parameters."
+}
+
+variable "root-admin-passwd" {
+  description = "This value will be substituted for any occurence of 'admin-password' in the cloud-init config template."
+}
+
+variable "root-admin-pub-key" {
+  description = "This value will be substituted for any occurence of 'admin-pub-key' in the cloud-init config template."
+}
+

modules/libvirt-nodes/main.tf

@@ -8,26 +8,19 @@ terraform {
   }
 }
 
-provider "libvirt" {
-  uri = var.libvirt-connection-url
-}
-
 resource "libvirt_volume" "node-images" {
-  name   = "${var.name-prefix}-${count.index}"
+  name   = "${var.name-prefix}-base"
   pool   = var.pool-name
   source = var.base-image
-  count  = var.num-nodes
   format = "qcow2"
 }
 
-data "template_file" "node-user-datas" {
+resource "libvirt_volume" "node-images-resized" {
-  template = file("${path.module}/cloud_init.cfg")
+  name           = "${var.name-prefix}-${count.index}-resized"
-  vars = {
+  pool           = var.pool-name
-    admin-passwd  = "${var.root-admin-passwd}"
+  base_volume_id = libvirt_volume.node-images.id
-    admin-pub-key = "${var.root-admin-pub-key}"
-    hostname      = "${var.name-prefix}-${count.index}"
-  }
   count          = var.num-nodes
+  size           = var.node-disk-size
 }
 
 data "template_file" "network-config" {
@@ -36,7 +29,7 @@ data "template_file" "network-config" {
 
 resource "libvirt_cloudinit_disk" "node-inits" {
   name           = "${var.name-prefix}-${count.index}-init"
-  user_data      = element(data.template_file.node-user-datas.*.rendered, count.index)
+  user_data      = element(var.user-datas.*.rendered, count.index)
   network_config = data.template_file.network-config.rendered
   pool           = var.pool-name
   count          = var.num-nodes
@@ -51,9 +44,9 @@ resource "libvirt_domain" "nodes" {
   cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index)
 
   network_interface {
-    network_name   = "default"
+    network_name   = var.network-name
     hostname       = "${var.name-prefix}-${count.index}"
-    wait_for_lease = true
+    # wait_for_lease = true
   }
 
   # IMPORTANT: this is a known bug on cloud images, since they expect a console
@@ -72,7 +65,7 @@ resource "libvirt_domain" "nodes" {
   }
 
   disk {
-    volume_id = element(libvirt_volume.node-images.*.id, count.index)
+    volume_id = element(libvirt_volume.node-images-resized.*.id, count.index)
   }
 
   graphics {

modules/libvirt-nodes/outpus.tf

@@ -0,0 +1,6 @@
+
+# This only works on the default network. They will not work using the bridged
+# network.
+# output "ips" {
+#   value = libvirt_domain.nodes.*.network_interface.0.addresses.0
+# }

modules/libvirt-nodes/variables.tf

@@ -12,6 +12,16 @@ variable "name-prefix" {
   description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"."
 }
 
+variable "network-name" {
+  default = "default"
+  description = "The name of a pre-existing virtual-network."
+}
+
+variable "node-disk-size" {
+  default = 4294967296
+  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
+}
+
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."
@@ -24,6 +34,10 @@ variable "node-vcpus" {
   type        = number
 }
 
+variable "user-datas" {
+  description = "A list of cloud-init configs that get applied to their corresponding node."
+}
+
 variable "num-nodes" {
   description = "The number of nodes to create with this config."
 }

modules/node/outpus.tf

@@ -1,4 +0,0 @@
-
-output "ips" {
-  value = libvirt_domain.nodes.*.network_interface.0.addresses.0
-}

providers.tf

@@ -0,0 +1,76 @@
+
+provider "aws" {
+  region = "us-gov-west-1"
+}
+
+
+terraform {
+  required_version = ">= 1.0.8"
+
+    backend "s3" {
+      
+      bucket            = "mss-terraform-state"
+      key               = "global/s3/terraform.tfstate"
+      region            = "us-gov-west-1"
+      
+      dynamodb_table    = "mss-terraform-state-lock"
+      encrypt           = true
+
+  }
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.6.11"
+    }
+  }
+}
+
+
+
+
+
+##
+#S3 bucket create to hold our TFState file so we can all share env settings
+resource "aws_s3_bucket" "terraform_state" {
+    bucket = "mss-terraform-state"
+
+    # enable versioning for the state files 
+    versioning {
+        enabled = true
+    }
+
+    #enable server-side encryption
+    server_side_encryption_configuration {
+      
+      rule {
+          apply_server_side_encryption_by_default {
+              sse_algorithm = "AES256"
+              }
+          }
+        } 
+}
+
+##
+# no sql database used so that we can lock the TFstate file in the S3 bucket to ensure two people 
+# are not running a terraform command at the same time 
+resource "aws_dynamodb_table" "terraform_locks" {
+    name            = "mss-terraform-state-lock"
+    billing_mode    = "PAY_PER_REQUEST"
+    hash_key        = "LockID"
+
+    attribute {
+      name = "LockID"
+      type = "S"
+    }
+
+
+}
+
+
+##
+# output variable to give details on the s3 bucket created
+#TODO: move to output.tf
+output "s3_bucket_arn" {
+  value       = aws_s3_bucket.terraform_state.arn
+  description = "The ARN of the S3 bucket"
+}

variables.tf

@@ -1,3 +1,27 @@
+variable "admin-ips" {
+  description = "A list of ips or cidr blocks that are allowed to connect to the nodes."
+  type = list(string)
+}
+
+variable "aws-ec2-instance-type" {
+  default = "t2.micro"
+  description = "The AWS instance type to use for all nodes."
+}
+
+variable "aws-region" {
+  default = "us-east-1"
+  description = "The AWS region to use."
+}
+
+variable "aws-subnet-cidr-block" {
+  default = "10.0.1.0/24"
+  description = "The address space to be used for this subnet."
+}
+
+variable "aws-vpc-cidr-block" {
+  default = "10.0.0.0/16"
+  description = "The address space to be used for the VPC that all the AWS nodes will be in."
+}
 
 variable "disk-image-dir" {
   description = "This is the location on the KVM hypervisor host where all the disk images will be kept."
@@ -12,6 +36,16 @@ variable "libvirt-connection-url" {
   description = "The libvirt connection URI, ie. qemu+ssh://<user>@<host>/system"
 }
 
+variable "libvirt-network-name" {
+  default = "default"
+  description = "The name of a pre-existing libvirt virtual-network."
+}
+
+variable "libvirt-node-disk-size" {
+  default = 4294967296
+  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
+}
+
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."
@@ -44,7 +78,7 @@ variable "worker-nodes" {
   type        = number
 }
 
-variable "ubuntu-image" {
+variable "base-image" {
   default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
 }
 
@@ -52,3 +86,61 @@ variable "vm-name-prefix" {
   default = "k8s-tf"
   description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0."
 }
+
+################################################################################
+# AWS AMI vars
+# These variables are really mor like constants. Using variables improves
+# readability. The defaults are manually updated. Use the aws-amis module to get
+# the latest for each distro.
+################################################################################
+
+variable "amzn2-ami" {
+  default     = "ami-0dd0ccab7e2801812"
+  description = "The AMI to use for Amazon Linux 2."
+}
+variable "ubuntu-ami" {
+  default = "ami-06c7d6c0987eaa46c"
+  description = "The AMI to use for Ubuntu."
+}
+variable "centos7-ami" {
+  default = "ami-00f8e2c955f7ffa9b"
+  description = "The AMI to use for CentOS 7."
+}
+variable "centos8-ami" {
+  default = "ami-057cacbfbbb471bb3"
+  description = "The AMI to use for CentOS 8."
+}
+variable "arch-ami" {
+  default = "ami-02653f06de985e3ba"
+  description = "The AMI to use for Arch Linux."
+}
+variable "rhel7-ami" {
+  default = "ami-0a509b3c2a4d05b3f"
+  description = "The AMI to use for RHEL 7."
+}
+variable "rhel8-ami" {
+  default = "ami-0d871ca8a77af2948"
+  description = "The AMI to use for RHEL 8."
+}
+
+################################################################################
+# Libvirt Images
+# These variables are really mor like constants. Using variables improves
+# readability. The defaults are manually updated.
+################################################################################
+
+variable "ubuntu-img" {
+  default     = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
+  description = "The libvirt image tp use for Ubuntu."
+}
+
+variable "centos7-img" {
+  # Latest as of 2021-12-06.
+  default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+  description = "The libvirt image tp use for CentOS 7."
+}
+
+variable "centos8-img" {
+  default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2"
+  description = "The libvirt image tp use for CentOS 8."
+}