Recomment user-data from master.

- Moved network_config.cfg to libvirt-nodes.

.gitignore

@@ -44,7 +44,6 @@ terraform.rc
 ################################################################################
 
 k8s-key*
-*admin-key*
 STARTHERE
 inventory
 

README.md

@@ -1,53 +1,19 @@
-A terraform script that will setup VMs via AWS or KVM/QEMU (libvirt).
+A Terraform script to create k8s nodes. This script has modules for creating the
+nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS.
 
 The modules allow you create N VMs of a specific type. So you could create 1
 master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS
 VMs, or whatever fits your needs.
 
-This script creates the VMs and then uses Cloud-init for initial configuration
-of the VMs.
-
-libvirt vs. AWS
-----------------------------------------
-
-This script allows VMs to be created libvirt or AWS. You can choose either one,
-however, running both at the same time has not been tested. Therefore, when
-using one, you should comment out all code pertaining to the other provider.
-
 Cloud-Init
 ----------------------------------------
 
 Both the libvirt and aws modules use cloud-init for initial configuration of the
-VMs. Cloud-init is used to create the admin user and add the mss-admin keys.
+VMs.
-Cloud-init is also used to install some initial packages. The goal is to get
-these VMs setup enough so that they can be configured via Ansible.
-
-Environment Setup
-----------------------------------------
-
-1. From within the nodes-terraform directory run `terraform init`. You will a
-   see bunch if output as terraform grabs the dependencies it needs.
-
-2. Copy the shared `terraform.tfvars` from S3.
-   ```shell
-   aws s3 cp s3://mss-terraform-state/global/s3/terraform.tfvars .
-   ```
-   For instructions on getting AWS CLI to work correctly see `aws-api-setup.md`
-   in the `doc-ops` repo.
-
-   Take care to copy your `terraform.tfvars` back to the S3 bucket whenever you
-   make a change to it.
-
-3. Verify that you branch and variables match whats deployed by running
-   `terraform plan`. If everything is in sync you should see `No changes. Your
-   infrastructure matches the configuration.` _IF_ you do not see this message
-   make sure to sync with the other admins before continuing.
-
 
 Dependencies
 ----------------------------------------
 
-_UNFINISHED_
 TODO REM add libvirt provider
 libvirt provider depends on mkisofs
 

cloud_init.cfg

@@ -24,23 +24,9 @@ chpasswd:
   expire: false
 
 hostname: ${hostname}
-fqdn: ${hostname}
 
-package_update: true
+# Use this when it's determined that we need a bigger disk image.
-package_upgrade: true
+# This must be used in conjuction with 'size' in 'libvirt_volume'
-package_reboot_if_required: true
+# growpart:
-
+#   mode: auto
-%{ if install-qemu-agent }
+#   devices: ['/']
-packages:
-  # This are only necessary for libvirt.
-  - qemu-guest-agent
-runcmd:
-  # TODO At some point revisit this, this was added because it seemed like
-  # apparmor was causing dhclient to not get an IP address for ubuntu. This
-  # should be double checked.
-  - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient
-  # These are only necessary for libvirt.
-  - systemctl enable qemu-guest-agent
-  - systemctl start qemu-guest-agent
-  - systemctl status qemu-guest-agent
-%{ endif }

example.tfvars

@@ -1,4 +1,4 @@
-vm-name-prefix = "ansible-test"
+vm-name-prefix = "docker-ansible-test"
 
 # A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0'
 # equates to any ip address.
@@ -7,17 +7,18 @@ admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ]
 disk-image-dir = "/path/to/disk/pool/"
 libvirt-connection-url = "qemu+ssh://<user>@<host>/system"
 
+master-nodes = 1
+worker-nodes = 2
+
 node-memory = 2048
 node-vcpus = 2
 
-## libvirt disk size
+# 1 GiB, 1 vcpu, only one that is free.
-# 1 GiB = 1073741824
+# This one won't work with k8s because it requires at least 2 vcpus.
-# 4 GiB
+aws-ec2-instance-type = "t2.micro"
-# libvirt-node-disk-size = "${4 * 1073741824}"
+
-# 8 GiB
+# 4 GiB, 2 vcpus
-# libvirt-node-disk-size = "${8 * 1073741824}"
+# aws-ec2-instnce-type = "t2.medium"
-# 12 GiB
-# libvirt-node-disk-size = "${12 * 1073741824}"
 
 ################################################################################
 # AWS images (AMIs)
@@ -49,22 +50,8 @@ base-image = "ami-0dd0ccab7e2801812"
 ################################################################################
 
 # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
-# From https://cloud.centos.org/centos/7/images/ from	2020-12-06
+# From https://cloud.centos.org/centos/7/images/ from	2020-11-12 06:52
-# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2"
-
-## Arch
-# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
-# on 2021-11-28
-# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
-
-## Arch
-# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
-# on 2021-11-28
-# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
-
-################################################################################
-# Keys/Passwords
-################################################################################
 
 # Password hash created with:
 # python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))'

get-aws-node-status.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-aws ec2 describe-instance-status | \
-    jq '.InstanceStatuses[] | {id: .InstanceId, instance_status: .InstanceStatus.Status, system_status: .SystemStatus.Status}'

get-libvirt-bridge-ips.sh

@@ -1,81 +0,0 @@
-#!/bin/bash
-
-# This script will grab the IPs for libvirt VMs. This script is only needed when
-# using a bridge as the network for the VMs. This should only be needed while
-# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is
-# unresolved.
-
-# These are the network interfaces that this script will attempt to get the IP
-# address for.
-# Ubuntu 20.04    ens3
-# Centos 7 & 8    eth0
-NET_INTERFACES="eth0 ens3"
-
-LIBVIRT_CONNECTION_URL="libvirt-connection-url"
-VM_NAME_PREFIX="vm-name-prefix"
-
-INV_GROUPS="$( \
-cat terraform.tfstate | \
-    jq '.resources[] | select(.type=="libvirt_domain") | .module' | \
-    sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )"
-
-# Grab the connection URL and the vm name prefix. We do this by greping all
-# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the
-# last grep result, this way we make sure any value in terraform.tfvars will
-# take priority.
-CONN_URLS="$( \
-    find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \
-    grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)"
-
-CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
-
-NAME_PREFIXES="$( \
-    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \
-    grep "$VM_NAME_PREFIX" terraform.tfvars)"
-
-NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
-
-# These can be used for debugging.
-# echo "Using connection URL: $CONN_URL"
-# echo "Using prefix: $NAME_PREFIX"
-
-# Get the names of our VMs from libvirt.
-VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')"
-
-# Convert the lines of VM names to an array.
-OLD_IFS=$IFS
-IFS=$'\n'
-VMS=($VMS)
-IFS=$OLD_IFS
-
-# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the
-# result to VM_IP_PAIRS as <vm-name>:<ipv4-address>.
-VM_IP_PAIRS=""
-for VM in "${VMS[@]}"; do
-    for INTERFACE in $NET_INTERFACES; do
-        IP="$( \
-            virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \
-            jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \
-            sed 's/"//g')"
-        # Add the VM:IP pair if IP is not empty.
-        if [ ! -z "$IP" ]; then
-            VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP"
-        fi
-    done
-done
-
-# Write inventory
-cat /dev/null > inventory
-for GROUP in $INV_GROUPS; do
-    echo "[$GROUP]" >> inventory
-    echo "$VM_IP_PAIRS" | \
-        grep $GROUP | \
-        sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory
-done
-
-# Print vars
-echo "$VM_IP_PAIRS" | \
-    sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \
-    sed s/$NAME_PREFIX-//g | \
-    sed 's/-/_/g' | \
-    awk '{print toupper($1)}'

get-vm-ips.sh

@@ -1,98 +1,60 @@
-#!/bin/bash
+#!/bin/sh
 
-# This script will create environment variables for all of the output IPs. An
+# This script will create environment variables for all of the output IPs. It
-# anisble inventory file is created as well.
+# will also create a `ANSIBLE_INV` variable that will be a comma separated
+# string of all the IPs. A anisble inventory file called "inventory is created
+# as well.
 #
 # Use eval $(./get-vm-ips.sh) to set env vars for ips.
 
 terraform refresh > /dev/null
 
-# The file to write the inventory to. This file will be completely overridden.
+# All terraform outputs in json format.
-INVENTORY_FILE="inventory"
+OUTPUTS_JSON="$(
-
+    terraform show -json | \
-# Grab the the vm name prefix. We do this by greping all *.tfvars files making
+        jq '.values.outputs' | \
-# sure to cat terraform.tfvars last. Then we just grab the last grep result,
+        sed 's/-/_/g')"
-# this way we make sure any value in terraform.tfvars will take priority.
+# Just the IP address outputs in json format. Also all '-' characters are
-VM_NAME_PREFIX_VAR="vm-name-prefix"
+# replaced by '_' becuase '-' causes jq some problems.
-VM_NAME_PREFIXES="$( \
+IPS_JSON="$(
-    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \
+    echo $OUTPUTS_JSON | \
-    grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)"
+        jq 'to_entries | .[] | select(.key | contains("ips"))')"
-VM_NAME_PREFIX="$(
+# An array of all node "types"
-    echo "$VM_NAME_PREFIXES" | \
+NODE_TYPE_ARRAY="$(
-        tail -n 1 | \
+    echo $IPS_JSON | \
-        sed 's/^.*=\s*"\(.*\)"/\1/g')"
+        jq '.key' | \
-
-PUBLIC_IP_OUTPUT="groups_hostnames_ips"
-PRIVATE_IP_OUTPUT="groups_hostnames_private_ips"
-IP_TYPE="$PRIVATE_IP_OUTPUT"
-
-# This command stores the output data in the format below.
-# [
-#   {
-#     "group": "master",
-#     "vms": [
-#       {
-#         "hostname": "ansible-test-master-0",
-#         "ip": "52.14.114.48"
-#       }
-#     ]
-#   },
-#   {
-#     "group": "worker",
-#     "vms": [
-#       {
-#         "hostname": "ansible-test-worker-0",
-#         "ip": "3.145.121.159"
-#       },
-#       {
-#         "hostname": "ansible-test-worker-1",
-#         "ip": "18.217.112.176"
-#       }
-#     ]
-#   }
-# ]
-DATA="$(terraform show -json | \
-    jq '.values.outputs.'"$IP_TYPE"'.value | to_entries |
-        map({group: .key, vms:.value | to_entries |
-        map({hostname:.key,ip:.value})})')"
-
-# Pull out the groups from $DATA. The format is a single string with the groups
-# separated by spaces, ie. "group1 group2 group3".
-ANS_GROUPS="$(
-    echo $DATA | \
-        jq '.[] | .group' | \
         sed 's/"//g' | \
-        tr '\n' ' '
+        sed -z 's/\n/ /g;s/ $/\n/g')"
-    )"
 
-# Clear the inventory file.
+# Loop over all the node types and create an export line for each IP.
-cat /dev/null > $INVENTORY_FILE
+VM_IP_EXPORTS="$(
+    for TYPE in $NODE_TYPE_ARRAY; do
 
-# For each group, write the VM info to $INVENTORY_FILE and also print a variable
+        # Convert type, converts "master-ips" to "MASTER"
-# expression to stdout.
+        TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)"
-for GROUP in $ANS_GROUPS; do
+        echo "$OUTPUTS_JSON" | \
+            jq '.'"$TYPE"'.value[]' | \
+            # Add line numbers starting with 0.
+            nl -v 0 | \
+            # Print an export string with a type placeholder "__TYPE__".
+            awk '{print "export __TYPE__" $1 "=" $2}' | \
+            sed s/__TYPE__/$TYPE_UPPER/g
+    done)"
 
-    # Write the inventory file to $INVENTORY_FILE.
+ANSIBLE_INV="$(
-    echo "[$GROUP]" >> $INVENTORY_FILE
+    echo "$VM_IP_EXPORTS" | \
-    echo $DATA | \
+        sed 's/"//g' | \
-        jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
+        sed 's/^.*=//g' | \
-            "\(.hostname) ansible_host=\(.ip)"' | \
+        sed -z 's/\n/,/g;s/,$/\n/g')"
-        sed 's/"//g' \
-            >> $INVENTORY_FILE
 
-    # For this group, collect expressions into VARS. The format is:
+# Create an inventory file for ansible.
-    # HOSTNAME1=0.0.0.0
+echo "[k8s_nodes]" > inventory
-    # HOSTNAME2=0.0.0.0
+echo $VM_IP_EXPORTS | \
-    VARS="$(
+    sed 's/"//g' | \
-        echo -n $DATA | \
+    sed 's/export //g' | \
-            jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
+    sed 's/ /\n/g' | \
-                "\(.hostname)=\(.ip)"' | \
+    sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \
-            sed 's/"//g' | \
+        >> inventory
-            sed "s/$VM_NAME_PREFIX-//g" | \
+
-            sed 's/-/_/g'
+echo $VM_IP_EXPORTS | sed 's/" /"\n/g'
-        )"
+echo export ANSIBLE_INV=$ANSIBLE_INV
-    # Print the contents of $VARS converted to uppercase. If it's not expty.
-    if [ ! -z "$VARS" ]; then
-        echo "${VARS^^}"
-    fi
-done

main.tf

@@ -1,188 +1,85 @@
-
+terraform {
-locals {
+  required_version = ">= 0.13"
-  old-k8s-subnets-ids = [
+  required_providers {
-    # module.aws-network-from-scratch.subnet.id,
+    libvirt = {
-    module.aws-network-existing-managed.subnet-by-name["subnet_1"].id,
+      source = "dmacvicar/libvirt"
-    module.aws-network-existing-managed.subnet-by-name["subnet_3"].id,
+      version = "0.6.11"
-  ]
+    }
-  new-k8s-subnets-ids = [
-    module.aws-network-existing-mss-dev.subnet-by-name["mssdev Subnet 1"].id,
-    module.aws-network-existing-mss-dev.subnet-by-name["mssdev Subnet 3"].id,
-  ]
-  nfs-subnets = [
-    # module.aws-network-from-scratch.subnet,
-    module.aws-network-existing-managed.subnet-by-name["subnet_4"],
-  ]
-  aws-managed-security-group-id = module.aws-network-existing-managed.default-sg.id
-  aws-mss-dev-security-group-id = module.aws-network-existing-mss-dev.default-sg.id
-  # aws-managed-security-group-id = module.aws-network-from-scratch.default-security-group.id
-
-  # The names of these nodes is created by:
-  # "{var.vm-name-prefix}-{name from nodes-config}-{number}"
-  # The length of this full name must be < 16 characters. This is a limitaion of
-  # Active Directory? The "realm join" command failed when the name was too
-  # long, although it did not give the reason for the failure, the reason was
-  # surmised because it looked like it was truncating the name when attempting
-  # to join.
-  #
-  # NOTE: This naming restriction is only important if you plan on joining these
-  # VMs to Active Directory.
-  nodes-config = {
-    # TODO if the above comment about the name length is true, then this name is
-    # too long. IMPORTANT! If you change this then you need to change the k8s
-    # ansible role that assigns roles based on wether or not 'master' is in the
-    # name.
-    "k8s-master" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.t2-medium-4gib-2vcpu
-      subnet-ids      = local.old-k8s-subnets-ids
-      security-groups = [local.aws-managed-security-group-id]
-      num             = 1
-    },
-    "k8s-wrkr" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
-      subnet-ids      = local.old-k8s-subnets-ids
-      security-groups = [local.aws-managed-security-group-id]
-      disk-size       = 32
-      num             = 2
-    },
-    "worker" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
-      subnet-ids      = local.old-k8s-subnets-ids
-      security-groups = [local.aws-managed-security-group-id]
-      disk-size       = 64
-      num             = 3
-    },
-    "tst-master" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.t2-medium-4gib-2vcpu
-      subnet-ids      = local.old-k8s-subnets-ids
-      security-groups = [local.aws-managed-security-group-id]
-      num             = 0
-    },
-    "tst-wrkr" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
-      subnet-ids      = local.old-k8s-subnets-ids
-      security-groups = [local.aws-managed-security-group-id]
-      disk-size       = 32
-      num             = 0
-    },
-    "rancher-master" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
-      subnet-ids      = local.new-k8s-subnets-ids
-      security-groups = [local.aws-mss-dev-security-group-id]
-      disk-size       = 64
-      num             = 1
-    },
-    "rancher-worker" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
-      subnet-ids      = local.new-k8s-subnets-ids
-      security-groups = [local.aws-mss-dev-security-group-id]
-      disk-size       = 64
-      num             = 2
-    },
-    "dev" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
-      # subnet-ids    = [module.aws-network-from-scratch.subnet.id]
-      subnet-ids      = [module.aws-network-existing-managed.subnet-by-name["subnet_4"].id]
-      security-groups = [local.aws-managed-security-group-id]
-      disk-size       = 32
-      num             = 2
-    },
-    "nfs" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.t3a-medium-amd-4gib-2vcpu
-      # subnet-ids    = [module.aws-network-from-scratch.subnet.id]
-      subnet-ids      = [module.aws-network-existing-managed.subnet-by-name["subnet_4"].id]
-      security-groups = [local.aws-managed-security-group-id]
-      num             = 1
-      num-disks       = 1
-      # TODO Remove this, prefferred method is to use 'disks' and 'disk-mounts'
-      # like shown below.
-      zfs-disk-size   = 10
-    },
-    "proxy" = {
-      base-image      = module.aws-ami-constants.ubuntu-ami
-      aws-ec2-type    = module.ec2-types.t2-micro-1gib-1vcpu
-      # subnet-ids    = [module.aws-network-from-scratch.subnet.id]
-      subnet-ids      = [module.aws-network-existing-managed.subnet-by-name["subnet_4"].id]
-      security-groups = [local.aws-managed-security-group-id]
-      private-ips     = [var.aws-proxy-private-ip]
-      num             = 1
-    },
   }
-
-  disks = {
-    "zfs-64g" = {
-      num               = 1,
-      size              = 64
-      availability_zone = local.nfs-subnets[0].availability_zone
-    },
-    "zfs-256g" = {
-      num               = 1,
-      size              = 256
-      availability_zone = local.nfs-subnets[0].availability_zone
-    },
-  }
-
-  disk-mounts = [
-    {
-      # TODO make this attach field work.
-      attach        = false
-      ec2-id        = module.nodes["nfs"].nodes[0].id,
-      disk-group    = "zfs-64g"
-      # TODO also make sure that get drive letters or whetever still works. did
-      # it ever work?
-      # TODO We need to have a map of drive letters to device path in linux.
-      # This gets a little more complicated becuase the t2 instances map to
-      # /dev/xvdg while the t3a instance types map to /dev/nvme0n1 where 0 is
-      # the nth drive connected. The nth drive does not seem to map to a drive
-      # letter, ie. drive letter "f" does not map to 6, the nth drive simply
-      # increments by 1 for each drive attached regardless of the drive letter
-      # set.
-      drive-letters = ["g", "h", "i"]
-    },
-    {
-      # TODO make this attach field work.
-      attach        = false
-      ec2-id        = module.nodes["nfs"].nodes[0].id,
-      disk-group    = "zfs-256g"
-      # TODO also make sure that get drive letters or whetever still works. did
-      # it ever work?
-      # TODO We need to have a map of drive letters to device path in linux.
-      # This gets a little more complicated becuase the t2 instances map to
-      # /dev/xvdg while the t3a instance types map to /dev/nvme0n1 where 0 is
-      # the nth drive connected. The nth drive does not seem to map to a drive
-      # letter, ie. drive letter "f" does not map to 6, the nth drive simply
-      # increments by 1 for each drive attached regardless of the drive letter
-      # set.
-      drive-letters = ["j", "k", "l"]
-    },
-  ]
-
-  # This is only needed for libvirt.
-  install-qemu-agent = false
 }
 
 ################################################################################
 # cloud-init
 ################################################################################
 
-module "cloud-init-config" {
+# data "template_file" "master-node-user-datas" {
-  for_each            = local.nodes-config
+#   template = file("${path.module}/cloud_init.cfg")
-  source              = "./modules/cloud-init-config"
+#   vars = {
-  cloud-init-template = "${path.module}/cloud_init.cfg"
+#     admin-passwd  = "${var.root-admin-passwd}"
-  hostname-prefix     = "${var.vm-name-prefix}-${each.key}"
+#     admin-pub-key = "${var.root-admin-pub-key}"
-  num                 = each.value.num
+#     hostname      = "${var.vm-name-prefix}-master-${count.index}"
-  root-admin-passwd   = var.root-admin-passwd
+#   }
-  root-admin-pub-key  = var.root-admin-pub-key
+#   count = var.master-nodes
-  install-qemu-agent  = local.install-qemu-agent
+# }
+# 
+# data "template_file" "worker-node-user-datas" {
+#   template = file("${path.module}/cloud_init.cfg")
+#   vars = {
+#     admin-passwd  = "${var.root-admin-passwd}"
+#     admin-pub-key = "${var.root-admin-pub-key}"
+#     hostname      = "${var.vm-name-prefix}-worker-${count.index}"
+#   }
+#   count = var.worker-nodes
+# }
+
+data "template_file" "amzn2-node-user-datas" {
+  template = file("${path.module}/cloud_init.cfg")
+  vars = {
+    admin-passwd  = "${var.root-admin-passwd}"
+    admin-pub-key = "${var.root-admin-pub-key}"
+    hostname      = "${var.vm-name-prefix}-amzn2-${count.index}"
+  }
+  count = 1
+}
+
+data "template_file" "ubuntu-node-user-datas" {
+  template = file("${path.module}/cloud_init.cfg")
+  vars = {
+    admin-passwd  = "${var.root-admin-passwd}"
+    admin-pub-key = "${var.root-admin-pub-key}"
+    hostname      = "${var.vm-name-prefix}-ubuntu-${count.index}"
+  }
+  count = 1
+}
+
+data "template_file" "arch-node-user-datas" {
+  template = file("${path.module}/cloud_init.cfg")
+  vars = {
+    admin-passwd  = "${var.root-admin-passwd}"
+    admin-pub-key = "${var.root-admin-pub-key}"
+    hostname      = "${var.vm-name-prefix}-arch-${count.index}"
+  }
+  count = 1
+}
+
+data "template_file" "centos7-node-user-datas" {
+  template = file("${path.module}/cloud_init.cfg")
+  vars = {
+    admin-passwd  = "${var.root-admin-passwd}"
+    admin-pub-key = "${var.root-admin-pub-key}"
+    hostname      = "${var.vm-name-prefix}-centos7-${count.index}"
+  }
+  count = 1
+}
+
+data "template_file" "centos8-node-user-datas" {
+  template = file("${path.module}/cloud_init.cfg")
+  vars = {
+    admin-passwd  = "${var.root-admin-passwd}"
+    admin-pub-key = "${var.root-admin-pub-key}"
+    hostname      = "${var.vm-name-prefix}-centos8-${count.index}"
+  }
+  count = 1
 }
 
 ################################################################################
@@ -191,56 +88,22 @@ module "cloud-init-config" {
 # libvirt modules/resources.
 ################################################################################
 
-# This module sets the ec2-types constants.
+provider "aws" {
-module "ec2-types" {
+  region = "us-east-2"
-  source = "./modules/ec2-types"
 }
 
-# This module sets the AWS AMI constants.
+module "aws-amis" {
-module "aws-ami-constants" {
+  source = "./modules/aws-amis"
-  source = "./modules/aws-ami-constants"
 }
 
-# This module will grab the latest ami for a variety of distros. Uncomment to
+module "aws-network" {
-# get a list of the latest AMIs for our supported distros.
+  source = "./modules/aws-network"
-# module "aws-amis" {
+  name-prefix = var.vm-name-prefix
-#   source = "./modules/aws-amis"
+  vpc-cidr-block = var.aws-vpc-cidr-block
-# }
+  subnet-cidr-block = var.aws-subnet-cidr-block
-# output "amis" {
+  admin-ips = var.admin-ips
-#   value = module.aws-amis.amis
-# }
-
-################################################################################
-# AWS Networking
-# Use the 2 modules below to create resources for the AWS network.
-# aws-network-from-scratch will build the AWS network from scratch.
-# aws-network-existing will query AWS for an existing VPC.
-################################################################################
-
-# module "aws-network-from-scratch" {
-#   source            = "./modules/aws-network-from-scratch"
-#   name-prefix       = var.vm-name-prefix
-#   vpc-cidr-block    = var.aws-vpc-cidr-block
-#   subnet-cidr-block = var.aws-subnet-cidr-block
-#   admin-ips         = var.admin-ips
-# }
-
-module "aws-network-existing-managed" {
-  source                      = "./modules/aws-network-existing"
-  default-vpc-name            = var.aws-existing-managed-vpc-name
-  default-security-group-name = var.aws-existing-managed-sg-name
-  existing-subnet-names       = var.aws-existing-managed-subnet-names
 }
 
-module "aws-network-existing-mss-dev" {
-  source                      = "./modules/aws-network-existing"
-  default-vpc-name            = var.aws-existing-mss-dev-vpc-name
-  default-security-group-name = var.aws-existing-mss-dev-sg-name
-  existing-subnet-names       = var.aws-existing-mss-dev-subnet-names
-}
-
-################################################################################
-
 # This key pair is not actually used. Keys are added to the nodes via cloud-init
 # instead. We just add this here that this key will show up in the AWS console."
 resource "aws_key_pair" "key" {
@@ -251,74 +114,85 @@ resource "aws_key_pair" "key" {
   }
 }
 
-module "disks" {
+module "amzn2-nodes" {
-  for_each          = local.disks
-  source            = "./modules/aws-disks"
-  availability_zone = each.value.availability_zone
-  size              = each.value.size
-  num               = each.value.num
-  prefix            = each.key
-}
-
-module "disk-mounts" {
-  source = "./modules/aws-disk-mounts"
-  disks  = module.disks[element(local.disk-mounts, count.index).disk-group].disks
-  ec2-id = element(local.disk-mounts, count.index).ec2-id
-  drive-letters = try(element(local.disk-mounts, count.index).drive-letters, null)
-  count  = length(local.disk-mounts)
-}
-
-# TODO REM remove if unuesed.
-resource "aws_ebs_volume" "zfs" {
-  # TODO REM look at types.
-  availability_zone = local.nfs-subnets[0].availability_zone
-  size = local.nodes-config["nfs"].zfs-disk-size
-  encrypted = true
-  count = local.nodes-config["nfs"].num-disks
-  tags = {
-    Name = "zfs-disk-${count.index}"
-  }
-}
-
-resource "aws_volume_attachment" "mount-nfs-volume" {
-  device_name = "/dev/sd${element(var.aws-zfs-drive-letters, count.index)}"
-  instance_id = module.nodes["nfs"].nodes[0].id
-  count = local.nodes-config["nfs"].num-disks
-  volume_id = element(aws_ebs_volume.zfs, count.index).id
-}
-
-output "zfs-drive-letters" {
-  value = aws_volume_attachment.mount-nfs-volume.*.device_name
-}
-
-module "nodes" {
-  for_each           = local.nodes-config
   source             = "./modules/aws-nodes"
-  ec2-instance-type  = each.value.aws-ec2-type
+  ami                = var.base-image
-  ami                = each.value.base-image
+  ec2-instance-type  = var.aws-ec2-instance-type
-  subnet-ids         = each.value.subnet-ids
+  subnet-id          = module.aws-network.subnet.id
-  private-ips        = try(each.value.private-ips, [])
+  security-group-ids = [module.aws-network.default-security-group.id]
-  security-group-ids = each.value.security-groups
+  user-datas         = data.template_file.amzn2-node-user-datas
-  user-datas         = lookup(module.cloud-init-config, each.key, null).user-datas
+  num-nodes          = 1
-  disk-size          = try(each.value.disk-size, null)
+  name-prefix        = "${var.vm-name-prefix}-amzn2"
-  num-nodes          = each.value.num
-  name-prefix        = "${var.vm-name-prefix}-${each.key}"
-  # TODO add a input for the key so that it will show up as the key in the aws
-  # console.
 }
 
-# TODO an attempt to create a windows machine.
+module "ubuntu-nodes" {
-# module "nodes-win" {
+  source             = "./modules/aws-nodes"
+  ami                = "ami-0629230e074c580f2"
+  ec2-instance-type  = var.aws-ec2-instance-type
+  subnet-id          = module.aws-network.subnet.id
+  security-group-ids = [module.aws-network.default-security-group.id]
+  user-datas         = data.template_file.ubuntu-node-user-datas
+  num-nodes          = 1
+  name-prefix        = "${var.vm-name-prefix}-ubuntu"
+}
+
+module "arch-nodes" {
+  source             = "./modules/aws-nodes"
+  ami                = "ami-02653f06de985e3ba"
+  ec2-instance-type  = var.aws-ec2-instance-type
+  subnet-id          = module.aws-network.subnet.id
+  security-group-ids = [module.aws-network.default-security-group.id]
+  user-datas         = data.template_file.ubuntu-node-user-datas
+  num-nodes          = 1
+  name-prefix        = "${var.vm-name-prefix}-arch"
+}
+
+module "centos7-nodes" {
+  source             = "./modules/aws-nodes"
+  ami                = "ami-00f8e2c955f7ffa9b"
+  ec2-instance-type  = var.aws-ec2-instance-type
+  subnet-id          = module.aws-network.subnet.id
+  security-group-ids = [module.aws-network.default-security-group.id]
+  user-datas         = data.template_file.ubuntu-node-user-datas
+  num-nodes          = 1
+  name-prefix        = "${var.vm-name-prefix}-centos7"
+}
+
+module "centos8-nodes" {
+  source             = "./modules/aws-nodes"
+  ami                = "ami-057cacbfbbb471bb3"
+  ec2-instance-type  = var.aws-ec2-instance-type
+  subnet-id          = module.aws-network.subnet.id
+  security-group-ids = [module.aws-network.default-security-group.id]
+  user-datas         = data.template_file.ubuntu-node-user-datas
+  num-nodes          = 1
+  name-prefix        = "${var.vm-name-prefix}-centos8"
+}
+
+output "amis" {
+  value = module.aws-amis.amis
+}
+
+# module "master-nodes" {
 #   source             = "./modules/aws-nodes"
-#   ec2-instance-type  = module.ec2-types.t2-small-2gib-1vcpu
+#   ami                = var.base-image
-#   ami                = module.aws-ami-constants.win-srv-2019-ami
+#   ec2-instance-type  = var.aws-ec2-instance-type
-#   subnet-ids         = [module.aws-network-existing-managed.subnet-by-name["subnet_2"].id]
+#   subnet-id          = module.aws-network.subnet.id
-#   private-ips        = []
+#   security-group-ids = [module.aws-network.default-security-group.id]
-#   security-group-ids = [local.aws-managed-security-group-id]
+#   user-datas         = data.template_file.master-node-user-datas
-#   # TODO REM need to figure out how to not pass a user data.
+#   num-nodes          = var.master-nodes
-#   user-datas         = [null]
+#   name-prefix        = "${var.vm-name-prefix}-master"
-#   num-nodes          = 1
+# }
-#   name-prefix        = "${var.vm-name-prefix}-win-test"
+# 
+# module "worker-nodes" {
+#   source             = "./modules/aws-nodes"
+#   ami                = var.base-image
+#   ec2-instance-type  = var.aws-ec2-instance-type
+#   subnet-id          = module.aws-network.subnet.id
+#   security-group-ids = [module.aws-network.default-security-group.id]
+#   user-datas         = data.template_file.worker-node-user-datas
+#   num-nodes          = var.worker-nodes
+#   name-prefix        = "${var.vm-name-prefix}-worker"
 # }
 
 ################################################################################
@@ -334,28 +208,35 @@ module "nodes" {
 # provider "libvirt" {
 #   uri = var.libvirt-connection-url
 # }
-#
+# 
-# module "libvirt-images" {
+# module "master-nodes" {
-#   source = "./modules/libvirt-images"
-# }
-#
-# module "nodes" {
-#   for_each               = local.nodes-config
 #   source                 = "./modules/libvirt-nodes"
 #   pool-name              = libvirt_pool.images.name
-#   name-prefix            = "${var.vm-name-prefix}-${each.key}"
+#   name-prefix            = "${var.vm-name-prefix}-master"
-#   num-nodes              = each.value.num
+#   num-nodes              = var.master-nodes
 #   node-memory            = var.node-memory
 #   node-vcpus             = var.node-vcpus
-#   node-disk-size         = var.libvirt-node-disk-size
+#   base-image             = var.base-image
-#   base-image             = each.value.base-image
-#   network-name           = var.libvirt-network-name
 #   root-admin-passwd      = var.root-admin-passwd
 #   root-admin-pub-key     = var.root-admin-pub-key
 #   libvirt-connection-url = var.libvirt-connection-url
-#   user-datas             = lookup(module.cloud-init-config, each.key, null).user-datas
+#   user-datas             = data.template_file.master-node-user-datas
 # }
-#
+# 
+# module "worker-nodes" {
+#   source                 = "./modules/libvirt-nodes"
+#   pool-name              = libvirt_pool.images.name
+#   name-prefix            = "${var.vm-name-prefix}-worker"
+#   num-nodes              = var.worker-nodes
+#   node-memory            = var.node-memory
+#   node-vcpus             = var.node-vcpus
+#   base-image             = var.base-image
+#   root-admin-passwd      = var.root-admin-passwd
+#   root-admin-pub-key     = var.root-admin-pub-key
+#   libvirt-connection-url = var.libvirt-connection-url
+#   user-datas             = data.template_file.worker-node-user-datas
+# }
+# 
 # resource "libvirt_pool" "images" {
 #   name = var.disk-image-pool-name
 #   type = "dir"
@@ -366,14 +247,31 @@ module "nodes" {
 # end libvirt
 ################################################################################
 
-# This will outpus a map of group => [{hostname, ip}].
+output "amzn2-ips" {
-# TODO A 'names' output needs to be added to libvirt.
+  value = module.amzn2-nodes.ips
-output "groups_hostnames_ips" {
-  value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) }
 }
 
-# This will outpus a map of group => [{hostname, private_ip}].
+output "ubuntu-ips" {
-# TODO Figure out how what to do about private_ips for libvirt.
+  value = module.ubuntu-nodes.ips
-output "groups_hostnames_private_ips" {
-  value = { for type, node in module.nodes : type => zipmap(node.names, node.private_ips) }
 }
+
+output "arch-ips" {
+  value = module.arch-nodes.ips
+}
+
+output "centos7-ips" {
+  value = module.centos7-nodes.ips
+}
+
+output "centos8-ips" {
+  value = module.centos8-nodes.ips
+}
+
+# TODO REM move to other file?
+# output "master-ips" {
+#   value = module.master-nodes.ips
+# }
+# 
+# output "worker-ips" {
+#   value = module.worker-nodes.ips
+# }

modules/aws-ami-constants/outputs.tf

@@ -1,58 +0,0 @@
-################################################################################
-# AWS AMI vars
-# These outputs are really more like constants. Using outputs improves
-# readability. The values are manually updated. Use the aws-amis module to get
-# the latest for each distro. Keep in mind that AMIs are region specific this is
-# why you may see values and other regions commented out.
-################################################################################
-
-output "amzn2-ami" {
-  # us-east-2
-  # default     = "ami-0dd0ccab7e2801812"
-  # us-gov-west-1
-  value       = "ami-02ab588324a95cf31"
-  description = "The AMI to use for Amazon Linux 2."
-}
-output "ubuntu-ami" {
-  # us-east-2
-  # value = "ami-06c7d6c0987eaa46c"
-  # us-gov-west-1
-  value       = "ami-066189aeb91baa0ab"
-  description = "The AMI to use for Ubuntu."
-}
-output "centos7-ami" {
-  # us-east-2
-  value       = "ami-00f8e2c955f7ffa9b"
-  description = "The AMI to use for CentOS 7."
-}
-output "centos8-ami" {
-  # us-east-2
-  value       = "ami-057cacbfbbb471bb3"
-  description = "The AMI to use for CentOS 8."
-}
-output "arch-ami" {
-  # us-east-2
-  value       = "ami-02653f06de985e3ba"
-  description = "The AMI to use for Arch Linux."
-}
-output "rhel7-ami" {
-  # us-east-2
-  # value = "ami-0a509b3c2a4d05b3f"
-  # us-gov-west-1
-  value       = "ami-04ccdf5793086ea95"
-  description = "The AMI to use for RHEL 7."
-}
-output "rhel8-ami" {
-  # us-east-2
-  # value = "ami-0d871ca8a77af2948"
-  # us-gov-west-1
-  value       = "ami-0b1f10cd1cd107dd2"
-  description = "The AMI to use for RHEL 8."
-}
-output "win-srv-2019-ami" {
-  # us-east-2
-  # value = ???
-  # us-gov-west-1
-  value       = "ami-0f838c3c35ab60fc4"
-  description = "The AMI to use for Windows Server 2019."
-}

modules/aws-amis/main.tf

@@ -1,61 +1,33 @@
 locals {
   amis = {
     amzn2 = {
-      # us-east-2
+      owner-id = "137112412989"
-      # owner-id = "137112412989"
-      # us-gov-west-1
-      owner-id = "045324592363"
       name = "amzn2-ami-hvm-2*x86_64-gp2"
     },
     ubuntu = {
-      # us-east-2
+      owner-id = "099720109477"
-      # owner-id = "099720109477"
-      # us-gov-west-1
-      owner-id = "513442679011"
       name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
     },
-    # centos7 = {
+    centos7 = {
-    #   # us-east-2
+      owner-id = "125523088429"
-    #   # owner-id = "125523088429"
+      name = "CentOS 7.*x86_64"
-    #   # us-gov-west-1
+    },
-    #   # owner-id = THERE IS NO CENTOS7 IMAGE in us-gov-west-1!!
+    centos8 = {
-    #   name = "CentOS 7.*x86_64"
+      owner-id = "125523088429"
-    # },
+      name = "CentOS 8.*x86_64"
-    # centos8 = {
+    },
-    #   # us-east-2
+    arch = {
-    #   # owner-id = "125523088429"
+      owner-id = "093273469852"
-    #   # us-gov-west-1
+      name = "arch-linux-lts-hvm*x86_64-ebs"
-    #   # owner-id = THERE IS NO CENTOS8 IMAGE in us-gov-west-1!!
+    },
-    #   name = "CentOS 8.*x86_64"
-    # },
-    # arch = {
-    #   # us-east-2
-    #   # owner-id = "093273469852"
-    #   # us-gov-west-1
-    #   # owner-id = THERE IS NO ARCH IMAGE in us-gov-west-1!!
-    #   name = "arch-linux-lts-hvm*x86_64-ebs"
-    # },
     rhel7 = {
-      # us-east-2
+      owner-id = "309956199498"
-      # owner-id = "309956199498"
-      # us-gov-west-1
-      owner-id = "219670896067"
       name = "RHEL-7.*HVM*x86_64*GP2"
     },
     rhel8 = {
-      # us-east-2
+      owner-id = "309956199498"
-      # owner-id = "309956199498"
-      # us-gov-west-1
-      owner-id = "219670896067"
       name = "RHEL-8.*HVM*x86_64*GP2"
-    },
+    }
-    win-srv-2019 = {
-      # us-east-2
-      # owner-id = "???"
-      # us-gov-west-1
-      owner-id = "077303321853"
-      name = "Windows_Server-2019-English-Full-Base*"
-    },
   }
 }
 

modules/aws-disk-mounts/main.tf

@@ -1,6 +0,0 @@
-resource "aws_volume_attachment" "mount-volume" {
-  device_name = "/dev/sd${element(var.drive-letters, count.index)}"
-  instance_id = var.ec2-id
-  count = length(var.disks)
-  volume_id = element(var.disks, count.index).id
-}

modules/aws-disk-mounts/variables.tf

@@ -1,16 +0,0 @@
-variable "drive-letters" {
-  default = ["f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
-  description = "These drive letters will be used when arraching disks to EC2 instance with `ec2-id'"
-  type = list(string)
-  nullable = false
-}
-
-variable "disks" {
-  description = "An array of disks to attach to the EC2 isntance with `ec2-id`"
-  type = list
-}
-
-variable "ec2-id" {
-  description = "The ID of the EC@ instance to attach the drives to."
-  type = string
-}

modules/aws-disks/main.tf

@@ -1,10 +0,0 @@
-
-resource "aws_ebs_volume" "disks" {
-  availability_zone = var.availability_zone
-  size = var.size
-  count = var.num
-  encrypted = true
-  tags = {
-    Name = "${var.prefix}-${count.index}"
-  }
-}

modules/aws-disks/outputs.tf

@@ -1,3 +0,0 @@
-output "disks" {
-  value = aws_ebs_volume.disks
-}

modules/aws-disks/variables.tf

@@ -1,19 +0,0 @@
-variable "availability_zone" {
-  description = "The availability zone this disk will reside in. This AZ is applied to all 'num' disks."
-  type = string
-}
-
-variable "num" {
-  description = "The number of disks to be created with this config."
-  type = number
-}
-
-variable "prefix" {
-  description = "This prefix will be used in the name of each disk created with this module. If the prefix is 'zfs', then it will create disks with names, zfs-0 and zfs-1."
-  type = string
-}
-
-variable "size" {
-  description = "The size of the disk in GB."
-  type = number
-}

modules/aws-network-existing/main.tf

@@ -1,35 +0,0 @@
-locals {
-  az-to-subnets = {
-    for s in data.aws_subnet.subnets : s.availability_zone => s.id...
-  }
-}
-
-data "aws_vpc" "default" {
-  tags = {
-    Name = var.default-vpc-name
-  }
-}
-
-data "aws_subnets" "subnet-ids" {
-  filter {
-    name = "vpc-id"
-    values = [data.aws_vpc.default.id]
-  }
-}
-
-data "aws_subnet" "subnets" {
-  for_each = toset(data.aws_subnets.subnet-ids.ids)
-  id = each.key
-}
-
-data "aws_subnet" "subnet-by-name" {
-  for_each = toset(var.existing-subnet-names)
-  filter {
-    name   = "tag:Name"
-    values = [each.key]
-  }
-}
-
-data "aws_security_group" "default" {
-  name = var.default-security-group-name
-}

modules/aws-network-existing/outputs.tf

@@ -1,27 +0,0 @@
-output "default-vpc" {
-  value = data.aws_vpc.default
-}
-
-output "default-sg" {
-  value = data.aws_security_group.default
-}
-
-output "subnets" {
-  description = "An array of all subnets in default-vpc."
-  value = data.aws_subnet.subnets
-}
-
-output "one-subnet-per-az" {
-  description = "An array of subnets that selects 1 subnet per az."
-  value = [for k,v in local.az-to-subnets : v[0]]
-}
-
-output "subnet-by-name" {
-  description = "A map of subnet name to subnet resource."
-  value = data.aws_subnet.subnet-by-name
-}
-
-output "az-to-subnets" {
-  description = "A map of availability zone to array of subnets that are in thet availability zone."
-  value = local.az-to-subnets
-}

modules/aws-network-existing/variables.tf

@@ -1,13 +0,0 @@
-variable "default-security-group-name" {
-  description = "The name of the existing default security group. This module will query AWS for a security group with this name,"
-}
-
-variable "default-vpc-name" {
-  description = "The name of the existing default VPC. This module will query AWS for a VPC with this name,"
-}
-
-variable "existing-subnet-names" {
-  description = "A list of subnet names that already exist in default-vpc-name"
-  default     = []
-  type        = list(string)
-}

modules/aws-network-from-scratch/main.tf

@@ -1,67 +0,0 @@
-resource "aws_vpc" "vpc" {
-  cidr_block = var.vpc-cidr-block
-  tags = {
-      Name = "${var.name-prefix}-vpc"
-  }
-}
-
-resource "aws_subnet" "subnet" {
-  vpc_id = aws_vpc.vpc.id
-  cidr_block = var.subnet-cidr-block
-  # availability_zone = var.avail_zone
-  tags = {
-      Name = "${var.name-prefix}-subnet"
-  }
-}
-
-resource "aws_default_security_group" "sg" {
-  vpc_id = aws_vpc.vpc.id
-
-  ingress {
-    from_port   = 22
-    to_port     = 22
-    protocol    = "tcp"
-    cidr_blocks = var.admin-ips
-  }
-
-  egress {
-    from_port       = 0
-    to_port         = 0
-    protocol        = "-1"
-    cidr_blocks     = ["0.0.0.0/0"]
-    prefix_list_ids = []
-  }
-
-  tags = {
-    Name = "${var.name-prefix}-ssh-from-admins-sg"
-  }
-}
-
-resource "aws_internet_gateway" "igw" {
-	vpc_id = aws_vpc.vpc.id
-  tags = {
-    Name = "${var.name-prefix}-igw"
-  }
-}
-
-resource "aws_default_route_table" "route-table" {
-   default_route_table_id = aws_vpc.vpc.main_route_table_id
-
-   route {
-     cidr_block = "0.0.0.0/0"
-     gateway_id = aws_internet_gateway.igw.id
-   }
-
-   # default route, mapping VPC CIDR block to "local", created implicitly and
-   # cannot be specified.
-
-   tags = {
-     Name = "${var.name-prefix}-route-table"
-   }
-}
-
-# Associate subnet with Route Table
-resource "aws_route_table_association" "a-rtb-subnet" {
-  subnet_id      = aws_subnet.subnet.id
-  route_table_id = aws_default_route_table.route-table.id
-}

modules/aws-network-from-scratch/outputs.tf

@@ -1,11 +0,0 @@
-output "vpc" {
-  value = aws_vpc.vpc
-}
-
-output "subnet" {
-  value = aws_subnet.subnet
-}
-
-output "default-security-group" {
-  value = aws_default_security_group.sg
-}

modules/aws-network-from-scratch/variables.tf

@@ -1,23 +0,0 @@
-variable "admin-ips" {
-  description = "A list of ips or cidr blocks that are allowed to connect to the nodes."
-  type = list(string)
-}
-
-variable "name-prefix" {
-  default = "tf"
-  description = "This prefix will be used in all the names of the resources creates in our AWS network."
-  type = string
-}
-
-variable "subnet-cidr-block" {
-  default = "10.0.1.0/24"
-  description = "The address space to be used for this subnet."
-  type = string
-}
-
-variable "vpc-cidr-block" {
-  default = "10.0.0.0/16"
-  description = "The address space to be used for out networks VPC."
-  type = string
-}
-

modules/aws-network/main.tf

@@ -33,7 +33,7 @@ resource "aws_default_security_group" "sg" {
   }
 
   tags = {
-    Name = "${var.name-prefix}-ssh-from-admins-sg"
+    Name = "${var.name-prefix}-ssh-from-admins--sg"
   }
 }
 

modules/aws-nodes/main.tf

@@ -1,19 +1,12 @@
 resource "aws_instance" "nodes" {
   ami                         = var.ami
   instance_type               = var.ec2-instance-type
-  # TODO Make this a variable.
+  # key_name                    = aws_key_pair.debug1.key_name
   associate_public_ip_address = true
-  subnet_id = element(var.subnet-ids, count.index % length(var.subnet-ids))
+  subnet_id                   = var.subnet-id
-  # Set a private IP if var.private-ips is not empty, otherwise let AWS assign
-  # the IP.
-  private_ip = length(var.private-ips) == 0 ? null : element(var.private-ips, count.index)
   vpc_security_group_ids = var.security-group-ids
-  user_data              = element(var.user-datas.*.rendered, count.index)
+  user_data = element(var.user-datas.*.rendered, count.index)
-  root_block_device {
+  count                       = var.num-nodes
-    volume_size           = var.disk-size
-    delete_on_termination = true
-  }
-  count = var.num-nodes
 
   tags = {
     Name = "${var.name-prefix}-${count.index}"

modules/aws-nodes/outputs.tf

@@ -1,15 +1,3 @@
 output "ips" {
   value = aws_instance.nodes.*.public_ip
 }
-
-output "private_ips" {
-  value = aws_instance.nodes.*.private_ip
-}
-
-output "names" {
-  value = aws_instance.nodes.*.tags.Name
-}
-
-output "nodes" {
-  value = aws_instance.nodes
-}

modules/aws-nodes/variables.tf

@@ -1,48 +1,36 @@
 variable "ami" {
   description = "The AWS AMI to be used for all the nodes"
-  type        = string
+  type = string
-}
-
-variable "disk-size" {
-  default     = 8
-  description = "The size of the root FS disk in GB."
-  type        = number
 }
 
 variable "ec2-instance-type" {
-  default     = "t2.micro"
+  default = "t2.micro"
   description = "The AWS instance type to use for all nodes."
-  type        = string
+  type = string
 }
 
 variable "name-prefix" {
-  default     = "tf-node"
+  default = "tf-node"
   description = "This prefix will be applied to all names created by this module."
-  type        = string
+  type = string
 }
 
 variable "num-nodes" {
-  default     = 1
+  default = 1
   description = "The number of nodes to create from the given input parameters."
-  type        = number
+  type = number
-}
-
-variable "private-ips" {
-  default     = []
-  description = "A list of private IP addresses to use for the nodes. If the list is empty then each node will get an IP assigned from AWS."
-  type        = list(string)
 }
 
 variable "user-datas" {
   description = "A list of cloud-init configs that get applied to their corresponding node."
 }
 
-variable "subnet-ids" {
+variable "subnet-id" {
-  description = "An array of subnet ids. These subnets will be round robined as the subnet to use for each node."
+  description = "The ID of the subnet that all the nodes will be added to."
-  type        = list(string)
+  type = string
 }
 
 variable "security-group-ids" {
   description = "A list of security group IDs to be applied to all the nodes."
-  type        = list(string)
+  type = list(string)
 }

modules/cloud-init-config/main.tf

@@ -1,10 +0,0 @@
-data "template_file" "user-datas" {
-  template = file("${var.cloud-init-template}")
-  vars = {
-    admin-passwd       = "${var.root-admin-passwd}"
-    admin-pub-key      = "${var.root-admin-pub-key}"
-    hostname           = "${var.hostname-prefix}-${count.index}"
-    install-qemu-agent = var.install-qemu-agent
-  }
-  count = var.num
-}

modules/cloud-init-config/outputs.tf

@@ -1,3 +0,0 @@
-output "user-datas" {
-  value = data.template_file.user-datas
-}

modules/cloud-init-config/variables.tf

@@ -1,28 +0,0 @@
-variable "cloud-init-template" {
-  default     = "../../cloud_init.cfg"
-  description = "The path to the cloud-init config template."
-  type        = string
-}
-
-variable "hostname-prefix" {
-  description = "This prefix wil be applied as a prefix for the hostnames."
-}
-
-variable "install-qemu-agent" {
-  default     = false
-  description = "This flag determines whether or not qemu-agent is installed."
-  type        = bool
-}
-
-variable "num" {
-  description = "The number of user-datas to create with these parameters."
-}
-
-variable "root-admin-passwd" {
-  description = "This value will be substituted for any occurence of 'admin-password' in the cloud-init config template."
-}
-
-variable "root-admin-pub-key" {
-  description = "This value will be substituted for any occurence of 'admin-pub-key' in the cloud-init config template."
-}
-

modules/ec2-types/outputs.tf

@@ -1,45 +0,0 @@
-################################################################################
-# AWS EC2 types.
-# Thesee are only here to help readability. Using these outputs allows the user
-# to be able to see the resources of each type without having to look them up.
-################################################################################
-
-output "t2-micro-1gib-1vcpu" {
-  description = "t2.micro EC2 instance with 1 GiB mem and 1 vCPU."
-  value = "t2.micro"
-}
-
-output "t2-small-2gib-1vcpu" {
-  description = "t2.small EC2 instance with 2 GiB mem and 1 vCPUs."
-  value = "t2.small"
-}
-
-output "t2-medium-4gib-2vcpu" {
-  description = "t2.medium EC2 instance with 4 GiB mem and 2 vCPUs."
-  value = "t2.medium"
-}
-
-output "t2-large-8gib-2vcpu" {
-  description = "t2.large EC2 instance with 8 GiB mem and 2 vCPUs."
-  value = "t2.large"
-}
-
-output "t2-xlarge-16gib-4vcpu" {
-  description = "t2.xlarge EC2 instance with 16 GiB mem and 4 vCPUs."
-  value = "t2.xlarge"
-}
-
-output "t3a-medium-amd-4gib-2vcpu" {
-  description = "t3a.medium EC2 instance on AMD with GiB mem and 2 vCPUs."
-  value = "t3a.medium"
-}
-
-output "t3a-large-amd-8gib-2vcpu" {
-  description = "t3a.large EC2 instance on AMD with 8 GiB mem and 2 vCPUs."
-  value = "t3a.large"
-}
-
-output "c5a-xlarge-amd-8gib-4vcpu" {
-  description = "c5a.xlarge EC2 instance on AMD with 8 GiB mem and 4 vCPUs."
-  value = "c5a.xlarge"
-}

modules/libvirt-images/outputs.tf

@@ -1,22 +0,0 @@
-################################################################################
-# Libvirt Images
-# These outputs are really more like constants. Using outputs improves
-# readability. The values are manually updated.
-################################################################################
-
-output "ubuntu-img" {
-  value       = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
-  description = "The libvirt image to use for Ubuntu."
-}
-
-output "centos7-img" {
-  # Latest as of 2021-12-06.
-  value       = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
-  description = "The libvirt image to use for CentOS 7."
-}
-
-output "centos8-img" {
-  value       = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2"
-  description = "The libvirt image to use for CentOS 8."
-}
-

modules/libvirt-nodes/main.tf

@@ -8,19 +8,16 @@ terraform {
   }
 }
 
-resource "libvirt_volume" "node-images" {
+provider "libvirt" {
-  name   = "${var.name-prefix}-base"
+  uri = var.libvirt-connection-url
-  pool   = var.pool-name
-  source = var.base-image
-  format = "qcow2"
 }
 
-resource "libvirt_volume" "node-images-resized" {
+resource "libvirt_volume" "node-images" {
-  name           = "${var.name-prefix}-${count.index}-resized"
+  name   = "${var.name-prefix}-${count.index}"
-  pool           = var.pool-name
+  pool   = var.pool-name
-  base_volume_id = libvirt_volume.node-images.id
+  source = var.base-image
-  count          = var.num-nodes
+  count  = var.num-nodes
-  size           = var.node-disk-size
+  format = "qcow2"
 }
 
 data "template_file" "network-config" {
@@ -44,9 +41,9 @@ resource "libvirt_domain" "nodes" {
   cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index)
 
   network_interface {
-    network_name   = var.network-name
+    network_name   = "default"
     hostname       = "${var.name-prefix}-${count.index}"
-    # wait_for_lease = true
+    wait_for_lease = true
   }
 
   # IMPORTANT: this is a known bug on cloud images, since they expect a console
@@ -65,7 +62,7 @@ resource "libvirt_domain" "nodes" {
   }
 
   disk {
-    volume_id = element(libvirt_volume.node-images-resized.*.id, count.index)
+    volume_id = element(libvirt_volume.node-images.*.id, count.index)
   }
 
   graphics {

modules/libvirt-nodes/outpus.tf

@@ -1,6 +1,4 @@
 
-# This only works on the default network. They will not work using the bridged
+output "ips" {
-# network.
+  value = libvirt_domain.nodes.*.network_interface.0.addresses.0
-# output "ips" {
+}
-#   value = libvirt_domain.nodes.*.network_interface.0.addresses.0
-# }

modules/libvirt-nodes/variables.tf

@@ -12,16 +12,6 @@ variable "name-prefix" {
   description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"."
 }
 
-variable "network-name" {
-  default = "default"
-  description = "The name of a pre-existing virtual-network."
-}
-
-variable "node-disk-size" {
-  default = 4294967296
-  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
-}
-
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."

providers.tf

@@ -1,77 +0,0 @@
-
-provider "aws" {
-  region = var.aws-region
-  # profile = <aws cli profile>
-}
-
-
-terraform {
-  required_version = ">= 1.0.8"
-
-  backend "s3" {
-    
-    bucket            = "mss-terraform-state"
-    key               = "global/s3/terraform.tfstate"
-    region            = "us-gov-west-1"
-    
-    dynamodb_table    = "mss-terraform-state-lock"
-    encrypt           = true
-
-  }
-  required_providers {
-    libvirt = {
-      source  = "dmacvicar/libvirt"
-      version = "0.6.11"
-    }
-  }
-}
-
-
-
-
-
-##
-#S3 bucket create to hold our TFState file so we can all share env settings
-resource "aws_s3_bucket" "terraform_state" {
-    bucket = "mss-terraform-state"
-
-    # enable versioning for the state files 
-    versioning {
-        enabled = true
-    }
-
-    #enable server-side encryption
-    server_side_encryption_configuration {
-      
-      rule {
-          apply_server_side_encryption_by_default {
-              sse_algorithm = "AES256"
-              }
-          }
-        } 
-}
-
-##
-# no sql database used so that we can lock the TFstate file in the S3 bucket to ensure two people 
-# are not running a terraform command at the same time 
-resource "aws_dynamodb_table" "terraform_locks" {
-    name            = "mss-terraform-state-lock"
-    billing_mode    = "PAY_PER_REQUEST"
-    hash_key        = "LockID"
-
-    attribute {
-      name = "LockID"
-      type = "S"
-    }
-
-
-}
-
-
-##
-# output variable to give details on the s3 bucket created
-#TODO: move to output.tf
-output "s3_bucket_arn" {
-  value       = aws_s3_bucket.terraform_state.arn
-  description = "The ARN of the S3 bucket"
-}

variables.tf

@@ -1,69 +1,11 @@
 variable "admin-ips" {
-  default = ["0.0.0.0/0"]
+  description = "A list of ips or cidr blocks that are allowed to connect to the nodes."
-  description = "A list of ips or cidr blocks that are allowed to connect to the nodes. This variable is only used when using the aws-network-from-scratch module."
   type = list(string)
 }
 
-variable "aws-zfs-drive-letters" {
+variable "aws-ec2-instance-type" {
-  default     = ["f", "g", "h", "i", "j"]
+  default = "t2.micro"
-  description = "These are the drove letters to use when attaching EBS volumes, ie. /dev/sdf or /dev/sdg"
+  description = "The AWS instance type to use for all nodes."
-  type        = list(string)
-}
-
-################################################################################
-# Query existing infrastructure.
-# Variables used to grab existing resources from AWS.
-#
-# We have 2 sets of three variables in this section. 1 section has 'managed' in
-# the name and the other has 'mss-dev' in the name. There are not restrictions
-# as to what we can put into these VPCs. The names of the variable simply match
-# the names of the VPCs and have no meaning for what resources are allowed.
-################################################################################
-
-variable "aws-existing-managed-sg-name" {
-  default     = "change-me-if-using-aws-network-existing"
-  description = "The name of the existing security group when using aws-network-existing."
-}
-
-variable "aws-existing-managed-vpc-name" {
-  default     = "change-me-if-using-aws-network-existing"
-  description = "The name of the existing VPC when using aws-network-existing."
-}
-
-variable "aws-existing-managed-subnet-names" {
-  description = "A list of subnet names that already exist in aws-existing-managed-vpc-name"
-  default     = []
-  type        = list(string)
-}
-
-variable "aws-existing-mss-dev-sg-name" {
-  default     = "change-me-if-using-aws-network-existing"
-  description = "The name of the existing security group when using aws-network-existing with the mss-dev VPC."
-}
-
-variable "aws-existing-mss-dev-vpc-name" {
-  default     = "change-me-if-using-aws-network-existing"
-  description = "The name of the existing VPC when using aws-network-existing with the mss-dev VPC."
-}
-
-variable "aws-existing-mss-dev-subnet-names" {
-  description = "A list of subnet names that already exist in aws-existing-mss-dev-vpc-name"
-  default     = []
-  type        = list(string)
-}
-
-################################################################################
-# End query existing infrastructure.
-################################################################################
-
-variable "aws-proxy-private-ip" {
-  description = "The private IP to request for the proxy instance."
-  type        = string
-}
-
-variable "aws-region" {
-  default = "us-east-1"
-  description = "The AWS region to use."
 }
 
 variable "aws-subnet-cidr-block" {
@@ -86,20 +28,9 @@ variable "disk-image-pool-name" {
 }
 
 variable "libvirt-connection-url" {
-  default = "nobody@localhost"
   description = "The libvirt connection URI, ie. qemu+ssh://<user>@<host>/system"
 }
 
-variable "libvirt-network-name" {
-  default = "default"
-  description = "The name of a pre-existing libvirt virtual-network."
-}
-
-variable "libvirt-node-disk-size" {
-  default = 4294967296
-  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
-}
-
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."
@@ -120,6 +51,18 @@ variable "root-admin-pub-key" {
   description = "The public key to be added to authorized_keys for the root and admin accounts."
 }
 
+variable "master-nodes" {
+  default     = 1
+  description = "The number of master nodes to create."
+  type        = number
+}
+
+variable "worker-nodes" {
+  default     = 2
+  description = "The number of worker nodes to create."
+  type        = number
+}
+
 variable "base-image" {
   default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
 }