libvirt bridged network interface

- Added support for libvirt VMs that use a bridged network interface.

cloud_init.cfg

@@ -26,17 +26,15 @@ chpasswd:
 hostname: ${hostname}
 fqdn: ${hostname}
 
-%{ if install-qemu-agent }
 packages:
-  # This are only necessary for libvirt.
+  # TODO These are only necessary for libvirt.
   - qemu-guest-agent
 runcmd:
   # TODO At some point revisit this, this was added because it seemed like
   # apparmor was causing dhclient to not get an IP address for ubuntu. This
   # should be double checked.
   - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient
-  # These are only necessary for libvirt.
+  # TODO These are only necessary for libvirt.
   - systemctl enable qemu-guest-agent
   - systemctl start qemu-guest-agent
   - systemctl status qemu-guest-agent
-%{ endif }

example.tfvars

@@ -68,11 +68,6 @@ base-image = "ami-0dd0ccab7e2801812"
 # on 2021-11-28
 # base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
 
-## Arch
-# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
-# on 2021-11-28
-# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
-
 ################################################################################
 # Keys/Passwords
 ################################################################################

get-vm-ips.sh

@@ -1,92 +1,60 @@
-#!/bin/bash
+#!/bin/sh
 
-# This script will create environment variables for all of the output IPs. An
-# anisble inventory file is created as well.
+# This script will create environment variables for all of the output IPs. It
+# will also create a `ANSIBLE_INV` variable that will be a comma separated
+# string of all the IPs. A anisble inventory file called "inventory is created
+# as well.
 #
 # Use eval $(./get-vm-ips.sh) to set env vars for ips.
 
 terraform refresh > /dev/null
 
-# The file to write the inventory to. This file will be completely overridden.
-INVENTORY_FILE="inventory"
-
-# Grab the the vm name prefix. We do this by greping all *.tfvars files making
-# sure to cat terraform.tfvars last. Then we just grab the last grep result,
-# this way we make sure any value in terraform.tfvars will take priority.
-VM_NAME_PREFIX_VAR="vm-name-prefix"
-VM_NAME_PREFIXES="$( \
-    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \
-    grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)"
-VM_NAME_PREFIX="$(
-    echo "$VM_NAME_PREFIXES" | \
-        tail -n 1 | \
-        sed 's/^.*=\s*"\(.*\)"/\1/g')"
-
-# This command stores the output data in the format below.
-# [
-#   {
-#     "group": "master",
-#     "vms": [
-#       {
-#         "hostname": "ansible-test-master-0",
-#         "ip": "52.14.114.48"
-#       }
-#     ]
-#   },
-#   {
-#     "group": "worker",
-#     "vms": [
-#       {
-#         "hostname": "ansible-test-worker-0",
-#         "ip": "3.145.121.159"
-#       },
-#       {
-#         "hostname": "ansible-test-worker-1",
-#         "ip": "18.217.112.176"
-#       }
-#     ]
-#   }
-# ]
-DATA="$(terraform show -json | \
-    jq '.values.outputs.groups_hostnames_ips.value | to_entries |
-        map({group: .key, vms:.value | to_entries |
-        map({hostname:.key,ip:.value})})')"
-
-# Pull out the groups from $DATA. The format is a single string with the groups
-# separated by spaces, ie. "group1 group2 group3".
-ANS_GROUPS="$(
-    echo $DATA | \
-        jq '.[] | .group' | \
+# All terraform outputs in json format.
+OUTPUTS_JSON="$(
+    terraform show -json | \
+        jq '.values.outputs' | \
+        sed 's/-/_/g')"
+# Just the IP address outputs in json format. Also all '-' characters are
+# replaced by '_' becuase '-' causes jq some problems.
+IPS_JSON="$(
+    echo $OUTPUTS_JSON | \
+        jq 'to_entries | .[] | select(.key | contains("ips"))')"
+# An array of all node "types"
+NODE_TYPE_ARRAY="$(
+    echo $IPS_JSON | \
+        jq '.value.value | to_entries | .[] | .key' | \
         sed 's/"//g' | \
-        tr '\n' ' '
-    )"
+        sed -z 's/\n/ /g;s/ $/\n/g')"
 
-# Clear the inventory file.
-cat /dev/null > $INVENTORY_FILE
+# Loop over all the node types and create an export line for each IP.
+VM_IP_EXPORTS="$(
+    for TYPE in $NODE_TYPE_ARRAY; do
 
-# For each group, write the VM info to $INVENTORY_FILE and also print a variable
-# expression to stdout.
-for GROUP in $ANS_GROUPS; do
+        # Convert type, converts "master-ips" to "MASTER"
+        TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)"
+        echo "$IPS_JSON" | \
+            jq '.value.value.'"$TYPE"'[]' | \
+            # Add line numbers starting with 0.
+            nl -v 0 | \
+            # Print an export string with a type placeholder "__TYPE__".
+            awk '{print "export __TYPE___" $1 "=" $2}' | \
+            sed s/__TYPE__/$TYPE_UPPER/g
+    done)"
 
-    # Write the inventory file to $INVENTORY_FILE.
-    echo "[$GROUP]" >> $INVENTORY_FILE
-    echo $DATA | \
-        jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
-            "\(.hostname) ansible_host=\(.ip)"' | \
-        sed 's/"//g' \
-            >> $INVENTORY_FILE
-
-    # For this group, collect expressions into VARS. The format is:
-    # HOSTNAME1=0.0.0.0
-    # HOSTNAME2=0.0.0.0
-    VARS="$(
-        echo $DATA | \
-            jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
-                "\(.hostname)=\(.ip)"' | \
+ANSIBLE_INV="$(
+    echo "$VM_IP_EXPORTS" | \
         sed 's/"//g' | \
-            sed "s/$VM_NAME_PREFIX-//g" | \
-            sed 's/-/_/g'
-        )"
-    # Print the contents of $VARS converted to uppercase.
-    echo "${VARS^^}"
-done
+        sed 's/^.*=//g' | \
+        sed -z 's/\n/,/g;s/,$/\n/g')"
+
+# Create an inventory file for ansible.
+echo "[k8s_nodes]" > inventory
+echo $VM_IP_EXPORTS | \
+    sed 's/"//g' | \
+    sed 's/export //g' | \
+    sed 's/ /\n/g' | \
+    sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \
+        >> inventory
+
+echo $VM_IP_EXPORTS | sed 's/" /"\n/g'
+echo export ANSIBLE_INV=$ANSIBLE_INV

main.tf

@@ -1,16 +1,44 @@
+terraform {
+  required_version = ">= 1.0.8"
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.6.11"
+    }
+  }
+}
 
 locals {
   nodes-config = {
-    "master" = {
+    "amzn2" = {
+      base-image = var.amzn2-ami
+      num = 1
+    },
+    "ubuntu" = {
+      base-image = var.ubuntu-ami
+      num = 1
+    },
+    "arch" = {
+      base-image = var.arch-ami
+      num = 1
+    },
+    "centos7" = {
+      base-image = var.centos7-ami
+      num = 1
+    },
+    "centos8" = {
       base-image = var.centos8-ami
       num = 1
     },
-    "worker" = {
-      base-image = var.centos8-ami
-      num = 2
+    "rhel7" = {
+      base-image = var.rhel7-ami
+      num = 1
+    },
+    "rhel8" = {
+      base-image = var.rhel8-ami
+      num = 1
     }
   }
-  install-qemu-agent = false
 }
 
 ################################################################################
@@ -25,7 +53,6 @@ module "cloud-init-config" {
   num                 = each.value.num
   root-admin-passwd   = var.root-admin-passwd
   root-admin-pub-key  = var.root-admin-pub-key
-  install-qemu-agent  = local.install-qemu-agent
 }
 
 ################################################################################
@@ -34,6 +61,10 @@ module "cloud-init-config" {
 # libvirt modules/resources.
 ################################################################################
 
+provider "aws" {
+  region = "us-east-2"
+}
+
 # This module will grab the latest ami for a variety of distros. Uncomment to
 # get a list of the latest AMIs for our supported distros.
 # module "aws-amis" {
@@ -43,27 +74,14 @@ module "cloud-init-config" {
 #   value = module.aws-amis.amis
 # }
 
-################################################################################
-# AWS Networking
-# Use of the 2 modules below to create resources for the AWS network.
-# aws-network-from-scratch will build the AWS network from scratch.
-# aws-network-existing will query AWS for an existing VPC.
-################################################################################
-
-# module "aws-network-from-scratch" {
-#   source            = "./modules/aws-network-from-scratch"
-#   name-prefix       = var.vm-name-prefix
-#   vpc-cidr-block    = var.aws-vpc-cidr-block
-#   subnet-cidr-block = var.aws-subnet-cidr-block
-#   admin-ips         = var.admin-ips
-# }
-
-module "aws-network-existing" {
-  source = "./modules/aws-network-existing"
+module "aws-network" {
+  source            = "./modules/aws-network"
+  name-prefix       = var.vm-name-prefix
+  vpc-cidr-block    = var.aws-vpc-cidr-block
+  subnet-cidr-block = var.aws-subnet-cidr-block
+  admin-ips         = var.admin-ips
 }
 
-################################################################################
-
 # This key pair is not actually used. Keys are added to the nodes via cloud-init
 # instead. We just add this here that this key will show up in the AWS console."
 resource "aws_key_pair" "key" {
@@ -127,8 +145,6 @@ module "nodes" {
 # end libvirt
 ################################################################################
 
-# This will output a map of group => [{hostname, ip}].
-# TODO A 'names' output needs to be added to libvirt.
-output "groups_hostnames_ips" {
-  value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) }
+output "ips" {
+  value = { for type, node in module.nodes : type => node.ips }
 }

modules/aws-network-existing/main.tf

@@ -1,23 +0,0 @@
-locals {
-  az-to-subnets = {
-    for s in data.aws_subnet.subnets : s.availability_zone => s.id...
-  }
-}
-
-data "aws_vpc" "default" {
-  tags = {
-    Name = var.default-vpc-name
-  }
-}
-
-data "aws_subnets" "subnet-ids" {
-  filter {
-    name = "vpc-id"
-    values = [data.aws_vpc.default.id]
-  }
-}
-
-data "aws_subnet" "subnets" {
-  for_each = toset(data.aws_subnets.subnet-ids.ids)
-  id = each.key
-}

modules/aws-network-existing/outputs.tf

@@ -1,18 +0,0 @@
-output "default-vpc" {
-  value = data.aws_vpc.default
-}
-
-output "subnets" {
-  description = "An array of all subnets in default-vpc."
-  value = data.aws_subnet.subnets
-}
-
-output "k8s-subnets" {
-  description = "An array of subnets to be used for k8s VMs. These subnets were chosen by selecting a single subnet from each availability_zone."
-  value = [for k,v in local.az-to-subnets : v[0]]
-}
-
-output "az-to-subnets" {
-  description = "A map of availability zone to array of subnets that are in thet availability zone."
-  value = local.az-to-subnets
-}

modules/aws-network-existing/variables.tf

@@ -1,4 +0,0 @@
-variable "default-vpc-name" {
-  description = "The name of the existing default VPC. This module will query AWS for a VPC with this name,"
-  default = "Managed VPC"
-}

modules/aws-network/main.tf

@@ -33,7 +33,7 @@ resource "aws_default_security_group" "sg" {
   }
 
   tags = {
-    Name = "${var.name-prefix}-ssh-from-admins-sg"
+    Name = "${var.name-prefix}-ssh-from-admins--sg"
   }
 }
 

modules/aws-nodes/main.tf

@@ -1,7 +1,6 @@
 resource "aws_instance" "nodes" {
   ami                         = var.ami
   instance_type               = var.ec2-instance-type
-  # TODO REM double check this key.
   # key_name                    = aws_key_pair.debug1.key_name
   associate_public_ip_address = true
   subnet_id                   = var.subnet-id

modules/aws-nodes/outputs.tf

@@ -1,7 +1,3 @@
 output "ips" {
   value = aws_instance.nodes.*.public_ip
 }
-
-output "names" {
-  value = aws_instance.nodes.*.tags.Name
-}

modules/cloud-init-config/main.tf

@@ -4,7 +4,6 @@ data "template_file" "user-datas" {
     admin-passwd  = "${var.root-admin-passwd}"
     admin-pub-key = "${var.root-admin-pub-key}"
     hostname      = "${var.hostname-prefix}-${count.index}"
-    install-qemu-agent = var.install-qemu-agent
   }
   count = var.num
 }

modules/cloud-init-config/variables.tf

@@ -8,12 +8,6 @@ variable "hostname-prefix" {
   description = "This prefix wil be applied as a prefix for the hostnames."
 }
 
-variable "install-qemu-agent" {
-  default     = false
-  description = "This flag determines whether or not qemu-agent is installed."
-  type        = bool
-}
-
 variable "num" {
   description = "The number of user-datas to create with these parameters."
 }

providers.tf

@@ -1,76 +0,0 @@
-
-provider "aws" {
-  region = "us-gov-west-1"
-}
-
-
-terraform {
-  required_version = ">= 1.0.8"
-
-    backend "s3" {
-      
-      bucket            = "mss-terraform-state"
-      key               = "global/s3/terraform.tfstate"
-      region            = "us-gov-west-1"
-      
-      dynamodb_table    = "mss-terraform-state-lock"
-      encrypt           = true
-
-  }
-  required_providers {
-    libvirt = {
-      source  = "dmacvicar/libvirt"
-      version = "0.6.11"
-    }
-  }
-}
-
-
-
-
-
-##
-#S3 bucket create to hold our TFState file so we can all share env settings
-resource "aws_s3_bucket" "terraform_state" {
-    bucket = "mss-terraform-state"
-
-    # enable versioning for the state files 
-    versioning {
-        enabled = true
-    }
-
-    #enable server-side encryption
-    server_side_encryption_configuration {
-      
-      rule {
-          apply_server_side_encryption_by_default {
-              sse_algorithm = "AES256"
-              }
-          }
-        } 
-}
-
-##
-# no sql database used so that we can lock the TFstate file in the S3 bucket to ensure two people 
-# are not running a terraform command at the same time 
-resource "aws_dynamodb_table" "terraform_locks" {
-    name            = "mss-terraform-state-lock"
-    billing_mode    = "PAY_PER_REQUEST"
-    hash_key        = "LockID"
-
-    attribute {
-      name = "LockID"
-      type = "S"
-    }
-
-
-}
-
-
-##
-# output variable to give details on the s3 bucket created
-#TODO: move to output.tf
-output "s3_bucket_arn" {
-  value       = aws_s3_bucket.terraform_state.arn
-  description = "The ARN of the S3 bucket"
-}

variables.tf

@@ -8,11 +8,6 @@ variable "aws-ec2-instance-type" {
   description = "The AWS instance type to use for all nodes."
 }
 
-variable "aws-region" {
-  default = "us-east-1"
-  description = "The AWS region to use."
-}
-
 variable "aws-subnet-cidr-block" {
   default = "10.0.1.0/24"
   description = "The address space to be used for this subnet."