Use new VPC.

- Made the qemu-agent content in the cloud-init config to be controlled
  by a 'install-qemu-agent' flag.

.gitignore

@@ -44,6 +44,7 @@ terraform.rc
 ################################################################################
 
 k8s-key*
+*admin-key*
 STARTHERE
 inventory
 

README.md

@@ -1,19 +1,53 @@
-A Terraform script to create k8s nodes. This script has modules for creating the
-nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS.
+A terraform script that will setup VMs via AWS or KVM/QEMU (libvirt).
 
 The modules allow you create N VMs of a specific type. So you could create 1
 master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS
 VMs, or whatever fits your needs.
 
+This script creates the VMs and then uses Cloud-init for initial configuration
+of the VMs.
+
+libvirt vs. AWS
+----------------------------------------
+
+This script allows VMs to be created libvirt or AWS. You can choose either one,
+however, running both at the same time has not been tested. Therefore, when
+using one, you should comment out all code pertaining to the other provider.
+
 Cloud-Init
 ----------------------------------------
 
 Both the libvirt and aws modules use cloud-init for initial configuration of the
-VMs.
+VMs. Cloud-init is used to create the admin user and add the mss-admin keys.
+Cloud-init is also used to install some initial packages. The goal is to get
+these VMs setup enough so that they can be configured via Ansible.
+
+Environment Setup
+----------------------------------------
+
+1. From within the nodes-terraform directory run `terraform init`. You will a
+   see bunch if output as terraform grabs the dependencies it needs.
+
+2. Copy the shared `terraform.tfvars` from S3.
+   ```shell
+   aws s3 cp s3://mss-terraform-state/global/s3/terraform.tfvars .
+   ```
+   For instructions on getting AWS CLI to work correctly see `aws-api-setup.md`
+   in the `doc-ops` repo.
+
+   Take care to copy your `terraform.tfvars` back to the S3 bucket whenever you
+   make a change to it.
+
+3. Verify that you branch and variables match whats deployed by running
+   `terraform plan`. If everything is in sync you should see `No changes. Your
+   infrastructure matches the configuration.` _IF_ you do not see this message
+   make sure to sync with the other admins before continuing.
+
 
 Dependencies
 ----------------------------------------
 
+_UNFINISHED_
 TODO REM add libvirt provider
 libvirt provider depends on mkisofs
 

cloud_init.cfg

@@ -26,6 +26,10 @@ chpasswd:
 hostname: ${hostname}
 fqdn: ${hostname}
 
+package_update: true
+package_upgrade: true
+package_reboot_if_required: true
+
 %{ if install-qemu-agent }
 packages:
   # This are only necessary for libvirt.

example.tfvars

@@ -19,17 +19,6 @@ node-vcpus = 2
 # 12 GiB
 # libvirt-node-disk-size = "${12 * 1073741824}"
 
-################################################################################
-# AWS EC2 instance types
-################################################################################
-
-# 1 GiB, 1 vcpu, only one that is free.
-# This one won't work with k8s because it requires at least 2 vcpus.
-aws-ec2-instance-type = "t2.micro"
-
-# 4 GiB, 2 vcpus
-# aws-ec2-instnce-type = "t2.medium"
-
 ################################################################################
 # AWS images (AMIs)
 ################################################################################

get-vm-ips.sh

@@ -22,6 +22,10 @@ VM_NAME_PREFIX="$(
         tail -n 1 | \
         sed 's/^.*=\s*"\(.*\)"/\1/g')"
 
+PUBLIC_IP_OUTPUT="groups_hostnames_ips"
+PRIVATE_IP_OUTPUT="groups_hostnames_private_ips"
+IP_TYPE="$PRIVATE_IP_OUTPUT"
+
 # This command stores the output data in the format below.
 # [
 #   {
@@ -48,7 +52,7 @@ VM_NAME_PREFIX="$(
 #   }
 # ]
 DATA="$(terraform show -json | \
-    jq '.values.outputs.groups_hostnames_ips.value | to_entries |
+    jq '.values.outputs.'"$IP_TYPE"'.value | to_entries |
         map({group: .key, vms:.value | to_entries |
         map({hostname:.key,ip:.value})})')"
 
@@ -80,13 +84,15 @@ for GROUP in $ANS_GROUPS; do
     # HOSTNAME1=0.0.0.0
     # HOSTNAME2=0.0.0.0
     VARS="$(
-        echo $DATA | \
+        echo -n $DATA | \
             jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
                 "\(.hostname)=\(.ip)"' | \
             sed 's/"//g' | \
             sed "s/$VM_NAME_PREFIX-//g" | \
             sed 's/-/_/g'
         )"
-    # Print the contents of $VARS converted to uppercase.
+    # Print the contents of $VARS converted to uppercase. If it's not expty.
+    if [ ! -z "$VARS" ]; then
         echo "${VARS^^}"
+    fi
 done

main.tf

@@ -1,24 +1,172 @@
-terraform {
-  required_version = ">= 1.0.8"
-  required_providers {
-    libvirt = {
-      source  = "dmacvicar/libvirt"
-      version = "0.6.11"
-    }
-  }
-}
 
 locals {
+  old-k8s-subnets-ids = [
+    # module.aws-network-from-scratch.subnet.id,
+    module.aws-network-existing-managed.subnet-by-name["subnet_1"].id,
+    module.aws-network-existing-managed.subnet-by-name["subnet_3"].id,
+  ]
+  new-k8s-subnets-ids = [
+    module.aws-network-existing-mss-dev.subnet-by-name["mssdev Subnet 1"].id,
+    module.aws-network-existing-mss-dev.subnet-by-name["mssdev Subnet 3"].id,
+  ]
+  nfs-subnets = [
+    # module.aws-network-from-scratch.subnet,
+    module.aws-network-existing-managed.subnet-by-name["subnet_4"],
+  ]
+  aws-managed-security-group-id = module.aws-network-existing-managed.default-sg.id
+  aws-mss-dev-security-group-id = module.aws-network-existing-mss-dev.default-sg.id
+  # aws-managed-security-group-id = module.aws-network-from-scratch.default-security-group.id
+
+  # The names of these nodes is created by:
+  # "{var.vm-name-prefix}-{name from nodes-config}-{number}"
+  # The length of this full name must be < 16 characters. This is a limitaion of
+  # Active Directory? The "realm join" command failed when the name was too
+  # long, although it did not give the reason for the failure, the reason was
+  # surmised because it looked like it was truncating the name when attempting
+  # to join.
+  #
+  # NOTE: This naming restriction is only important if you plan on joining these
+  # VMs to Active Directory.
   nodes-config = {
-    "master" = {
-      base-image = var.centos8-ami
+    # TODO if the above comment about the name length is true, then this name is
+    # too long. IMPORTANT! If you change this then you need to change the k8s
+    # ansible role that assigns roles based on wether or not 'master' is in the
+    # name.
+    "k8s-master" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.t2-medium-4gib-2vcpu
+      subnet-ids      = local.old-k8s-subnets-ids
+      security-groups = [local.aws-managed-security-group-id]
       num             = 1
     },
-    "worker" = {
-      base-image = var.centos8-ami
+    "k8s-wrkr" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
+      subnet-ids      = local.old-k8s-subnets-ids
+      security-groups = [local.aws-managed-security-group-id]
+      disk-size       = 32
       num             = 2
+    },
+    "worker" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
+      subnet-ids      = local.old-k8s-subnets-ids
+      security-groups = [local.aws-managed-security-group-id]
+      disk-size       = 64
+      num             = 3
+    },
+    "tst-master" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.t2-medium-4gib-2vcpu
+      subnet-ids      = local.old-k8s-subnets-ids
+      security-groups = [local.aws-managed-security-group-id]
+      num             = 0
+    },
+    "tst-wrkr" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
+      subnet-ids      = local.old-k8s-subnets-ids
+      security-groups = [local.aws-managed-security-group-id]
+      disk-size       = 32
+      num             = 0
+    },
+    "rancher-master" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
+      subnet-ids      = local.new-k8s-subnets-ids
+      security-groups = [local.aws-mss-dev-security-group-id]
+      disk-size       = 64
+      num             = 1
+    },
+    "rancher-worker" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
+      subnet-ids      = local.new-k8s-subnets-ids
+      security-groups = [local.aws-mss-dev-security-group-id]
+      disk-size       = 64
+      num             = 2
+    },
+    "dev" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.c5a-xlarge-amd-8gib-4vcpu
+      # subnet-ids    = [module.aws-network-from-scratch.subnet.id]
+      subnet-ids      = [module.aws-network-existing-managed.subnet-by-name["subnet_4"].id]
+      security-groups = [local.aws-managed-security-group-id]
+      disk-size       = 32
+      num             = 2
+    },
+    "nfs" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.t3a-medium-amd-4gib-2vcpu
+      # subnet-ids    = [module.aws-network-from-scratch.subnet.id]
+      subnet-ids      = [module.aws-network-existing-managed.subnet-by-name["subnet_4"].id]
+      security-groups = [local.aws-managed-security-group-id]
+      num             = 1
+      num-disks       = 1
+      # TODO Remove this, prefferred method is to use 'disks' and 'disk-mounts'
+      # like shown below.
+      zfs-disk-size   = 10
+    },
+    "proxy" = {
+      base-image      = module.aws-ami-constants.ubuntu-ami
+      aws-ec2-type    = module.ec2-types.t2-micro-1gib-1vcpu
+      # subnet-ids    = [module.aws-network-from-scratch.subnet.id]
+      subnet-ids      = [module.aws-network-existing-managed.subnet-by-name["subnet_4"].id]
+      security-groups = [local.aws-managed-security-group-id]
+      private-ips     = [var.aws-proxy-private-ip]
+      num             = 1
+    },
   }
+
+  disks = {
+    "zfs-64g" = {
+      num               = 1,
+      size              = 64
+      availability_zone = local.nfs-subnets[0].availability_zone
+    },
+    "zfs-256g" = {
+      num               = 1,
+      size              = 256
+      availability_zone = local.nfs-subnets[0].availability_zone
+    },
   }
+
+  disk-mounts = [
+    {
+      # TODO make this attach field work.
+      attach        = false
+      ec2-id        = module.nodes["nfs"].nodes[0].id,
+      disk-group    = "zfs-64g"
+      # TODO also make sure that get drive letters or whetever still works. did
+      # it ever work?
+      # TODO We need to have a map of drive letters to device path in linux.
+      # This gets a little more complicated becuase the t2 instances map to
+      # /dev/xvdg while the t3a instance types map to /dev/nvme0n1 where 0 is
+      # the nth drive connected. The nth drive does not seem to map to a drive
+      # letter, ie. drive letter "f" does not map to 6, the nth drive simply
+      # increments by 1 for each drive attached regardless of the drive letter
+      # set.
+      drive-letters = ["g", "h", "i"]
+    },
+    {
+      # TODO make this attach field work.
+      attach        = false
+      ec2-id        = module.nodes["nfs"].nodes[0].id,
+      disk-group    = "zfs-256g"
+      # TODO also make sure that get drive letters or whetever still works. did
+      # it ever work?
+      # TODO We need to have a map of drive letters to device path in linux.
+      # This gets a little more complicated becuase the t2 instances map to
+      # /dev/xvdg while the t3a instance types map to /dev/nvme0n1 where 0 is
+      # the nth drive connected. The nth drive does not seem to map to a drive
+      # letter, ie. drive letter "f" does not map to 6, the nth drive simply
+      # increments by 1 for each drive attached regardless of the drive letter
+      # set.
+      drive-letters = ["j", "k", "l"]
+    },
+  ]
+
+  # This is only needed for libvirt.
   install-qemu-agent = false
 }
 
@@ -43,8 +191,14 @@ module "cloud-init-config" {
 # libvirt modules/resources.
 ################################################################################
 
-provider "aws" {
-  region = "us-east-2"
+# This module sets the ec2-types constants.
+module "ec2-types" {
+  source = "./modules/ec2-types"
+}
+
+# This module sets the AWS AMI constants.
+module "aws-ami-constants" {
+  source = "./modules/aws-ami-constants"
 }
 
 # This module will grab the latest ami for a variety of distros. Uncomment to
@@ -56,14 +210,37 @@ provider "aws" {
 #   value = module.aws-amis.amis
 # }
 
-module "aws-network" {
-  source            = "./modules/aws-network"
-  name-prefix       = var.vm-name-prefix
-  vpc-cidr-block    = var.aws-vpc-cidr-block
-  subnet-cidr-block = var.aws-subnet-cidr-block
-  admin-ips         = var.admin-ips
+################################################################################
+# AWS Networking
+# Use the 2 modules below to create resources for the AWS network.
+# aws-network-from-scratch will build the AWS network from scratch.
+# aws-network-existing will query AWS for an existing VPC.
+################################################################################
+
+# module "aws-network-from-scratch" {
+#   source            = "./modules/aws-network-from-scratch"
+#   name-prefix       = var.vm-name-prefix
+#   vpc-cidr-block    = var.aws-vpc-cidr-block
+#   subnet-cidr-block = var.aws-subnet-cidr-block
+#   admin-ips         = var.admin-ips
+# }
+
+module "aws-network-existing-managed" {
+  source                      = "./modules/aws-network-existing"
+  default-vpc-name            = var.aws-existing-managed-vpc-name
+  default-security-group-name = var.aws-existing-managed-sg-name
+  existing-subnet-names       = var.aws-existing-managed-subnet-names
 }
 
+module "aws-network-existing-mss-dev" {
+  source                      = "./modules/aws-network-existing"
+  default-vpc-name            = var.aws-existing-mss-dev-vpc-name
+  default-security-group-name = var.aws-existing-mss-dev-sg-name
+  existing-subnet-names       = var.aws-existing-mss-dev-subnet-names
+}
+
+################################################################################
+
 # This key pair is not actually used. Keys are added to the nodes via cloud-init
 # instead. We just add this here that this key will show up in the AWS console."
 resource "aws_key_pair" "key" {
@@ -74,18 +251,76 @@ resource "aws_key_pair" "key" {
   }
 }
 
+module "disks" {
+  for_each          = local.disks
+  source            = "./modules/aws-disks"
+  availability_zone = each.value.availability_zone
+  size              = each.value.size
+  num               = each.value.num
+  prefix            = each.key
+}
+
+module "disk-mounts" {
+  source = "./modules/aws-disk-mounts"
+  disks  = module.disks[element(local.disk-mounts, count.index).disk-group].disks
+  ec2-id = element(local.disk-mounts, count.index).ec2-id
+  drive-letters = try(element(local.disk-mounts, count.index).drive-letters, null)
+  count  = length(local.disk-mounts)
+}
+
+# TODO REM remove if unuesed.
+resource "aws_ebs_volume" "zfs" {
+  # TODO REM look at types.
+  availability_zone = local.nfs-subnets[0].availability_zone
+  size = local.nodes-config["nfs"].zfs-disk-size
+  encrypted = true
+  count = local.nodes-config["nfs"].num-disks
+  tags = {
+    Name = "zfs-disk-${count.index}"
+  }
+}
+
+resource "aws_volume_attachment" "mount-nfs-volume" {
+  device_name = "/dev/sd${element(var.aws-zfs-drive-letters, count.index)}"
+  instance_id = module.nodes["nfs"].nodes[0].id
+  count = local.nodes-config["nfs"].num-disks
+  volume_id = element(aws_ebs_volume.zfs, count.index).id
+}
+
+output "zfs-drive-letters" {
+  value = aws_volume_attachment.mount-nfs-volume.*.device_name
+}
+
 module "nodes" {
   for_each           = local.nodes-config
   source             = "./modules/aws-nodes"
+  ec2-instance-type  = each.value.aws-ec2-type
   ami                = each.value.base-image
-  ec2-instance-type  = var.aws-ec2-instance-type
-  subnet-id          = module.aws-network.subnet.id
-  security-group-ids = [module.aws-network.default-security-group.id]
+  subnet-ids         = each.value.subnet-ids
+  private-ips        = try(each.value.private-ips, [])
+  security-group-ids = each.value.security-groups
   user-datas         = lookup(module.cloud-init-config, each.key, null).user-datas
+  disk-size          = try(each.value.disk-size, null)
   num-nodes          = each.value.num
   name-prefix        = "${var.vm-name-prefix}-${each.key}"
+  # TODO add a input for the key so that it will show up as the key in the aws
+  # console.
 }
 
+# TODO an attempt to create a windows machine.
+# module "nodes-win" {
+#   source             = "./modules/aws-nodes"
+#   ec2-instance-type  = module.ec2-types.t2-small-2gib-1vcpu
+#   ami                = module.aws-ami-constants.win-srv-2019-ami
+#   subnet-ids         = [module.aws-network-existing-managed.subnet-by-name["subnet_2"].id]
+#   private-ips        = []
+#   security-group-ids = [local.aws-managed-security-group-id]
+#   # TODO REM need to figure out how to not pass a user data.
+#   user-datas         = [null]
+#   num-nodes          = 1
+#   name-prefix        = "${var.vm-name-prefix}-win-test"
+# }
+
 ################################################################################
 # end aws
 ################################################################################
@@ -100,6 +335,10 @@ module "nodes" {
 #   uri = var.libvirt-connection-url
 # }
 #
+# module "libvirt-images" {
+#   source = "./modules/libvirt-images"
+# }
+#
 # module "nodes" {
 #   for_each               = local.nodes-config
 #   source                 = "./modules/libvirt-nodes"
@@ -132,3 +371,9 @@ module "nodes" {
 output "groups_hostnames_ips" {
   value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) }
 }
+
+# This will outpus a map of group => [{hostname, private_ip}].
+# TODO Figure out how what to do about private_ips for libvirt.
+output "groups_hostnames_private_ips" {
+  value = { for type, node in module.nodes : type => zipmap(node.names, node.private_ips) }
+}

modules/aws-ami-constants/outputs.tf

@@ -0,0 +1,58 @@
+################################################################################
+# AWS AMI vars
+# These outputs are really more like constants. Using outputs improves
+# readability. The values are manually updated. Use the aws-amis module to get
+# the latest for each distro. Keep in mind that AMIs are region specific this is
+# why you may see values and other regions commented out.
+################################################################################
+
+output "amzn2-ami" {
+  # us-east-2
+  # default     = "ami-0dd0ccab7e2801812"
+  # us-gov-west-1
+  value       = "ami-02ab588324a95cf31"
+  description = "The AMI to use for Amazon Linux 2."
+}
+output "ubuntu-ami" {
+  # us-east-2
+  # value = "ami-06c7d6c0987eaa46c"
+  # us-gov-west-1
+  value       = "ami-066189aeb91baa0ab"
+  description = "The AMI to use for Ubuntu."
+}
+output "centos7-ami" {
+  # us-east-2
+  value       = "ami-00f8e2c955f7ffa9b"
+  description = "The AMI to use for CentOS 7."
+}
+output "centos8-ami" {
+  # us-east-2
+  value       = "ami-057cacbfbbb471bb3"
+  description = "The AMI to use for CentOS 8."
+}
+output "arch-ami" {
+  # us-east-2
+  value       = "ami-02653f06de985e3ba"
+  description = "The AMI to use for Arch Linux."
+}
+output "rhel7-ami" {
+  # us-east-2
+  # value = "ami-0a509b3c2a4d05b3f"
+  # us-gov-west-1
+  value       = "ami-04ccdf5793086ea95"
+  description = "The AMI to use for RHEL 7."
+}
+output "rhel8-ami" {
+  # us-east-2
+  # value = "ami-0d871ca8a77af2948"
+  # us-gov-west-1
+  value       = "ami-0b1f10cd1cd107dd2"
+  description = "The AMI to use for RHEL 8."
+}
+output "win-srv-2019-ami" {
+  # us-east-2
+  # value = ???
+  # us-gov-west-1
+  value       = "ami-0f838c3c35ab60fc4"
+  description = "The AMI to use for Windows Server 2019."
+}

modules/aws-amis/main.tf

@@ -1,33 +1,61 @@
 locals {
   amis = {
     amzn2 = {
-      owner-id = "137112412989"
+      # us-east-2
+      # owner-id = "137112412989"
+      # us-gov-west-1
+      owner-id = "045324592363"
       name = "amzn2-ami-hvm-2*x86_64-gp2"
     },
     ubuntu = {
-      owner-id = "099720109477"
+      # us-east-2
+      # owner-id = "099720109477"
+      # us-gov-west-1
+      owner-id = "513442679011"
       name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
     },
-    centos7 = {
-      owner-id = "125523088429"
-      name = "CentOS 7.*x86_64"
-    },
-    centos8 = {
-      owner-id = "125523088429"
-      name = "CentOS 8.*x86_64"
-    },
-    arch = {
-      owner-id = "093273469852"
-      name = "arch-linux-lts-hvm*x86_64-ebs"
-    },
+    # centos7 = {
+    #   # us-east-2
+    #   # owner-id = "125523088429"
+    #   # us-gov-west-1
+    #   # owner-id = THERE IS NO CENTOS7 IMAGE in us-gov-west-1!!
+    #   name = "CentOS 7.*x86_64"
+    # },
+    # centos8 = {
+    #   # us-east-2
+    #   # owner-id = "125523088429"
+    #   # us-gov-west-1
+    #   # owner-id = THERE IS NO CENTOS8 IMAGE in us-gov-west-1!!
+    #   name = "CentOS 8.*x86_64"
+    # },
+    # arch = {
+    #   # us-east-2
+    #   # owner-id = "093273469852"
+    #   # us-gov-west-1
+    #   # owner-id = THERE IS NO ARCH IMAGE in us-gov-west-1!!
+    #   name = "arch-linux-lts-hvm*x86_64-ebs"
+    # },
     rhel7 = {
-      owner-id = "309956199498"
+      # us-east-2
+      # owner-id = "309956199498"
+      # us-gov-west-1
+      owner-id = "219670896067"
       name = "RHEL-7.*HVM*x86_64*GP2"
     },
     rhel8 = {
-      owner-id = "309956199498"
+      # us-east-2
+      # owner-id = "309956199498"
+      # us-gov-west-1
+      owner-id = "219670896067"
       name = "RHEL-8.*HVM*x86_64*GP2"
-    }
+    },
+    win-srv-2019 = {
+      # us-east-2
+      # owner-id = "???"
+      # us-gov-west-1
+      owner-id = "077303321853"
+      name = "Windows_Server-2019-English-Full-Base*"
+    },
   }
 }
 

modules/aws-disk-mounts/main.tf

@@ -0,0 +1,6 @@
+resource "aws_volume_attachment" "mount-volume" {
+  device_name = "/dev/sd${element(var.drive-letters, count.index)}"
+  instance_id = var.ec2-id
+  count = length(var.disks)
+  volume_id = element(var.disks, count.index).id
+}

modules/aws-disk-mounts/variables.tf

@@ -0,0 +1,16 @@
+variable "drive-letters" {
+  default = ["f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"]
+  description = "These drive letters will be used when arraching disks to EC2 instance with `ec2-id'"
+  type = list(string)
+  nullable = false
+}
+
+variable "disks" {
+  description = "An array of disks to attach to the EC2 isntance with `ec2-id`"
+  type = list
+}
+
+variable "ec2-id" {
+  description = "The ID of the EC@ instance to attach the drives to."
+  type = string
+}

modules/aws-disks/main.tf

@@ -0,0 +1,10 @@
+
+resource "aws_ebs_volume" "disks" {
+  availability_zone = var.availability_zone
+  size = var.size
+  count = var.num
+  encrypted = true
+  tags = {
+    Name = "${var.prefix}-${count.index}"
+  }
+}

modules/aws-disks/outputs.tf

@@ -0,0 +1,3 @@
+output "disks" {
+  value = aws_ebs_volume.disks
+}

modules/aws-disks/variables.tf

@@ -0,0 +1,19 @@
+variable "availability_zone" {
+  description = "The availability zone this disk will reside in. This AZ is applied to all 'num' disks."
+  type = string
+}
+
+variable "num" {
+  description = "The number of disks to be created with this config."
+  type = number
+}
+
+variable "prefix" {
+  description = "This prefix will be used in the name of each disk created with this module. If the prefix is 'zfs', then it will create disks with names, zfs-0 and zfs-1."
+  type = string
+}
+
+variable "size" {
+  description = "The size of the disk in GB."
+  type = number
+}

modules/aws-network-existing/main.tf

@@ -0,0 +1,35 @@
+locals {
+  az-to-subnets = {
+    for s in data.aws_subnet.subnets : s.availability_zone => s.id...
+  }
+}
+
+data "aws_vpc" "default" {
+  tags = {
+    Name = var.default-vpc-name
+  }
+}
+
+data "aws_subnets" "subnet-ids" {
+  filter {
+    name = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
+}
+
+data "aws_subnet" "subnets" {
+  for_each = toset(data.aws_subnets.subnet-ids.ids)
+  id = each.key
+}
+
+data "aws_subnet" "subnet-by-name" {
+  for_each = toset(var.existing-subnet-names)
+  filter {
+    name   = "tag:Name"
+    values = [each.key]
+  }
+}
+
+data "aws_security_group" "default" {
+  name = var.default-security-group-name
+}

modules/aws-network-existing/outputs.tf

@@ -0,0 +1,27 @@
+output "default-vpc" {
+  value = data.aws_vpc.default
+}
+
+output "default-sg" {
+  value = data.aws_security_group.default
+}
+
+output "subnets" {
+  description = "An array of all subnets in default-vpc."
+  value = data.aws_subnet.subnets
+}
+
+output "one-subnet-per-az" {
+  description = "An array of subnets that selects 1 subnet per az."
+  value = [for k,v in local.az-to-subnets : v[0]]
+}
+
+output "subnet-by-name" {
+  description = "A map of subnet name to subnet resource."
+  value = data.aws_subnet.subnet-by-name
+}
+
+output "az-to-subnets" {
+  description = "A map of availability zone to array of subnets that are in thet availability zone."
+  value = local.az-to-subnets
+}

modules/aws-network-existing/variables.tf

@@ -0,0 +1,13 @@
+variable "default-security-group-name" {
+  description = "The name of the existing default security group. This module will query AWS for a security group with this name,"
+}
+
+variable "default-vpc-name" {
+  description = "The name of the existing default VPC. This module will query AWS for a VPC with this name,"
+}
+
+variable "existing-subnet-names" {
+  description = "A list of subnet names that already exist in default-vpc-name"
+  default     = []
+  type        = list(string)
+}

modules/aws-network-from-scratch/main.tf

@@ -0,0 +1,67 @@
+resource "aws_vpc" "vpc" {
+  cidr_block = var.vpc-cidr-block
+  tags = {
+      Name = "${var.name-prefix}-vpc"
+  }
+}
+
+resource "aws_subnet" "subnet" {
+  vpc_id = aws_vpc.vpc.id
+  cidr_block = var.subnet-cidr-block
+  # availability_zone = var.avail_zone
+  tags = {
+      Name = "${var.name-prefix}-subnet"
+  }
+}
+
+resource "aws_default_security_group" "sg" {
+  vpc_id = aws_vpc.vpc.id
+
+  ingress {
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = var.admin-ips
+  }
+
+  egress {
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+    cidr_blocks     = ["0.0.0.0/0"]
+    prefix_list_ids = []
+  }
+
+  tags = {
+    Name = "${var.name-prefix}-ssh-from-admins-sg"
+  }
+}
+
+resource "aws_internet_gateway" "igw" {
+	vpc_id = aws_vpc.vpc.id
+  tags = {
+    Name = "${var.name-prefix}-igw"
+  }
+}
+
+resource "aws_default_route_table" "route-table" {
+   default_route_table_id = aws_vpc.vpc.main_route_table_id
+
+   route {
+     cidr_block = "0.0.0.0/0"
+     gateway_id = aws_internet_gateway.igw.id
+   }
+
+   # default route, mapping VPC CIDR block to "local", created implicitly and
+   # cannot be specified.
+
+   tags = {
+     Name = "${var.name-prefix}-route-table"
+   }
+}
+
+# Associate subnet with Route Table
+resource "aws_route_table_association" "a-rtb-subnet" {
+  subnet_id      = aws_subnet.subnet.id
+  route_table_id = aws_default_route_table.route-table.id
+}

modules/aws-network-from-scratch/outputs.tf

@@ -0,0 +1,11 @@
+output "vpc" {
+  value = aws_vpc.vpc
+}
+
+output "subnet" {
+  value = aws_subnet.subnet
+}
+
+output "default-security-group" {
+  value = aws_default_security_group.sg
+}

modules/aws-network-from-scratch/variables.tf

@@ -0,0 +1,23 @@
+variable "admin-ips" {
+  description = "A list of ips or cidr blocks that are allowed to connect to the nodes."
+  type = list(string)
+}
+
+variable "name-prefix" {
+  default = "tf"
+  description = "This prefix will be used in all the names of the resources creates in our AWS network."
+  type = string
+}
+
+variable "subnet-cidr-block" {
+  default = "10.0.1.0/24"
+  description = "The address space to be used for this subnet."
+  type = string
+}
+
+variable "vpc-cidr-block" {
+  default = "10.0.0.0/16"
+  description = "The address space to be used for out networks VPC."
+  type = string
+}
+

modules/aws-nodes/main.tf

@@ -1,11 +1,18 @@
 resource "aws_instance" "nodes" {
   ami                         = var.ami
   instance_type               = var.ec2-instance-type
-  # key_name                    = aws_key_pair.debug1.key_name
+  # TODO Make this a variable.
   associate_public_ip_address = true
-  subnet_id                   = var.subnet-id
+  subnet_id = element(var.subnet-ids, count.index % length(var.subnet-ids))
+  # Set a private IP if var.private-ips is not empty, otherwise let AWS assign
+  # the IP.
+  private_ip = length(var.private-ips) == 0 ? null : element(var.private-ips, count.index)
   vpc_security_group_ids = var.security-group-ids
   user_data              = element(var.user-datas.*.rendered, count.index)
+  root_block_device {
+    volume_size           = var.disk-size
+    delete_on_termination = true
+  }
   count = var.num-nodes
 
   tags = {

modules/aws-nodes/outputs.tf

@@ -2,6 +2,14 @@ output "ips" {
   value = aws_instance.nodes.*.public_ip
 }
 
+output "private_ips" {
+  value = aws_instance.nodes.*.private_ip
+}
+
 output "names" {
   value = aws_instance.nodes.*.tags.Name
 }
+
+output "nodes" {
+  value = aws_instance.nodes
+}

modules/aws-nodes/variables.tf

@@ -3,6 +3,12 @@ variable "ami" {
   type        = string
 }
 
+variable "disk-size" {
+  default     = 8
+  description = "The size of the root FS disk in GB."
+  type        = number
+}
+
 variable "ec2-instance-type" {
   default     = "t2.micro"
   description = "The AWS instance type to use for all nodes."
@@ -21,13 +27,19 @@ variable "num-nodes" {
   type        = number
 }
 
+variable "private-ips" {
+  default     = []
+  description = "A list of private IP addresses to use for the nodes. If the list is empty then each node will get an IP assigned from AWS."
+  type        = list(string)
+}
+
 variable "user-datas" {
   description = "A list of cloud-init configs that get applied to their corresponding node."
 }
 
-variable "subnet-id" {
-  description = "The ID of the subnet that all the nodes will be added to."
-  type = string
+variable "subnet-ids" {
+  description = "An array of subnet ids. These subnets will be round robined as the subnet to use for each node."
+  type        = list(string)
 }
 
 variable "security-group-ids" {

modules/ec2-types/outputs.tf

@@ -0,0 +1,45 @@
+################################################################################
+# AWS EC2 types.
+# Thesee are only here to help readability. Using these outputs allows the user
+# to be able to see the resources of each type without having to look them up.
+################################################################################
+
+output "t2-micro-1gib-1vcpu" {
+  description = "t2.micro EC2 instance with 1 GiB mem and 1 vCPU."
+  value = "t2.micro"
+}
+
+output "t2-small-2gib-1vcpu" {
+  description = "t2.small EC2 instance with 2 GiB mem and 1 vCPUs."
+  value = "t2.small"
+}
+
+output "t2-medium-4gib-2vcpu" {
+  description = "t2.medium EC2 instance with 4 GiB mem and 2 vCPUs."
+  value = "t2.medium"
+}
+
+output "t2-large-8gib-2vcpu" {
+  description = "t2.large EC2 instance with 8 GiB mem and 2 vCPUs."
+  value = "t2.large"
+}
+
+output "t2-xlarge-16gib-4vcpu" {
+  description = "t2.xlarge EC2 instance with 16 GiB mem and 4 vCPUs."
+  value = "t2.xlarge"
+}
+
+output "t3a-medium-amd-4gib-2vcpu" {
+  description = "t3a.medium EC2 instance on AMD with GiB mem and 2 vCPUs."
+  value = "t3a.medium"
+}
+
+output "t3a-large-amd-8gib-2vcpu" {
+  description = "t3a.large EC2 instance on AMD with 8 GiB mem and 2 vCPUs."
+  value = "t3a.large"
+}
+
+output "c5a-xlarge-amd-8gib-4vcpu" {
+  description = "c5a.xlarge EC2 instance on AMD with 8 GiB mem and 4 vCPUs."
+  value = "c5a.xlarge"
+}

modules/libvirt-images/outputs.tf

@@ -0,0 +1,22 @@
+################################################################################
+# Libvirt Images
+# These outputs are really more like constants. Using outputs improves
+# readability. The values are manually updated.
+################################################################################
+
+output "ubuntu-img" {
+  value       = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
+  description = "The libvirt image to use for Ubuntu."
+}
+
+output "centos7-img" {
+  # Latest as of 2021-12-06.
+  value       = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+  description = "The libvirt image to use for CentOS 7."
+}
+
+output "centos8-img" {
+  value       = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2"
+  description = "The libvirt image to use for CentOS 8."
+}
+

providers.tf

@@ -0,0 +1,77 @@
+
+provider "aws" {
+  region = var.aws-region
+  # profile = <aws cli profile>
+}
+
+
+terraform {
+  required_version = ">= 1.0.8"
+
+  backend "s3" {
+    
+    bucket            = "mss-terraform-state"
+    key               = "global/s3/terraform.tfstate"
+    region            = "us-gov-west-1"
+    
+    dynamodb_table    = "mss-terraform-state-lock"
+    encrypt           = true
+
+  }
+  required_providers {
+    libvirt = {
+      source  = "dmacvicar/libvirt"
+      version = "0.6.11"
+    }
+  }
+}
+
+
+
+
+
+##
+#S3 bucket create to hold our TFState file so we can all share env settings
+resource "aws_s3_bucket" "terraform_state" {
+    bucket = "mss-terraform-state"
+
+    # enable versioning for the state files 
+    versioning {
+        enabled = true
+    }
+
+    #enable server-side encryption
+    server_side_encryption_configuration {
+      
+      rule {
+          apply_server_side_encryption_by_default {
+              sse_algorithm = "AES256"
+              }
+          }
+        } 
+}
+
+##
+# no sql database used so that we can lock the TFstate file in the S3 bucket to ensure two people 
+# are not running a terraform command at the same time 
+resource "aws_dynamodb_table" "terraform_locks" {
+    name            = "mss-terraform-state-lock"
+    billing_mode    = "PAY_PER_REQUEST"
+    hash_key        = "LockID"
+
+    attribute {
+      name = "LockID"
+      type = "S"
+    }
+
+
+}
+
+
+##
+# output variable to give details on the s3 bucket created
+#TODO: move to output.tf
+output "s3_bucket_arn" {
+  value       = aws_s3_bucket.terraform_state.arn
+  description = "The ARN of the S3 bucket"
+}

variables.tf

@@ -1,11 +1,69 @@
 variable "admin-ips" {
-  description = "A list of ips or cidr blocks that are allowed to connect to the nodes."
+  default = ["0.0.0.0/0"]
+  description = "A list of ips or cidr blocks that are allowed to connect to the nodes. This variable is only used when using the aws-network-from-scratch module."
   type = list(string)
 }
 
-variable "aws-ec2-instance-type" {
-  default = "t2.micro"
-  description = "The AWS instance type to use for all nodes."
+variable "aws-zfs-drive-letters" {
+  default     = ["f", "g", "h", "i", "j"]
+  description = "These are the drove letters to use when attaching EBS volumes, ie. /dev/sdf or /dev/sdg"
+  type        = list(string)
+}
+
+################################################################################
+# Query existing infrastructure.
+# Variables used to grab existing resources from AWS.
+#
+# We have 2 sets of three variables in this section. 1 section has 'managed' in
+# the name and the other has 'mss-dev' in the name. There are not restrictions
+# as to what we can put into these VPCs. The names of the variable simply match
+# the names of the VPCs and have no meaning for what resources are allowed.
+################################################################################
+
+variable "aws-existing-managed-sg-name" {
+  default     = "change-me-if-using-aws-network-existing"
+  description = "The name of the existing security group when using aws-network-existing."
+}
+
+variable "aws-existing-managed-vpc-name" {
+  default     = "change-me-if-using-aws-network-existing"
+  description = "The name of the existing VPC when using aws-network-existing."
+}
+
+variable "aws-existing-managed-subnet-names" {
+  description = "A list of subnet names that already exist in aws-existing-managed-vpc-name"
+  default     = []
+  type        = list(string)
+}
+
+variable "aws-existing-mss-dev-sg-name" {
+  default     = "change-me-if-using-aws-network-existing"
+  description = "The name of the existing security group when using aws-network-existing with the mss-dev VPC."
+}
+
+variable "aws-existing-mss-dev-vpc-name" {
+  default     = "change-me-if-using-aws-network-existing"
+  description = "The name of the existing VPC when using aws-network-existing with the mss-dev VPC."
+}
+
+variable "aws-existing-mss-dev-subnet-names" {
+  description = "A list of subnet names that already exist in aws-existing-mss-dev-vpc-name"
+  default     = []
+  type        = list(string)
+}
+
+################################################################################
+# End query existing infrastructure.
+################################################################################
+
+variable "aws-proxy-private-ip" {
+  description = "The private IP to request for the proxy instance."
+  type        = string
+}
+
+variable "aws-region" {
+  default = "us-east-1"
+  description = "The AWS region to use."
 }
 
 variable "aws-subnet-cidr-block" {
@@ -28,6 +86,7 @@ variable "disk-image-pool-name" {
 }
 
 variable "libvirt-connection-url" {
+  default = "nobody@localhost"
   description = "The libvirt connection URI, ie. qemu+ssh://<user>@<host>/system"
 }
 
@@ -61,18 +120,6 @@ variable "root-admin-pub-key" {
   description = "The public key to be added to authorized_keys for the root and admin accounts."
 }
 
-variable "master-nodes" {
-  default     = 1
-  description = "The number of master nodes to create."
-  type        = number
-}
-
-variable "worker-nodes" {
-  default     = 2
-  description = "The number of worker nodes to create."
-  type        = number
-}
-
 variable "base-image" {
   default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
 }
@@ -81,61 +128,3 @@ variable "vm-name-prefix" {
   default = "k8s-tf"
   description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0."
 }
-
-################################################################################
-# AWS AMI vars
-# These variables are really mor like constants. Using variables improves
-# readability. The defaults are manually updated. Use the aws-amis module to get
-# the latest for each distro.
-################################################################################
-
-variable "amzn2-ami" {
-  default     = "ami-0dd0ccab7e2801812"
-  description = "The AMI to use for Amazon Linux 2."
-}
-variable "ubuntu-ami" {
-  default = "ami-06c7d6c0987eaa46c"
-  description = "The AMI to use for Ubuntu."
-}
-variable "centos7-ami" {
-  default = "ami-00f8e2c955f7ffa9b"
-  description = "The AMI to use for CentOS 7."
-}
-variable "centos8-ami" {
-  default = "ami-057cacbfbbb471bb3"
-  description = "The AMI to use for CentOS 8."
-}
-variable "arch-ami" {
-  default = "ami-02653f06de985e3ba"
-  description = "The AMI to use for Arch Linux."
-}
-variable "rhel7-ami" {
-  default = "ami-0a509b3c2a4d05b3f"
-  description = "The AMI to use for RHEL 7."
-}
-variable "rhel8-ami" {
-  default = "ami-0d871ca8a77af2948"
-  description = "The AMI to use for RHEL 8."
-}
-
-################################################################################
-# Libvirt Images
-# These variables are really mor like constants. Using variables improves
-# readability. The defaults are manually updated.
-################################################################################
-
-variable "ubuntu-img" {
-  default     = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
-  description = "The libvirt image tp use for Ubuntu."
-}
-
-variable "centos7-img" {
-  # Latest as of 2021-12-06.
-  default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
-  description = "The libvirt image tp use for CentOS 7."
-}
-
-variable "centos8-img" {
-  default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2"
-  description = "The libvirt image tp use for CentOS 8."
-}