Added proxy and ZFS drives.

cloud_init.cfg

@@ -26,6 +26,10 @@ chpasswd:
 hostname: ${hostname}
 fqdn: ${hostname}
 
+package_update: true
+package_upgrade: true
+package_reboot_if_required: true
+
 %{ if install-qemu-agent }
 packages:
   # This are only necessary for libvirt.

example.tfvars

@@ -19,17 +19,6 @@ node-vcpus = 2
 # 12 GiB
 # libvirt-node-disk-size = "${12 * 1073741824}"
 
-################################################################################
-# AWS EC2 instance types
-################################################################################
-
-# 1 GiB, 1 vcpu, only one that is free.
-# This one won't work with k8s because it requires at least 2 vcpus.
-aws-ec2-instance-type = "t2.micro"
-
-# 4 GiB, 2 vcpus
-# aws-ec2-instnce-type = "t2.medium"
-
 ################################################################################
 # AWS images (AMIs)
 ################################################################################

get-vm-ips.sh

@@ -84,13 +84,15 @@ for GROUP in $ANS_GROUPS; do
     # HOSTNAME1=0.0.0.0
     # HOSTNAME2=0.0.0.0
     VARS="$(
-        echo $DATA | \
+        echo -n $DATA | \
             jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | 
                 "\(.hostname)=\(.ip)"' | \
             sed 's/"//g' | \
             sed "s/$VM_NAME_PREFIX-//g" | \
             sed 's/-/_/g'
         )"
-    # Print the contents of $VARS converted to uppercase.
-    echo "${VARS^^}"
+    # Print the contents of $VARS converted to uppercase. If it's not expty.
+    if [ ! -z "$VARS" ]; then
+        echo "${VARS^^}"
+    fi
 done

main.tf

@@ -1,14 +1,53 @@
 
 locals {
+  k8s-subnets-ids = [
+    # module.aws-network-from-scratch.subnet.id,
+    module.aws-network-existing.subnet-by-name["subnet_1"].id,
+    module.aws-network-existing.subnet-by-name["subnet_3"].id,
+  ]
+  nfs-subnets = [
+    # module.aws-network-from-scratch.subnet,
+    module.aws-network-existing.subnet-by-name["subnet_4"],
+  ]
+  aws-security-group-id = module.aws-network-existing.default-sg.id
+  # aws-security-group-id = module.aws-network-from-scratch.default-security-group.id
   nodes-config = {
-    "master" = {
+    "k8s-master" = {
       base-image = var.ubuntu-ami
+      aws-ec2-type = var.t2-medium-4gib-2vcpu
+      subnet-ids = local.k8s-subnets-ids
+      num = 0
+    },
+    "k8s-worker" = {
+      base-image = var.ubuntu-ami
+      aws-ec2-type = var.t2-medium-4gib-2vcpu
+      subnet-ids = local.k8s-subnets-ids
+      num = 0
+    },
+    "ansible-test" = {
+      base-image = var.ubuntu-ami
+      aws-ec2-type = var.t2-micro-1gib-1vcpu
+      # subnet-ids = [module.aws-network-from-scratch.subnet.id]
+      subnet-ids = [module.aws-network-existing.subnet-by-name["subnet_2"].id]
+      num = 0
+    },
+    "nfs" = {
+      base-image = var.ubuntu-ami
+      aws-ec2-type = var.t2-micro-1gib-1vcpu
+      # subnet-ids = [module.aws-network-from-scratch.subnet.id]
+      subnet-ids = [module.aws-network-existing.subnet-by-name["subnet_4"].id]
+      num = 1
+      num-disks = 1
+      disk-size = 10
+    },
+    "proxy" = {
+      base-image = var.ubuntu-ami
+      aws-ec2-type = var.t2-micro-1gib-1vcpu
+      # subnet-ids = [module.aws-network-from-scratch.subnet.id]
+      subnet-ids = [module.aws-network-existing.subnet-by-name["subnet_4"].id]
+      private-ips = [var.aws-proxy-private-ip]
       num = 1
     },
-    "worker" = {
-      base-image = var.ubuntu-ami
-      num = 2
-    }
   }
   install-qemu-agent = false
 }
@@ -62,6 +101,7 @@ module "aws-network-existing" {
   source                      = "./modules/aws-network-existing"
   default-vpc-name            = var.aws-existing-vpc-name
   default-security-group-name = var.aws-existing-sg-name
+  existing-subnet-names       = var.aws-existing-subnet-names
 }
 
 ################################################################################
@@ -76,16 +116,41 @@ resource "aws_key_pair" "key" {
   }
 }
 
+resource "aws_ebs_volume" "zfs" {
+  # TODO REM look at types.
+  availability_zone = local.nfs-subnets[0].availability_zone
+  size = local.nodes-config["nfs"].disk-size
+  encrypted = true
+  count = local.nodes-config["nfs"].num-disks
+  tags = {
+    Name = "zfs-disk-${count.index}"
+  }
+}
+
+resource "aws_volume_attachment" "mount-nfs-volume" {
+  device_name = "/dev/sd${element(var.aws-zfs-drive-letters, count.index)}"
+  instance_id = module.nodes["nfs"].nodes[0].id
+  count = local.nodes-config["nfs"].num-disks
+  volume_id = element(aws_ebs_volume.zfs, count.index).id
+}
+
+output "zfs-drive-letters" {
+  value = aws_volume_attachment.mount-nfs-volume.*.device_name
+}
+
 module "nodes" {
   for_each           = local.nodes-config
   source             = "./modules/aws-nodes"
+  ec2-instance-type  = each.value.aws-ec2-type
   ami                = each.value.base-image
-  ec2-instance-type  = var.aws-ec2-instance-type
-  subnet-id          = module.aws-network-existing.k8s-subnets-ids[0]
-  security-group-ids = [module.aws-network-existing.default-sg.id]
+  subnet-ids         = each.value.subnet-ids
+  private-ips        = try(each.value.private-ips, [])
+  security-group-ids = [local.aws-security-group-id]
   user-datas         = lookup(module.cloud-init-config, each.key, null).user-datas
   num-nodes          = each.value.num
   name-prefix        = "${var.vm-name-prefix}-${each.key}"
+  # TODO add a input for the key so that it will show up as the key in the aws
+  # console.
 }
 
 ################################################################################
@@ -101,7 +166,7 @@ module "nodes" {
 # provider "libvirt" {
 #   uri = var.libvirt-connection-url
 # }
-# 
+#
 # module "nodes" {
 #   for_each               = local.nodes-config
 #   source                 = "./modules/libvirt-nodes"
@@ -118,7 +183,7 @@ module "nodes" {
 #   libvirt-connection-url = var.libvirt-connection-url
 #   user-datas             = lookup(module.cloud-init-config, each.key, null).user-datas
 # }
-# 
+#
 # resource "libvirt_pool" "images" {
 #   name = var.disk-image-pool-name
 #   type = "dir"

modules/aws-network-existing/main.tf

@@ -22,6 +22,14 @@ data "aws_subnet" "subnets" {
   id = each.key
 }
 
+data "aws_subnet" "subnet-by-name" {
+  for_each = toset(var.existing-subnet-names)
+  filter {
+    name   = "tag:Name"
+    values = [each.key]
+  }
+}
+
 data "aws_security_group" "default" {
   name = var.default-security-group-name
 }

modules/aws-network-existing/outputs.tf

@@ -11,11 +11,16 @@ output "subnets" {
   value = data.aws_subnet.subnets
 }
 
-output "k8s-subnets-ids" {
-  description = "An array of subnets to be used for k8s VMs. These subnets were chosen by selecting a single subnet from each availability_zone."
+output "one-subnet-per-az" {
+  description = "An array of subnets that selects 1 subnet per az."
   value = [for k,v in local.az-to-subnets : v[0]]
 }
 
+output "subnet-by-name" {
+  description = "A map of subnet name to subnet resource."
+  value = data.aws_subnet.subnet-by-name
+}
+
 output "az-to-subnets" {
   description = "A map of availability zone to array of subnets that are in thet availability zone."
   value = local.az-to-subnets

modules/aws-network-existing/variables.tf

@@ -5,3 +5,9 @@ variable "default-security-group-name" {
 variable "default-vpc-name" {
   description = "The name of the existing default VPC. This module will query AWS for a VPC with this name,"
 }
+
+variable "existing-subnet-names" {
+  description = "A list of subnet names that already exist in default-vpc-name"
+  default     = []
+  type        = list(string)
+}

modules/aws-nodes/main.tf

@@ -3,7 +3,10 @@ resource "aws_instance" "nodes" {
   instance_type               = var.ec2-instance-type
   # TODO Make this a variable.
   associate_public_ip_address = true
-  subnet_id                   = var.subnet-id
+  subnet_id = element(var.subnet-ids, count.index % length(var.subnet-ids))
+  # Set a private IP if var.private-ips is not empty, otherwise let AWS assign
+  # the IP.
+  private_ip = length(var.private-ips) == 0 ? null : element(var.private-ips, count.index)
   vpc_security_group_ids = var.security-group-ids
   user_data = element(var.user-datas.*.rendered, count.index)
   count                       = var.num-nodes

modules/aws-nodes/outputs.tf

@@ -9,3 +9,7 @@ output "private_ips" {
 output "names" {
   value = aws_instance.nodes.*.tags.Name
 }
+
+output "nodes" {
+  value = aws_instance.nodes
+}

modules/aws-nodes/variables.tf

@@ -21,13 +21,19 @@ variable "num-nodes" {
   type = number
 }
 
+variable "private-ips" {
+  default = []
+  description = "A list of private IP addresses to use for the nodes. If the list is empty then each node will get an IP assigned from AWS."
+  type = list(string)
+}
+
 variable "user-datas" {
   description = "A list of cloud-init configs that get applied to their corresponding node."
 }
 
-variable "subnet-id" {
-  description = "The ID of the subnet that all the nodes will be added to."
-  type = string
+variable "subnet-ids" {
+  description = "An array of subnet ids. These subnets will be round robined as the subnet to use for each node."
+  type = list(string)
 }
 
 variable "security-group-ids" {

providers.tf

@@ -1,20 +1,21 @@
 
 provider "aws" {
-  region = "us-gov-west-1"
+  region = var.aws-region
+  # profile = <aws cli profile>
 }
 
 
 terraform {
   required_version = ">= 1.0.8"
 
-    backend "s3" {
-      
-      bucket            = "mss-terraform-state"
-      key               = "global/s3/terraform.tfstate"
-      region            = "us-gov-west-1"
-      
-      dynamodb_table    = "mss-terraform-state-lock"
-      encrypt           = true
+  backend "s3" {
+    
+    bucket            = "mss-terraform-state"
+    key               = "global/s3/terraform.tfstate"
+    region            = "us-gov-west-1"
+    
+    dynamodb_table    = "mss-terraform-state-lock"
+    encrypt           = true
 
   }
   required_providers {
@@ -73,4 +74,4 @@ resource "aws_dynamodb_table" "terraform_locks" {
 output "s3_bucket_arn" {
   value       = aws_s3_bucket.terraform_state.arn
   description = "The ARN of the S3 bucket"
-}
+}

variables.tf

@@ -4,21 +4,33 @@ variable "admin-ips" {
   type = list(string)
 }
 
-variable "aws-ec2-instance-type" {
-  default = "t2.micro"
-  description = "The AWS instance type to use for all nodes."
+variable "aws-zfs-drive-letters" {
+  default     = ["f", "g", "h", "i", "j"]
+  description = "These are the drove letters to use when attaching EBS volumes, ie. /dev/sdf or /dev/sdg"
+  type        = list(string)
 }
 
 variable "aws-existing-sg-name" {
-  default = "change-me-if-using-aws-network-existing"
+  default     = "change-me-if-using-aws-network-existing"
   description = "The name of the existing security group when using aws-network-existing."
 }
 
 variable "aws-existing-vpc-name" {
-  default = "change-me-if-using-aws-network-existing"
+  default     = "change-me-if-using-aws-network-existing"
   description = "The name of the existing VPC when using aws-network-existing."
 }
 
+variable "aws-existing-subnet-names" {
+  description = "A list of subnet names that already exist in aws-existing-vpc-name"
+  default     = []
+  type        = list(string)
+}
+
+variable "aws-proxy-private-ip" {
+  description = "The private IP to request for the proxy instance."
+  type        = string
+}
+
 variable "aws-region" {
   default = "us-east-1"
   description = "The AWS region to use."
@@ -101,7 +113,7 @@ variable "vm-name-prefix" {
 
 ################################################################################
 # AWS AMI vars
-# These variables are really mor like constants. Using variables improves
+# These variables are really more like constants. Using variables improves
 # readability. The defaults are manually updated. Use the aws-amis module to get
 # the latest for each distro.
 ################################################################################
@@ -150,9 +162,35 @@ variable "rhel8-ami" {
   description = "The AMI to use for RHEL 8."
 }
 
+################################################################################
+# AWS EC2 types.
+# These variables are really more like constants. Using variables improves
+# readability.
+################################################################################
+
+variable "t2-micro-1gib-1vcpu" {
+  description = "t2.micro EC2 instance with 1 GiB mem and 1 vCPU."
+  default = "t2.micro"
+}
+
+variable "t2-medium-4gib-2vcpu" {
+  description = "t2.medium EC2 instance with 4 GiB mem and 2 vCPUs."
+  default = "t2.medium"
+}
+
+variable "t2-large-8gib-2vcpu" {
+  description = "t2.large EC2 instance with 8 GiB mem and 2 vCPUs."
+  default = "t2.large"
+}
+
+variable "t2-xlarge-16gib-4vcpu" {
+  description = "t2.xlarge EC2 instance with 16 GiB mem and 4 vCPUs."
+  default = "t2.xlarge"
+}
+
 ################################################################################
 # Libvirt Images
-# These variables are really mor like constants. Using variables improves
+# These variables are really more like constants. Using variables improves
 # readability. The defaults are manually updated.
 ################################################################################