From 3bb1cb4a951d2b37f7e538238aa495991e01ae89 Mon Sep 17 00:00:00 2001 From: Curtis Wilson Date: Fri, 10 Dec 2021 11:00:45 -0500 Subject: [PATCH] Added support for bridge network interfaces for libvirt. --- k8s-nodes/cloud_init.cfg | 20 ++- k8s-nodes/example.tfvars | 23 ++- k8s-nodes/get-libvirt-bridge-ips.sh | 81 ++++++++++ k8s-nodes/get-vm-ips.sh | 138 +++++++++++------- k8s-nodes/main.tf | 14 +- k8s-nodes/modules/aws-network/main.tf | 2 +- k8s-nodes/modules/aws-nodes/outputs.tf | 4 + k8s-nodes/modules/cloud-init-config/main.tf | 7 +- .../modules/cloud-init-config/variables.tf | 6 + k8s-nodes/modules/libvirt-nodes/main.tf | 17 ++- k8s-nodes/modules/libvirt-nodes/outpus.tf | 8 +- k8s-nodes/modules/libvirt-nodes/variables.tf | 10 ++ k8s-nodes/variables.tf | 32 ++++ 13 files changed, 286 insertions(+), 76 deletions(-) create mode 100755 k8s-nodes/get-libvirt-bridge-ips.sh diff --git a/k8s-nodes/cloud_init.cfg b/k8s-nodes/cloud_init.cfg index 1023821..5266096 100644 --- a/k8s-nodes/cloud_init.cfg +++ b/k8s-nodes/cloud_init.cfg @@ -24,9 +24,19 @@ chpasswd: expire: false hostname: ${hostname} +fqdn: ${hostname} -# Use this when it's determined that we need a bigger disk image. -# This must be used in conjuction with 'size' in 'libvirt_volume' -# growpart: -# mode: auto -# devices: ['/'] +%{ if install-qemu-agent } +packages: + # This are only necessary for libvirt. + - qemu-guest-agent +runcmd: + # TODO At some point revisit this, this was added because it seemed like + # apparmor was causing dhclient to not get an IP address for ubuntu. This + # should be double checked. + - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient + # These are only necessary for libvirt. + - systemctl enable qemu-guest-agent + - systemctl start qemu-guest-agent + - systemctl status qemu-guest-agent +%{ endif } diff --git a/k8s-nodes/example.tfvars b/k8s-nodes/example.tfvars index 9d7f2b9..2131e2e 100644 --- a/k8s-nodes/example.tfvars +++ b/k8s-nodes/example.tfvars @@ -10,6 +10,15 @@ libvirt-connection-url = "qemu+ssh://@/system" node-memory = 2048 node-vcpus = 2 +## libvirt disk size +# 1 GiB = 1073741824 +# 4 GiB +# libvirt-node-disk-size = "${4 * 1073741824}" +# 8 GiB +# libvirt-node-disk-size = "${8 * 1073741824}" +# 12 GiB +# libvirt-node-disk-size = "${12 * 1073741824}" + ################################################################################ # AWS EC2 instance types ################################################################################ @@ -51,8 +60,18 @@ base-image = "ami-0dd0ccab7e2801812" ################################################################################ # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" -# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 -# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" +# From https://cloud.centos.org/centos/7/images/ from 2020-12-06 +# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + +## Arch +# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output +# on 2021-11-28 +# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" + +## Arch +# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output +# on 2021-11-28 +# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" ################################################################################ # Keys/Passwords diff --git a/k8s-nodes/get-libvirt-bridge-ips.sh b/k8s-nodes/get-libvirt-bridge-ips.sh new file mode 100755 index 0000000..ef82c8f --- /dev/null +++ b/k8s-nodes/get-libvirt-bridge-ips.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# This script will grab the IPs for libvirt VMs. This script is only needed when +# using a bridge as the network for the VMs. This should only be needed while +# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is +# unresolved. + +# These are the network interfaces that this script will attempt to get the IP +# address for. +# Ubuntu 20.04 ens3 +# Centos 7 & 8 eth0 +NET_INTERFACES="eth0 ens3" + +LIBVIRT_CONNECTION_URL="libvirt-connection-url" +VM_NAME_PREFIX="vm-name-prefix" + +INV_GROUPS="$( \ +cat terraform.tfstate | \ + jq '.resources[] | select(.type=="libvirt_domain") | .module' | \ + sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )" + +# Grab the connection URL and the vm name prefix. We do this by greping all +# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the +# last grep result, this way we make sure any value in terraform.tfvars will +# take priority. +CONN_URLS="$( \ + find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \ + grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)" + +CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +NAME_PREFIXES="$( \ + find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \ + grep "$VM_NAME_PREFIX" terraform.tfvars)" + +NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +# These can be used for debugging. +# echo "Using connection URL: $CONN_URL" +# echo "Using prefix: $NAME_PREFIX" + +# Get the names of our VMs from libvirt. +VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')" + +# Convert the lines of VM names to an array. +OLD_IFS=$IFS +IFS=$'\n' +VMS=($VMS) +IFS=$OLD_IFS + +# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the +# result to VM_IP_PAIRS as :. +VM_IP_PAIRS="" +for VM in "${VMS[@]}"; do + for INTERFACE in $NET_INTERFACES; do + IP="$( \ + virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \ + jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \ + sed 's/"//g')" + # Add the VM:IP pair if IP is not empty. + if [ ! -z "$IP" ]; then + VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP" + fi + done +done + +# Write inventory +cat /dev/null > inventory +for GROUP in $INV_GROUPS; do + echo "[$GROUP]" >> inventory + echo "$VM_IP_PAIRS" | \ + grep $GROUP | \ + sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory +done + +# Print vars +echo "$VM_IP_PAIRS" | \ + sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \ + sed s/$NAME_PREFIX-//g | \ + sed 's/-/_/g' | \ + awk '{print toupper($1)}' diff --git a/k8s-nodes/get-vm-ips.sh b/k8s-nodes/get-vm-ips.sh index 230b5cf..b8ab1df 100755 --- a/k8s-nodes/get-vm-ips.sh +++ b/k8s-nodes/get-vm-ips.sh @@ -1,60 +1,92 @@ -#!/bin/sh +#!/bin/bash -# This script will create environment variables for all of the output IPs. It -# will also create a `ANSIBLE_INV` variable that will be a comma separated -# string of all the IPs. A anisble inventory file called "inventory is created -# as well. +# This script will create environment variables for all of the output IPs. An +# anisble inventory file is created as well. # # Use eval $(./get-vm-ips.sh) to set env vars for ips. terraform refresh > /dev/null -# All terraform outputs in json format. -OUTPUTS_JSON="$( - terraform show -json | \ - jq '.values.outputs' | \ - sed 's/-/_/g')" -# Just the IP address outputs in json format. Also all '-' characters are -# replaced by '_' becuase '-' causes jq some problems. -IPS_JSON="$( - echo $OUTPUTS_JSON | \ - jq 'to_entries | .[] | select(.key | contains("ips"))')" -# An array of all node "types" -NODE_TYPE_ARRAY="$( - echo $IPS_JSON | \ - jq '.value.value | to_entries | .[] | .key' | \ - sed 's/"//g' | \ - sed -z 's/\n/ /g;s/ $/\n/g')" - -# Loop over all the node types and create an export line for each IP. -VM_IP_EXPORTS="$( - for TYPE in $NODE_TYPE_ARRAY; do - - # Convert type, converts "master-ips" to "MASTER" - TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)" - echo "$IPS_JSON" | \ - jq '.value.value.'"$TYPE"'[]' | \ - # Add line numbers starting with 0. - nl -v 0 | \ - # Print an export string with a type placeholder "__TYPE__". - awk '{print "export __TYPE___" $1 "=" $2}' | \ - sed s/__TYPE__/$TYPE_UPPER/g - done)" - -ANSIBLE_INV="$( - echo "$VM_IP_EXPORTS" | \ +# The file to write the inventory to. This file will be completely overridden. +INVENTORY_FILE="inventory" + +# Grab the the vm name prefix. We do this by greping all *.tfvars files making +# sure to cat terraform.tfvars last. Then we just grab the last grep result, +# this way we make sure any value in terraform.tfvars will take priority. +VM_NAME_PREFIX_VAR="vm-name-prefix" +VM_NAME_PREFIXES="$( \ + find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \ + grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)" +VM_NAME_PREFIX="$( + echo "$VM_NAME_PREFIXES" | \ + tail -n 1 | \ + sed 's/^.*=\s*"\(.*\)"/\1/g')" + +# This command stores the output data in the format below. +# [ +# { +# "group": "master", +# "vms": [ +# { +# "hostname": "ansible-test-master-0", +# "ip": "52.14.114.48" +# } +# ] +# }, +# { +# "group": "worker", +# "vms": [ +# { +# "hostname": "ansible-test-worker-0", +# "ip": "3.145.121.159" +# }, +# { +# "hostname": "ansible-test-worker-1", +# "ip": "18.217.112.176" +# } +# ] +# } +# ] +DATA="$(terraform show -json | \ + jq '.values.outputs.groups_hostnames_ips.value | to_entries | + map({group: .key, vms:.value | to_entries | + map({hostname:.key,ip:.value})})')" + +# Pull out the groups from $DATA. The format is a single string with the groups +# separated by spaces, ie. "group1 group2 group3". +ANS_GROUPS="$( + echo $DATA | \ + jq '.[] | .group' | \ sed 's/"//g' | \ - sed 's/^.*=//g' | \ - sed -z 's/\n/,/g;s/,$/\n/g')" - -# Create an inventory file for ansible. -echo "[k8s_nodes]" > inventory -echo $VM_IP_EXPORTS | \ - sed 's/"//g' | \ - sed 's/export //g' | \ - sed 's/ /\n/g' | \ - sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \ - >> inventory - -echo $VM_IP_EXPORTS | sed 's/" /"\n/g' -echo export ANSIBLE_INV=$ANSIBLE_INV + tr '\n' ' ' + )" + +# Clear the inventory file. +cat /dev/null > $INVENTORY_FILE + +# For each group, write the VM info to $INVENTORY_FILE and also print a variable +# expression to stdout. +for GROUP in $ANS_GROUPS; do + + # Write the inventory file to $INVENTORY_FILE. + echo "[$GROUP]" >> $INVENTORY_FILE + echo $DATA | \ + jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | + "\(.hostname) ansible_host=\(.ip)"' | \ + sed 's/"//g' \ + >> $INVENTORY_FILE + + # For this group, collect expressions into VARS. The format is: + # HOSTNAME1=0.0.0.0 + # HOSTNAME2=0.0.0.0 + VARS="$( + echo $DATA | \ + jq '.[] | select(.group=="'"$GROUP"'") | .vms[] | + "\(.hostname)=\(.ip)"' | \ + sed 's/"//g' | \ + sed "s/$VM_NAME_PREFIX-//g" | \ + sed 's/-/_/g' + )" + # Print the contents of $VARS converted to uppercase. + echo "${VARS^^}" +done diff --git a/k8s-nodes/main.tf b/k8s-nodes/main.tf index 81e0da1..64849b6 100644 --- a/k8s-nodes/main.tf +++ b/k8s-nodes/main.tf @@ -22,14 +22,15 @@ terraform { locals { nodes-config = { "master" = { - base-image = var.amzn2-ami + base-image = var.centos8-ami num = 1 }, "worker" = { - base-image = var.amzn2-ami + base-image = var.centos8-ami num = 2 } } + install-qemu-agent = false } ################################################################################ @@ -44,6 +45,7 @@ module "cloud-init-config" { num = each.value.num root-admin-passwd = var.root-admin-passwd root-admin-pub-key = var.root-admin-pub-key + install-qemu-agent = local.install-qemu-agent } ################################################################################ @@ -117,7 +119,9 @@ module "nodes" { # num-nodes = each.value.num # node-memory = var.node-memory # node-vcpus = var.node-vcpus +# node-disk-size = var.libvirt-node-disk-size # base-image = each.value.base-image +# network-name = var.libvirt-network-name # root-admin-passwd = var.root-admin-passwd # root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url @@ -134,6 +138,8 @@ module "nodes" { # end libvirt ################################################################################ -output "ips" { - value = { for type, node in module.nodes : type => node.ips } +# This will outpus a map of group => [{hostname, ip}]. +# TODO A 'names' output needs to be added to libvirt. +output "groups_hostnames_ips" { + value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) } } diff --git a/k8s-nodes/modules/aws-network/main.tf b/k8s-nodes/modules/aws-network/main.tf index be001ac..6056fc0 100644 --- a/k8s-nodes/modules/aws-network/main.tf +++ b/k8s-nodes/modules/aws-network/main.tf @@ -33,7 +33,7 @@ resource "aws_default_security_group" "sg" { } tags = { - Name = "${var.name-prefix}-ssh-from-admins--sg" + Name = "${var.name-prefix}-ssh-from-admins-sg" } } diff --git a/k8s-nodes/modules/aws-nodes/outputs.tf b/k8s-nodes/modules/aws-nodes/outputs.tf index d6faaf0..0b4fe7c 100644 --- a/k8s-nodes/modules/aws-nodes/outputs.tf +++ b/k8s-nodes/modules/aws-nodes/outputs.tf @@ -1,3 +1,7 @@ output "ips" { value = aws_instance.nodes.*.public_ip } + +output "names" { + value = aws_instance.nodes.*.tags.Name +} diff --git a/k8s-nodes/modules/cloud-init-config/main.tf b/k8s-nodes/modules/cloud-init-config/main.tf index e1c75c4..6b5beed 100644 --- a/k8s-nodes/modules/cloud-init-config/main.tf +++ b/k8s-nodes/modules/cloud-init-config/main.tf @@ -1,9 +1,10 @@ data "template_file" "user-datas" { template = file("${var.cloud-init-template}") vars = { - admin-passwd = "${var.root-admin-passwd}" - admin-pub-key = "${var.root-admin-pub-key}" - hostname = "${var.hostname-prefix}-${count.index}" + admin-passwd = "${var.root-admin-passwd}" + admin-pub-key = "${var.root-admin-pub-key}" + hostname = "${var.hostname-prefix}-${count.index}" + install-qemu-agent = var.install-qemu-agent } count = var.num } diff --git a/k8s-nodes/modules/cloud-init-config/variables.tf b/k8s-nodes/modules/cloud-init-config/variables.tf index fc4f437..a1b0f2d 100644 --- a/k8s-nodes/modules/cloud-init-config/variables.tf +++ b/k8s-nodes/modules/cloud-init-config/variables.tf @@ -8,6 +8,12 @@ variable "hostname-prefix" { description = "This prefix wil be applied as a prefix for the hostnames." } +variable "install-qemu-agent" { + default = false + description = "This flag determines whether or not qemu-agent is installed." + type = bool +} + variable "num" { description = "The number of user-datas to create with these parameters." } diff --git a/k8s-nodes/modules/libvirt-nodes/main.tf b/k8s-nodes/modules/libvirt-nodes/main.tf index cbd0dac..d258c46 100644 --- a/k8s-nodes/modules/libvirt-nodes/main.tf +++ b/k8s-nodes/modules/libvirt-nodes/main.tf @@ -9,13 +9,20 @@ terraform { } resource "libvirt_volume" "node-images" { - name = "${var.name-prefix}-${count.index}" + name = "${var.name-prefix}-base" pool = var.pool-name source = var.base-image - count = var.num-nodes format = "qcow2" } +resource "libvirt_volume" "node-images-resized" { + name = "${var.name-prefix}-${count.index}-resized" + pool = var.pool-name + base_volume_id = libvirt_volume.node-images.id + count = var.num-nodes + size = var.node-disk-size +} + data "template_file" "network-config" { template = file("${path.module}/network_config.cfg") } @@ -37,9 +44,9 @@ resource "libvirt_domain" "nodes" { cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) network_interface { - network_name = "default" + network_name = var.network-name hostname = "${var.name-prefix}-${count.index}" - wait_for_lease = true + # wait_for_lease = true } # IMPORTANT: this is a known bug on cloud images, since they expect a console @@ -58,7 +65,7 @@ resource "libvirt_domain" "nodes" { } disk { - volume_id = element(libvirt_volume.node-images.*.id, count.index) + volume_id = element(libvirt_volume.node-images-resized.*.id, count.index) } graphics { diff --git a/k8s-nodes/modules/libvirt-nodes/outpus.tf b/k8s-nodes/modules/libvirt-nodes/outpus.tf index 4d399a6..62af47b 100644 --- a/k8s-nodes/modules/libvirt-nodes/outpus.tf +++ b/k8s-nodes/modules/libvirt-nodes/outpus.tf @@ -1,4 +1,6 @@ -output "ips" { - value = libvirt_domain.nodes.*.network_interface.0.addresses.0 -} +# This only works on the default network. They will not work using the bridged +# network. +# output "ips" { +# value = libvirt_domain.nodes.*.network_interface.0.addresses.0 +# } diff --git a/k8s-nodes/modules/libvirt-nodes/variables.tf b/k8s-nodes/modules/libvirt-nodes/variables.tf index 4253b62..101321d 100644 --- a/k8s-nodes/modules/libvirt-nodes/variables.tf +++ b/k8s-nodes/modules/libvirt-nodes/variables.tf @@ -12,6 +12,16 @@ variable "name-prefix" { description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"." } +variable "network-name" { + default = "default" + description = "The name of a pre-existing virtual-network." +} + +variable "node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." diff --git a/k8s-nodes/variables.tf b/k8s-nodes/variables.tf index 3719268..cdba553 100644 --- a/k8s-nodes/variables.tf +++ b/k8s-nodes/variables.tf @@ -31,6 +31,16 @@ variable "libvirt-connection-url" { description = "The libvirt connection URI, ie. qemu+ssh://@/system" } +variable "libvirt-network-name" { + default = "default" + description = "The name of a pre-existing libvirt virtual-network." +} + +variable "libvirt-node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." @@ -107,3 +117,25 @@ variable "rhel8-ami" { default = "ami-0d871ca8a77af2948" description = "The AMI to use for RHEL 8." } + +################################################################################ +# Libvirt Images +# These variables are really mor like constants. Using variables improves +# readability. The defaults are manually updated. +################################################################################ + +variable "ubuntu-img" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" + description = "The libvirt image tp use for Ubuntu." +} + +variable "centos7-img" { + # Latest as of 2021-12-06. + default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + description = "The libvirt image tp use for CentOS 7." +} + +variable "centos8-img" { + default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2" + description = "The libvirt image tp use for CentOS 8." +}