libvirt bridged network interface

- Added support for libvirt VMs that use a bridged network interface.

cloud_init.cfg

@@ -24,9 +24,17 @@ chpasswd:
   expire: false
 
 hostname: ${hostname}
+fqdn: ${hostname}
 
-# Use this when it's determined that we need a bigger disk image.
+packages:
-# This must be used in conjuction with 'size' in 'libvirt_volume'
+  # TODO These are only necessary for libvirt.
-# growpart:
+  - qemu-guest-agent
-#   mode: auto
+runcmd:
-#   devices: ['/']
+  # TODO At some point revisit this, this was added because it seemed like
+  # apparmor was causing dhclient to not get an IP address for ubuntu. This
+  # should be double checked.
+  - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient
+  # TODO These are only necessary for libvirt.
+  - systemctl enable qemu-guest-agent
+  - systemctl start qemu-guest-agent
+  - systemctl status qemu-guest-agent

example.tfvars

@@ -10,6 +10,15 @@ libvirt-connection-url = "qemu+ssh://<user>@<host>/system"
 node-memory = 2048
 node-vcpus = 2
 
+## libvirt disk size
+# 1 GiB = 1073741824
+# 4 GiB
+# libvirt-node-disk-size = "${4 * 1073741824}"
+# 8 GiB
+# libvirt-node-disk-size = "${8 * 1073741824}"
+# 12 GiB
+# libvirt-node-disk-size = "${12 * 1073741824}"
+
 ################################################################################
 # AWS EC2 instance types
 ################################################################################
@@ -51,8 +60,13 @@ base-image = "ami-0dd0ccab7e2801812"
 ################################################################################
 
 # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
-# From https://cloud.centos.org/centos/7/images/ from	2020-11-12 06:52
+# From https://cloud.centos.org/centos/7/images/ from	2020-12-06
-# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2"
+# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+
+## Arch
+# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
+# on 2021-11-28
+# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
 
 ################################################################################
 # Keys/Passwords

get-libvirt-bridge-ips.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+
+# This script will grab the IPs for libvirt VMs. This script is only needed when
+# using a bridge as the network for the VMs. This should only be needed while
+# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is
+# unresolved.
+
+# These are the network interfaces that this script will attempt to get the IP
+# address for.
+# Ubuntu 20.04    ens3
+# Centos 7 & 8    eth0
+NET_INTERFACES="eth0 ens3"
+
+LIBVIRT_CONNECTION_URL="libvirt-connection-url"
+VM_NAME_PREFIX="vm-name-prefix"
+
+INV_GROUPS="$( \
+cat terraform.tfstate | \
+    jq '.resources[] | select(.type=="libvirt_domain") | .module' | \
+    sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )"
+
+# Grab the connection URL and the vm name prefix. We do this by greping all
+# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the
+# last grep result, this way we make sure any value in terraform.tfvars will
+# take priority.
+CONN_URLS="$( \
+    find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \
+    grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)"
+
+CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
+
+NAME_PREFIXES="$( \
+    find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \
+    grep "$VM_NAME_PREFIX" terraform.tfvars)"
+
+NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
+
+# These can be used for debugging.
+# echo "Using connection URL: $CONN_URL"
+# echo "Using prefix: $NAME_PREFIX"
+
+# Get the names of our VMs from libvirt.
+VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')"
+
+# Convert the lines of VM names to an array.
+OLD_IFS=$IFS
+IFS=$'\n'
+VMS=($VMS)
+IFS=$OLD_IFS
+
+# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the
+# result to VM_IP_PAIRS as <vm-name>:<ipv4-address>.
+VM_IP_PAIRS=""
+for VM in "${VMS[@]}"; do
+    for INTERFACE in $NET_INTERFACES; do
+        IP="$( \
+            virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \
+            jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \
+            sed 's/"//g')"
+        # Add the VM:IP pair if IP is not empty.
+        if [ ! -z "$IP" ]; then
+            VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP"
+        fi
+    done
+done
+
+# Write inventory
+cat /dev/null > inventory
+for GROUP in $INV_GROUPS; do
+    echo "[$GROUP]" >> inventory
+    echo "$VM_IP_PAIRS" | \
+        grep $GROUP | \
+        sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory
+done
+
+# Print vars
+echo "$VM_IP_PAIRS" | \
+    sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \
+    sed s/$NAME_PREFIX-//g | \
+    sed 's/-/_/g' | \
+    awk '{print toupper($1)}'

main.tf

@@ -126,7 +126,9 @@ module "nodes" {
 #   num-nodes              = each.value.num
 #   node-memory            = var.node-memory
 #   node-vcpus             = var.node-vcpus
+#   node-disk-size         = var.libvirt-node-disk-size
 #   base-image             = each.value.base-image
+#   network-name           = var.libvirt-network-name
 #   root-admin-passwd      = var.root-admin-passwd
 #   root-admin-pub-key     = var.root-admin-pub-key
 #   libvirt-connection-url = var.libvirt-connection-url

modules/libvirt-nodes/main.tf

@@ -9,13 +9,20 @@ terraform {
 }
 
 resource "libvirt_volume" "node-images" {
-  name   = "${var.name-prefix}-${count.index}"
+  name   = "${var.name-prefix}-base"
   pool   = var.pool-name
   source = var.base-image
-  count  = var.num-nodes
   format = "qcow2"
 }
 
+resource "libvirt_volume" "node-images-resized" {
+  name           = "${var.name-prefix}-${count.index}-resized"
+  pool           = var.pool-name
+  base_volume_id = libvirt_volume.node-images.id
+  count          = var.num-nodes
+  size           = var.node-disk-size
+}
+
 data "template_file" "network-config" {
   template = file("${path.module}/network_config.cfg")
 }
@@ -37,9 +44,9 @@ resource "libvirt_domain" "nodes" {
   cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index)
 
   network_interface {
-    network_name   = "default"
+    network_name   = var.network-name
     hostname       = "${var.name-prefix}-${count.index}"
-    wait_for_lease = true
+    # wait_for_lease = true
   }
 
   # IMPORTANT: this is a known bug on cloud images, since they expect a console
@@ -58,7 +65,7 @@ resource "libvirt_domain" "nodes" {
   }
 
   disk {
-    volume_id = element(libvirt_volume.node-images.*.id, count.index)
+    volume_id = element(libvirt_volume.node-images-resized.*.id, count.index)
   }
 
   graphics {

modules/libvirt-nodes/outpus.tf

@@ -1,4 +1,6 @@
 
-output "ips" {
+# This only works on the default network. They will not work using the bridged
-  value = libvirt_domain.nodes.*.network_interface.0.addresses.0
+# network.
-}
+# output "ips" {
+#   value = libvirt_domain.nodes.*.network_interface.0.addresses.0
+# }

modules/libvirt-nodes/variables.tf

@@ -12,6 +12,16 @@ variable "name-prefix" {
   description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"."
 }
 
+variable "network-name" {
+  default = "default"
+  description = "The name of a pre-existing virtual-network."
+}
+
+variable "node-disk-size" {
+  default = 4294967296
+  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
+}
+
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."

variables.tf

@@ -31,6 +31,16 @@ variable "libvirt-connection-url" {
   description = "The libvirt connection URI, ie. qemu+ssh://<user>@<host>/system"
 }
 
+variable "libvirt-network-name" {
+  default = "default"
+  description = "The name of a pre-existing libvirt virtual-network."
+}
+
+variable "libvirt-node-disk-size" {
+  default = 4294967296
+  description = "The size of the disk to be used for libvirt nodes. (in bytes)"
+}
+
 variable "node-memory" {
   default     = "2048"
   description = "The amount of memory to be used for all the nodes."
@@ -107,3 +117,25 @@ variable "rhel8-ami" {
   default = "ami-0d871ca8a77af2948"
   description = "The AMI to use for RHEL 8."
 }
+
+################################################################################
+# Libvirt Images
+# These variables are really mor like constants. Using variables improves
+# readability. The defaults are manually updated.
+################################################################################
+
+variable "ubuntu-img" {
+  default     = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
+  description = "The libvirt image tp use for Ubuntu."
+}
+
+variable "centos7-img" {
+  # Latest as of 2021-12-06.
+  default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
+  description = "The libvirt image tp use for CentOS 7."
+}
+
+variable "centos8-img" {
+  default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2"
+  description = "The libvirt image tp use for CentOS 8."
+}