diff --git a/cloud_init.cfg b/cloud_init.cfg index 1023821..4b4f11c 100644 --- a/cloud_init.cfg +++ b/cloud_init.cfg @@ -24,9 +24,17 @@ chpasswd: expire: false hostname: ${hostname} +fqdn: ${hostname} -# Use this when it's determined that we need a bigger disk image. -# This must be used in conjuction with 'size' in 'libvirt_volume' -# growpart: -# mode: auto -# devices: ['/'] +packages: + # TODO These are only necessary for libvirt. + - qemu-guest-agent +runcmd: + # TODO At some point revisit this, this was added because it seemed like + # apparmor was causing dhclient to not get an IP address for ubuntu. This + # should be double checked. + - echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient + # TODO These are only necessary for libvirt. + - systemctl enable qemu-guest-agent + - systemctl start qemu-guest-agent + - systemctl status qemu-guest-agent diff --git a/example.tfvars b/example.tfvars index 10c8f4b..2131e2e 100644 --- a/example.tfvars +++ b/example.tfvars @@ -10,6 +10,15 @@ libvirt-connection-url = "qemu+ssh://@/system" node-memory = 2048 node-vcpus = 2 +## libvirt disk size +# 1 GiB = 1073741824 +# 4 GiB +# libvirt-node-disk-size = "${4 * 1073741824}" +# 8 GiB +# libvirt-node-disk-size = "${8 * 1073741824}" +# 12 GiB +# libvirt-node-disk-size = "${12 * 1073741824}" + ################################################################################ # AWS EC2 instance types ################################################################################ @@ -51,8 +60,13 @@ base-image = "ami-0dd0ccab7e2801812" ################################################################################ # base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" -# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52 -# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2" +# From https://cloud.centos.org/centos/7/images/ from 2020-12-06 +# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + +## Arch +# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output +# on 2021-11-28 +# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box" ## Arch # From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output diff --git a/get-libvirt-bridge-ips.sh b/get-libvirt-bridge-ips.sh new file mode 100755 index 0000000..ef82c8f --- /dev/null +++ b/get-libvirt-bridge-ips.sh @@ -0,0 +1,81 @@ +#!/bin/bash + +# This script will grab the IPs for libvirt VMs. This script is only needed when +# using a bridge as the network for the VMs. This should only be needed while +# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is +# unresolved. + +# These are the network interfaces that this script will attempt to get the IP +# address for. +# Ubuntu 20.04 ens3 +# Centos 7 & 8 eth0 +NET_INTERFACES="eth0 ens3" + +LIBVIRT_CONNECTION_URL="libvirt-connection-url" +VM_NAME_PREFIX="vm-name-prefix" + +INV_GROUPS="$( \ +cat terraform.tfstate | \ + jq '.resources[] | select(.type=="libvirt_domain") | .module' | \ + sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )" + +# Grab the connection URL and the vm name prefix. We do this by greping all +# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the +# last grep result, this way we make sure any value in terraform.tfvars will +# take priority. +CONN_URLS="$( \ + find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \ + grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)" + +CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +NAME_PREFIXES="$( \ + find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \ + grep "$VM_NAME_PREFIX" terraform.tfvars)" + +NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')" + +# These can be used for debugging. +# echo "Using connection URL: $CONN_URL" +# echo "Using prefix: $NAME_PREFIX" + +# Get the names of our VMs from libvirt. +VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')" + +# Convert the lines of VM names to an array. +OLD_IFS=$IFS +IFS=$'\n' +VMS=($VMS) +IFS=$OLD_IFS + +# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the +# result to VM_IP_PAIRS as :. +VM_IP_PAIRS="" +for VM in "${VMS[@]}"; do + for INTERFACE in $NET_INTERFACES; do + IP="$( \ + virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \ + jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \ + sed 's/"//g')" + # Add the VM:IP pair if IP is not empty. + if [ ! -z "$IP" ]; then + VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP" + fi + done +done + +# Write inventory +cat /dev/null > inventory +for GROUP in $INV_GROUPS; do + echo "[$GROUP]" >> inventory + echo "$VM_IP_PAIRS" | \ + grep $GROUP | \ + sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory +done + +# Print vars +echo "$VM_IP_PAIRS" | \ + sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \ + sed s/$NAME_PREFIX-//g | \ + sed 's/-/_/g' | \ + awk '{print toupper($1)}' diff --git a/main.tf b/main.tf index caa213c..10cb32b 100644 --- a/main.tf +++ b/main.tf @@ -106,7 +106,9 @@ module "nodes" { # num-nodes = each.value.num # node-memory = var.node-memory # node-vcpus = var.node-vcpus +# node-disk-size = var.libvirt-node-disk-size # base-image = each.value.base-image +# network-name = var.libvirt-network-name # root-admin-passwd = var.root-admin-passwd # root-admin-pub-key = var.root-admin-pub-key # libvirt-connection-url = var.libvirt-connection-url diff --git a/modules/libvirt-nodes/main.tf b/modules/libvirt-nodes/main.tf index cbd0dac..d258c46 100644 --- a/modules/libvirt-nodes/main.tf +++ b/modules/libvirt-nodes/main.tf @@ -9,13 +9,20 @@ terraform { } resource "libvirt_volume" "node-images" { - name = "${var.name-prefix}-${count.index}" + name = "${var.name-prefix}-base" pool = var.pool-name source = var.base-image - count = var.num-nodes format = "qcow2" } +resource "libvirt_volume" "node-images-resized" { + name = "${var.name-prefix}-${count.index}-resized" + pool = var.pool-name + base_volume_id = libvirt_volume.node-images.id + count = var.num-nodes + size = var.node-disk-size +} + data "template_file" "network-config" { template = file("${path.module}/network_config.cfg") } @@ -37,9 +44,9 @@ resource "libvirt_domain" "nodes" { cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index) network_interface { - network_name = "default" + network_name = var.network-name hostname = "${var.name-prefix}-${count.index}" - wait_for_lease = true + # wait_for_lease = true } # IMPORTANT: this is a known bug on cloud images, since they expect a console @@ -58,7 +65,7 @@ resource "libvirt_domain" "nodes" { } disk { - volume_id = element(libvirt_volume.node-images.*.id, count.index) + volume_id = element(libvirt_volume.node-images-resized.*.id, count.index) } graphics { diff --git a/modules/libvirt-nodes/outpus.tf b/modules/libvirt-nodes/outpus.tf index 4d399a6..62af47b 100644 --- a/modules/libvirt-nodes/outpus.tf +++ b/modules/libvirt-nodes/outpus.tf @@ -1,4 +1,6 @@ -output "ips" { - value = libvirt_domain.nodes.*.network_interface.0.addresses.0 -} +# This only works on the default network. They will not work using the bridged +# network. +# output "ips" { +# value = libvirt_domain.nodes.*.network_interface.0.addresses.0 +# } diff --git a/modules/libvirt-nodes/variables.tf b/modules/libvirt-nodes/variables.tf index 4253b62..101321d 100644 --- a/modules/libvirt-nodes/variables.tf +++ b/modules/libvirt-nodes/variables.tf @@ -12,6 +12,16 @@ variable "name-prefix" { description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"." } +variable "network-name" { + default = "default" + description = "The name of a pre-existing virtual-network." +} + +variable "node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." diff --git a/variables.tf b/variables.tf index 3719268..cdba553 100644 --- a/variables.tf +++ b/variables.tf @@ -31,6 +31,16 @@ variable "libvirt-connection-url" { description = "The libvirt connection URI, ie. qemu+ssh://@/system" } +variable "libvirt-network-name" { + default = "default" + description = "The name of a pre-existing libvirt virtual-network." +} + +variable "libvirt-node-disk-size" { + default = 4294967296 + description = "The size of the disk to be used for libvirt nodes. (in bytes)" +} + variable "node-memory" { default = "2048" description = "The amount of memory to be used for all the nodes." @@ -107,3 +117,25 @@ variable "rhel8-ami" { default = "ami-0d871ca8a77af2948" description = "The AMI to use for RHEL 8." } + +################################################################################ +# Libvirt Images +# These variables are really mor like constants. Using variables improves +# readability. The defaults are manually updated. +################################################################################ + +variable "ubuntu-img" { + default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img" + description = "The libvirt image tp use for Ubuntu." +} + +variable "centos7-img" { + # Latest as of 2021-12-06. + default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2" + description = "The libvirt image tp use for CentOS 7." +} + +variable "centos8-img" { + default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2" + description = "The libvirt image tp use for CentOS 8." +}