Added support for bridge network interfaces for libvirt.

master
shnee 4 years ago
parent 10d01ff839
commit 7de2ba1728

@ -24,9 +24,19 @@ chpasswd:
expire: false
hostname: ${hostname}
fqdn: ${hostname}
# Use this when it's determined that we need a bigger disk image.
# This must be used in conjuction with 'size' in 'libvirt_volume'
# growpart:
# mode: auto
# devices: ['/']
%{ if install-qemu-agent }
packages:
# This are only necessary for libvirt.
- qemu-guest-agent
runcmd:
# TODO At some point revisit this, this was added because it seemed like
# apparmor was causing dhclient to not get an IP address for ubuntu. This
# should be double checked.
- echo "/proc/*/task/*/comm wr," | tee -a /etc/apparmor.d/local/sbin.dhclient
# These are only necessary for libvirt.
- systemctl enable qemu-guest-agent
- systemctl start qemu-guest-agent
- systemctl status qemu-guest-agent
%{ endif }

@ -10,6 +10,15 @@ libvirt-connection-url = "qemu+ssh://<user>@<host>/system"
node-memory = 2048
node-vcpus = 2
## libvirt disk size
# 1 GiB = 1073741824
# 4 GiB
# libvirt-node-disk-size = "${4 * 1073741824}"
# 8 GiB
# libvirt-node-disk-size = "${8 * 1073741824}"
# 12 GiB
# libvirt-node-disk-size = "${12 * 1073741824}"
################################################################################
# AWS EC2 instance types
################################################################################
@ -51,8 +60,18 @@ base-image = "ami-0dd0ccab7e2801812"
################################################################################
# base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52
# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2"
# From https://cloud.centos.org/centos/7/images/ from 2020-12-06
# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
## Arch
# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
# on 2021-11-28
# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
## Arch
# From https://gitlab.archlinux.org/archlinux/arch-boxes/-/jobs/40102/artifacts/browse/output
# on 2021-11-28
# base-image = "/media/nas/software/isos/Arch-Linux-x86_64-libvirt-20211128.40102.box"
################################################################################
# Keys/Passwords

@ -0,0 +1,81 @@
#!/bin/bash
# This script will grab the IPs for libvirt VMs. This script is only needed when
# using a bridge as the network for the VMs. This should only be needed while
# https://github.com/dmacvicar/terraform-provider-libvirt/issues/891 is
# unresolved.
# These are the network interfaces that this script will attempt to get the IP
# address for.
# Ubuntu 20.04 ens3
# Centos 7 & 8 eth0
NET_INTERFACES="eth0 ens3"
LIBVIRT_CONNECTION_URL="libvirt-connection-url"
VM_NAME_PREFIX="vm-name-prefix"
INV_GROUPS="$( \
cat terraform.tfstate | \
jq '.resources[] | select(.type=="libvirt_domain") | .module' | \
sed 's/".*\[\\"\(.*\)\\.*$/\1/g' )"
# Grab the connection URL and the vm name prefix. We do this by greping all
# *.tfvars files making sure to cat terraform.tfvars last. Then we just grab the
# last grep result, this way we make sure any value in terraform.tfvars will
# take priority.
CONN_URLS="$( \
find . -name "*.tfvars" -exec grep "$LIBVIRT_CONNECTION_URL" {} \; && \
grep "$LIBVIRT_CONNECTION_URL" terraform.tfvars)"
CONN_URL="$(echo "$CONN_URLS" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
NAME_PREFIXES="$( \
find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX" {} \; && \
grep "$VM_NAME_PREFIX" terraform.tfvars)"
NAME_PREFIX="$(echo "$NAME_PREFIXES" | tail -n 1 | sed 's/^.*=\s*"\(.*\)"/\1/g')"
# These can be used for debugging.
# echo "Using connection URL: $CONN_URL"
# echo "Using prefix: $NAME_PREFIX"
# Get the names of our VMs from libvirt.
VMS="$(virsh -c $CONN_URL list --all | grep $NAME_PREFIX | awk '{print $2}')"
# Convert the lines of VM names to an array.
OLD_IFS=$IFS
IFS=$'\n'
VMS=($VMS)
IFS=$OLD_IFS
# Loop over our VM array and grab the ipv4 IP address from libvirt. Then add the
# result to VM_IP_PAIRS as <vm-name>:<ipv4-address>.
VM_IP_PAIRS=""
for VM in "${VMS[@]}"; do
for INTERFACE in $NET_INTERFACES; do
IP="$( \
virsh -c $CONN_URL qemu-agent-command $VM '{"execute": "guest-network-get-interfaces"}' | \
jq '.return[] | select(.name=="'"$INTERFACE"'") | ."ip-addresses"[] | select(."ip-address-type"=="ipv4") | ."ip-address"' | \
sed 's/"//g')"
# Add the VM:IP pair if IP is not empty.
if [ ! -z "$IP" ]; then
VM_IP_PAIRS="$VM_IP_PAIRS"$'\n'"$VM:$IP"
fi
done
done
# Write inventory
cat /dev/null > inventory
for GROUP in $INV_GROUPS; do
echo "[$GROUP]" >> inventory
echo "$VM_IP_PAIRS" | \
grep $GROUP | \
sed 's/^\(.*\):\(.*\)$/\1 ansible_host=\2/g' >> inventory
done
# Print vars
echo "$VM_IP_PAIRS" | \
sed 's/^\(.*\):\(.*\)$/\1=\2/g' | \
sed s/$NAME_PREFIX-//g | \
sed 's/-/_/g' | \
awk '{print toupper($1)}'

@ -1,60 +1,92 @@
#!/bin/sh
#!/bin/bash
# This script will create environment variables for all of the output IPs. It
# will also create a `ANSIBLE_INV` variable that will be a comma separated
# string of all the IPs. A anisble inventory file called "inventory is created
# as well.
# This script will create environment variables for all of the output IPs. An
# anisble inventory file is created as well.
#
# Use eval $(./get-vm-ips.sh) to set env vars for ips.
terraform refresh > /dev/null
# All terraform outputs in json format.
OUTPUTS_JSON="$(
terraform show -json | \
jq '.values.outputs' | \
sed 's/-/_/g')"
# Just the IP address outputs in json format. Also all '-' characters are
# replaced by '_' becuase '-' causes jq some problems.
IPS_JSON="$(
echo $OUTPUTS_JSON | \
jq 'to_entries | .[] | select(.key | contains("ips"))')"
# An array of all node "types"
NODE_TYPE_ARRAY="$(
echo $IPS_JSON | \
jq '.value.value | to_entries | .[] | .key' | \
sed 's/"//g' | \
sed -z 's/\n/ /g;s/ $/\n/g')"
# Loop over all the node types and create an export line for each IP.
VM_IP_EXPORTS="$(
for TYPE in $NODE_TYPE_ARRAY; do
# Convert type, converts "master-ips" to "MASTER"
TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)"
echo "$IPS_JSON" | \
jq '.value.value.'"$TYPE"'[]' | \
# Add line numbers starting with 0.
nl -v 0 | \
# Print an export string with a type placeholder "__TYPE__".
awk '{print "export __TYPE___" $1 "=" $2}' | \
sed s/__TYPE__/$TYPE_UPPER/g
done)"
ANSIBLE_INV="$(
echo "$VM_IP_EXPORTS" | \
sed 's/"//g' | \
sed 's/^.*=//g' | \
sed -z 's/\n/,/g;s/,$/\n/g')"
# The file to write the inventory to. This file will be completely overridden.
INVENTORY_FILE="inventory"
# Grab the the vm name prefix. We do this by greping all *.tfvars files making
# sure to cat terraform.tfvars last. Then we just grab the last grep result,
# this way we make sure any value in terraform.tfvars will take priority.
VM_NAME_PREFIX_VAR="vm-name-prefix"
VM_NAME_PREFIXES="$( \
find . -name "*.tfvars" -exec grep "$VM_NAME_PREFIX_VAR" {} \; && \
grep "$VM_NAME_PREFIX_VAR" terraform.tfvars)"
VM_NAME_PREFIX="$(
echo "$VM_NAME_PREFIXES" | \
tail -n 1 | \
sed 's/^.*=\s*"\(.*\)"/\1/g')"
# This command stores the output data in the format below.
# [
# {
# "group": "master",
# "vms": [
# {
# "hostname": "ansible-test-master-0",
# "ip": "52.14.114.48"
# }
# ]
# },
# {
# "group": "worker",
# "vms": [
# {
# "hostname": "ansible-test-worker-0",
# "ip": "3.145.121.159"
# },
# {
# "hostname": "ansible-test-worker-1",
# "ip": "18.217.112.176"
# }
# ]
# }
# ]
DATA="$(terraform show -json | \
jq '.values.outputs.groups_hostnames_ips.value | to_entries |
map({group: .key, vms:.value | to_entries |
map({hostname:.key,ip:.value})})')"
# Create an inventory file for ansible.
echo "[k8s_nodes]" > inventory
echo $VM_IP_EXPORTS | \
# Pull out the groups from $DATA. The format is a single string with the groups
# separated by spaces, ie. "group1 group2 group3".
ANS_GROUPS="$(
echo $DATA | \
jq '.[] | .group' | \
sed 's/"//g' | \
sed 's/export //g' | \
sed 's/ /\n/g' | \
sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \
>> inventory
tr '\n' ' '
)"
echo $VM_IP_EXPORTS | sed 's/" /"\n/g'
echo export ANSIBLE_INV=$ANSIBLE_INV
# Clear the inventory file.
cat /dev/null > $INVENTORY_FILE
# For each group, write the VM info to $INVENTORY_FILE and also print a variable
# expression to stdout.
for GROUP in $ANS_GROUPS; do
# Write the inventory file to $INVENTORY_FILE.
echo "[$GROUP]" >> $INVENTORY_FILE
echo $DATA | \
jq '.[] | select(.group=="'"$GROUP"'") | .vms[] |
"\(.hostname) ansible_host=\(.ip)"' | \
sed 's/"//g' \
>> $INVENTORY_FILE
# For this group, collect expressions into VARS. The format is:
# HOSTNAME1=0.0.0.0
# HOSTNAME2=0.0.0.0
VARS="$(
echo $DATA | \
jq '.[] | select(.group=="'"$GROUP"'") | .vms[] |
"\(.hostname)=\(.ip)"' | \
sed 's/"//g' | \
sed "s/$VM_NAME_PREFIX-//g" | \
sed 's/-/_/g'
)"
# Print the contents of $VARS converted to uppercase.
echo "${VARS^^}"
done

@ -11,14 +11,15 @@ terraform {
locals {
nodes-config = {
"master" = {
base-image = var.amzn2-ami
base-image = var.centos8-ami
num = 1
},
"worker" = {
base-image = var.amzn2-ami
base-image = var.centos8-ami
num = 2
}
}
install-qemu-agent = false
}
################################################################################
@ -33,6 +34,7 @@ module "cloud-init-config" {
num = each.value.num
root-admin-passwd = var.root-admin-passwd
root-admin-pub-key = var.root-admin-pub-key
install-qemu-agent = local.install-qemu-agent
}
################################################################################
@ -106,7 +108,9 @@ module "nodes" {
# num-nodes = each.value.num
# node-memory = var.node-memory
# node-vcpus = var.node-vcpus
# node-disk-size = var.libvirt-node-disk-size
# base-image = each.value.base-image
# network-name = var.libvirt-network-name
# root-admin-passwd = var.root-admin-passwd
# root-admin-pub-key = var.root-admin-pub-key
# libvirt-connection-url = var.libvirt-connection-url
@ -123,6 +127,8 @@ module "nodes" {
# end libvirt
################################################################################
output "ips" {
value = { for type, node in module.nodes : type => node.ips }
# This will outpus a map of group => [{hostname, ip}].
# TODO A 'names' output needs to be added to libvirt.
output "groups_hostnames_ips" {
value = { for type, node in module.nodes : type => zipmap(node.names, node.ips) }
}

@ -33,7 +33,7 @@ resource "aws_default_security_group" "sg" {
}
tags = {
Name = "${var.name-prefix}-ssh-from-admins--sg"
Name = "${var.name-prefix}-ssh-from-admins-sg"
}
}

@ -1,3 +1,7 @@
output "ips" {
value = aws_instance.nodes.*.public_ip
}
output "names" {
value = aws_instance.nodes.*.tags.Name
}

@ -4,6 +4,7 @@ data "template_file" "user-datas" {
admin-passwd = "${var.root-admin-passwd}"
admin-pub-key = "${var.root-admin-pub-key}"
hostname = "${var.hostname-prefix}-${count.index}"
install-qemu-agent = var.install-qemu-agent
}
count = var.num
}

@ -8,6 +8,12 @@ variable "hostname-prefix" {
description = "This prefix wil be applied as a prefix for the hostnames."
}
variable "install-qemu-agent" {
default = false
description = "This flag determines whether or not qemu-agent is installed."
type = bool
}
variable "num" {
description = "The number of user-datas to create with these parameters."
}

@ -9,13 +9,20 @@ terraform {
}
resource "libvirt_volume" "node-images" {
name = "${var.name-prefix}-${count.index}"
name = "${var.name-prefix}-base"
pool = var.pool-name
source = var.base-image
count = var.num-nodes
format = "qcow2"
}
resource "libvirt_volume" "node-images-resized" {
name = "${var.name-prefix}-${count.index}-resized"
pool = var.pool-name
base_volume_id = libvirt_volume.node-images.id
count = var.num-nodes
size = var.node-disk-size
}
data "template_file" "network-config" {
template = file("${path.module}/network_config.cfg")
}
@ -37,9 +44,9 @@ resource "libvirt_domain" "nodes" {
cloudinit = element(libvirt_cloudinit_disk.node-inits.*.id, count.index)
network_interface {
network_name = "default"
network_name = var.network-name
hostname = "${var.name-prefix}-${count.index}"
wait_for_lease = true
# wait_for_lease = true
}
# IMPORTANT: this is a known bug on cloud images, since they expect a console
@ -58,7 +65,7 @@ resource "libvirt_domain" "nodes" {
}
disk {
volume_id = element(libvirt_volume.node-images.*.id, count.index)
volume_id = element(libvirt_volume.node-images-resized.*.id, count.index)
}
graphics {

@ -1,4 +1,6 @@
output "ips" {
value = libvirt_domain.nodes.*.network_interface.0.addresses.0
}
# This only works on the default network. They will not work using the bridged
# network.
# output "ips" {
# value = libvirt_domain.nodes.*.network_interface.0.addresses.0
# }

@ -12,6 +12,16 @@ variable "name-prefix" {
description = "This will be a prefix for all resource names, ie. domains will be created suck as \"k8s-node-2\"."
}
variable "network-name" {
default = "default"
description = "The name of a pre-existing virtual-network."
}
variable "node-disk-size" {
default = 4294967296
description = "The size of the disk to be used for libvirt nodes. (in bytes)"
}
variable "node-memory" {
default = "2048"
description = "The amount of memory to be used for all the nodes."

@ -31,6 +31,16 @@ variable "libvirt-connection-url" {
description = "The libvirt connection URI, ie. qemu+ssh://<user>@<host>/system"
}
variable "libvirt-network-name" {
default = "default"
description = "The name of a pre-existing libvirt virtual-network."
}
variable "libvirt-node-disk-size" {
default = 4294967296
description = "The size of the disk to be used for libvirt nodes. (in bytes)"
}
variable "node-memory" {
default = "2048"
description = "The amount of memory to be used for all the nodes."
@ -107,3 +117,25 @@ variable "rhel8-ami" {
default = "ami-0d871ca8a77af2948"
description = "The AMI to use for RHEL 8."
}
################################################################################
# Libvirt Images
# These variables are really mor like constants. Using variables improves
# readability. The defaults are manually updated.
################################################################################
variable "ubuntu-img" {
default = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
description = "The libvirt image tp use for Ubuntu."
}
variable "centos7-img" {
# Latest as of 2021-12-06.
default = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2111.qcow2"
description = "The libvirt image tp use for CentOS 7."
}
variable "centos8-img" {
default = "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2"
description = "The libvirt image tp use for CentOS 8."
}

Loading…
Cancel
Save