shnee/k8s-terraform
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Simplified main. Added AMI module.

Browse Source
This commit is contained in:
shnee
2021-12-13 20:26:00 -05:00
parent 7ca4db8737
commit 86acf7736d
14 changed files with 390 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
k8s-nodes/.gitignore
+49
View File
@@ -0,0 +1,49 @@
################################################################################
# Pulled from github/gitignore 2021-11-10 commit 1a84870
################################################################################
# Local .terraform directories
**/.terraform/*
# .tfstate files
*.tfstate
*.tfstate.*
# Crash log files
crash.log
# Added to original pulled from github/gitignore
crash*.log
# Exclude all .tfvars files, which are likely to contain sentitive data, such as
# password, private keys, and other secrets. These should not be part of version
# control as they are data points which are potentially sensitive and subject
# to change depending on the environment.
#
*.tfvars
# Ignore override files as they are usually used to override resources locally and so
# are not checked in
override.tf
override.tf.json
*_override.tf
*_override.tf.json
# Include override files you do wish to add to version control using negated pattern
#
# !example_override.tf
# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan
# example: *tfplan*
# Ignore CLI configuration files
.terraformrc
terraform.rc
################################################################################
# end Pulled from github/gitignore 2021-11-10 commit 1a84870
################################################################################
k8s-key*
STARTHERE
inventory
k8s-nodes/.terraform.lock.hcl
Generated
+57
View File
@@ -0,0 +1,57 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/dmacvicar/libvirt" {
version = "0.6.11"
hashes = [
"h1:6QzHY/7aNdaaDxJZKygotWnM5uHoS2gs/03CzUCJX60=",
"zh:15300a1c3c294eccade4c8a678412d81602ab041dc0a5aab72fee5425d778e89",
"zh:1605806de0d3b86b7e94b5d04a7ad9b6ac695781f9672ab6002c23caef43b98e",
"zh:21efc5937d89f9ec96bc626d2ce3621c0919b3da97ab63b4e520c37d3f5c7357",
"zh:2c143a6909917fd11191447de4c496f084c7da5200beb9f512791a80a1f33e7c",
"zh:3ca369718cc49feefc3a6ffa795a9055e60de33989a9f1c72b6db16048a181fa",
"zh:71db1d1cf2c06984bba408ad5dc9b4e25285684ee5c530a61583b202cff21b96",
"zh:a67adfc988311d34adcc119500c2ef048a45d632b00bb5a15ea6d3ffdc1c3d1c",
"zh:a83448cbcc194e3b52af9b89b9273a116082d83f2c966035bf8a8c5d5606ca9c",
"zh:a9c5a818dd2606460d4d6f33af7cb387f3e984d631fc233aaec0dda4e0756c2b",
"zh:af3263e66cf9138361d6d7408533edd6de8498e67c88cf0084421ae31fe89054",
"zh:b9596cb26c1e391172472de4ada9b3b0a08e4777e41327db8e021454cc6aae20",
"zh:e28124b9ee0b8c18b6f776eb6523d8935f3072c47cd803ea2f1a06206effaa48",
"zh:edd00638d8c088b8a38e7ab8b9e8ab3bd710f7357d0f6b4a38e0028bd49d8460",
]
}
provider "registry.terraform.io/hashicorp/aws" {
version = "3.64.2"
hashes = [
"h1:oFsgTmmt/eJ8KLo59PSmYu/irUHJiPnQEey0VqaOOck=",
"zh:0b029a2282beabfe410eb2969e18ca773d3473415e442be4dc8ce0eb6d1cd8c5",
"zh:3209de3266a1138f1ccb09f094fdd98b6f55afc06e291db0abe092ec5dbe7640",
"zh:40648266551631cbc15f8a76e80faf300510e3b38c2544d43fc25e37e6802727",
"zh:483c8af92ae70146f2790a70c1a810251e7135aa912b66e769c934eddceebe32",
"zh:4d106d8d415d8df342f3f85e58c35418e6c55e3cb7f02897f832cefac4dca68c",
"zh:972626a6ddb31d5216606d12ab5c30fbf8d51ed2bbe0efcdd7cffa68c1141557",
"zh:a230d55ec52b1695148d40296877ee23e0b302e817154f9b838eb117c87b13fa",
"zh:c95fddfbd7f870db949da0601323e866e0f0fb0d4a93e96725ae5b88029e84d5",
"zh:ea0c7f568074f835f22273c8e7e61e87f5277e32004c72122915fd3c8df49ccc",
"zh:f96d25887e6e2d2ae47659e2c586efea2167995b59a479ae65a02b097da86474",
"zh:fe7502d8e52d3b5ccb2b3c178e7ea894344783093aa71ffb20e978914c976182",
]
}
provider "registry.terraform.io/hashicorp/template" {
version = "2.2.0"
hashes = [
"h1:94qn780bi1qjrbC3uQtjJh3Wkfwd5+tTtJHOb7KTg9w=",
"zh:01702196f0a0492ec07917db7aaa595843d8f171dc195f4c988d2ffca2a06386",
"zh:09aae3da826ba3d7df69efeb25d146a1de0d03e951d35019a0f80e4f58c89b53",
"zh:09ba83c0625b6fe0a954da6fbd0c355ac0b7f07f86c91a2a97849140fea49603",
"zh:0e3a6c8e16f17f19010accd0844187d524580d9fdb0731f675ffcf4afba03d16",
"zh:45f2c594b6f2f34ea663704cc72048b212fe7d16fb4cfd959365fa997228a776",
"zh:77ea3e5a0446784d77114b5e851c970a3dde1e08fa6de38210b8385d7605d451",
"zh:8a154388f3708e3df5a69122a23bdfaf760a523788a5081976b3d5616f7d30ae",
"zh:992843002f2db5a11e626b3fc23dc0c87ad3729b3b3cff08e32ffb3df97edbde",
"zh:ad906f4cebd3ec5e43d5cd6dc8f4c5c9cc3b33d2243c89c5fc18f97f7277b51d",
"zh:c979425ddb256511137ecd093e23283234da0154b7fa8b21c2687182d9aea8b2",
]
}
k8s-nodes/README.md
+12
View File
@@ -1,3 +1,15 @@
A Terraform script to create k8s nodes. This script has modules for creating the
nodes on a KVM/QEMU (libvirt) hypervisor or creating the nodes via AWS.
The modules allow you create N VMs of a specific type. So you could create 1
master node and 3 worker nodes or you could create 3 Ubuntu VMs and 5 CentOS
VMs, or whatever fits your needs.
Cloud-Init
----------------------------------------
Both the libvirt and aws modules use cloud-init for initial configuration of the
VMs.
Dependencies
----------------------------------------
k8s-nodes/example.tfvars
+35 -3
View File
@@ -1,3 +1,5 @@
vm-name-prefix = "ansible-test"
# A CIDR block ending in '/32' equates to a single IP address, '0.0.0.0/0'
# equates to any ip address.
admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ]
@@ -5,12 +7,13 @@ admin-ips = [ "8.8.8.8/32", "0.0.0.0/0" ]
disk-image-dir = "/path/to/disk/pool/"
libvirt-connection-url = "qemu+ssh://<user>@<host>/system"
master-nodes = 1
worker-nodes = 2
node-memory = 2048
node-vcpus = 2
################################################################################
# AWS EC2 instance types
################################################################################
# 1 GiB, 1 vcpu, only one that is free.
# This one won't work with k8s because it requires at least 2 vcpus.
aws-ec2-instance-type = "t2.micro"
@@ -18,14 +21,43 @@ aws-ec2-instance-type = "t2.micro"
# 4 GiB, 2 vcpus
# aws-ec2-instnce-type = "t2.medium"
################################################################################
# AWS images (AMIs)
################################################################################
## Amazon Linux 2
# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - Oregon - 2021.11.11 - free
# base-image = "ami-00be885d550dcee43"
# AWS Amazon Linux 2 AMI (HVM), SSD Volume Type - us-east-2 - 2021.11.12 - free
base-image = "ami-0dd0ccab7e2801812"
## CentOS
# CentOS 7.9.2009 x86_64 - us-east-2 - 2021-11-15
# base-image = "ami-00f8e2c955f7ffa9b"
# CentOS 8.4.2105 x86_64 - us-east-2 - 2021-11015
# base-image = "ami-057cacbfbbb471bb3"
## Ubuntu
# Ubuntu Server 20.04 LTS (HVM), SSD Volume Type
# us-east-2 - (64-bit x86) - 2021.11.12 - free
# base-image = "ami-0629230e074c580f2"
## Arch linux
# arch-linux-lts-hvm-2021.06.02.x86_64-ebs - us-east-2
# base-image = "ami-02653f06de985e3ba"
################################################################################
# libvirt images
################################################################################
# base-image = "https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64-disk-kvm.img"
# From https://cloud.centos.org/centos/7/images/ from 2020-11-12 06:52
# base-image = "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2"
################################################################################
# Keys/Passwords
################################################################################
# Password hash created with:
# python3 -c 'import crypt; print(crypt.crypt("linux", crypt.mksalt(crypt.METHOD_SHA512)))'
# where "linux" is the password.
k8s-nodes/get-aws-node-status.sh
Executable
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
aws ec2 describe-instance-status | \
jq '.InstanceStatuses[] | {id: .InstanceId, instance_status: .InstanceStatus.Status, system_status: .SystemStatus.Status}'
k8s-nodes/get-vm-ips.sh
+52 -11
View File
@@ -1,19 +1,60 @@
#!/bin/sh
# This script will create environment variables for all of the output IPs. It
# will also create a `ANSIBLE_INV` variable that will be a comma separated
# string of all the IPs. A anisble inventory file called "inventory is created
# as well.
#
# Use eval $(./get-vm-ips.sh) to set env vars for ips.
terraform refresh > /dev/null
IPS_JSON="$(terraform show -json | jq '.values.outputs')"
# All terraform outputs in json format.
OUTPUTS_JSON="$(
terraform show -json | \
jq '.values.outputs' | \
sed 's/-/_/g')"
# Just the IP address outputs in json format. Also all '-' characters are
# replaced by '_' becuase '-' causes jq some problems.
IPS_JSON="$(
echo $OUTPUTS_JSON | \
jq 'to_entries | .[] | select(.key | contains("ips"))')"
# An array of all node "types"
NODE_TYPE_ARRAY="$(
echo $IPS_JSON | \
jq '.value.value | to_entries | .[] | .key' | \
sed 's/"//g' | \
sed -z 's/\n/ /g;s/ $/\n/g')"
echo $IPS_JSON | \
jq '."master-ips".value[]' | \
nl -v 0 | \
awk '{print "export MASTER" $1 "=" $2}' | \
sed 's/"//g'
# Loop over all the node types and create an export line for each IP.
VM_IP_EXPORTS="$(
for TYPE in $NODE_TYPE_ARRAY; do
echo $IPS_JSON | \
jq '."worker-ips".value[]' | \
nl -v 0 | \
awk '{print "export WORKER" $1 "=" $2}' | \
sed 's/"//g'
# Convert type, converts "master-ips" to "MASTER"
TYPE_UPPER="$(echo ${TYPE^^} | sed s/_.*$//g)"
echo "$IPS_JSON" | \
jq '.value.value.'"$TYPE"'[]' | \
# Add line numbers starting with 0.
nl -v 0 | \
# Print an export string with a type placeholder "__TYPE__".
awk '{print "export __TYPE___" $1 "=" $2}' | \
sed s/__TYPE__/$TYPE_UPPER/g
done)"
ANSIBLE_INV="$(
echo "$VM_IP_EXPORTS" | \
sed 's/"//g' | \
sed 's/^.*=//g' | \
sed -z 's/\n/,/g;s/,$/\n/g')"
# Create an inventory file for ansible.
echo "[k8s_nodes]" > inventory
echo $VM_IP_EXPORTS | \
sed 's/"//g' | \
sed 's/export //g' | \
sed 's/ /\n/g' | \
sed 's/^\(.*\)\(=.*\)$/\1 ansible_host\2/g' \
>> inventory
echo $VM_IP_EXPORTS | sed 's/" /"\n/g'
echo export ANSIBLE_INV=$ANSIBLE_INV
k8s-nodes/main.tf
+50 -66
View File
@@ -1,35 +1,38 @@
terraform {
required_version = ">= 0.13"
required_version = ">= 1.0.8"
required_providers {
libvirt = {
source = "dmacvicar/libvirt"
source = "dmacvicar/libvirt"
version = "0.6.11"
}
}
}
locals {
nodes-config = {
"master" = {
base-image = var.amzn2-ami
num = 1
},
"worker" = {
base-image = var.amzn2-ami
num = 2
}
}
}
################################################################################
# cloud-init
################################################################################
data "template_file" "master-node-user-datas" {
template = file("${path.module}/cloud_init.cfg")
vars = {
admin-passwd = "${var.root-admin-passwd}"
admin-pub-key = "${var.root-admin-pub-key}"
hostname = "${var.vm-name-prefix}-master-${count.index}"
}
count = var.master-nodes
}
data "template_file" "worker-node-user-datas" {
template = file("${path.module}/cloud_init.cfg")
vars = {
admin-passwd = "${var.root-admin-passwd}"
admin-pub-key = "${var.root-admin-pub-key}"
hostname = "${var.vm-name-prefix}-worker-${count.index}"
}
count = var.worker-nodes
module "cloud-init-config" {
for_each = local.nodes-config
source = "./modules/cloud-init-config"
cloud-init-template = "${path.module}/cloud_init.cfg"
hostname-prefix = "${var.vm-name-prefix}-${each.key}"
num = each.value.num
root-admin-passwd = var.root-admin-passwd
root-admin-pub-key = var.root-admin-pub-key
}
################################################################################
@@ -42,12 +45,21 @@ provider "aws" {
region = "us-east-2"
}
# This module will grab the latest ami for a variety of distros. Uncomment to
# get a list of the latest AMIs for our supported distros.
# module "aws-amis" {
# source = "./modules/aws-amis"
# }
# output "amis" {
# value = module.aws-amis.amis
# }
module "aws-network" {
source = "./modules/aws-network"
name-prefix = var.vm-name-prefix
vpc-cidr-block = var.aws-vpc-cidr-block
source = "./modules/aws-network"
name-prefix = var.vm-name-prefix
vpc-cidr-block = var.aws-vpc-cidr-block
subnet-cidr-block = var.aws-subnet-cidr-block
admin-ips = var.admin-ips
admin-ips = var.admin-ips
}
# This key pair is not actually used. Keys are added to the nodes via cloud-init
@@ -60,26 +72,16 @@ resource "aws_key_pair" "key" {
}
}
module "master-nodes" {
module "nodes" {
for_each = local.nodes-config
source = "./modules/aws-nodes"
ami = var.base-image
ami = each.value.base-image
ec2-instance-type = var.aws-ec2-instance-type
subnet-id = module.aws-network.subnet.id
security-group-ids = [module.aws-network.default-security-group.id]
user-datas = data.template_file.master-node-user-datas
num-nodes = var.master-nodes
name-prefix = "${var.vm-name-prefix}-master"
}
module "worker-nodes" {
source = "./modules/aws-nodes"
ami = var.base-image
ec2-instance-type = var.aws-ec2-instance-type
subnet-id = module.aws-network.subnet.id
security-group-ids = [module.aws-network.default-security-group.id]
user-datas = data.template_file.worker-node-user-datas
num-nodes = var.worker-nodes
name-prefix = "${var.vm-name-prefix}-worker"
user-datas = lookup(module.cloud-init-config, each.key, null).user-datas
num-nodes = each.value.num
name-prefix = "${var.vm-name-prefix}-${each.key}"
}
################################################################################
@@ -96,32 +98,19 @@ module "worker-nodes" {
# uri = var.libvirt-connection-url
# }
#
# module "master-nodes" {
# module "nodes" {
# for_each = local.nodes-config
# source = "./modules/libvirt-nodes"
# pool-name = libvirt_pool.images.name
# name-prefix = "${var.vm-name-prefix}-master"
# num-nodes = var.master-nodes
# name-prefix = "${var.vm-name-prefix}-${each.key}"
# num-nodes = each.value.num
# node-memory = var.node-memory
# node-vcpus = var.node-vcpus
# base-image = var.base-image
# base-image = each.value.base-image
# root-admin-passwd = var.root-admin-passwd
# root-admin-pub-key = var.root-admin-pub-key
# libvirt-connection-url = var.libvirt-connection-url
# user-datas = data.template_file.master-node-user-datas
# }
#
# module "worker-nodes" {
# source = "./modules/libvirt-nodes"
# pool-name = libvirt_pool.images.name
# name-prefix = "${var.vm-name-prefix}-worker"
# num-nodes = var.worker-nodes
# node-memory = var.node-memory
# node-vcpus = var.node-vcpus
# base-image = var.base-image
# root-admin-passwd = var.root-admin-passwd
# root-admin-pub-key = var.root-admin-pub-key
# libvirt-connection-url = var.libvirt-connection-url
# user-datas = data.template_file.worker-node-user-datas
# user-datas = lookup(module.cloud-init-config, each.key, null).user-datas
# }
#
# resource "libvirt_pool" "images" {
@@ -134,11 +123,6 @@ module "worker-nodes" {
# end libvirt
################################################################################
# TODO REM move to other file?
output "master-ips" {
value = module.master-nodes.ips
}
output "worker-ips" {
value = module.worker-nodes.ips
output "ips" {
value = { for type, node in module.nodes : type => node.ips }
}
k8s-nodes/modules/aws-amis/main.tf
+58
View File
@@ -0,0 +1,58 @@
locals {
amis = {
amzn2 = {
owner-id = "137112412989"
name = "amzn2-ami-hvm-2*x86_64-gp2"
},
ubuntu = {
owner-id = "099720109477"
name = "ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"
},
centos7 = {
owner-id = "125523088429"
name = "CentOS 7.*x86_64"
},
centos8 = {
owner-id = "125523088429"
name = "CentOS 8.*x86_64"
},
arch = {
owner-id = "093273469852"
name = "arch-linux-lts-hvm*x86_64-ebs"
},
rhel7 = {
owner-id = "309956199498"
name = "RHEL-7.*HVM*x86_64*GP2"
},
rhel8 = {
owner-id = "309956199498"
name = "RHEL-8.*HVM*x86_64*GP2"
}
}
}
data "aws_ami" "amis" {
for_each = local.amis
most_recent = true
owners = [each.value.owner-id]
filter {
name = "name"
values = [each.value.name]
}
filter {
name = "virtualization-type"
values = ["hvm"]
}
filter {
name = "architecture"
values = ["x86_64"]
}
filter {
name = "root-device-type"
values = ["ebs"]
}
}
k8s-nodes/modules/aws-amis/outputs.tf
+3
View File
@@ -0,0 +1,3 @@
output "amis" {
value = tomap({ for type, ami in data.aws_ami.amis : type => ami.id })
}
k8s-nodes/modules/cloud-init-config/main.tf
+9
View File
@@ -0,0 +1,9 @@
data "template_file" "user-datas" {
template = file("${var.cloud-init-template}")
vars = {
admin-passwd = "${var.root-admin-passwd}"
admin-pub-key = "${var.root-admin-pub-key}"
hostname = "${var.hostname-prefix}-${count.index}"
}
count = var.num
}
k8s-nodes/modules/cloud-init-config/outputs.tf
+3
View File
@@ -0,0 +1,3 @@
output "user-datas" {
value = data.template_file.user-datas
}
k8s-nodes/modules/cloud-init-config/variables.tf
+22
View File
@@ -0,0 +1,22 @@
variable "cloud-init-template" {
default = "../../cloud_init.cfg"
description = "The path to the cloud-init config template."
type = string
}
variable "hostname-prefix" {
description = "This prefix wil be applied as a prefix for the hostnames."
}
variable "num" {
description = "The number of user-datas to create with these parameters."
}
variable "root-admin-passwd" {
description = "This value will be substituted for any occurence of 'admin-password' in the cloud-init config template."
}
variable "root-admin-pub-key" {
description = "This value will be substituted for any occurence of 'admin-pub-key' in the cloud-init config template."
}
k8s-nodes/modules/libvirt-nodes/main.tf
-4
View File
@@ -8,10 +8,6 @@ terraform {
}
}
provider "libvirt" {
uri = var.libvirt-connection-url
}
resource "libvirt_volume" "node-images" {
name = "${var.name-prefix}-${count.index}"
pool = var.pool-name
k8s-nodes/variables.tf
+36
View File
@@ -71,3 +71,39 @@ variable "vm-name-prefix" {
default = "k8s-tf"
description = "This prefix will appear before all VM names and hostnames, ie. k8s-tf-master-0."
}
################################################################################
# AWS AMI vars
# These variables are really mor like constants. Using variables improves
# readability. The defaults are manually updated. Use the aws-amis module to get
# the latest for each distro.
################################################################################
variable "amzn2-ami" {
default = "ami-0dd0ccab7e2801812"
description = "The AMI to use for Amazon Linux 2."
}
variable "ubuntu-ami" {
default = "ami-06c7d6c0987eaa46c"
description = "The AMI to use for Ubuntu."
}
variable "centos7-ami" {
default = "ami-00f8e2c955f7ffa9b"
description = "The AMI to use for CentOS 7."
}
variable "centos8-ami" {
default = "ami-057cacbfbbb471bb3"
description = "The AMI to use for CentOS 8."
}
variable "arch-ami" {
default = "ami-02653f06de985e3ba"
description = "The AMI to use for Arch Linux."
}
variable "rhel7-ami" {
default = "ami-0a509b3c2a4d05b3f"
description = "The AMI to use for RHEL 7."
}
variable "rhel8-ami" {
default = "ami-0d871ca8a77af2948"
description = "The AMI to use for RHEL 8."
}