Cleaned up.

defaults/main.yml

@@ -1,11 +1,17 @@
 ---
 k8s_version: "1.22"
+k8s_rhel_version: "1.22.4"
 
 pod_network_cidr: "10.10.0.0/16"
 kubeadm_init_command: "kubeadm init --pod-network-cidr={{ pod_network_cidr }}"
 
+# This flag controls whether or not the kubectl config file is copied from a
+# master node to the local machine running the ansible playbook.
+copy_kubectl_config_to_local: false
+
 apt_gpg_key_url: "https://packages.cloud.google.com/apt/doc/apt-key.gpg"
 apt_gpg_key_fingerprint: "59FE0256827269DC81578F928B57C5C2836F4BEB"
+# This Ubuntu URL should also work for Debian.
 apt_repo: "deb https://apt.kubernetes.io/ kubernetes-xenial main"
 
 rh_yum_base_url: "https://packages.cloud.google.com/\
@@ -15,3 +21,12 @@ rh_yum_gpg_key:
   - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
 
 calico_manifest_url: "https://docs.projectcalico.org/manifests/calico.yaml"
+
+# k8s static pods
+kube_apiserver: 'k8s.gcr.io/kube-apiserver'
+kube_controller_manager: 'k8s.gcr.io/kube-controller-manager'
+kube_scheduler: 'k8s.gcr.io/kube-scheduler'
+kube_proxy: 'k8s.gcr.io/kube-proxy'
+etcd: 'k8s.gcr.io/etcd'
+coredns: 'k8s.gcr.io/coredns'
+pause: 'k8s.gcr.io/pause'

tasks/calico.yml

@@ -0,0 +1,18 @@
+---
+################################################################################
+# calico
+# https://projectcalico.docs.tigera.io/getting-started/kubernetes/self-managed-onprem/onpremises
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#pod-network
+################################################################################
+
+- name: Check if calico is already deployed by getting all pods.
+  command: kubectl get pods --all-namespaces
+  ignore_errors: true
+  changed_when: false
+  register: pods
+
+- name: Deploy Calico if not already installed.
+  command: "kubectl apply -f {{ calico_manifest_url }}"
+  register: calico_result
+  changed_when: "'created' in calico_result.stdout"
+  when: "'calico' not in pods.stdout"

tasks/configure_docker.yml

@@ -1,5 +1,9 @@
 ---
+################################################################################
+# Congifure docker for k8s.
 # https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
+################################################################################
+
 - name: Install docker daemon config.
   template:
     src: docker-daemon-config.json
@@ -17,6 +21,6 @@
   become: true
   when: docker_daemon_config.changed
   # ansible-lint complains that this task should be a handler but we need this
-  # daemon_reload and service restarted before kublet is installed, other wise
+  # daemon_reload and service restarted before kublet is installed, otherwise
   # the kibelet service has issues.
   tags: ['skip_ansible_lint']

tasks/debian_family.yml

@@ -0,0 +1,44 @@
+---
+################################################################################
+# Install kublet, kudeadm, and kubectl
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl
+################################################################################
+
+- name: Install kubectl dependencies for Debian-like distros.
+  package:
+    name:
+      - apt-transport-https
+      - ca-certificates
+      - curl
+    state: present
+  become: true
+
+- name: Add GPG key to apt for Debian-like distro.
+  apt_key:
+    url: "{{ apt_gpg_key_url }}"
+    id: "{{ apt_gpg_key_fingerprint }}"
+    state: present
+  become: true
+
+- name: Add k8s repository for Debian-like distro.
+  apt_repository:
+    repo: "{{ apt_repo }}"
+    state: present
+    update_cache: true
+  become: true
+
+- name: Add k8s apt preferences file to pin a version.
+  template:
+    src: apt-preferences-k8s.j2
+    dest: /etc/apt/preferences.d/kubernetes
+    mode: 0644
+  become: true
+
+- name: Install kubectl Debian-like distro.
+  package:
+    name:
+      - kubectl
+      - kubeadm
+      - kubelet
+    state: present
+  become: true

tasks/disable_swap.yml

@@ -0,0 +1,27 @@
+---
+################################################################################
+# Disable swap
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin
+################################################################################
+
+- name: Get swap status.
+  command: swapon -s
+  # It's unclear why you need to be root for this commend. This command works
+  # fine manually as the admin user, but the only way I could get this role to
+  # work was to become root.
+  become: true
+  changed_when: false
+  register: swap_status
+
+- name: Disable swap if needed.
+  command: swapoff -a
+  become: true
+  changed_when: true
+  when: swap_status.stdout | length > 0
+
+- name: comment out swap in fstab.
+  replace:
+    path: /etc/fstab
+    regexp: '^([^#].*swap.*)$'
+    replace: '# \1'
+  become: true

tasks/join_nodes.yml

@@ -0,0 +1,26 @@
+---
+################################################################################
+# Join nodes to cluster.
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#join-nodes 
+################################################################################
+
+- name: Get the kubeadm join command from the Kubernetes master.
+  command: kubeadm token create --print-join-command
+  changed_when: false
+  when: kubernetes_role == 'master'
+  register: kubernetes_join_command_result
+
+- name: Set the kubeadm join command globally.
+  set_fact:
+    kubernetes_join_command: "{{ kubernetes_join_command_result.stdout }}"
+  when: kubernetes_join_command_result.stdout is defined
+  delegate_to: "{{ item }}"
+  delegate_facts: true
+  with_items: "{{ groups['all'] }}"
+
+- name: Join node to cluster.
+  command:
+    cmd: "{{ kubernetes_join_command }}"
+    creates: /etc/kubernetes/kubelet.conf
+  become: true
+  when: kubernetes_role == 'worker'

tasks/main.yml

@@ -1,122 +1,18 @@
 ---
-# Configure docker for k8s.
+# Congifure docker for k8s.
+# https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker
 - include_tasks: configure_docker.yml
 
-################################################################################
 # Install kublet, kudeadm, and kubectl
-################################################################################
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl
-
+- include_tasks: debian_family.yml
-- name: Install kubectl dependencies.
-  package:
-    name:
-      - apt-transport-https
-      - ca-certificates
-      - curl
-    state: present
-  become: true
   when: ansible_os_family == "Debian"
+- include_tasks: redhat_family.yml
+  when: ansible_os_family == "RedHat"
 
-- name: Add GPG key to apt for Ubuntu.
+# Configure sysctl kernel modules
-  apt_key:
-    url: "{{ apt_gpg_key_url }}"
-    id: "{{ apt_gpg_key_fingerprint }}"
-    state: present
-  become: true
-  when: ansible_os_family == "Debian"
-
-- name: Add k8s repository for Debian-like distro.
-  apt_repository:
-    repo: "{{ apt_repo }}"
-    state: present
-    update_cache: true
-  become: true
-  when: ansible_os_family == "Debian"
-
-- name: Add k8s apt preferences file to pin a version.
-  template:
-    src: apt-preferences-k8s.j2
-    dest: /etc/apt/preferences.d/kubernetes
-    mode: 0644
-  become: true
-  when: ansible_os_family == "Debian"
-
-- name: Install kubectl.
-  package:
-    name:
-      - kubectl
-      - kubeadm
-      - kubelet
-    state: present
-  become: true
-  when: ansible_os_family == "Debian"
-
-- name: Add k8s repo for RedHat-like distro.
-  yum_repository:
-    name: kubernetes
-    description: kubernetes
-    baseurl: "{{ rh_yum_base_url }}"
-    enabled: true
-    gpgcheck: true
-    repo_gpgcheck: true
-    gpgkey: "{{ rh_yum_gpg_key }}"
-    exclude: kubelet kudeadm kubectl
-  become: true
-  register: yum_repo
-  when: ansible_os_family == 'RedHat'
-
-- name: debug
-  debug:
-    var: kubernetes_role
-
-- name: Set SELinux to permissive mode.
-  selinux:
-    policy: targeted
-    state: permissive
-  become: true
-  when: ansible_os_family == 'RedHat'
-
-- name: Update SELinux config to set it to permissive mode at boot.
-  replace:
-    path: /etc/selinux/config
-    regexp: '^SELINUX=enforcing$'
-    replace: 'SELINUX=permissive'
-  become: true
-  when: ansible_os_family == 'RedHat'
-
-- name: Install kubelet, kubectl, and kubeadm.
-  yum:
-    name:
-      - kubelet
-      - kubectl
-      - kubeadm
-    state: present
-    disable_excludes: kubernetes
-  become: true
-  when: ansible_os_family == 'RedHat'
-
-################################################################################
-# sysctl stuff
 # https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic
-################################################################################
+- include_tasks: sysctl_and_kernel_modules.yml
-
-- name: Force kernel modules to load.
-  copy:
-    dest: /etc/modules-load.d/k8s.conf
-    content: br_netfilter
-    mode: 0644
-  become: true
-
-- name: Let iptables see bridged traffic.
-  sysctl:
-    name: "{{ item }}"
-    value: '1'
-    state: present
-  become: true
-  loop:
-    - net.bridge.bridge-nf-call-iptables
-    - net.bridge.bridge-nf-call-ip6tables
-
-################################################################################
 
 - name: Ensure kubelet is started and enabled at boot.
   service:
@@ -125,36 +21,12 @@
     enabled: true
   become: true
 
-################################################################################
+# Disable swap
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin
 
-# STARTHERE This command is failing on centos7 machine even though the command
+- include_tasks: disable_swap.yml
-# works manually.
-#- name: Get swap status w/ shell.
-#  shell: swapon -s
-- name: Get swap status.
-  command: swapon -s
-  # It's unclear why you need to be root for this commend. This command works
-  # fine manually as the admin user, but the only way I could get this script to
-  # work was to become root.
-  become: true
-  changed_when: false
-  register: swap_status
-- name: Disable swap if needed.
-  command: swapoff -a
-  become: true
-  changed_when: true
-  when: swap_status.stdout | length > 0
-- name: comment out swap in fstab.
-  replace:
-    path: /etc/fstab
-    regexp: '^([^#].*swap.*)$'
-    replace: '# \1'
-  become: true
-
-################################################################################
-# master
-################################################################################
 
+# Pre-pull k8s status pods.
 - name: Check the docker images currently on machine.
   command: docker images
   # We do this as root because the docker admin will not be able to until we
@@ -163,8 +35,6 @@
   register: docker_images
   changed_when: false
 
-# TODO REM make these images full paths. and vars
-# TODO REM check nodes id they hae these images also.
 - name: Pre-grab the container images for the static pods.
   command: kubeadm config images pull
   # We do this as root because the docker admin will not be able to until we
@@ -172,96 +42,24 @@
   become: true
   changed_when: true
   when: >
-    ( 'kube-apiserver' not in docker_images.stdout or
+    ( kube_apiserver not in docker_images.stdout or
-      'kube-controller-manager' not in docker_images.stdout or
+      kube_controller_manager not in docker_images.stdout or
-      'kube-scheduler' not in docker_images.stdout or
+      kube_scheduler not in docker_images.stdout or
-      'kube-proxy' not in docker_images.stdout or
+      kube_proxy not in docker_images.stdout or
-      'etcd' not in docker_images.stdout or
+      etcd not in docker_images.stdout or
-      'coredns' not in docker_images.stdout or
+      coredns not in docker_images.stdout or
-      'pause' not in docker_images.stdout )
+      pause not in docker_images.stdout )
 
-- name: Initialize Kubernetes master with kubeadm init
+# master
-  command:
+- include_tasks: master.yml
-    cmd: "{{ kubeadm_init_command }}"
-    creates: /etc/kubernetes/admin.conf
-  become: true
-  # TODO REM do we need this register?
-  register: kubeadmin_init
-  ignore_errors: true
   when: kubernetes_role == 'master'
 
-- name: Debug kubectl init.
+# calico
-  debug:
+# https://projectcalico.docs.tigera.io/getting-started/kubernetes/self-managed-onprem/onpremises
-    var: kubeadmin_init
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#pod-network
+- include_tasks: calico.yml
   when: kubernetes_role == 'master'
 
-# TODO REM find a way to copy this file to local machine.
-- name: Ensure .kube directory exists.
-  file:
-    path: ~/.kube
-    state: directory
-    mode: 0700
-  when: kubernetes_role == 'master'
-
-- name: Copy the kubectl admin.conf to ~/.kube/conf.
-  copy:
-    src: /etc/kubernetes/admin.conf
-    remote_src: true
-    dest: "/home/{{ ansible_user }}/.kube/config"
-    mode: 0600
-    owner: "{{ ansible_user }}"
-  become: true
-  when: kubernetes_role == 'master'
-
-################################################################################
 # Join nodes to cluster.
-################################################################################
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#join-nodes 
-
+- include_tasks: join_nodes.yml
-- name: Print the role.
-  debug:
-    var: kubernetes_role
-- name: Get the kubeadm join command from the Kubernetes master.
-  command: kubeadm token create --print-join-command
-  changed_when: false
-  when: kubernetes_role == 'master'
-  register: kubernetes_join_command_result
-
-# TODO REM remove this
-- name: debug
-  debug:
-    var: kubernetes_join_command_result
-
-# STARTHERE try to open firewall stuff on centos 7.
-# This is not how we want to do this just grab the join command from a mater no
-# need to set a fact.
-#- name: Set the kubeadm join command globally.
-#  set_fact:
-#    kubernetes_join_command: "{{ kubernetes_join_command_result.stdout }}"
-#  when: kubernetes_join_command_result.stdout is defined
-#  delegate_to: "{{ item }}"
-#  delegate_facts: true
-#  with_items: "{{ groups['all'] }}"
-#
-#- name: Join node to cluster.
-#  command:
-#    cmd: "{{ kubernetes_join_command }}"
-#    creates: /etc/kubernetes/kubelet.conf
-#  become: true
-#  when: kubernetes_role == 'worker'
-#
-#################################################################################
-## calico
-#################################################################################
-#
-#- name: Check if calico is already deployed by getting all pods.
-#  command: kubectl get pods --all-namespaces
-#  ignore_errors: true
-#  changed_when: false
-#  register: pods
-#  when: kubernetes_role == 'master'
-#
-#- name: Deploy Calico if not already installed.
-#  command: "kubectl apply -f {{ calico_manifest_url }}"
-#  register: calico_result
-#  changed_when: "'created' in calico_result.stdout"
-#  when: kubernetes_role == 'master' and 'calico' not in pods.stdout

tasks/master.yml

@@ -0,0 +1,33 @@
+---
+# Steps for the master node.
+
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#initializing-your-control-plane-node
+- name: Initialize Kubernetes master with kubeadm init.
+  command:
+    cmd: "{{ kubeadm_init_command }}"
+    creates: /etc/kubernetes/admin.conf
+  become: true
+
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#more-information
+- name: Ensure .kube directory exists.
+  file:
+    path: ~/.kube
+    state: directory
+    mode: 0700
+
+- name: Copy the kubectl admin.conf to ~/.kube/conf.
+  copy:
+    src: /etc/kubernetes/admin.conf
+    remote_src: true
+    dest: "/home/{{ ansible_user }}/.kube/config"
+    mode: 0600
+    owner: "{{ ansible_user }}"
+  become: true
+
+- name: Copy the kubectl config to this local machine.
+  fetch:
+    src: "/home/{{ ansible_user }}/.kube/config"
+    dest: ./kube_config
+    flat: true
+  changed_when: false
+  when: copy_kubectl_config_to_local

tasks/redhat_family.yml

@@ -0,0 +1,40 @@
+---
+################################################################################
+# Install kublet, kudeadm, and kubectl
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl
+################################################################################
+
+- name: Add k8s repo for RedHat-like distro.
+  yum_repository:
+    name: kubernetes
+    description: kubernetes
+    baseurl: "{{ rh_yum_base_url }}"
+    enabled: true
+    gpgcheck: true
+    repo_gpgcheck: true
+    gpgkey: "{{ rh_yum_gpg_key }}"
+    exclude: kubelet kudeadm kubectl
+  become: true
+
+- name: Set SELinux to permissive mode.
+  selinux:
+    policy: targeted
+    state: permissive
+  become: true
+
+- name: Update SELinux config to set it to permissive mode at boot.
+  replace:
+    path: /etc/selinux/config
+    regexp: '^SELINUX=enforcing$'
+    replace: 'SELINUX=permissive'
+  become: true
+
+- name: Install kubelet, kubectl, and kubeadm.
+  yum:
+    name:
+      - kubelet-{{ k8s_rhel_version }}-0
+      - kubectl-{{ k8s_rhel_version }}-0
+      - kubeadm-{{ k8s_rhel_version }}-0
+    state: present
+    disable_excludes: kubernetes
+  become: true

tasks/sysctl_and_kernel_modules.yml

@@ -0,0 +1,26 @@
+---
+################################################################################
+# sysctl stuff
+# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic
+################################################################################
+
+- name: Force kernel modules to load.
+  copy:
+    dest: /etc/modules-load.d/k8s.conf
+    content: br_netfilter
+    mode: 0644
+  become: true
+
+- name: Let iptables see bridged traffic.
+  sysctl:
+    name: "{{ item }}"
+    value: '1'
+    state: present
+  become: true
+  loop:
+    - net.bridge.bridge-nf-call-iptables
+    - net.bridge.bridge-nf-call-ip6tables
+
+# TODO Add task to open ports.
+# Currently this role depends on the hosts having these ports open already.
+# https://kubernetes.io/docs/reference/ports-and-protocols/