diff --git a/defaults/main.yml b/defaults/main.yml index e353ea6..084cc7f 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,11 +1,17 @@ --- k8s_version: "1.22" +k8s_rhel_version: "1.22.4" pod_network_cidr: "10.10.0.0/16" kubeadm_init_command: "kubeadm init --pod-network-cidr={{ pod_network_cidr }}" +# This flag controls whether or not the kubectl config file is copied from a +# master node to the local machine running the ansible playbook. +copy_kubectl_config_to_local: false + apt_gpg_key_url: "https://packages.cloud.google.com/apt/doc/apt-key.gpg" apt_gpg_key_fingerprint: "59FE0256827269DC81578F928B57C5C2836F4BEB" +# This Ubuntu URL should also work for Debian. apt_repo: "deb https://apt.kubernetes.io/ kubernetes-xenial main" rh_yum_base_url: "https://packages.cloud.google.com/\ @@ -15,3 +21,12 @@ rh_yum_gpg_key: - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg calico_manifest_url: "https://docs.projectcalico.org/manifests/calico.yaml" + +# k8s static pods +kube_apiserver: 'k8s.gcr.io/kube-apiserver' +kube_controller_manager: 'k8s.gcr.io/kube-controller-manager' +kube_scheduler: 'k8s.gcr.io/kube-scheduler' +kube_proxy: 'k8s.gcr.io/kube-proxy' +etcd: 'k8s.gcr.io/etcd' +coredns: 'k8s.gcr.io/coredns' +pause: 'k8s.gcr.io/pause' diff --git a/tasks/calico.yml b/tasks/calico.yml new file mode 100644 index 0000000..56b7f5b --- /dev/null +++ b/tasks/calico.yml @@ -0,0 +1,18 @@ +--- +################################################################################ +# calico +# https://projectcalico.docs.tigera.io/getting-started/kubernetes/self-managed-onprem/onpremises +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#pod-network +################################################################################ + +- name: Check if calico is already deployed by getting all pods. + command: kubectl get pods --all-namespaces + ignore_errors: true + changed_when: false + register: pods + +- name: Deploy Calico if not already installed. + command: "kubectl apply -f {{ calico_manifest_url }}" + register: calico_result + changed_when: "'created' in calico_result.stdout" + when: "'calico' not in pods.stdout" diff --git a/tasks/configure_docker.yml b/tasks/configure_docker.yml index 13133a4..43b767e 100644 --- a/tasks/configure_docker.yml +++ b/tasks/configure_docker.yml @@ -1,5 +1,9 @@ --- +################################################################################ +# Congifure docker for k8s. # https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker +################################################################################ + - name: Install docker daemon config. template: src: docker-daemon-config.json @@ -17,6 +21,6 @@ become: true when: docker_daemon_config.changed # ansible-lint complains that this task should be a handler but we need this - # daemon_reload and service restarted before kublet is installed, other wise + # daemon_reload and service restarted before kublet is installed, otherwise # the kibelet service has issues. tags: ['skip_ansible_lint'] diff --git a/tasks/debian_family.yml b/tasks/debian_family.yml new file mode 100644 index 0000000..d59bdb1 --- /dev/null +++ b/tasks/debian_family.yml @@ -0,0 +1,44 @@ +--- +################################################################################ +# Install kublet, kudeadm, and kubectl +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl +################################################################################ + +- name: Install kubectl dependencies for Debian-like distros. + package: + name: + - apt-transport-https + - ca-certificates + - curl + state: present + become: true + +- name: Add GPG key to apt for Debian-like distro. + apt_key: + url: "{{ apt_gpg_key_url }}" + id: "{{ apt_gpg_key_fingerprint }}" + state: present + become: true + +- name: Add k8s repository for Debian-like distro. + apt_repository: + repo: "{{ apt_repo }}" + state: present + update_cache: true + become: true + +- name: Add k8s apt preferences file to pin a version. + template: + src: apt-preferences-k8s.j2 + dest: /etc/apt/preferences.d/kubernetes + mode: 0644 + become: true + +- name: Install kubectl Debian-like distro. + package: + name: + - kubectl + - kubeadm + - kubelet + state: present + become: true diff --git a/tasks/disable_swap.yml b/tasks/disable_swap.yml new file mode 100644 index 0000000..17b29f5 --- /dev/null +++ b/tasks/disable_swap.yml @@ -0,0 +1,27 @@ +--- +################################################################################ +# Disable swap +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin +################################################################################ + +- name: Get swap status. + command: swapon -s + # It's unclear why you need to be root for this commend. This command works + # fine manually as the admin user, but the only way I could get this role to + # work was to become root. + become: true + changed_when: false + register: swap_status + +- name: Disable swap if needed. + command: swapoff -a + become: true + changed_when: true + when: swap_status.stdout | length > 0 + +- name: comment out swap in fstab. + replace: + path: /etc/fstab + regexp: '^([^#].*swap.*)$' + replace: '# \1' + become: true diff --git a/tasks/join_nodes.yml b/tasks/join_nodes.yml new file mode 100644 index 0000000..06fdb4c --- /dev/null +++ b/tasks/join_nodes.yml @@ -0,0 +1,26 @@ +--- +################################################################################ +# Join nodes to cluster. +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#join-nodes +################################################################################ + +- name: Get the kubeadm join command from the Kubernetes master. + command: kubeadm token create --print-join-command + changed_when: false + when: kubernetes_role == 'master' + register: kubernetes_join_command_result + +- name: Set the kubeadm join command globally. + set_fact: + kubernetes_join_command: "{{ kubernetes_join_command_result.stdout }}" + when: kubernetes_join_command_result.stdout is defined + delegate_to: "{{ item }}" + delegate_facts: true + with_items: "{{ groups['all'] }}" + +- name: Join node to cluster. + command: + cmd: "{{ kubernetes_join_command }}" + creates: /etc/kubernetes/kubelet.conf + become: true + when: kubernetes_role == 'worker' diff --git a/tasks/main.yml b/tasks/main.yml index 41f8f0e..fdb1960 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,122 +1,18 @@ --- -# Configure docker for k8s. +# Congifure docker for k8s. +# https://kubernetes.io/docs/setup/production-environment/container-runtimes/#docker - include_tasks: configure_docker.yml -################################################################################ # Install kublet, kudeadm, and kubectl -################################################################################ - -- name: Install kubectl dependencies. - package: - name: - - apt-transport-https - - ca-certificates - - curl - state: present - become: true - when: ansible_os_family == "Debian" - -- name: Add GPG key to apt for Ubuntu. - apt_key: - url: "{{ apt_gpg_key_url }}" - id: "{{ apt_gpg_key_fingerprint }}" - state: present - become: true - when: ansible_os_family == "Debian" - -- name: Add k8s repository for Debian-like distro. - apt_repository: - repo: "{{ apt_repo }}" - state: present - update_cache: true - become: true - when: ansible_os_family == "Debian" - -- name: Add k8s apt preferences file to pin a version. - template: - src: apt-preferences-k8s.j2 - dest: /etc/apt/preferences.d/kubernetes - mode: 0644 - become: true - when: ansible_os_family == "Debian" - -- name: Install kubectl. - package: - name: - - kubectl - - kubeadm - - kubelet - state: present - become: true +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl +- include_tasks: debian_family.yml when: ansible_os_family == "Debian" +- include_tasks: redhat_family.yml + when: ansible_os_family == "RedHat" -- name: Add k8s repo for RedHat-like distro. - yum_repository: - name: kubernetes - description: kubernetes - baseurl: "{{ rh_yum_base_url }}" - enabled: true - gpgcheck: true - repo_gpgcheck: true - gpgkey: "{{ rh_yum_gpg_key }}" - exclude: kubelet kudeadm kubectl - become: true - register: yum_repo - when: ansible_os_family == 'RedHat' - -- name: debug - debug: - var: kubernetes_role - -- name: Set SELinux to permissive mode. - selinux: - policy: targeted - state: permissive - become: true - when: ansible_os_family == 'RedHat' - -- name: Update SELinux config to set it to permissive mode at boot. - replace: - path: /etc/selinux/config - regexp: '^SELINUX=enforcing$' - replace: 'SELINUX=permissive' - become: true - when: ansible_os_family == 'RedHat' - -- name: Install kubelet, kubectl, and kubeadm. - yum: - name: - - kubelet - - kubectl - - kubeadm - state: present - disable_excludes: kubernetes - become: true - when: ansible_os_family == 'RedHat' - -################################################################################ -# sysctl stuff +# Configure sysctl kernel modules # https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic -################################################################################ - -- name: Force kernel modules to load. - copy: - dest: /etc/modules-load.d/k8s.conf - content: br_netfilter - mode: 0644 - become: true - -- name: Let iptables see bridged traffic. - sysctl: - name: "{{ item }}" - value: '1' - state: present - become: true - loop: - - net.bridge.bridge-nf-call-iptables - - net.bridge.bridge-nf-call-ip6tables - -################################################################################ +- include_tasks: sysctl_and_kernel_modules.yml - name: Ensure kubelet is started and enabled at boot. service: @@ -125,36 +21,12 @@ enabled: true become: true -################################################################################ - -# STARTHERE This command is failing on centos7 machine even though the command -# works manually. -#- name: Get swap status w/ shell. -# shell: swapon -s -- name: Get swap status. - command: swapon -s - # It's unclear why you need to be root for this commend. This command works - # fine manually as the admin user, but the only way I could get this script to - # work was to become root. - become: true - changed_when: false - register: swap_status -- name: Disable swap if needed. - command: swapoff -a - become: true - changed_when: true - when: swap_status.stdout | length > 0 -- name: comment out swap in fstab. - replace: - path: /etc/fstab - regexp: '^([^#].*swap.*)$' - replace: '# \1' - become: true +# Disable swap +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#before-you-begin -################################################################################ -# master -################################################################################ +- include_tasks: disable_swap.yml +# Pre-pull k8s status pods. - name: Check the docker images currently on machine. command: docker images # We do this as root because the docker admin will not be able to until we @@ -163,8 +35,6 @@ register: docker_images changed_when: false -# TODO REM make these images full paths. and vars -# TODO REM check nodes id they hae these images also. - name: Pre-grab the container images for the static pods. command: kubeadm config images pull # We do this as root because the docker admin will not be able to until we @@ -172,96 +42,24 @@ become: true changed_when: true when: > - ( 'kube-apiserver' not in docker_images.stdout or - 'kube-controller-manager' not in docker_images.stdout or - 'kube-scheduler' not in docker_images.stdout or - 'kube-proxy' not in docker_images.stdout or - 'etcd' not in docker_images.stdout or - 'coredns' not in docker_images.stdout or - 'pause' not in docker_images.stdout ) + ( kube_apiserver not in docker_images.stdout or + kube_controller_manager not in docker_images.stdout or + kube_scheduler not in docker_images.stdout or + kube_proxy not in docker_images.stdout or + etcd not in docker_images.stdout or + coredns not in docker_images.stdout or + pause not in docker_images.stdout ) -- name: Initialize Kubernetes master with kubeadm init - command: - cmd: "{{ kubeadm_init_command }}" - creates: /etc/kubernetes/admin.conf - become: true - # TODO REM do we need this register? - register: kubeadmin_init - ignore_errors: true - when: kubernetes_role == 'master' - -- name: Debug kubectl init. - debug: - var: kubeadmin_init - when: kubernetes_role == 'master' - -# TODO REM find a way to copy this file to local machine. -- name: Ensure .kube directory exists. - file: - path: ~/.kube - state: directory - mode: 0700 +# master +- include_tasks: master.yml when: kubernetes_role == 'master' -- name: Copy the kubectl admin.conf to ~/.kube/conf. - copy: - src: /etc/kubernetes/admin.conf - remote_src: true - dest: "/home/{{ ansible_user }}/.kube/config" - mode: 0600 - owner: "{{ ansible_user }}" - become: true +# calico +# https://projectcalico.docs.tigera.io/getting-started/kubernetes/self-managed-onprem/onpremises +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#pod-network +- include_tasks: calico.yml when: kubernetes_role == 'master' -################################################################################ # Join nodes to cluster. -################################################################################ - -- name: Print the role. - debug: - var: kubernetes_role -- name: Get the kubeadm join command from the Kubernetes master. - command: kubeadm token create --print-join-command - changed_when: false - when: kubernetes_role == 'master' - register: kubernetes_join_command_result - -# TODO REM remove this -- name: debug - debug: - var: kubernetes_join_command_result - -# STARTHERE try to open firewall stuff on centos 7. -# This is not how we want to do this just grab the join command from a mater no -# need to set a fact. -#- name: Set the kubeadm join command globally. -# set_fact: -# kubernetes_join_command: "{{ kubernetes_join_command_result.stdout }}" -# when: kubernetes_join_command_result.stdout is defined -# delegate_to: "{{ item }}" -# delegate_facts: true -# with_items: "{{ groups['all'] }}" -# -#- name: Join node to cluster. -# command: -# cmd: "{{ kubernetes_join_command }}" -# creates: /etc/kubernetes/kubelet.conf -# become: true -# when: kubernetes_role == 'worker' -# -################################################################################# -## calico -################################################################################# -# -#- name: Check if calico is already deployed by getting all pods. -# command: kubectl get pods --all-namespaces -# ignore_errors: true -# changed_when: false -# register: pods -# when: kubernetes_role == 'master' -# -#- name: Deploy Calico if not already installed. -# command: "kubectl apply -f {{ calico_manifest_url }}" -# register: calico_result -# changed_when: "'created' in calico_result.stdout" -# when: kubernetes_role == 'master' and 'calico' not in pods.stdout +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#join-nodes +- include_tasks: join_nodes.yml diff --git a/tasks/master.yml b/tasks/master.yml new file mode 100644 index 0000000..5e0e7ae --- /dev/null +++ b/tasks/master.yml @@ -0,0 +1,33 @@ +--- +# Steps for the master node. + +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#initializing-your-control-plane-node +- name: Initialize Kubernetes master with kubeadm init. + command: + cmd: "{{ kubeadm_init_command }}" + creates: /etc/kubernetes/admin.conf + become: true + +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/create-cluster-kubeadm/#more-information +- name: Ensure .kube directory exists. + file: + path: ~/.kube + state: directory + mode: 0700 + +- name: Copy the kubectl admin.conf to ~/.kube/conf. + copy: + src: /etc/kubernetes/admin.conf + remote_src: true + dest: "/home/{{ ansible_user }}/.kube/config" + mode: 0600 + owner: "{{ ansible_user }}" + become: true + +- name: Copy the kubectl config to this local machine. + fetch: + src: "/home/{{ ansible_user }}/.kube/config" + dest: ./kube_config + flat: true + changed_when: false + when: copy_kubectl_config_to_local diff --git a/tasks/redhat_family.yml b/tasks/redhat_family.yml new file mode 100644 index 0000000..e314053 --- /dev/null +++ b/tasks/redhat_family.yml @@ -0,0 +1,40 @@ +--- +################################################################################ +# Install kublet, kudeadm, and kubectl +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-kubeadm-kubelet-and-kubectl +################################################################################ + +- name: Add k8s repo for RedHat-like distro. + yum_repository: + name: kubernetes + description: kubernetes + baseurl: "{{ rh_yum_base_url }}" + enabled: true + gpgcheck: true + repo_gpgcheck: true + gpgkey: "{{ rh_yum_gpg_key }}" + exclude: kubelet kudeadm kubectl + become: true + +- name: Set SELinux to permissive mode. + selinux: + policy: targeted + state: permissive + become: true + +- name: Update SELinux config to set it to permissive mode at boot. + replace: + path: /etc/selinux/config + regexp: '^SELINUX=enforcing$' + replace: 'SELINUX=permissive' + become: true + +- name: Install kubelet, kubectl, and kubeadm. + yum: + name: + - kubelet-{{ k8s_rhel_version }}-0 + - kubectl-{{ k8s_rhel_version }}-0 + - kubeadm-{{ k8s_rhel_version }}-0 + state: present + disable_excludes: kubernetes + become: true diff --git a/tasks/sysctl_and_kernel_modules.yml b/tasks/sysctl_and_kernel_modules.yml new file mode 100644 index 0000000..c6cc311 --- /dev/null +++ b/tasks/sysctl_and_kernel_modules.yml @@ -0,0 +1,26 @@ +--- +################################################################################ +# sysctl stuff +# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#letting-iptables-see-bridged-traffic +################################################################################ + +- name: Force kernel modules to load. + copy: + dest: /etc/modules-load.d/k8s.conf + content: br_netfilter + mode: 0644 + become: true + +- name: Let iptables see bridged traffic. + sysctl: + name: "{{ item }}" + value: '1' + state: present + become: true + loop: + - net.bridge.bridge-nf-call-iptables + - net.bridge.bridge-nf-call-ip6tables + +# TODO Add task to open ports. +# Currently this role depends on the hosts having these ports open already. +# https://kubernetes.io/docs/reference/ports-and-protocols/