initial commit

This commit is contained in:
Bartal Læarsson
2025-02-27 21:41:38 +00:00
parent da3f780fd2
commit ccdc52070b
27 changed files with 27 additions and 4 deletions
+245
View File
@@ -0,0 +1,245 @@
# Kubernetes Cluster Deployment with Ansible
This repository contains Ansible playbooks to set up a Kubernetes cluster on existing Ubuntu 24.04 VMs.
## Prerequisites
1. Three Ubuntu 24.04 (Noble Numbat) VMs running
2. SSH access to all VMs
3. Ansible installed on your control machine
4. Sudo access configured on all VMs
## Repository Structure
```
kubernetes-ansible/
├── inventory.yml # Inventory file with VM information
├── main.yml # Main playbook with task orchestration
├── tasks/
│ ├── setup-hosts.yml # Configure /etc/hosts
│ ├── update-dns.yml # Configure DNS
│ ├── ssh.yml # Configure SSH settings
│ ├── setup-kernel.yml # Configure kernel modules and sysctl
│ ├── node-setup.yml # Install Kubernetes components
│ ├── control-plane-init.yml # Initialize control plane
│ └── worker-join.yml # Join worker nodes to cluster
└── README.md
```
## Configuration Steps
1. Clone this repository:
```bash
git clone <repository-url>
cd kubernetes-ansible
```
2. Install required Ansible collections:
```bash
ansible-galaxy collection install community.general
```
3. Edit `inventory.yml` and update the IP addresses of your VMs:
```yaml
controlplane:
ansible_host: 192.168.1.X # Replace with your control plane IP
node01:
ansible_host: 192.168.1.Y # Replace with your first worker node IP
node02:
ansible_host: 192.168.1.Z # Replace with your second worker node IP
```
4. Update the SSH credentials in the inventory file:
```yaml
vars:
ansible_user: ubuntu # Replace with your SSH user
# Use one of these depending on your authentication method:
# ansible_ssh_private_key_file: ~/.ssh/id_rsa
# ansible_ssh_pass: your_password
```
## Deployment
You can deploy the entire cluster at once:
```bash
ansible-playbook -i inventory.yml main.yml
```
Or run deployment incrementally using tags:
```bash
# 1. Configure hosts, DNS, SSH, and kernel
ansible-playbook -i inventory.yml main.yml --tags common
# 2. Install Kubernetes components on all nodes
ansible-playbook -i inventory.yml main.yml --tags k8s-setup
# 3. Initialize control plane
ansible-playbook -i inventory.yml main.yml --tags control-plane
# 4. Join worker nodes
ansible-playbook -i inventory.yml main.yml --tags workers
```
## Iterative Testing Approach
To methodically test the deployment step by step:
### 1. Test Basic Connectivity
```bash
# Verify SSH access to all nodes
ansible -i inventory.yml all -m ping
```
### 2. Test Basic System Configuration
```bash
# Apply common configuration to control plane only
ansible-playbook -i inventory.yml main.yml --tags common --limit controlplane
# Verify hosts file configuration
ansible -i inventory.yml controlplane -m command -a "cat /etc/hosts"
# Verify DNS configuration
ansible -i inventory.yml controlplane -m command -a "systemctl status systemd-resolved"
```
### 3. Test Kernel Configuration
```bash
# Verify kernel modules
ansible -i inventory.yml controlplane -m command -a "lsmod | grep -E 'overlay|br_netfilter'"
# Verify sysctl settings
ansible -i inventory.yml controlplane -m command -a "sysctl net.bridge.bridge-nf-call-iptables net.ipv4.ip_forward"
```
### 4. Test Kubernetes Components
```bash
# Install Kubernetes components on control plane
ansible-playbook -i inventory.yml main.yml --tags k8s-setup --limit controlplane
# Verify containerd is running
ansible -i inventory.yml controlplane -m command -a "systemctl status containerd"
# Verify Kubernetes binaries
ansible -i inventory.yml controlplane -m command -a "kubectl version --client"
```
### 5. Initialize Control Plane
```bash
# Initialize control plane
ansible-playbook -i inventory.yml main.yml --tags control-plane
# Verify control plane initialization
ansible -i inventory.yml controlplane -m command -a "kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes"
```
### 6. Join Worker Nodes One by One
```bash
# Join first worker node
ansible-playbook -i inventory.yml main.yml --tags workers --limit node01
# Verify first worker joined
ansible -i inventory.yml controlplane -m command -a "kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes"
# Join second worker node
ansible-playbook -i inventory.yml main.yml --tags workers --limit node02
# Verify all nodes
ansible -i inventory.yml controlplane -m command -a "kubectl --kubeconfig=/etc/kubernetes/admin.conf get nodes"
```
### 7. Test Network Plugin
```bash
# Check Flannel pods
ansible -i inventory.yml controlplane -m command -a "kubectl --kubeconfig=/etc/kubernetes/admin.conf get pods -n kube-system | grep flannel"
# Deploy test application
ansible -i inventory.yml controlplane -m command -a "kubectl --kubeconfig=/etc/kubernetes/admin.conf create deployment nginx --image=nginx"
ansible -i inventory.yml controlplane -m command -a "kubectl --kubeconfig=/etc/kubernetes/admin.conf expose deployment nginx --port=80 --type=NodePort"
```
### 8. Debugging Common Issues
If any step fails, try these debugging commands:
```bash
# Check service logs
ansible -i inventory.yml <node> -m command -a "journalctl -xeu <service-name>"
# Example for kubelet
ansible -i inventory.yml node01 -m command -a "journalctl -xeu kubelet"
# Check network connectivity
ansible -i inventory.yml controlplane -m command -a "ping -c 1 {{ hostvars['node01']['ansible_host'] }}"
```
### 9. Reset if Needed
To reset a node and start fresh:
```bash
# Reset kubeadm
ansible -i inventory.yml <node> -m command -a "kubeadm reset -f"
# Clean up directories
ansible -i inventory.yml <node> -m command -a "rm -rf /etc/kubernetes /var/lib/kubelet /var/lib/etcd ~/.kube"
```
## Key Components
- **Kernel Configuration**: Sets up required kernel modules (overlay, br_netfilter) and sysctl settings
- **Container Runtime**: Installs and configures containerd with systemd cgroup driver
- **Kubernetes**: Installs kubelet, kubeadm, and kubectl
- **Networking**: Configures Flannel as the CNI plugin with pod CIDR 10.244.0.0/16
## Ubuntu 24.04 Specific Notes
- This playbook has been adjusted for Ubuntu 24.04 (Noble Numbat)
- The containerd installation includes a fallback to the Docker repository if the default package isn't available
- Kernel module loading includes error handling for modules that might be built into the kernel
- DNS configuration is specifically adjusted for systemd-resolved in Ubuntu 24.04
## Validation
After deployment, verify your cluster:
```bash
ssh <your-user>@<controlplane-ip>
kubectl get nodes
```
You should see all nodes in the `Ready` state:
```
NAME STATUS ROLES AGE VERSION
controlplane Ready control-plane 10m v1.28.x
node01 Ready <none> 8m v1.28.x
node02 Ready <none> 8m v1.28.x
```
## Troubleshooting
If nodes don't join the cluster or show issues:
1. Check connectivity between nodes: `ping <node-ip>`
2. Verify firewall allows required ports (6443, 10250)
3. Review logs on problematic nodes: `journalctl -xeu kubelet`
4. Verify DNS resolution: `nslookup controlplane`
## Cleanup
To reset a node and remove Kubernetes:
```bash
sudo kubeadm reset
sudo apt-get purge -y kubeadm kubectl kubelet kubernetes-cni
sudo apt-get autoremove -y
sudo rm -rf ~/.kube /etc/kubernetes /var/lib/kubelet /var/lib/etcd
```
+22
View File
@@ -0,0 +1,22 @@
---
all:
children:
k8s_cluster:
children:
control_plane:
hosts:
controlplane:
ansible_host: 10.203.146.213
workers:
hosts:
node01:
ansible_host: 10.203.146.194
node02:
ansible_host: 10.203.146.61
node03:
ansible_host: 10.203.146.193
vars:
ansible_user: ubuntu
ansible_become: yes
ansible_ssh_private_key_file: ~/.ssh/id_rsa
+45
View File
@@ -0,0 +1,45 @@
---
- name: Common configuration for all nodes
hosts: k8s_cluster
become: yes
tags:
- common
tasks:
- name: Include hosts setup
include_tasks: tasks/setup-hosts.yml
- name: Include DNS setup
include_tasks: tasks/update-dns.yml
- name: Include SSH setup
include_tasks: tasks/ssh.yml
- name: Include kernel setup
include_tasks: tasks/setup-kernel.yml
- name: Kubernetes setup for all nodes
hosts: k8s_cluster
become: yes
tags:
- k8s-setup
tasks:
- name: Include Kubernetes node setup
include_tasks: tasks/node-setup.yml
- name: Initialize Kubernetes control plane
hosts: control_plane
become: yes
tags:
- control-plane
tasks:
- name: Initialize control plane
include_tasks: tasks/control-plane-init.yml
- name: Join worker nodes to cluster
hosts: workers
become: yes
tags:
- workers
tasks:
- name: Join worker nodes
include_tasks: tasks/worker-join.yml
@@ -0,0 +1,98 @@
---
# tasks/control-plane-init.yml
- name: Set Kubernetes network CIDRs for Calico
set_fact:
pod_cidr: "192.168.0.0/16" # Calico default CIDR
service_cidr: "10.96.0.0/16"
- name: Initialize Kubernetes control plane
command: >
kubeadm init
--pod-network-cidr={{ pod_cidr }}
--service-cidr={{ service_cidr }}
--apiserver-advertise-address={{ ansible_default_ipv4.address }}
register: kubeadm_init
args:
creates: /etc/kubernetes/admin.conf
- name: Create .kube directory for current user
file:
path: "/home/{{ ansible_user }}/.kube"
state: directory
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: '0755'
- name: Copy Kubernetes admin.conf to user's .kube/config
copy:
src: /etc/kubernetes/admin.conf
dest: "/home/{{ ansible_user }}/.kube/config"
remote_src: yes
owner: "{{ ansible_user }}"
group: "{{ ansible_user }}"
mode: '0644'
- name: Install Calico Operator
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.1/manifests/tigera-operator.yaml
register: calico_operator_result
changed_when: "'created' in calico_operator_result.stdout"
failed_when:
- calico_operator_result.rc != 0
- "'already exists' not in calico_operator_result.stderr"
- name: Install Calico Custom Resources
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.1/manifests/custom-resources.yaml
register: calico_resources_result
changed_when: "'created' in calico_resources_result.stdout"
failed_when:
- calico_resources_result.rc != 0
- "'already exists' not in calico_resources_result.stderr"
- name: Wait for calico-system namespace to be created
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf get namespace | grep calico-system
register: calico_ns
until: calico_ns.rc == 0
retries: 30
delay: 10
changed_when: false
- name: Wait for Calico pods to be ready
shell: |
# Give pods some time to be created
sleep 30
kubectl --kubeconfig=/etc/kubernetes/admin.conf -n calico-system wait --for=condition=ready pods --all --timeout=300s
register: calico_pods_ready
changed_when: false
failed_when: false # Don't fail if timeout occurs, we'll verify the node status later
- name: Wait for CoreDNS to be ready
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf -n kube-system wait --for=condition=ready pods -l k8s-app=kube-dns --timeout=120s
register: coredns_ready
changed_when: false
failed_when: false
- name: Wait for node to be ready
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf wait --for=condition=ready nodes --all --timeout=120s
register: node_ready
changed_when: false
- name: Remove taint from control plane node
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/control-plane-
register: taint_result
changed_when: "'untainted' in taint_result.stdout"
failed_when: false # This might fail if the taint doesn't exist, which is fine
- name: Remove load balancer label from all nodes
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf label nodes --all node.kubernetes.io/exclude-from-external-load-balancers-
register: label_result
changed_when: "'labeled' in label_result.stdout"
failed_when: false # This might fail if the label doesn't exist, which is fine
- name: Get join command
command: kubeadm token create --print-join-command
register: join_command
changed_when: false
- name: Store join command
set_fact:
join_command: "{{ join_command.stdout }}"
@@ -0,0 +1,169 @@
---
# tasks/node-setup.yml
- name: Update apt cache
apt:
update_cache: yes
- name: Install apt transport packages
apt:
name:
- apt-transport-https
- ca-certificates
- curl
- gnupg
state: present
# Add Docker GPG key
- name: Add Docker GPG key
apt_key:
url: https://download.docker.com/linux/ubuntu/gpg
state: present
- name: Add Docker repository
apt_repository:
repo: "deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
state: present
filename: docker
- name: Install containerd from Docker repository
apt:
name: containerd.io
state: present
update_cache: yes
- name: Create containerd directory
file:
path: /etc/containerd
state: directory
mode: '0755'
- name: Check if containerd config command works
command: which containerd
register: containerd_check
changed_when: false
failed_when: false
- name: Generate default containerd config (if containerd exists)
command: containerd config default
register: containerd_config
changed_when: false
when: containerd_check.rc == 0
- name: Create a basic containerd config (if command failed)
copy:
content: |
version = 2
[plugins]
[plugins."io.containerd.grpc.v1.cri"]
[plugins."io.containerd.grpc.v1.cri".containerd]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
dest: /etc/containerd/config.toml
owner: root
group: root
mode: '0644'
when: containerd_check.rc != 0
- name: Write containerd config from generated output
copy:
content: "{{ containerd_config.stdout }}"
dest: /etc/containerd/config.toml
owner: root
group: root
mode: '0644'
when: containerd_check.rc == 0 and containerd_config.stdout is defined and containerd_config.stdout != ""
- name: Enable SystemdCgroup in containerd config
replace:
path: /etc/containerd/config.toml
regexp: 'SystemdCgroup = false'
replace: 'SystemdCgroup = true'
when: containerd_check.rc == 0
- name: Restart containerd
systemd:
name: containerd
state: restarted
enabled: yes
- name: Get latest Kubernetes version
shell: curl -L -s https://dl.k8s.io/release/stable.txt | awk 'BEGIN { FS="." } { printf "%s.%s", $1, $2 }'
register: kube_latest
changed_when: false
- name: Create keyrings directory
file:
path: /etc/apt/keyrings
state: directory
mode: '0755'
- name: Download Kubernetes apt key
get_url:
url: "https://pkgs.k8s.io/core:/stable:/{{ kube_latest.stdout }}/deb/Release.key"
dest: /tmp/kubernetes-release.key
mode: '0644'
- name: Add Kubernetes apt key
shell: cat /tmp/kubernetes-release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
args:
creates: /etc/apt/keyrings/kubernetes-apt-keyring.gpg
- name: Add Kubernetes apt repository
copy:
content: "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/{{ kube_latest.stdout }}/deb/ /"
dest: /etc/apt/sources.list.d/kubernetes.list
owner: root
group: root
mode: '0644'
- name: Update apt cache
apt:
update_cache: yes
- name: Install Kubernetes components
apt:
name:
- kubelet
- kubeadm
- kubectl
state: present
- name: Hold Kubernetes packages
dpkg_selections:
name: "{{ item }}"
selection: hold
loop:
- kubelet
- kubeadm
- kubectl
- name: Check if crictl is installed
command: which crictl
register: crictl_check
changed_when: false
failed_when: false
- name: Configure crictl
shell: |
crictl config \
--set runtime-endpoint=unix:///run/containerd/containerd.sock \
--set image-endpoint=unix:///run/containerd/containerd.sock
changed_when: false
when: crictl_check.rc == 0
- name: Get primary network interface IP
shell: ip -4 addr show {{ ansible_default_ipv4.interface }} | grep "inet" | head -1 | awk '{print $2}' | cut -d/ -f1
register: node_ip
changed_when: false
- name: Configure kubelet node IP
copy:
content: "KUBELET_EXTRA_ARGS='--node-ip {{ node_ip.stdout }}'"
dest: /etc/default/kubelet
owner: root
group: root
mode: '0644'
@@ -0,0 +1,33 @@
---
- name: Remove unwanted entries from /etc/hosts
lineinfile:
path: /etc/hosts
regexp: "{{ item }}"
state: absent
loop:
- "^.*ubuntu-jammy.*$"
- "^.*{{ ansible_hostname }}.*$"
- name: Get primary IP address
shell: "ip route | grep default | grep -Pv '10\\.\\d+\\.\\d+\\.\\d+' | awk '{ print $9 }'"
register: primary_ip
changed_when: false
- name: Set PRIMARY_IP environment variable
lineinfile:
path: /etc/environment
line: "PRIMARY_IP={{ primary_ip.stdout }}"
state: present
- name: Add controlplane entry to /etc/hosts
lineinfile:
path: /etc/hosts
line: "{{ hostvars['controlplane']['ansible_host'] }} controlplane"
state: present
- name: Add worker nodes to /etc/hosts
lineinfile:
path: /etc/hosts
line: "{{ hostvars[item]['ansible_host'] }} {{ item }}"
state: present
loop: "{{ groups['workers'] }}"
@@ -0,0 +1,44 @@
---
# tasks/setup-kernel.yml
- name: Create Kubernetes kernel modules config file
copy:
content: |
overlay
br_netfilter
dest: /etc/modules-load.d/k8s.conf
owner: root
group: root
mode: '0644'
- name: Attempt to load overlay module
shell: modprobe overlay || true
changed_when: false
- name: Attempt to load br_netfilter module
shell: modprobe br_netfilter || true
changed_when: false
- name: Create Kubernetes sysctl config file
copy:
content: |
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
dest: /etc/sysctl.d/k8s.conf
owner: root
group: root
mode: '0644'
register: sysctl_conf
- name: Apply sysctl changes
command: sysctl --system
when: sysctl_conf.changed
- name: Check loaded modules
shell: "lsmod | grep -E 'overlay|br_netfilter' || echo 'Modules may be built into kernel'"
register: modules_check
changed_when: false
- name: Display module status
debug:
var: modules_check.stdout_lines
+18
View File
@@ -0,0 +1,18 @@
---
# tasks/ssh.yml
- name: Enable password authentication in sshd_config
replace:
path: /etc/ssh/sshd_config
regexp: '#PasswordAuthentication'
replace: 'PasswordAuthentication'
- name: Enable KbdInteractiveAuthentication in sshd_config
replace:
path: /etc/ssh/sshd_config
regexp: 'KbdInteractiveAuthentication no'
replace: 'KbdInteractiveAuthentication yes'
- name: Restart SSH service
systemd:
name: ssh
state: restarted
@@ -0,0 +1,18 @@
---
- name: Ensure systemd-resolved is running
systemd:
name: systemd-resolved
state: started
enabled: yes
- name: Configure systemd-resolved to use Google DNS
lineinfile:
path: /etc/systemd/resolved.conf
regexp: '^#?DNS='
line: 'DNS=8.8.8.8'
state: present
- name: Restart systemd-resolved service
systemd:
name: systemd-resolved
state: restarted
@@ -0,0 +1,13 @@
---
# tasks/worker-join.yml
- name: Get join command from control plane
delegate_to: controlplane
command: kubeadm token create --print-join-command
register: join_command_result
changed_when: false
run_once: true
- name: Join worker node to the cluster
command: "{{ join_command_result.stdout }}"
args:
creates: /etc/kubernetes/kubelet.conf