commit c14452a5fe473c0224345df1aafc917e8db31616 Author: Bartal Læarsson Date: Tue Feb 4 21:38:54 2025 +0000 initial commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/multipass-kubeadm-cluster/cloudinit.yaml b/multipass-kubeadm-cluster/cloudinit.yaml new file mode 100644 index 0000000..0684f68 --- /dev/null +++ b/multipass-kubeadm-cluster/cloudinit.yaml @@ -0,0 +1,73 @@ +#cloud-config +package_update: true +package_upgrade: true + +write_files: + - path: /etc/modules-load.d/containerd.conf + content: | + overlay + br_netfilter + - path: /etc/sysctl.d/99-kubernetes-cri.conf + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + +runcmd: + # Disable swap + - swapoff -a + - sed -i '/swap/d' /etc/fstab + + # Load necessary modules + - modprobe overlay + - modprobe br_netfilter + + # Apply sysctl params + - sysctl --system + + # Remove conflicting packages + - for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do apt-get remove -y $pkg; done + + # Install required packages + - apt-get install -y ca-certificates curl gpg apt-transport-https + + # Add Docker GPG key + - mkdir -p -m 0755 /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + - chmod a+r /etc/apt/keyrings/docker.asc + + # Add Docker repository + - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo ${UBUNTU_CODENAME:-$VERSION_CODENAME}) stable" | tee /etc/apt/sources.list.d/docker.list + + # Update package lists + - apt-get update + + # Install containerd + - apt-get install -y containerd.io + + # Configure containerd + - mkdir -p /etc/containerd + - containerd config default | tee /etc/containerd/config.toml + - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + - systemctl restart containerd + - systemctl enable containerd + + # Add Kubernetes GPG key + - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + + # Add Kubernetes repository + - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + + # Update package lists + - apt-get update + + # Install kubelet, kubeadm, and kubectl + - apt-get install -y kubelet kubeadm kubectl + - apt-mark hold kubelet kubeadm kubectl + + # Enable kubelet service + - systemctl enable --now kubelet + + # pull kubeadm related image + - sudo kubeadm config images pull + diff --git a/multipass-kubeadm-cluster/containerd-install-and-cheat-sheet.md b/multipass-kubeadm-cluster/containerd-install-and-cheat-sheet.md new file mode 100644 index 0000000..5a6c22a --- /dev/null +++ b/multipass-kubeadm-cluster/containerd-install-and-cheat-sheet.md @@ -0,0 +1,97 @@ +# Installation +```bash +# Add Docker's official GPG key: +sudo apt-get update +sudo apt-get install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +``` + +```bash +sudo apt-get install containerd.io +``` + +--- + +# Cheat Sheet +## **1. Images (Pull, List, Remove)** +| Action | Docker | containerd (`ctr`) | +|--------|--------|------------------| +| Pull an image | `docker pull busybox` | `sudo ctr images pull docker.io/library/busybox:latest` | +| List images | `docker images` | `sudo ctr images list` | +| Remove an image | `docker rmi busybox` | `sudo ctr images rm docker.io/library/busybox:latest` | + +--- + +## **2. Containers (Run, List, Stop, Remove)** +| Action | Docker | containerd (`ctr`) | +|--------|--------|------------------| +| Run a container (interactive) | `docker run -it busybox sh` | `sudo ctr run --rm --tty docker.io/library/busybox:latest my-container sh` | +| Run a container (detached) | `docker run -d busybox` | `sudo ctr run -d --tty docker.io/library/busybox:latest my-container sh` | +| List running containers | `docker ps` | `sudo ctr tasks ls` | +| Stop a running container | `docker stop my-container` | `sudo ctr tasks kill my-container` | +| Remove a stopped container | `docker rm my-container` | `sudo ctr containers delete my-container` | + +--- + +## **3. Container Execution (Attach, Exec, Logs)** +| Action | Docker | containerd (`ctr`) | +|--------|--------|------------------| +| Attach to a running container | `docker attach my-container` | `sudo ctr tasks exec --tty my-container sh` | +| Execute a command inside a container | `docker exec -it my-container sh` | `sudo ctr tasks exec --tty my-container sh` | +| View container logs | `docker logs my-container` | **Not directly available in `ctr`** (use `crictl logs ` if using Kubernetes) | + +--- + +## **4. Container Lifecycle Management** +| Action | Docker | containerd (`ctr`) | +|--------|--------|------------------| +| Restart a container | `docker restart my-container` | `sudo ctr tasks kill -s SIGTERM my-container && sudo ctr run -d --tty docker.io/library/busybox:latest my-container sh` | +| Pause a container | `docker pause my-container` | **Not directly supported** | +| Unpause a container | `docker unpause my-container` | **Not directly supported** | + +--- + +## **5. Volumes & Networking** +Unlike Docker, `containerd` does **not** manage networking or volumes directly. You need to use **CNI (Container Network Interface)** for networking and manually mount host volumes when running a container. + +For example, in Docker: +```sh +docker run -v /host/path:/container/path busybox +``` +In `containerd`, you must specify the mount manually: +```sh +sudo ctr run --mount type=bind,src=/host/path,dst=/container/path,options=rbind:rw docker.io/library/busybox:latest my-container sh +``` + +--- + +## **6. Checking System Info** +| Action | Docker | containerd (`ctr`) | +|--------|--------|------------------| +| Show system info | `docker info` | `sudo ctr version && sudo ctr plugins ls` | + +--- + +## **7. Removing Everything (Clean Up)** +| Action | Docker | containerd (`ctr`) | +|--------|--------|------------------| +| Remove all stopped containers | `docker container prune` | `sudo ctr containers list -q | xargs -r sudo ctr containers delete` | +| Remove all images | `docker image prune -a` | `sudo ctr images list -q | xargs -r sudo ctr images rm` | + +--- + +### **Summary** +- `ctr` is more **low-level** than Docker, so it lacks built-in networking, volume management, and logging. +- You must manually handle networking using **CNI**. +- For Kubernetes workloads, you usually use `crictl` instead of `ctr`. +- `containerd` focuses purely on running containers; it does not replace Docker Compose or Swarm. + diff --git a/multipass-kubeadm-cluster/install-kubeadm.md b/multipass-kubeadm-cluster/install-kubeadm.md new file mode 100644 index 0000000..30dbe61 --- /dev/null +++ b/multipass-kubeadm-cluster/install-kubeadm.md @@ -0,0 +1,138 @@ +# Install kubeadm for Kubernetes version 1.32 + +## Prerquisites +- A compatible Linux host +- 2 GB or more of RAM per machine +- 2 CPUs or more for control plane machines. +- Full network connectivity between all machines in the cluster (public or private network is fine). +- Unique hostname, MAC address, and product_uuid for every node. + - Verify the `MAC address` and `product_uuid` are unique for every node + You can get the `MAC address` of the network interfaces using the command `ip link` or `ifconfig -a` + The `product_uuid `can be checked by using the command `sudo cat /sys/class/dmi/id/product_uuid` + It is very likely that hardware devices will have unique addresses, although some virtual machines may have identical values. + Kubernetes uses these values to uniquely identify the nodes in the cluster. + If these values are not unique to each node, the installation process may fail. +- Certain ports are open on your machines: + +### Control plane + +|Protocol|Direction|Port Range|Purpose|Used By| +|---|---|---|---|---| +|TCP|Inbound|6443|Kubernetes API server|All| +|TCP|Inbound|2379-2380|etcd server client API|kube-apiserver, etcd| +|TCP|Inbound|10250|Kubelet API|Self, Control plane| +|TCP|Inbound|10259|kube-scheduler|Self| +|TCP|Inbound|10257|kube-controller-manager|Self| +Although etcd ports are included in control plane section, +you can also host your own etcd cluster externally or on custom ports. + +### Worker node(s) +|Protocol|Direction|Port Range|Purpose| Used By | +|---|---|---|---|---| +|TCP|Inbound|10250|Kubelet API|Self, Control plane | +|TCP|Inbound|10256|kube-proxy|Self, Load balancers | +|TCP|Inbound|30000-32767|NodePort Services†|All| +† Default port range for NodePort Services. + +All default port numbers can be overridden. +When custom ports are used those ports need to be open instead of defaults mentioned here. + +One common example is API server port that is sometimes switched to 443. +Alternatively, the default port is kept as is and API server is put behind a load balancer that listens on 443 +and routes the requests to API server on the default port. + +### Check required ports +These required ports need to be open in order for Kubernetes components to communicate with each other. +You can use tools like netcat to check if a port is open. For example: +```bash +nc 127.0.0.1 6443 -v +``` +The pod network plugin you use may also require certain ports to be open. +Since this differs with each pod network plugin, +please see the documentation for the plugins about what port(s) those need. + +## Swap Configuration +The default behavior of a kubelet is to fail to start if swap memory is detected on a node. +This means that swap should either be disabled or tolerated by kubelet. + +- To tolerate swap, add `failSwapOn:` false to kubelet configuration or as a command line argument. + Note: even if `failSwapOn: false` is provided, workloads wouldn't have swap access by default. + This can be changed by setting a `swapBehavior`, again in the kubelet configuration file. + To use swap, set a `swapBehavior` other than the default `NoSwap ` setting. + +- To disable swap, `sudo swapoff -a` can be used to disable swapping temporarily. + To make this change persistent across reboots, make sure swap is disabled in config files like `/etc/fstab`, `systemd.swap` + depending how it was configured on your system + +## Install Container Runtime +Use containerd + +Make sure to remove anything docker/container related installations +```bash +for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do sudo apt-get remove $pkg; done +``` + +```bash +# Add Docker's official GPG key: +sudo apt-get update +sudo apt-get install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null +sudo apt-get update +``` + +```bash +sudo apt-get install containerd.io +``` + +## Install kubeadm, kubelet, and kubectl +You will install these packages on all of your machines: + +- `kubeadm`: the command to bootstrap the cluster. +- `kubelet`: the component that runs on all of the machines in your cluster and does things like starting pods and containers. +- `kubectl`: the command line util to talk to your cluster. + +1. Update the apt package index and install packages needed to use the Kubernetes apt repository: + +```bash +sudo apt-get update +# apt-transport-https may be a dummy package; if so, you can skip that package +sudo apt-get install -y apt-transport-https ca-certificates curl gpg +``` + +2. Download the public signing key for the Kubernetes package repositories. +The same signing key is used for all repositories so you can disregard the version in the URL: + +```bash +# If the directory `/etc/apt/keyrings` does not exist, it should be created before the curl command, read the note below. +# sudo mkdir -p -m 755 /etc/apt/keyrings +curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg +``` +3. Add the appropriate Kubernetes apt repository. + Please note that this repository have packages only for Kubernetes 1.32; + for other Kubernetes minor versions, you need to change the Kubernetes minor version in the URL to match your desired minor version + (you should also check that you are reading the documentation for the version of Kubernetes that you plan to install). + +```bash +# This overwrites any existing configuration in /etc/apt/sources.list.d/kubernetes.list +echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list +``` +4. Update the apt package index, install kubelet, kubeadm and kubectl, and pin their version: +```bash +sudo apt-get update +sudo apt-get install -y kubelet kubeadm kubectl +sudo apt-mark hold kubelet kubeadm kubectl +``` + +(Optional) Enable the kubelet service before running kubeadm: +```bash +sudo systemctl enable --now kubelet +``` +The kubelet is now restarting every few seconds, as it waits in a crashloop for kubeadm to tell it what to do. diff --git a/multipass-kubeadm-cluster/set-up-cluster-instructions.md b/multipass-kubeadm-cluster/set-up-cluster-instructions.md new file mode 100644 index 0000000..5df3787 --- /dev/null +++ b/multipass-kubeadm-cluster/set-up-cluster-instructions.md @@ -0,0 +1,224 @@ +# Install Multipass + +```bash +sudo apt update +sudo apt upgrade +sudo snap install multipass +``` + +# Launch +## Launch basic instance +```bash +multipass launch --name worker-01 +``` + +## Launch instance with defined resources +```bash +multipass launch --name worker-02 --disk 20G --memory 8G --cpus 2 +``` + +## Launch instance with defined resources and cloudinit +create a yaml file which defines the intial state of the ubuntu instance +for example `cloudinit.yaml` + +and run a launch command like this after you have created the subsequent file +```bash +multipass launch --name cp1 --cloud-init cloudinit.yaml --disk 20G --memory 8G --cpus 2 +``` + +```yaml +#cloud-config +package_update: true +package_upgrade: true + +write_files: + - path: /etc/modules-load.d/containerd.conf + content: | + overlay + br_netfilter + - path: /etc/sysctl.d/99-kubernetes-cri.conf + content: | + net.bridge.bridge-nf-call-iptables = 1 + net.bridge.bridge-nf-call-ip6tables = 1 + net.ipv4.ip_forward = 1 + +runcmd: + # Disable swap + - swapoff -a + - sed -i '/swap/d' /etc/fstab + + # Load necessary modules + - modprobe overlay + - modprobe br_netfilter + + # Apply sysctl params + - sysctl --system + + # Remove conflicting packages + - for pkg in docker.io docker-doc docker-compose docker-compose-v2 podman-docker containerd runc; do apt-get remove -y $pkg; done + + # Install required packages + - apt-get install -y ca-certificates curl gpg apt-transport-https + + # Add Docker GPG key + - mkdir -p -m 0755 /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc + - chmod a+r /etc/apt/keyrings/docker.asc + + # Add Docker repository + - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo ${UBUNTU_CODENAME:-$VERSION_CODENAME}) stable" | tee /etc/apt/sources.list.d/docker.list + + # Update package lists + - apt-get update + + # Install containerd + - apt-get install -y containerd.io + + # Configure containerd + - mkdir -p /etc/containerd + - containerd config default | tee /etc/containerd/config.toml + - sed -i 's/SystemdCgroup = false/SystemdCgroup = true/' /etc/containerd/config.toml + - systemctl restart containerd + - systemctl enable containerd + + # Add Kubernetes GPG key + - curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.32/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + + # Add Kubernetes repository + - echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.32/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + + # Update package lists + - apt-get update + + # Install kubelet, kubeadm, and kubectl + - apt-get install -y kubelet kubeadm kubectl + - apt-mark hold kubelet kubeadm kubectl + + # Enable kubelet service + - systemctl enable --now kubelet + + # pull kubeadm related image + - sudo kubeadm config images pull +``` + +When the control plane instance is running, you have to install a network plugin +from a third party provider. + +# Install and Configure Network Plugin 'Calico' + +Shell into the control plane instance with `multipass shell ` and run + +```bash +sudo kubeadm init --pod-network-cidr=192.168.0.0/16 +mkdir -p $HOME/.kube +sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config +sudo chown $(id -u):$(id -g) $HOME/.kube/config +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.1/manifests/tigera-operator.yaml +kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.29.1/manifests/custom-resources.yaml +watch kubectl get pods -n calico-system +``` +Wait until all the pods are up and running. + +Afterwards run +```bash +kubectl taint nodes --all node-role.kubernetes.io/control-plane- +``` +It should return something like +```bash +node/ untainted +``` + +Confirm that you have a node running in your cluster +```bash +kubectl get nodes -o wide +``` +It should return something like this +```bash +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME + Ready master 52m v1.12.2 10.128.0.28 Ubuntu 18.04.1 LTS 4.15.0-1023-gcp docker://18.6.1 +``` +Perhaps remove the load balancer label from the list of backend servers +```bash +kubectl label nodes --all node.kubernetes.io/exclude-from-external-load-balancers- +``` + +# Adding Linux Worker Nodes +Run the same `cloudinit.yaml` file as the control plane instance. +For example +```bash +multipass launch --name worker1 --cloud-init cloudinit.yaml --disk 20G --memory 8G --cpus 2 +``` +When the instance is ready, shell into the instance with `multipass shell worker1` and run +```bash +sudo kubeadm join --token : --discovery-token-ca-cert-hash sha256: +``` +The information/token was generated with the kubeadm init command. +If you need to recover the information, shell into `multipass shell cp1` +and run +```bash +# Run this on a control plane node +sudo kubeadm token list +``` + +If you do not want to shell into the control plane you should be able to run (from dev pc) +```bash +multipass exec cp1 sudo kubeadm token list +``` +which should return something like this +```bash +TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS +evv51i.5p35yitsiva6sjer 23h 2025-02-05T20:52:23Z authentication,signing The default bootstrap token generated by 'kubeadm init'. system:bootstrappers:kubeadm:default-node-token +``` + +By default, node join tokens expire after 24 hours. +If you are joining a node to the cluster after the current token has expired, +you can create a new token by running the following command on the control plane node: +```bash +# Run this on a control plane node +sudo kubeadm token create +``` +The output is similar to this: +```bash +5didvk.d09sbcov8ph2amjw +``` + +If you don't have the value of `--discovery-token-ca-cert-hash`, +you can get it by running the following commands on the control plane node: + +```bash +# Run this on a control plane node +sudo cat /etc/kubernetes/pki/ca.crt | openssl x509 -pubkey | openssl rsa -pubin -outform der 2>/dev/null | \ + openssl dgst -sha256 -hex | sed 's/^.* //' +``` + +The output is similar to: +```bash +8cb2de97839780a412b93877f8507ad6c94f73add17d5d7058e91741c9d5ec78 +``` + +The output of the kubeadm join command should look something like: +```bash +[preflight] Running pre-flight checks + +... (log output of join workflow) ... + +Node join complete: +* Certificate signing request sent to control-plane and response + received. +* Kubelet informed of new secure connection details. + + +Run 'kubectl get nodes' on control-plane to see this machine join. +``` +A few seconds later, you should notice this node in the output from kubectl get nodes. +(for example, run kubectl on a control plane node). + +[Note] +As the cluster nodes are usually initialized sequentially, +the CoreDNS Pods are likely to all run on the first control plane node. +To provide higher availability, +please rebalance the CoreDNS Pods with +```bash +kubectl -n kube-system rollout restart deployment coredns +``` +after at least one new node is joined.