Build a Highly Available Kubernetes Cluster with Dashboard, Nginx HA & Harbor

1. Introduction

Official Kubernetes documentation: https://kubernetes.io/

2. Base Environment Deployment

1) Preliminary preparation (all nodes)

# On 192.168.0.113
hostnamectl set-hostname k8s-master-168-0-113
# On 192.168.0.114
hostnamectl set-hostname k8s-node1-168-0-114
# On 192.168.0.115
hostnamectl set-hostname k8s-node2-168-0-115

Update /etc/hosts with the above IP‑hostname mappings.

2) SSH trust

ssh-keygen
ssh-copy-id -i ~/.ssh/id_rsa.pub root@k8s-master-168-0-113
ssh-copy-id -i ~/.ssh/id_rsa.pub root@k8s-node1-168-0-114
ssh-copy-id -i ~/.ssh/id_rsa.pub root@k8s-node2-168-0-115

3) Time synchronization

yum install chrony -y
systemctl start chronyd
systemctl enable chronyd
chronyc sources

4) Disable firewall and swap, set SELinux to permissive

systemctl stop firewalld && systemctl disable firewalld
swapoff -a && sed -ri 's/.*swap.*/#&/' /etc/fstab
setenforce 0 && sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config

5) Enable iptables bridge traffic

cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
overlay
br_netfilter
EOF
modprobe overlay && modprobe br_netfilter
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.ipv4.ip_forward = 1
EOF
sysctl --system

6) Install Docker (all nodes)

# Configure yum repo
cd /etc/yum.repos.d && mkdir bak && mv CentOS-Linux-* bak/
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum -y install yum-utils
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce
systemctl start docker && systemctl enable docker
docker --version

7) Configure Docker registry mirrors

cat >/etc/docker/daemon.json <<EOF
{
  "registry-mirrors": ["http://hub-mirror.c.163.com"]
}
EOF
systemctl reload docker

8) Install containerd and set cgroup driver to systemd

containerd config default > /etc/containerd/config.toml
sed -i 's#SystemdCgroup = false#SystemdCgroup = true#g' /etc/containerd/config.toml
systemctl restart containerd

9) Install kubeadm, kubelet and kubectl (master node)

yum install -y kubelet-1.24.1 kubeadm-1.24.1 kubectl-1.24.1 --disableexcludes=kubernetes
systemctl enable --now kubelet
systemctl status kubelet

10) Initialize the cluster

kubeadm init \
  --apiserver-advertise-address=192.168.0.113 \
  --image-repository registry.aliyuncs.com/google_containers \
  --control-plane-endpoint=cluster-endpoint \
  --kubernetes-version v1.24.1 \
  --service-cidr=10.1.0.0/16 \
  --pod-network-cidr=10.244.0.0/16 \
  --v=5

After init, set KUBECONFIG:

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

11) Install Pod network (Flannel)

docker pull quay.io/coreos/flannel:v0.14.0
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

12) Join worker nodes

# On master, get token and join command
kubeadm token create --print-join-command
# Run the printed command on each node

13) Configure IPVS for kube-proxy

modprobe ip_vs
modprobe ip_vs_sh
modprobe ip_vs_rr
modprobe ip_vs_wrr
kubectl edit configmap -n kube-system kube-proxy   # set mode: ipvs

3. High Availability Configuration

Two stacked control‑plane nodes (master + master2) sharing the same etcd.

1) Add second master (192.168.0.116)

# Repeat hostname, hosts, ssh‑key, time sync, firewall, swap, SELinux, bridge settings on the new master.
# Then run kubeadm join with <code>--control-plane</code> and the same <code>--certificate-key</code> printed from the first master.

2) Deploy Nginx + Keepalived load balancer

# Install packages on both masters
yum install nginx keepalived -y
# Nginx config (stream block) forwards port 16443 to the two API servers (192.168.0.113:6443 and 192.168.0.116:6443)
cat >/etc/nginx/nginx.conf <<"EOF"
stream {
  upstream k8s-apiserver {
    server 192.168.0.113:6443;
    server 192.168.0.116:6443;
  }
  server { listen 16443; proxy_pass k8s-apiserver; }
}
EOF
# Keepalived config defines virtual IP 192.168.0.120
cat >/etc/keepalived/keepalived.conf <<"EOF"
global_defs { router_id NGIX_MASTER }
vrrp_instance VI_1 {
  state MASTER
  interface ens33
  virtual_router_id 51
  priority 100
  advert_int 1
  authentication { auth_type PASS auth_pass 1111 }
  virtual_ipaddress { 192.168.0.120/24 }
  track_script { check_nginx }
}
EOF
# Simple script /etc/keepalived/check_nginx.sh returns 1 if nginx not running.
chmod +x /etc/keepalived/check_nginx.sh
systemctl daemon-reload && systemctl restart nginx keepalived && systemctl enable nginx keepalived

4. Deploy Kubernetes Dashboard

kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.6.0/aio/deploy/recommended.yaml
# Expose via NodePort 31443
kubectl patch svc kubernetes-dashboard -n kubernetes-dashboard -p '{"spec": {"type": "NodePort", "ports": [{"port":443,"targetPort":8443,"nodePort":31443}]}}'
# Create admin ServiceAccount and ClusterRoleBinding
cat >ServiceAccount.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard
EOF
kubectl apply -f ServiceAccount.yaml
# Get login token
kubectl -n kubernetes-dashboard create token admin-user

Access the UI at https://cluster-endpoint:31443 using the token.

5. Deploy Harbor Image Registry (HTTPS)

1) Install Helm

mkdir -p /opt/k8s/helm && cd /opt/k8s/helm
wget https://get.helm.sh/helm-v3.9.0-rc.1-linux-amd64.tar.gz
tar -xf helm-v3.9.0-rc.1-linux-amd64.tar.gz
ln -s /opt/k8s/helm/linux-amd64/helm /usr/bin/helm

2) Prepare TLS certificates (myharbor.com)

# Generate CA and server certs (see original steps). Resulting files: myharbor.com.key, myharbor.com.crt
kubectl create secret tls myharbor.com --key myharbor.com.key --cert myharbor.com.crt -n harbor

3) Add Harbor Helm repo and install

helm repo add harbor https://helm.goharbor.io
helm install myharbor harbor/harbor \
  --namespace harbor \
  --set expose.ingress.hosts.core=myharbor.com \
  --set expose.ingress.hosts.notary=notary.myharbor.com \
  --set-string expose.ingress.annotations.'nginx\.org/client-max-body-size'="1024m" \
  --set expose.tls.secretName=myharbor.com \
  --set persistence.persistentVolumeClaim.registry.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.jobservice.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.database.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.redis.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.trivy.storageClass=nfs-client \
  --set persistence.persistentVolumeClaim.chartmuseum.storageClass=nfs-client \
  --set persistence.enabled=true \
  --set externalURL=https://myharbor.com \
  --set harborAdminPassword=Harbor12345

After deployment, access https://myharbor.com (admin/Harbor12345).

6. NFS Shared Storage and Dynamic Provisioning

1) Install NFS utilities on all nodes

yum -y install nfs-utils rpcbind</nfs-utils>
systemctl start rpcbind && systemctl enable rpcbind
systemctl start nfs-server && systemctl enable nfs-server

2) On the master, create export directory

mkdir /opt/nfsdata
chmod 777 /opt/nfsdata
cat >/etc/exports <<EOF
/opt/nfsdata *(rw,no_root_squash,no_all_squash,sync)
EOF
exportfs -r

3) Mount on worker nodes

mkdir -p /mnt/nfsdata
echo "192.168.0.120:/opt/nfsdata /mnt/nfsdata nfs defaults 0 1" >> /etc/fstab
mount -a

4) Deploy nfs‑subdir‑external‑provisioner via Helm

helm repo add nfs-subdir-external-provisioner https://kubernetes-sigs.github.io/nfs-subdir-external-provisioner/
helm install nfs-subdir-external-provisioner nfs-subdir-external-provisioner/nfs-subdir-external-provisioner \
  --namespace nfs-provisioner --create-namespace \
  --set image.repository=willdockerhub/nfs-subdir-external-provisioner \
  --set image.tag=v4.0.2 \
  --set replicaCount=2 \
  --set storageClass.name=nfs-client \
  --set storageClass.defaultClass=true \
  --set nfs.server=192.168.0.120 \
  --set nfs.path=/opt/nfsdata

7. Configure Containerd to Trust Harbor Registry

mkdir -p /etc/containerd/myharbor.com
cp ca.crt /etc/containerd/myharbor.com/
cat >/etc/containerd/config.toml <<EOF
[plugins."io.containerd.grpc.v1.cri".registry]
  config_path = ""
  [plugins."io.containerd.grpc.v1.cri".registry.configs]
    [plugins."io.containerd.grpc.v1.cri".registry.configs."myharbor.com".tls]
      ca_file = "/etc/containerd/myharbor.com/ca.crt"
    [plugins."io.containerd.grpc.v1.cri".registry.configs."myharbor.com".auth]
      username = "admin"
      password = "Harbor12345"
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
    [plugins."io.containerd.grpc.v1.cri".registry.mirrors."myharbor.com"]
      endpoint = ["https://myharbor.com"]
EOF
systemctl daemon-reload && systemctl restart containerd

Conclusion

The guide provides a full end‑to‑end setup of a multi‑node, highly available Kubernetes cluster with dashboard access, a resilient Nginx + Keepalived load balancer, a private Harbor registry backed by NFS storage, and containerd configuration for secure image pulls.