su root
sudo yum install -y netcat 
nc 127.0.0.1 6443

sudo systemctl stop firewalld
sudo systemctl disable firewalld
# 关闭selinux
setenforce 0
# 永久
sed -i 's/enforcing/disabled/' /etc/selinux/config  
# 关闭swap（k8s禁止虚拟内存以提高性能）
swapoff -a 
#永久
sed -ri 's/.*swap.*/#&/' /etc/fstab 
# 所有节点添加 host
cat >> /etc/hosts << EOF
192.168.2.158 host158
192.168.2.159 host159
192.168.2.160 host160
192.168.2.161 host161
192.168.2.240 host240
192.168.2.240 host241
192.168.2.158 master-158
192.168.2.159 master-159
192.168.2.160 master-160
192.168.2.161 node-161
EOF
cat /etc/hosts

# 需要挂载 ISO 镜像
sudo mkdir /mnt/cdrom
sudo mount /dev/cdrom /mnt/cdrom/
sudo yum install -y yum-utils
sudo yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo
    
# 或者 阿里云
sudo yum-config-manager \
    --add-repo \
    http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 或者 清华大学源
sudo yum-config-manager \
    --add-repo \
    https://mirrors.tuna.tsinghua.edu.cn/docker-ce/linux/centos/docker-ce.repo

# 由于官网未开放同步方式, 可能会有索引gpg检查失败的情况
# sudo yum clean all && sudo yum makecache

# 卸载 docker
sudo yum remove -y docker-ce docker-ce-cli docker-compose-plugin
sudo yum remove -y kubelet kubeadm kubectl
#  SELinux 设置为 permissive 模式（将其禁用）
sudo setenforce 0
# 永久
sed -i 's/enforcing/disabled/' /etc/selinux/config  
# 所有 kubernetes 安装
sudo yum install -y kubelet kubeadm kubectl
# 安装可能会签名失败（ signature could not be verified for kubernetes ），用下面不校验 gpg
# sudo yum install -y --nogpgcheck kubelet kubeadm kubectl
sudo systemctl daemon-reload
sudo systemctl disable kubelet 
sudo systemctl enable kubelet 
sudo systemctl start kubelet
# 这里会因为缺失/var/lib/kubelet/config.yaml 而停止服务，但一直会尝试重启
sudo systemctl status kubelet -l
# 查看安装后的版本
kubectl version --client
kubectl version --client --output=yaml

# 确保 br_netfilter 模块被加载
lsmod | grep br_netfilter
# 显式加载该模块
sudo modprobe br_netfilter
#为了让你的 Linux 节点上的 iptables 能够正确地查看桥接流量
echo "1" > /proc/sys/net/bridge/bridge-nf-call-iptables
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF

cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward                 = 1
EOF
sudo sysctl --system

#关闭swap
swapoff -a
#永久
sed -ri 's/.*swap.*/#&/' /etc/fstab 
#
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF

sudo modprobe overlay
sudo modprobe br_netfilter
#验证 br_netfilter 模块是否已加载
lsmod | grep br_netfilter
# 设置必需的 sysctl 参数，这些参数在重新启动后仍然存在。
cat <<EOF | sudo tee /etc/sysctl.d/99-kubernetes-cri.conf
net.bridge.bridge-nf-call-iptables  = 1
net.ipv4.ip_forward                 = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF

# 应用 sysctl 参数而无需重新启动
sudo sysctl --system

# 写入配置
cat > ~/kubeadm-config-init.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta3
kind: InitConfiguration
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: fk3wpg.gs0mcv4twx3tz2mc
  ttl: 240h0m0s
  usages:
  - signing
  - authentication
  description: "描述设置了一个人性化的消息，为什么这个令牌存在以及它的用途" 
# NodeRegistration 包含与将新控制平面节点注册到集群相关的字段
nodeRegistration:
  name: master-158
  criSocket: unix:///var/run/containerd/containerd.sock
  ignorePreflightErrors:
    - IsPrivilegedUser
# LocalAPIEndpoint表示部署在这个控制平面节点上的API服务器实例的端点。
localAPIEndpoint:
  advertiseAddress: 192.168.2.158
  bindPort: 6443
---
apiVersion: kubeadm.k8s.io/v1beta3
kind: ClusterConfiguration
clusterName: k8s-cluster
etcd:
  external:
    endpoints:
      - "https://192.168.2.158:2379"
      - "https://192.168.2.159:2379"
      - "https://192.168.2.160:2379"
    caFile: "/etc/certs/etcd/ca.pem"
    certFile: "/etc/certs/etcd/etcd-158.pem"
    keyFile: "/etc/certs/etcd/etcd-158-key.pem"
# 网络持有集群的网络拓扑结构的配置
networking:
  dnsDomain: cluster.158
  serviceSubnet: 10.96.0.0/16
  podSubnet: "10.244.0.0/16"
kubernetesVersion: 1.24.1
controlPlaneEndpoint: "192.168.2.158:6443"
# APIServer 包含 API 服务器控制平面组件的额外设置
apiServer:
  extraArgs:
    bind-address: 0.0.0.0
    authorization-mode: "Node,RBAC"
    #service-cluster-ip-range: 10.96.0.0/16
    #service-node-port-range: 30000-32767
  timeoutForControlPlane: 4m0s
  certSANs:
  - "localhost"
  - "cluster.158"
  - "127.0.0.1"
  - "master-158"
  - "master-159"
  - "master-160"
  - "node-161"
  - "10.96.0.1"
  - "10.244.0.1"
  - "192.168.2.158"
  - "192.168.2.159"
  - "192.168.2.160"
  - "192.168.2.161"
  - "host158"
  - "host159"
  - "host160"
  - "host161"
#  包含控制器管理器控制平面组件的额外设置
controllerManager:
  extraArgs:
    bind-address: 0.0.0.0
    #"node-cidr-mask-size": "20"
    # 这里与 KubeProxyConfiguration.clusterCIDR 一致
    #cluster-cidr: 10.244.0.0/16
    #service-cluster-ip-range: 10.96.0.0/16
    #config: /etc/kubernetes/scheduler-config.yaml 
  #extraVolumes:
  #  - name: schedulerconfig
  #    hostPath: /home/johndoe/schedconfig.yaml
  #    mountPath: /etc/kubernetes/scheduler-config.yaml
  #    readOnly: true
  #    pathType: "File"
  #    调度程序包含调度程序控制平面组件的额外设置
scheduler:
  extraArgs:
    bind-address: 0.0.0.0
    #config: /etc/kubernetes/kubescheduler-config.yaml
  #extraVolumes:
  #  - hostPath: /etc/kubernetes/kubescheduler-config.yaml
  #    mountPath: /etc/kubernetes/kubescheduler-config.yaml
  #    name: kubescheduler-config
  #    readOnly: true
#      DNS 定义集群中安装的 DNS 插件的选项。
dns: {}
certificatesDir: /etc/kubernetes/pki
#imageRepository: k8s.gcr.io
imageRepository: registry.aliyuncs.com/google_containers
#用户启用的 FeatureGates。
#featureGates:
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
bindAddress: 0.0.0.0
bindAddressHardFail: false
clientConnection:
  acceptContentTypes: ""
  burst: 0
  contentType: ""
  kubeconfig: /var/lib/kube-proxy/kubeconfig.conf
  qps: 0
clusterCIDR: 10.244.0.0/16
configSyncPeriod: 2s 
conntrack: 
  maxPerCore: null 
  min: null 
  tcpCloseWaitTimeout: 60s 
  tcpEstablishedTimeout: 2s
  # 默认 LocalModeClusterCIDR
detectLocalMode: ""
detectLocal:
  bridgeInterface: ""
  interfaceNamePrefix: "" 
enableProfiling: false
healthzBindAddress: "0.0.0.0:10256"
hostnameOverride: "kube-proxy-158"
ipvs:
  excludeCIDRs: null
  minSyncPeriod: 1m
  scheduler: ""
  strictARP: true
  syncPeriod: 1m
  tcpFinTimeout: 0s
  tcpTimeout: 0s
  udpTimeout: 0s
metricsBindAddress: "127.0.0.1:10249"
mode: "ipvs"
nodePortAddresses: null
oomScoreAdj: null
portRange: ""
showHiddenMetricsForVersion: ""
udpIdleTimeout: 0s
winkernel:
  enableDSR: false
  forwardHealthCheckVip: false
  networkName: ""
  rootHnsEndpointName: ""
  sourceVip: ""
EOF

sudo kubeadm init \
--node-name master-158 \
# 用于指定`kube-apiserver`监听的`ip`地址，就是`master`本机`IP`地址
--apiserver-advertise-address=192.168.2.158 \
## 指定阿里云镜像仓库地址
--image-repository registry.aliyuncs.com/google_containers \ 
# 用于指定`k8s`版本(`kubeadm config images list`查看的)
--kubernetes-version v1.24.1 \ 
# 用于指定`SVC`的网络范围
--service-cidr=10.96.0.0/16 \
# 用于指定`Pod`的网络范围：`10.244.0.0/16`
--pod-network-cidr=10.244.0.0/16

# 普通用户
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# root 可以直接执行
export KUBECONFIG=/etc/kubernetes/admin.conf

# 在159上执行：添加加 master-159 节点
kubeadm join 192.168.2.158:6443 --node-name master-159 --token fk3wpg.gs0mcv4twx3tz2mc \
        --discovery-token-ca-cert-hash sha256:533e488affe662b66310ed8a353b1fba17c354e4c0df5285b95a01a33e986219 \
        --control-plane --v=5

# 在160上执行：添加 master-160 节点
kubeadm join 192.168.2.158:6443 --node-name master-160 --token fk3wpg.gs0mcv4twx3tz2mc \
        --discovery-token-ca-cert-hash sha256:533e488affe662b66310ed8a353b1fba17c354e4c0df5285b95a01a33e986219 \
        --control-plane --v=5

kubectl get nodes

kubeadm token list

kubeadm token create

#默认证书 /etc/kubernetes/pki/ca.crt
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | \
openssl dgst -sha256 -hex | sed 's/^.* //'

cd ~
# 可选参数 --no-check-certificate
wget https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml -O kube-flannel.yml
# 或者,直接使用没有指定网卡，可能会使 flannel处在CrashLoopBackOff状态
kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml

kubectl get pods --all-namespaces
# 
kubectl get pods -n kube-system

kubectl logs -n kube-system kube-flannel-ds-88xsf -f

kubectl get nodes

cd ~
wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml -O recommended.yaml --no-check-certificate
kubectl apply -f recommended.yaml

kubectl proxy

vim ~/dashboard-admin.yaml 

vim ~/dashboard-admin-bind-cluster-role.yaml 

kubectl create -f ~/dashboard-admin.yaml
kubectl create -f ~/dashboard-admin-bind-cluster-role.yaml

kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret | grep dashboard-admin | awk '{print $1}')

kubectl drain <node name> --delete-emptydir-data --force --ignore-daemonsets

echo "y" | kubeadm reset

iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X

ipvsadm -C
ipvsadm --clear

kubectl delete node <node name>

# 备份
mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
# 下载源文件
wget -O /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
# 或者
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
# 生成缓存
yum makecache

# 需要挂载 ISO 镜像
sudo mkdir /mnt/cdrom
sudo mount /dev/cdrom /mnt/cdrom/
sudo yum install -y yum-utils
sudo yum-config-manager \
    --add-repo \
    https://download.docker.com/linux/centos/docker-ce.repo

# step 1: 安装必要的一些系统工具
sudo yum install -y yum-utils device-mapper-persistent-data lvm2
# Step 2: 添加软件源信息
sudo yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# Step 3
sudo sed -i 's+download.docker.com+mirrors.aliyun.com/docker-ce+' /etc/yum.repos.d/docker-ce.repo
# Step 4: 更新并安装Docker-CE
sudo yum makecache fast