Sovia' Blog
Kubernetes之Kubeadm安装k8s
LiuSovia 化神
  2024-11-21 19:09:48   2023-01-18 03:12:10    2.6k 字

Kubernetes之Kubeadm安装k8s

1.初始化服务器相关环境

1
2
3
# 关闭防火墙 master及node节点设置
systemctl stop firewalld
systemctl disable firewalld
1
2
3
4
5
# 关闭swap master及node节点设置
# 临时关闭
swapoff -a 
sed -i '/swap/s/^/#/' /etc/fstab
cat /etc/fstab 
1
2
3
4
5
6
# 关闭selinux master及node节点设置
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
# 或者
vim /etc/selinux/config
SELINUX=disabled
1
2
3
4
5
# 同步时间 master及node节点设置
yum -y install ntpdate
ntpdate cn.pool.ntp.org
# 安装常用包 master及node节点设置
yum install vim bash-completion net-tools gcc wget curl lrzsz -y
1
2
3
4
5
6
7
# host文件添加主机名和对应IP master及node节点设置
cat >> /etc/hosts << EOF
192.168.150.11 k8s-master1
192.168.150.12 k8s-master2
192.168.150.13 K8s-master3
192.168.150.14 k8s-node1
EOF
1
2
3
4
5
6
7
8
# 解决路由异常问题 master及node节点设置
# 配置内核参数，将桥接的IPv4流量传递到iptables的链
cat >> /etc/sysctl.d/k8s.cof <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl --system
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
cat >ssh-kye.sh <<EOF
#!/bin/sh
#####################################################################
# 此脚本在master节点使用
# 设置免密登录
#####################################################################
pass=root123
iplist='192.168.150.11 192.168.150.12 192.168.150.13 192.168.150.14'
yum -y install expect
echo "y"|ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa

for i in \ ${iplist};do
expect -c  "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@\${i}
expect {
    "yes/no*" { send "yes"\r;exp_continue}
    "password*" { send "\${pass}"\r;exp_continue}
    timeout { }
}"
done
exit 0
EOF
1
2
3
4
5
# 设置主机名称 master及node节点设置
ssh 192.168.150.11 "hostnamectl set-hostname k8s-master1" &&
ssh 192.168.150.12 "hostnamectl set-hostname k8s-master2" &&
ssh 192.168.150.13 "hostnamectl set-hostname k8s-master3"  &&
ssh 192.168.150.14 "hostnamectl set-hostname k8s-node1"
1
2
3
4
5
6
7
8
9
10
#  添加阿里云YUM软件源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

2.安装相关服务器及docker

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# master及node节点安装
# 安装docker
# 1.更新yum包
yum update
# 2.卸载旧版本
yum remove docker
# 3.安装依赖
yum install -y yum-utils device-mapper-persistent-data lvm2
# 4.设置yum源
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
# 5.安装docker
yum -y install docker-ce docker-ce-cli containerd.io
# 6.启动docker
systemctl daemon-reload
systemctl restart docker
systemctl enable docker
# 7.验证安装是否成功
docker version
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 方式1 替换国内源 master及node节点设置
cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": ["http://hub-mirror.c.163.com"]
}
EOF

# 方式2 设置cgroupdriver为systemd(推荐)
cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": ["https://v16stybc.mirror.aliyuncs.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  # 本地仓库需要添加此项(默认80端口可不写)
  "insecure-registries":["192.168.150.21:81"]  
}
EOF

# 重启docker
systemctl daemon-reload
systemctl restart docker

3.安装kubelet、kubeadm、kubectl

1
2
3
4
5
6
7
8
9
10
# 方式1 选择版本号下载 master及node节点设置
yum -y install kubeadm-1.17.3 kubelet-1.17.3 kubectl-1.17.3
# 方式2 默认最新版下载
yum -y install kubeadm kubelet kubectl
# 方式3 选择本地rpm包安装

# 设置kubelet开机自启 master及node节点设置
systemctl daemon-reload
systemctl restart kubelet
systemctl enable kubelet

4.初始化kubernetes

方式1 kubeadm安装 

1
2
3
4
5
6
7
8
# master执行(单个master)
kubeadm init \
--apiserver-advertise-address=192.168.150.11 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.17.3 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 \
--apiserver-cert-extra-sans=192.168.150.11
1
2
3
4
5
6
7
# 释：
 --kubernetes-version：指定kubeadm版本
 --pod-network-cidr：指定pod所属网络
 --service-cidr：指定service网段
 --ignore-preflight-errors=Swap/all：忽略 swap/所有 报错
# 注：
　　因为kubeadm需要拉取必要的镜像，这些镜像需要“科学上网”；所以可以先在docker hub或其他镜像仓库拉取kube-proxy、kube-scheduler、kube-apiserver、kube-controller-manager、etcd、pause镜像；并加上 --ignore-preflight-errors=all 忽略所有报错即可。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# 自定义kubeadm配置文件（多个master集群）
kubeadm init --config=kubeadm-config.yaml
cat > kubeadm-config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
# 版本号
kubernetesVersion: v1.17.3
apiServer:
  #填写所有kube-apiserver节点的hostname、IP、VIP
  certSANs:
  - k8s-master1
  - k8s-master2
  - k8s-master3
  - k8s-node1
  - k8s-node2
  - k8s-node3
  - 192.168.150.21
  - 192.168.150.22
  - 192.168.150.23
  - 192.168.150.24
  - 192.168.150.25
  - 192.168.150.26
  # VIP地址
  - 192.168.150.20 
  # 地址
controlPlaneEndpoint: "192.168.150.20:6443"
networking:
  podSubnet: "10.244.0.0/16"
EOF
1
2
3
4
5
# 配置授权信息 master1执行

mkdir -p	$HOME/.kube
cp -i /etc/kubernetes/admin.conf	$HOME/.kube/config
chown $(id -u):$(id -g)	$HOME/.kube/config
1
2
3
4
5
6
7
8
# 复制文件到其他节点 master1执行

for host in k8s-master2 k8s-master3; do
  echo "---$host ---"
  ssh $host "mkdir -p $HOME/.kube"
  scp /etc/kubernetes/admin.conf $host:$HOME/.kube/config
  ssh $host "chown $(id -u):$(id -g) $HOME/.kube/config"
done

方式2 sealos安装 

1
./sealos init --passwd root123 --master 192.168.2.30 --master 192.168.2.31 --master 192.168.2.32 --node 192.168.2.33 --node 192.168.2.34 --node 192.168.2.35 --node 192.168.2.36 --pkg-url /data/kube1.17.3.tar.gz --version v1.17.3

5.查看集群状态

1
2
3
4
# 验证Kubernetes集群是否安装成功
kubectl  get pods -n kube-system
# 查看节点是否都Ready
kubectl get nodes

6.增加集群master与node节点

1
2
3
4
# 添加master
kubeadm join 192.168.150.20:6443 --token iwn4dd.5t69f9bcntalegep \
    --discovery-token-ca-cert-hash sha256:5e1a771521879a48a51fb73f8011f0d0c6802b6a7eac8e5d36f50bab55c4cd3d \
    --control-plane
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 增加集群master与node节点(初始化完成后有加入节点的信息，注意查看!!!)
# 以下命令问样例：
kubeadm join 192.168.150.10:6443 --token 74ru5i.6lfau1y9lcil9nr3 \
    --discovery-token-ca-cert-hash sha256:0f064460f74982d03a4219371323d905fb877b23654801d7ff2dc44f2c97e973

# token 过期时：
# 列出token
kubeadm token list
TOKEN                     TTL         EXPIRES                     USAGES                  DESCRIPTION                                                EXTRA GROUPS
riwrv7.wnpsc5610pre0od9   19h         2020-11-10T13:33:00+08:00   authentication,signing   <none>                                                     system:bootstrappers:kubeadm:default-node-token
# 若无token，则创建token
kubeadm token create

# 获取ca证书sha256编码hash值
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

# 增加control-plane命令(集群master)
kubeadm join ${masterUrl} --token --token 获取的token值 \ 
  --discovery-token-ca-cert-hash sha256:获取ca证书sha256编码hash值 \
  --control-plane
  
# 增加node节点
kubeadm join 192.168.150.10:6443 --token 获取的token值  \
	--discovery-token-ca-cert-hash sha256:获取ca证书sha256编码hash值
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#!/usr/bin/env bash

##############################################################
# 脚本名称: get_token.sh
# 脚本功能: 获取加入节点命令信息
###############################################################


# 生成token
kubeadm token create
# 获取token
Token=$(kubeadm token list|awk '{print $1}'|tail -n 1)
# 获取masterUrl
masterUrl=$(kubectl cluster-info|grep master |awk -F"https://" '{print $2}')
# 显示token
echo "Token=${Token}"

if [ ! -f /etc/kubernetes/pki/ca.crt ];then
  echo "not find /etc/kubernetes/pki/ca.crt , Please Check!"
  exit
fi
# 生成密钥
Pubkey=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')
certificate_key=$(kubeadm init phase upload-certs --upload-certs|awk '{print $1}'|tail -n 1)
# 返回信息
# 返回信息
echo "添加control-plane命令"
echo "kubeadm join ${masterUrl} --token ${Token} \\
--discovery-token-ca-cert-hash sha256:${Pubkey} \\
--control-plane --certificate-key ${certificate_key}" 
  
echo "添加node命令"  
echo "kubeadm join ${masterUrl} --token ${Token} \ 
--discovery-token-ca-cert-hash sha256:${Pubkey}"
exit

7.配置CNI网络插件(flannel,weave,calico)

1
2
3
# 配置flannel网络插件,可能失败多试几次
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
# 或者kubectl apply -f  *.yml文件
1
2
3
# 配置weave网络插件
sysctl net.bridge.bridge-nf-call-iptables=1
kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
1
2
3
4
5
# 配置calico，需要 kubeadm init 时设置 --pod-network-cidr=192.168.0.0/16
kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml 
kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml 

kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml

安装完成后权限配置，使api网关微服务正常工作

1
kubectl create clusterrolebinding k8s-api-admin --clusterrole=cluster-admin --user=root --user=kubelet --group=system:serviceaccounts

8.其他命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# 停止k8s相关服务
systemctl stop kube-apiserver
systemctl stop kube-scheduler
systemctl stop flanneld
systemctl stop kube-controller-manager
# 重启k8s相关服务
systemctl restart kube-apiserver
systemctl restart kube-scheduler
systemctl restart flanneld
systemctl restart kube-controller-manager
#更改ROLES状态
kubectl label node k8s-master2 node-role.kubernetes.io/worker=worker
# 重置
kubeadm reset
ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
# 执行命令及scp命令
for host in k8s-master1 k8s-master2 k8s-master3 k8s-node1; do
  echo "---$host---"
  ssh $host "mkdir -p $HOME/.kube"
  scp /etc/kubernetes/admin.conf $host:/etc/kubernetes/
  ssh $host "chown $(id -u):$(id -g) $HOME/.kube/config"
done
# 删除node节点
kubectl delete node k8s-master2
# 重启pod
kubectl get pod {podname} -n {namespace} -o yaml | kubectl replace --force -f -
# 查看状态
kubectl describe pod kuboard-59bdf4d5fb-k7d4f -n kube-system
# 查看目前所有的pod
kubectl get pod -A
# 查看目前所有的replica set
kubectl get rs -A
# 查看目前所有的deployment
kubectl get deployment 
# 查看my-nginx pod的详细状态
kubectl describe po my-nginx
# 查看my-nginx replica set的详细状态
kubectl describe rs my-nginx
# 查看my-nginx deployment的详细状态
kubectl describe deployment my-nginx

9.集群清理

1
2
3
4
5
6
7
8
9
10
11
12
13
# kubeadm安装
kubeadm reset -f
rm -rf ~/.kube/
rm -rf /etc/kubernetes/
rm -rf /etc/systemd/system/kubelet.service.d
rm -rf /etc/systemd/system/kubelet.service
rm -rf /usr/bin/kube*
rm -rf /etc/cni
rm -rf /opt/cni
rm -rf /var/lib/etcd

# sealos安装
sealos clean --all -f

10.问题排查

问题1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# 如果在运行 kubeadm init 命令时，遇到以下的警告
[preflight] WARNING: ebtables not found in system path
[preflight] WARNING: ethtool not found in system path
那么或许在你的节点上缺失 ebtables、ethtool 或者类似的可执行文件。 你可以使用以下命令安装它们：
对于 Ubuntu/Debian 用户，运行 apt install ebtables ethtool 命令
对于 CentOS/Fedora 用户，运行 yum install ebtables ethtool 命令

# 报错："open /run/flannel/subnet.env: no such file or directory"
cat >/run/flannel/subnet.env <<EOF
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
EOF

问题2

1
2
3
4
5
6
# 报错信息
Normal   Scheduled    <unknown>              default-scheduler   Successfully assigned kubesphere-monitoring-system/prometheus-k8s-0 to k8s-node2
  Warning  FailedMount  5m10s                  kubelet, k8s-node2  Unable to attach or mount volumes: unmounted volumes=[secret-kube-etcd-client-certs], unattached volumes=[prometheus-k8s-token-cndz7 config config-out tls-assets prometheus-k8s-db prometheus-k8s-rulefiles-0 secret-kube-etcd-client-certs]: timed out waiting for the condition
  Warning  FailedMount  2m54s (x2 over 7m24s)  kubelet, k8s-node2  Unable to attach or mount volumes: unmounted volumes=[secret-kube-etcd-client-certs], unattached volumes=[prometheus-k8s-db prometheus-k8s-rulefiles-0 secret-kube-etcd-client-certs prometheus-k8s-token-cndz7 config config-out tls-assets]: timed out waiting for the condition
  Warning  FailedMount  73s (x12 over 9m27s)   kubelet, k8s-node2  MountVolume.SetUp failed for volume "secret-kube-etcd-client-certs" : secret "kube-etcd-client-certs" not found`

1
2
3
4
5
# 解决
kubectl -n kubesphere-monitoring-system create  secret generic kube-etcd-client-certs \
 --from-file=etcd-client-ca.crt=/etc/kubernetes/pki/etcd/ca.crt \
 --from-file=etcd-client.crt=/etc/kubernetes/pki/etcd/healthcheck-client.crt \
 --from-file=etcd-client.key=/etc/kubernetes/pki/etcd/healthcheck-client.key

问题3

1
2
# 报错
# "cannot use 0.0.0.0 as the bind address for API Server"
1
2
3
# 解决
# 发现是没有设置默认网关的原因，按照以下命令，结合自己的网段，设置默认网关即可。
route add default gw 192.168.2.1
 评论
评论插件加载失败
正在加载评论插件
  1. Kubernetes之Kubeadm安装k8s
    1. 1.初始化服务器相关环境
    2. 2.安装相关服务器及docker
    3. 3.安装kubelet、kubeadm、kubectl
    4. 4.初始化kubernetes
    5. 5.查看集群状态
    6. 6.增加集群master与node节点
    7. 7.配置CNI网络插件(flannel,weave,calico)
    8. 8.其他命令
    9. 9.集群清理
    10. 10.问题排查
  1. Kubernetes之Kubeadm安装k8s
    1. 1.初始化服务器相关环境
    2. 2.安装相关服务器及docker
    3. 3.安装kubelet、kubeadm、kubectl
    4. 4.初始化kubernetes
    5. 5.查看集群状态
    6. 6.增加集群master与node节点
    7. 7.配置CNI网络插件(flannel,weave,calico)
    8. 8.其他命令
    9. 9.集群清理
    10. 10.问题排查