Kubernetes之Kubeadm安装k8s
LiuSw Lv6

Kubernetes之Kubeadm安装k8s

1.初始化服务器相关环境

1
2
3
# 关闭防火墙 master及node节点设置
systemctl stop firewalld
systemctl disable firewalld
1
2
3
4
5
# 关闭swap master及node节点设置
# 临时关闭
swapoff -a
sed -i '/swap/s/^/#/' /etc/fstab
cat /etc/fstab
1
2
3
4
5
6
# 关闭selinux master及node节点设置
setenforce 0
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
# 或者
vim /etc/selinux/config
SELINUX=disabled
1
2
3
4
5
# 同步时间 master及node节点设置
yum -y install ntpdate
ntpdate cn.pool.ntp.org
# 安装常用包 master及node节点设置
yum install vim bash-completion net-tools gcc wget curl lrzsz -y
1
2
3
4
5
6
7
# host文件添加主机名和对应IP master及node节点设置
cat >> /etc/hosts << EOF
192.168.150.11 k8s-master1
192.168.150.12 k8s-master2
192.168.150.13 K8s-master3
192.168.150.14 k8s-node1
EOF
1
2
3
4
5
6
7
8
# 解决路由异常问题 master及node节点设置
# 配置内核参数,将桥接的IPv4流量传递到iptables的链
cat >> /etc/sysctl.d/k8s.cof <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF

sysctl --system
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
cat >ssh-kye.sh <<EOF
#!/bin/sh
#####################################################################
# 此脚本在master节点使用
# 设置免密登录
#####################################################################
pass=root123
iplist='192.168.150.11 192.168.150.12 192.168.150.13 192.168.150.14'
yum -y install expect
echo "y"|ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa

for i in \ ${iplist};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@\${i}
expect {
"yes/no*" { send "yes"\r;exp_continue}
"password*" { send "\${pass}"\r;exp_continue}
timeout { }
}"
done
exit 0
EOF
1
2
3
4
5
# 设置主机名称 master及node节点设置
ssh 192.168.150.11 "hostnamectl set-hostname k8s-master1" &&
ssh 192.168.150.12 "hostnamectl set-hostname k8s-master2" &&
ssh 192.168.150.13 "hostnamectl set-hostname k8s-master3" &&
ssh 192.168.150.14 "hostnamectl set-hostname k8s-node1"
1
2
3
4
5
6
7
8
9
10
#  添加阿里云YUM软件源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

2.安装相关服务器及docker

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# master及node节点安装
# 安装docker
# 1.更新yum包
yum update
# 2.卸载旧版本
yum remove docker
# 3.安装依赖
yum install -y yum-utils device-mapper-persistent-data lvm2
# 4.设置yum源
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
# 5.安装docker
yum -y install docker-ce docker-ce-cli containerd.io
# 6.启动docker
systemctl daemon-reload
systemctl restart docker
systemctl enable docker
# 7.验证安装是否成功
docker version
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# 方式1 替换国内源 master及node节点设置
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors": ["http://hub-mirror.c.163.com"]
}
EOF

# 方式2 设置cgroupdriver为systemd(推荐)
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors": ["https://v16stybc.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
# 本地仓库需要添加此项(默认80端口可不写)
"insecure-registries":["192.168.150.21:81"]
}
EOF

# 重启docker
systemctl daemon-reload
systemctl restart docker

3.安装kubelet、kubeadm、kubectl

1
2
3
4
5
6
7
8
9
10
# 方式1 选择版本号下载 master及node节点设置
yum -y install kubeadm-1.17.3 kubelet-1.17.3 kubectl-1.17.3
# 方式2 默认最新版下载
yum -y install kubeadm kubelet kubectl
# 方式3 选择本地rpm包安装

# 设置kubelet开机自启 master及node节点设置
systemctl daemon-reload
systemctl restart kubelet
systemctl enable kubelet

4.初始化kubernetes

方式1 kubeadm安装

1
2
3
4
5
6
7
8
# master执行(单个master)
kubeadm init \
--apiserver-advertise-address=192.168.150.11 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.17.3 \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 \
--apiserver-cert-extra-sans=192.168.150.11
1
2
3
4
5
6
7
# 释:
--kubernetes-version:指定kubeadm版本
--pod-network-cidr:指定pod所属网络
--service-cidr:指定service网段
--ignore-preflight-errors=Swap/all:忽略 swap/所有 报错
# 注:
  因为kubeadm需要拉取必要的镜像,这些镜像需要“科学上网”;所以可以先在docker hub或其他镜像仓库拉取kube-proxy、kube-scheduler、kube-apiserver、kube-controller-manager、etcd、pause镜像;并加上 --ignore-preflight-errors=all 忽略所有报错即可。
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
# 自定义kubeadm配置文件(多个master集群)
kubeadm init --config=kubeadm-config.yaml
cat > kubeadm-config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
# 版本号
kubernetesVersion: v1.17.3
apiServer:
#填写所有kube-apiserver节点的hostname、IP、VIP
certSANs:
- k8s-master1
- k8s-master2
- k8s-master3
- k8s-node1
- k8s-node2
- k8s-node3
- 192.168.150.21
- 192.168.150.22
- 192.168.150.23
- 192.168.150.24
- 192.168.150.25
- 192.168.150.26
# VIP地址
- 192.168.150.20
# 地址
controlPlaneEndpoint: "192.168.150.20:6443"
networking:
podSubnet: "10.244.0.0/16"
EOF
1
2
3
4
5
# 配置授权信息 master1执行

mkdir -p $HOME/.kube
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
chown $(id -u):$(id -g) $HOME/.kube/config
1
2
3
4
5
6
7
8
# 复制文件到其他节点 master1执行

for host in k8s-master2 k8s-master3; do
echo "---$host ---"
ssh $host "mkdir -p $HOME/.kube"
scp /etc/kubernetes/admin.conf $host:$HOME/.kube/config
ssh $host "chown $(id -u):$(id -g) $HOME/.kube/config"
done

方式2 sealos安装

1
./sealos init --passwd root123 --master 192.168.2.30 --master 192.168.2.31 --master 192.168.2.32 --node 192.168.2.33 --node 192.168.2.34 --node 192.168.2.35 --node 192.168.2.36 --pkg-url /data/kube1.17.3.tar.gz --version v1.17.3

5.查看集群状态

1
2
3
4
# 验证Kubernetes集群是否安装成功
kubectl get pods -n kube-system
# 查看节点是否都Ready
kubectl get nodes

6.增加集群master与node节点

1
2
3
4
# 添加master
kubeadm join 192.168.150.20:6443 --token iwn4dd.5t69f9bcntalegep \
--discovery-token-ca-cert-hash sha256:5e1a771521879a48a51fb73f8011f0d0c6802b6a7eac8e5d36f50bab55c4cd3d \
--control-plane
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 增加集群master与node节点(初始化完成后有加入节点的信息,注意查看!!!)
# 以下命令问样例:
kubeadm join 192.168.150.10:6443 --token 74ru5i.6lfau1y9lcil9nr3 \
--discovery-token-ca-cert-hash sha256:0f064460f74982d03a4219371323d905fb877b23654801d7ff2dc44f2c97e973

# token 过期时:
# 列出token
kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
riwrv7.wnpsc5610pre0od9 19h 2020-11-10T13:33:00+08:00 authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
# 若无token,则创建token
kubeadm token create

# 获取ca证书sha256编码hash
openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //'

# 增加control-plane命令(集群master)
kubeadm join ${masterUrl} --token --token 获取的token值 \
--discovery-token-ca-cert-hash sha256:获取ca证书sha256编码hash值 \
--control-plane

# 增加node节点
kubeadm join 192.168.150.10:6443 --token 获取的token值 \
--discovery-token-ca-cert-hash sha256:获取ca证书sha256编码hash值
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
#!/usr/bin/env bash

##############################################################
# 脚本名称: get_token.sh
# 脚本功能: 获取加入节点命令信息
###############################################################


# 生成token
kubeadm token create
# 获取token
Token=$(kubeadm token list|awk '{print $1}'|tail -n 1)
# 获取masterUrl
masterUrl=$(kubectl cluster-info|grep master |awk -F"https://" '{print $2}')
# 显示token
echo "Token=${Token}"

if [ ! -f /etc/kubernetes/pki/ca.crt ];then
echo "not find /etc/kubernetes/pki/ca.crt , Please Check!"
exit
fi
# 生成密钥
Pubkey=$(openssl x509 -pubkey -in /etc/kubernetes/pki/ca.crt | openssl rsa -pubin -outform der 2>/dev/null | openssl dgst -sha256 -hex | sed 's/^.* //')
certificate_key=$(kubeadm init phase upload-certs --upload-certs|awk '{print $1}'|tail -n 1)
# 返回信息
# 返回信息
echo "添加control-plane命令"
echo "kubeadm join ${masterUrl} --token ${Token} \\
--discovery-token-ca-cert-hash sha256:${Pubkey} \\
--control-plane --certificate-key ${certificate_key}"

echo "添加node命令"
echo "kubeadm join ${masterUrl} --token ${Token} \
--discovery-token-ca-cert-hash sha256:${Pubkey}"
exit

7.配置CNI网络插件(flannel,weave,calico)

1
2
3
# 配置flannel网络插件,可能失败多试几次
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
# 或者kubectl apply -f *.yml文件
1
2
3
# 配置weave网络插件
sysctl net.bridge.bridge-nf-call-iptables=1
kubectl apply -f "https://cloud.weave.works/k8s/net?k8s-version=$(kubectl version | base64 | tr -d '\n')"
1
2
3
4
5
# 配置calico,需要 kubeadm init 时设置 --pod-network-cidr=192.168.0.0/16
kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/rbac-kdd.yaml
kubectl apply -f https://docs.projectcalico.org/v3.1/getting-started/kubernetes/installation/hosted/kubernetes-datastore/calico-networking/1.7/calico.yaml

kubectl apply -f http://docs.projectcalico.org/v2.4/getting-started/kubernetes/installation/hosted/kubeadm/1.6/calico.yaml

安装完成后权限配置,使api网关微服务正常工作

1
kubectl create clusterrolebinding k8s-api-admin --clusterrole=cluster-admin --user=root --user=kubelet --group=system:serviceaccounts

8.其他命令

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# 停止k8s相关服务
systemctl stop kube-apiserver
systemctl stop kube-scheduler
systemctl stop flanneld
systemctl stop kube-controller-manager
# 重启k8s相关服务
systemctl restart kube-apiserver
systemctl restart kube-scheduler
systemctl restart flanneld
systemctl restart kube-controller-manager
#更改ROLES状态
kubectl label node k8s-master2 node-role.kubernetes.io/worker=worker
# 重置
kubeadm reset
ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
# 执行命令及scp命令
for host in k8s-master1 k8s-master2 k8s-master3 k8s-node1; do
echo "---$host---"
ssh $host "mkdir -p $HOME/.kube"
scp /etc/kubernetes/admin.conf $host:/etc/kubernetes/
ssh $host "chown $(id -u):$(id -g) $HOME/.kube/config"
done
# 删除node节点
kubectl delete node k8s-master2
# 重启pod
kubectl get pod {podname} -n {namespace} -o yaml | kubectl replace --force -f -
# 查看状态
kubectl describe pod kuboard-59bdf4d5fb-k7d4f -n kube-system
# 查看目前所有的pod
kubectl get pod -A
# 查看目前所有的replica set
kubectl get rs -A
# 查看目前所有的deployment
kubectl get deployment
# 查看my-nginx pod的详细状态
kubectl describe po my-nginx
# 查看my-nginx replica set的详细状态
kubectl describe rs my-nginx
# 查看my-nginx deployment的详细状态
kubectl describe deployment my-nginx

9.集群清理

1
2
3
4
5
6
7
8
9
10
11
12
13
# kubeadm安装
kubeadm reset -f
rm -rf ~/.kube/
rm -rf /etc/kubernetes/
rm -rf /etc/systemd/system/kubelet.service.d
rm -rf /etc/systemd/system/kubelet.service
rm -rf /usr/bin/kube*
rm -rf /etc/cni
rm -rf /opt/cni
rm -rf /var/lib/etcd

# sealos安装
sealos clean --all -f

10.问题排查

问题1

1
2
3
4
5
6
7
8
9
10
11
12
13
14
# 如果在运行 kubeadm init 命令时,遇到以下的警告
[preflight] WARNING: ebtables not found in system path
[preflight] WARNING: ethtool not found in system path
那么或许在你的节点上缺失 ebtables、ethtool 或者类似的可执行文件。 你可以使用以下命令安装它们:
对于 Ubuntu/Debian 用户,运行 apt install ebtables ethtool 命令
对于 CentOS/Fedora 用户,运行 yum install ebtables ethtool 命令

# 报错:"open /run/flannel/subnet.env: no such file or directory"
cat >/run/flannel/subnet.env <<EOF
FLANNEL_NETWORK=10.244.0.0/16
FLANNEL_SUBNET=10.244.0.1/24
FLANNEL_MTU=1450
FLANNEL_IPMASQ=true
EOF

问题2

1
2
3
4
5
6
# 报错信息
Normal Scheduled <unknown> default-scheduler Successfully assigned kubesphere-monitoring-system/prometheus-k8s-0 to k8s-node2
Warning FailedMount 5m10s kubelet, k8s-node2 Unable to attach or mount volumes: unmounted volumes=[secret-kube-etcd-client-certs], unattached volumes=[prometheus-k8s-token-cndz7 config config-out tls-assets prometheus-k8s-db prometheus-k8s-rulefiles-0 secret-kube-etcd-client-certs]: timed out waiting for the condition
Warning FailedMount 2m54s (x2 over 7m24s) kubelet, k8s-node2 Unable to attach or mount volumes: unmounted volumes=[secret-kube-etcd-client-certs], unattached volumes=[prometheus-k8s-db prometheus-k8s-rulefiles-0 secret-kube-etcd-client-certs prometheus-k8s-token-cndz7 config config-out tls-assets]: timed out waiting for the condition
Warning FailedMount 73s (x12 over 9m27s) kubelet, k8s-node2 MountVolume.SetUp failed for volume "secret-kube-etcd-client-certs" : secret "kube-etcd-client-certs" not found`

1
2
3
4
5
# 解决
kubectl -n kubesphere-monitoring-system create secret generic kube-etcd-client-certs \
--from-file=etcd-client-ca.crt=/etc/kubernetes/pki/etcd/ca.crt \
--from-file=etcd-client.crt=/etc/kubernetes/pki/etcd/healthcheck-client.crt \
--from-file=etcd-client.key=/etc/kubernetes/pki/etcd/healthcheck-client.key

问题3

1
2
# 报错
# "cannot use 0.0.0.0 as the bind address for API Server"
1
2
3
# 解决
# 发现是没有设置默认网关的原因,按照以下命令,结合自己的网段,设置默认网关即可。
route add default gw 192.168.2.1
 评论