kubernetes【k8s】adm方式安装[超级详细]

集群规划【1master节点2node节点】

主机名          IP地址               推荐配置
k8s-master      172.16.115.237     1C2G40G
k8s-node1       172.16.115.238     1C2G40G
k8s-node2       172.16.115.239     1C2G40G

网络规划

network         IP地址段              解释
NodeIP         172.16.115.0/24    #对外提供用户访问
PodIP          10.2.0.0/16        #集群内部IP，可以动态感知后面的POD IP
ClusterIP      10.1.0.0/16        #POD的IP

1.k8s环境准备【机器标准化】所有节点执行

1.1配置hosts解析

cat >> /etc/hosts << EOF
172.16.115.237 k8s-master
172.16.115.238 k8s-node1
172.16.115.239 k8s-node2
EOF

1.2关闭防火墙

systemctl  stop  firewalld   NetworkManager
systemctl  disable  firewalld   NetworkManager

1.3关闭SELinux

setenforce 0
sed -i 's#SELINUX=disabled#SELINUX=disabled#g' /etc/selinux/config
getenforce

1.4关闭SWAP分区

swapoff -a
sed -i '/swap/d' /etc/fstab
free -h

1.5配置时间同步

yum install chrony -y
systemctl start chronyd
systemctl enable chronyd
date

1.6下载Docker CE Yum 源

yum-config-manager \--add-repo https://download.docker.com/linux/centos/docker-ce.repo

1.7安装开源版Docker CE

yum install docker-ce-19.03.6 docker-ce-cli-19.03.6 containerd.io

1.8启动Docker

service docker start
chkconfig docker on     #开机启动

1.9查看Docker版本

docker version
docker info

1.10配置镜像加速器

mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{"registry-mirrors": ["https://plqjafsr.mirror.aliyuncs.com"]
}
EOF
systemctl daemon-reload
systemctl restart docker

2.Kubernetes 1.15 安装

2.1初始化工具安装

yum install net-tools vim wget lrzsz git -y

2.2设置免密码登录

yum install -y expect
ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa
export mypass=123456
name=(k8s-master k8s-node1 k8s-node2)
for i in ${name[@]};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$iexpect {\"*yes/no*\" {send \"yes\r\"; exp_continue}\"*password*\" {send \"$mypass\r\"; exp_continue}\"*Password*\" {send \"$mypass\r\";}}"
done

2.3连接测试

ssh k8s-node1
ssh k8s-node2

2.4优化内核参数

cat >>/etc/sysctl.conf<<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
fs.file-max=52706963
fs.nr_open=52706963
EOF

2.5应用内核配置

modprobe br_netfilter
sysctl -p

3配置证书

3.1下载自签名证书生成工具

mkdir /soft && cd /soft
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

3.2生成ETCD证书

CA 证书配置

mkdir /root/etcd && cd /root/etcd
cat << EOF | tee ca-config.json
{"signing": {"default": {"expiry": "87600h"},"profiles": {"www": {"expiry": "87600h","usages": ["signing","key encipherment","server auth","client auth"]}}}
}
EOF

创建CA证书请求文件

cat << EOF | tee ca-csr.json
{"CN": "etcd CA","key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Beijing","ST": "Beijing"}]
}
EOF

创建ETCD证书请求文件

#可以把所有的master IP 加入到csr文件中（Master-1）

cat << EOF | tee server-csr.json
{"CN": "etcd","hosts": ["k8s-master","k8s-master2","k8s-master3","172.16.115.237",],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Beijing","ST": "Beijing"}]
}
EOF

生成 ETCD CA 证书和ETCD公私钥（Master-1）

cd /root/etcd/
生成ca证书（Master-1）
cfssl gencert -initca ca-csr.json | cfssljson -bare ca –
ll
total 24
-rw-r--r-- 1 root root  287 Apr  5 11:23 ca-config.json      #ca 的配置文件
-rw-r--r-- 1 root root  956 Apr  5 11:26 ca.csr           #ca 证书生成文件
-rw-r--r-- 1 root root  209 Apr  5 11:23 ca-csr.json          #ca 证书请求文件
-rw------- 1 root root 1679 Apr  5 11:26 ca-key.pem       #ca 证书key
-rw-r--r-- 1 root root 1265 Apr  5 11:26 ca.pem           #ca 证书
-rw-r--r-- 1 root root  338 Apr  5 11:26 server-csr.json

生成etcd证书（Master-1）

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
ll
total 36
-rw-r--r-- 1 root root  287 Apr  5 11:23 ca-config.json
-rw-r--r-- 1 root root  956 Apr  5 11:26 ca.csr
-rw-r--r-- 1 root root  209 Apr  5 11:23 ca-csr.json
-rw------- 1 root root 1679 Apr  5 11:26 ca-key.pem
-rw-r--r-- 1 root root 1265 Apr  5 11:26 ca.pem
-rw-r--r-- 1 root root 1054 Apr  5 11:31 server.csr
-rw-r--r-- 1 root root  338 Apr  5 11:26 server-csr.json
-rw------- 1 root root 1675 Apr  5 11:31 server-key.pem #etcd客户端使用
-rw-r--r-- 1 root root 1379 Apr  5 11:31 server.pem

创建 Kubernetes 相关证书

#此证书用于Kubernetes节点直接的通信, 与之前的ETCD证书不同. （Master-1）

配置ca 文件（Master-1）

mkdir /root/kubernetes/ && cd /root/kubernetes/
cat << EOF | tee ca-config.json
{"signing": {"default": {"expiry": "87600h"},"profiles": {"kubernetes": {"expiry": "87600h","usages": ["signing","key encipherment","server auth","client auth"]}}}
}
EOF

创建ca证书申请文件（Master-1）

 cat << EOF | tee ca-csr.json
{"CN": "kubernetes","key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Beijing","ST": "Beijing","O": "k8s","OU": "System"}]
}
EOF

生成API SERVER证书申请文件（Master-1）

 cat << EOF | tee server-csr.json
{"CN": "kubernetes","hosts": ["10.0.0.1","127.0.0.1",
"10.0.0.2",
"172.16.115.237",
"172.16.115.238",
"172.16.115.239",
"k8s-master",
"k8s-node1",
"k8s-node2","kubernetes","kubernetes.default","kubernetes.default.svc","kubernetes.default.svc.cluster","kubernetes.default.svc.cluster.local"],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Beijing","ST": "Beijing","O": "k8s","OU": "System"}]
}
EOF

创建 Kubernetes Proxy 证书申请文件（Master-1）

cat << EOF | tee kube-proxy-csr.json
{"CN": "system:kube-proxy","hosts": [],"key": {"algo": "rsa","size": 2048},"names": [{"C": "CN","L": "Beijing","ST": "Beijing","O": "k8s","OU": "System"}]
}
EOF

生成 kubernetes CA 证书和公私钥

生成ca证书（Master-1）

cfssl gencert -initca ca-csr.json | cfssljson -bare ca –

生成 api-server 证书（Master-1）

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

生成 kube-proxy 证书（Master-1）

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

部署ETCD

下载etcd二进制安装文件

mkdir -p /soft && cd /soft
wget https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz
tar -xvf etcd-v3.3.10-linux-amd64.tar.gz
cd etcd-v3.3.10-linux-amd64/
cp etcd etcdctl /usr/local/bin/

编辑etcd配置文件

#注意修改每个节点的ETCD_NAME
#注意修改每个节点的监听地址

mkdir -p /etc/etcd/{cfg,ssl}
cat  >/etc/etcd/cfg/etcd.conf<<EOFL
#[Member]
ETCD_NAME="master-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.115.237:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.115.237:2379,http://172.16.115.237:2390"#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.115.237:2380" #如果有多个可以用，号分割
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.115.237:2379"
ETCD_INITIAL_CLUSTER="master-1=https://172.16.115.237:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOFL

创建ETCD的系统启动服务（所有master）

cat > /usr/lib/systemd/system/etcd.service<<EOFL
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target[Service]
Type=notify
EnvironmentFile=/etc/etcd/cfg/etcd.conf
ExecStart=/usr/local/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--peer-cert-file=/etc/etcd/ssl/server.pem \
--peer-key-file=/etc/etcd/ssl/server-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536[Install]
WantedBy=multi-user.target
EOFL

复制etcd证书到指定目录

mkdir -p /etc/etcd/ssl/
cp /root/etcd/*pem /etc/etcd/ssl/ -rf

复制etcd证书到每个节点

for i in  k8s-node1 k8s-node2;do ssh $i mkdir -p /etc/etcd/{cfg,ssl};done
for i in  k8s-node1 k8s-node2;do scp /etc/etcd/ssl/* $i:/etc/etcd/ssl/;done
for i in  k8s-node1 k8s-node2;do echo $i "------>"; ssh $i ls /etc/etcd/ssl;done

启动etcd (所有节点)

chkconfig etcd on
service etcd start
service etcd status

检查etcd 集群是否运行正常

etcdctl --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem --endpoints="https://k8s-master:2379"  cluster-health

创建Docker所需分配POD 网段 (任意master节点)

etcdctl --ca-file=/etc/etcd/ssl/ca.pem \
--cert-file=/etc/etcd/ssl/server.pem --key-file=/etc/etcd/ssl/server-key.pem \
--endpoints="https://k8s-master:2379" \set /coreos.com/network/config  \'{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'

检查是否建立网段

etcdctl \
--endpoints=https://k8s-master:2379 \
--ca-file=/etc/etcd/ssl/ca.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--cert-file=/etc/etcd/ssl/server.pem \
get /coreos.com/network/config

4部署Flannel

下载Flannel二进制包

所有的节点,下载到master-1

mkdir /soft ; cd /soft
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
tar xvf flannel-v0.10.0-linux-amd64.tar.gz
mv flanneld mk-docker-opts.sh /usr/local/bin/

复制flanneld到其他的所有节点

for i in k8s-node1 k8s-node2;do scp /usr/local/bin/flanneld $i:/usr/local/bin/;done
for i in k8s-node1 k8s-node2;do scp /usr/local/bin/mk-docker-opts.sh $i:/usr/local/bin/;done

配置Flannel (所有节点)

mkdir -p /etc/flannel
cat > /etc/flannel/flannel.cfg<<EOF
FLANNEL_OPTIONS="-etcd-endpoints=https://172.16.115.237:2379 -etcd-cafile=/etc/etcd/ssl/ca.pem -etcd-certfile=/etc/etcd/ssl/server.pem  -etcd-keyfile=/etc/etcd/ssl/server-key.pem  --healthz-ip=0.0.0.0 --healthz-port=7100"
EOF
#多个ETCD: -etcd-endpoints=https://172.16.115.237:2379,https://172.16.115.238:2379,https://172.16.115.239:2379

配置Flannel配置文件

cat > /usr/lib/systemd/system/flanneld.service <<EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service[Service]
Type=notify
EnvironmentFile=/etc/flannel/flannel.cfg
ExecStart=/usr/local/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure[Install]
WantedBy=multi-user.target
EOF

启动Flannel

service flanneld start
chkconfig flanneld on
service flanneld status

所有的节点都需要有172.17.0.0/16 网段IP

ip a | grep flannel

节点停止flanneld

service flanneld stop

修改Docker启动文件

cat >/usr/lib/systemd/system/docker.service<<EOFL
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd  \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s[Install]
WantedBy=multi-user.target
EOFL

重启Docker服务

systemctl daemon-reload
service flanneld restart
service docker restart

检查IP地址, docker 与flanneld 是同一个网段

ip a

在每个Node节点Ping其他的节点, 网段都是通的。

[root@k8s-node1 ~]# ping 172.17.95.1
PING 172.17.95.1 (172.17.95.1) 56(84) bytes of data.
64 bytes from 172.17.95.1: icmp_seq=1 ttl=64 time=0.322 ms
64 bytes from 172.17.95.1: icmp_seq=2 ttl=64 time=0.225 ms
172.17.95.164 bytes from 172.17.95.1: icmp_seq=3 ttl=64 time=0.232 ms
64 bytes from 172.17.95.1: icmp_seq=4 ttl=64 time=0.245 ms

5安装Master 组件

Master端需要安装的组件如下:
kube-apiserver
kube-scheduler
kube-controller-manager

安装Api Server服务

下载Kubernetes二进制包(1.15.1)（master-1）

cd /soft
wget https://dl.k8s.io/v1.15.10/kubernetes-server-linux-amd64.tar.gz
tar xvf kubernetes-server-linux-amd64.tar.gz
cd kubernetes/server/bin/
cp kube-scheduler kube-apiserver kube-controller-manager kubectl /usr/local/bin/

配置Kubernetes证书

#Kubernetes各个组件之间通信需要证书,需要复制个每个master节点（master-1）

mkdir -p /etc/kubernetes/{cfg,ssl}
cp /root/kubernetes/*.pem /etc/kubernetes/ssl/

复制到其他的节点

for i in k8s-node1 k8s-node2;do ssh $i mkdir -p /etc/kubernetes/{cfg,ssl};done
for i in k8s-node1 k8s-node2;do scp /etc/kubernetes/ssl/* $i:/etc/kubernetes/ssl/;done
for i in k8s-node1 k8s-node2;do echo $i "---------->"; ssh $i ls /etc/kubernetes/ssl;done

创建 TLS Bootstrapping Token

# TLS bootstrapping 功能就是让 kubelet 先使用一个预定的低权限用户连接到 apiserver，
然后向 apiserver 申请证书，kubelet 的证书由 apiserver 动态签署
#Token可以是任意的包涵128 bit的字符串，可以使用安全的随机数发生器生成

head -c 16 /dev/urandom | od -An -t x | tr -d ' '
b78d1a70a76180a848090a178c806229

编辑Token 文件(master-1)

#b78d1a70a76180a848090a178c806229:随机字符串,自定义生成; kubelet-bootstrap:用户名; 10001:UID; system:kubelet-bootstrap：用户组

echo b78d1a70a76180a848090a178c806229,kubelet-bootstrap,10001,"system:kubelet-bootstrap" /etc/kubernetes/cfg/token.csv
#如有其他msater节点请复制到其他的master节点

创建Apiserver配置文件(所有的master节点)

#配置文件内容基本相同, 如果有多个节点, 那么需要修改IP地址即可

cat >/etc/kubernetes/cfg/kube-apiserver.cfg <<EOFL
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--insecure-bind-address=0.0.0.0 \
--insecure-port=8080 \
--etcd-servers=https://172.16.115.237:2379 \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=0.0.0.0 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth \
--token-auth-file=/etc/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/etc/kubernetes/ssl/server.pem  \
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/server.pem \
--etcd-keyfile=/etc/etcd/ssl/server-key.pem"
EOFL

#参数说明
--logtostderr                           启用日志
---v                                    日志等级
--etcd-servers                      etcd 集群地址
--etcd-servers=https://172.16.115.237:2379
--bind-address                      监听地址
--secure-port https                     安全端口
--advertise-address                     集群通告地址
--allow-privileged                      启用授权
--service-cluster-ip-range Service          虚拟IP地址段
--enable-admission-plugins              准入控制模块
--authorization-mode                    认证授权,启用RBAC授权
--enable-bootstrap-token-auth           启用TLS bootstrap功能
--token-auth-ﬁle                        token 文件
--service-node-port-range               Service Node类型默认分配端口范围

配置kube-apiserver 启动文件(所有的master节点)

cat >/usr/lib/systemd/system/kube-apiserver.service<<EOFL
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-apiserver.cfg
ExecStart=/usr/local/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure[Install]
WantedBy=multi-user.target
EOFL

启动kube-apiserver服务

service kube-apiserver start
chkconfig kube-apiserver on
service kube-apiserver status

查看加密的端口是否已经启动

netstat -anltup | grep 6443

查看加密的端口是否已经启动（node节点）

telnet 172.16.115.237 6443

部署kube-scheduler 服务

#创建kube-scheduler配置文件（所有的master节点）

cat >/etc/kubernetes/cfg/kube-scheduler.cfg<<EOFL
KUBE_SCHEDULER_OPTS="--logtostderr=true --v=4 --bind-address=0.0.0.0 --master=127.0.0.1:8080 --leader-elect"
EOFL

查看配置文件

cat  /etc/kubernetes/cfg/kube-scheduler.cfg

创建kube-scheduler 启动文件

#创建kube-scheduler systemd unit 文件（所有的master节点）

cat >/usr/lib/systemd/system/kube-scheduler.service<<EOFL
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-scheduler.cfg
ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure[Install]
WantedBy=multi-user.target
EOFL

启动kube-scheduler服务（所有的master节点）

service kube-scheduler restart
chkconfig kube-scheduler on

查看Master节点组件状态（任意一台master）

kubectl get cs

部署kube-controller-manager

创建kube-controller-manager配置文件(所有节点)

cat >/etc/kubernetes/cfg/kube-controller-manager.cfg<<EOFL
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \
--v=4 \
--master=127.0.0.1:8080 \
--leader-elect=true \
--address=0.0.0.0 \
--service-cluster-ip-range=10.0.0.0/24 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem  \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem"
EOFL

参数说明

--master=127.0.0.1:8080  #指定Master地址
--leader-elect           #竞争选举机制产生一个 leader 节点，其它节点为阻塞状态。
--service-cluster-ip-range #kubernetes service 指定的IP地址范围。

创建kube-controller-manager 启动文件

cat  >/usr/lib/systemd/system/kube-controller-manager.service<<EOFL
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-controller-manager.cfg
ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure[Install]
WantedBy=multi-user.target
EOFL

启动kube-controller-manager服务

chkconfig kube-controller-manager on
service kube-controller-manager start
service kube-controller-manager status

查看Master 节点组件状态

必须要在各个节点组件正常的情况下, 才去部署Node节点组件.（master节点）

kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok
scheduler            Healthy   ok
etcd-0               Healthy   {"health":"true"}

6部署Node节点组件

部署 kubelet 组件

从Master节点复制Kubernetes 文件到Node

cd /soft
for i in k8s-node1 k8s-node2;do scp kubernetes/server/bin/kubelet kubernetes/server/bin/kube-proxy $i:/usr/local/bin/;done

创建kubelet bootstrap.kubeconfig 文件

Maste-1节点

mkdir /root/config ; cd /root/config
cat >environment.sh<<EOFL
# 创建kubelet bootstrapping kubeconfig
BOOTSTRAP_TOKEN=b78d1a70a76180a848090a178c806229
KUBE_APISERVER="https://172.16.115.237:6443"
# 设置集群参数
kubectl config set-cluster kubernetes \--certificate-authority=/etc/kubernetes/ssl/ca.pem \--embed-certs=true \--server=\${KUBE_APISERVER} \--kubeconfig=bootstrap.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \--token=\${BOOTSTRAP_TOKEN} \--kubeconfig=bootstrap.kubeconfig
# 设置上下文参数
kubectl config set-context default \--cluster=kubernetes \--user=kubelet-bootstrap \--kubeconfig=bootstrap.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
#通过 bash environment.sh获取 bootstrap.kubeconfig 配置文件。
EOFL

执行脚本

sh environment.sh

创建kube-proxy kubeconfig文件（master-1）

cat  >env_proxy.sh<<EOF
# 创建kube-proxy kubeconfig文件
BOOTSTRAP_TOKEN=b78d1a70a76180a848090a178c806229
KUBE_APISERVER="https://172.16.115.237:6443"kubectl config set-cluster kubernetes \--certificate-authority=/etc/kubernetes/ssl/ca.pem \--embed-certs=true \--server=\${KUBE_APISERVER} \--kubeconfig=kube-proxy.kubeconfigkubectl config set-credentials kube-proxy \--client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \--client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \--embed-certs=true \--kubeconfig=kube-proxy.kubeconfigkubectl config set-context default \--cluster=kubernetes \--user=kube-proxy \--kubeconfig=kube-proxy.kubeconfigkubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
EOF

执行脚本

sh env_proxy.sh

复制kubeconfig文件与证书到所有Node节点

将bootstrap kubeconfig kube-proxy.kubeconfig 文件复制到所有Node节点
远程创建目录 (master-1)

for i in k8s-node1 k8s-node2;do ssh $i "mkdir -p /etc/kubernetes/{cfg,ssl}";done

复制证书文件ssl (master-1)

for i in k8s-node1 k8s-node2;do scp /etc/kubernetes/ssl/* $i:/etc/kubernetes/ssl/;done

复制kubeconfig文件 (master-1)

for i in k8s-node1 k8s-node2;do scp -rp bootstrap.kubeconfig kube-proxy.kubeconfig $i:/etc/kubernetes/cfg/;done

创建kubelet参数配置文件

不同的Node节点, 需要修改IP地址（node节点操作）

cat >/etc/kubernetes/cfg/kubelet.config<<EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 172.16.115.238
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS: ["10.0.0.2"]
clusterDomain: cluster.local.
failSwapOn: false
authentication:anonymous:enabled: true
EOF

创建kubelet配置文件

不同的Node节点, 需要修改IP地址

/etc/kubernetes/cfg/kubelet.kubeconfig 文件自动生成

cat >/etc/kubernetes/cfg/kubelet<<EOF
KUBELET_OPTS="--logtostderr=true \
--v=4 \
--hostname-override=172.16.115.239 \
--kubeconfig=/etc/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/etc/kubernetes/cfg/bootstrap.kubeconfig \
--config=/etc/kubernetes/cfg/kubelet.config \
--cert-dir=/etc/kubernetes/ssl \
--pod-infra-container-image=docker.io/kubernetes/pause:latest"
EOF

创建kubelet系统启动文件(node节点)

cat >/usr/lib/systemd/system/kubelet.service<<EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service[Service]
EnvironmentFile=/etc/kubernetes/cfg/kubelet
ExecStart=/usr/local/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
KillMode=process[Install]
WantedBy=multi-user.target
EOF

将kubelet-bootstrap用户绑定到系统集群角色

master-1节点操作

kubectl create clusterrolebinding kubelet-bootstrap \--clusterrole=system:node-bootstrapper \--user=kubelet-bootstrap

启动kubelet服务（node节点）

chkconfig kubelet on
service kubelet start
service kubelet status

服务端批准与查看CSR请求

查看CSR请求
Maste-1节点操作

kubectl get csr
NAME                                                   AGE   REQUESTOR           CONDITION
node-csr-elVwxGEknQrBQ3IowikGxUfygraO3MIwlaibMkfMFoM   32s   kubelet-bootstrap   Pending
node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM   30s   kubelet-bootstrap   Pending

批准请求

Master节点操作

kubectl certificate approve node-csr-elVwxGEknQrBQ3IowikGxUfygraO3MIwlaibMkfMFoM
kubectl certificate approve node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM

节点重名处理

如果出现节点重名, 可以先删除证书, 然后重新申请
Master节点删除csr

kubectl delete csr node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM

Node节点删除kubelet.kubeconfig

客户端重启kubelet服务, 再重新申请证书

rm -rf /etc/kubernetes/cfg/kubelet.kubeconfig

查看节点状态

所有的Node节点状态必须为Ready （master）

kubectl get nodesNAME
STATUS   ROLES    AGE     VERSION
172.16.115.238   Ready    <none>   2m37s   v1.15.10
172.16.115.239   Ready    <none>   2m23s   v1.15.10

部署kube-proxy 组件

kube-proxy 运行在所有Node节点上, 监听Apiserver 中 Service 和 Endpoint 的变化情况，创建路由规则来进行服务负载均衡。

创建kube-proxy配置文件

注意修改hostname-override地址, 不同的节点则不同。

cat >/etc/kubernetes/cfg/kube-proxy<<EOF
KUBE_PROXY_OPTS="--logtostderr=true \
--v=4 \
--metrics-bind-address=0.0.0.0 \
--hostname-override=172.16.115.239 \
--cluster-cidr=10.0.0.0/24 \
--kubeconfig=/etc/kubernetes/cfg/kube-proxy.kubeconfig"
EOF

创建kube-proxy systemd unit 文件

cat >/usr/lib/systemd/system/kube-proxy.service<<EOF
[Unit]
Description=Kubernetes Proxy
After=network.target[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure[Install]
WantedBy=multi-user.target
EOF

启动kube-proxy 服务

chkconfig kube-proxy on
service kube-proxy start
service kube-proxy status

运行Demo项目

kubectl run nginx --image=nginx --replicas=2
kubectl run --generator=deployment/apps.v1 is DEPRECATED and will be removed in a future version. Use kubectl run --generator=run-pod/v1 or kubectl create instead.
deployment.apps/nginx created

获取容器IP与运行节点

kubectl get pods -o wide

创建容器svc端口

kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort

查看容器状态

kubectl describe pod nginx-7bb7cd8db5-7ndfk

查看SVC

kubectl get svc
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.0.0.1     <none>        443/TCP        3h14m
nginx        NodePort    10.0.0.129   <none>        88:34947/TCP   94s

访问web

curl http://172.16.115.238:34947
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p><p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p><p><em>Thank you for using nginx.</em></p>
</body>
</html>

删除项目

kubectl delete deployment nginx
kubectl delete pods nginx
kubectl delete svc -l run=nginx
kubectl delete deployment.apps/nginx

服务启动顺序

启动Master节点

service keepalived start
service etcd start
service kube-scheduler start
service kube-controller-manager start
service kube-apiserver  restart
kubectl get cs

启动Node节点

service flanneld start
service docker start
service kubelet start
service kube-proxy start

停止Node节点

service kubelet stop
service kube-proxy stop
service docker stop
service flanneld stop

停止Master 节点

service kube-controller-manager stop
service kube-scheduler stop
service etcd stop
service keepalived stop

7部署DNS

部署coredns

cd /soft/kubernetes
tar xf kubernetes-src.tar.gz
mkdir /root/dns && cd /root/dns
cp /soft/kubernetes/cluster/addons/dns/coredns/coredns.yaml.sed /root/dns/coredns.yaml
修改一些参数：
修改参数有3个地方，一个是ip6.arpa 指定，一个是更改成国内镜像源，一个是定义clusterIP，具体如下
ip6.arpa修改为kubernetes cluster.local. in-addr.arpa ip6.arpa
国内镜像修改为 coredns/coredns:1.2.6
clusterIP修改为自己集群设置的IP范围内的，我集群的是 10.0.0.0/24,所以设置为10.0.0.2（且不为已使用的IP）
修改resources:limits:memory: 1Girequests:cpu: 1028mmemory: 500Mi具体的yaml是(我们只需修改clusterIP的IP为自己集群的IP范围内，且不重复的)作者：凤非飞
链接：https://www.jianshu.com/p/d655e35bf54a
来源：简书
著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。
kubectl apply -f coredns.yaml

查询所有ns中的pod

kubectl get pod -A

查询指定ns中的pod

kubectl get pod -n kube-system

查看启动进程

kubectl get pod -n kube-system coredns-76fb6cf764-c5tln -n kube-system

查看SVC

kubectl get svc -o wide -n=kube-system
NAME       TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE   SELECTOR
kube-dns   ClusterIP   10.0.0.254   <none>        53/UDP,53/TCP,9153/TCP   87s   k8s-app=kube-dns

验证DNS是否有效

删除之前创建的nginx demo

kubectl delete deployment nginx
kubectl delete pods nginx
kubectl delete svc -l run=nginx
kubectl delete deployment.apps/nginx

启动新容器

kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools

#出现错误
error: unable to upgrade connection: Forbidden (user=system:anonymous, verb=create, resource=nodes, subresource=proxy)
#解决方法

kubectl create clusterrolebinding system:anonymous --clusterrole=cluster-admin --user=system:anonymous
kubectl delete pod  dnstools
kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools

创建Nginx 容器

kubectl run nginx --image=nginx --replicas=2

创建svc (cluster IP)

kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort

查看SVC

kubectl get svc -A
NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
default       kubernetes   ClusterIP   10.0.0.1     <none>        443/TCP                  4h50m
default       nginx        NodePort    10.0.0.221   <none>        88:42616/TCP             32m
kube-system   kube-dns     ClusterIP   10.0.0.2     <none>        53/UDP,53/TCP,9153/TCP   4s

测试解析Nginx

dns 解析的名称是svc (service 名称, 非pod名称)

kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools
If you don't see a command prompt, try pressing enter.
dnstools# nslookup nginx
Server:         10.0.0.2
Address:        10.0.0.2#53Name:   nginx.default.svc.cluster.local
Address: 10.0.0.221

案例:容器的网络访问不区分命名空间（kubernetes ns）

在default ns 可以访问到kube-system ns 服务nginx

kubectl run nginx-n1 --image=nginx --replicas=1 -n kube-system

查看容器状态（指定命名空间）

kubectl get pods -n kube-system

查看容器状态（显示所有的命名空间）

kubectl get pod,svc -A
kubectl expose deployment nginx-n1 --port=99 --target-port=80 -n kube-system

跨ns访问服务

kubectl get svc -n kube-system | grep nginx-n1
nginx-n1   ClusterIP   10.0.0.68    <none>        99/TCP                   10s

访问服务

curl 10.0.0.68

解析不成功

nslookup nginx-n1
Server:         10.0.0.2
Address:        10.0.0.2#53** server can't find nginx-n1: NXDOMAIN

解决方法(默认解析为default空间)

nslookup nginx-n1.kube-system.svc.cluster.local
Server:         10.0.0.2
Address:        10.0.0.2#53Name:   nginx-n1.kube-system.svc.cluster.local
Address: 10.0.0.68

部署Dashboard

下载文件

创建目录(master-1节点)

mkdir /root/dashboard
wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml

修改端口

修改为nodeport端口50000
注意镜像地址无法下载, 使用另外的镜像替换

mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.10.1
sed -i '/targetPort:/a\ \ \ \ \ \ nodePort: 50000\n\ \ type: NodePort' kubernetes-dashboard.1.10.yaml

部署

kubectl apply -f kubernetes-dashboard.yaml

查看服务端口

kubectl get services -n kube-system

创建用户授权

kubectl create serviceaccount  dashboard-admin -n kube-system
kubectl create clusterrolebinding  \
dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin

获取Token

kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')
Name:         dashboard-admin-token-ptxn8
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: dashboard-adminkubernetes.io/service-account.uid: 2c17e5ee-c11b-4fde-bd0b-02235f433ee0Type:  kubernetes.io/service-account-tokenData
====
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW
4tdG9rZW4tcHR4bjgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiMmMxN2U1ZWUtYzExYi00ZmRlLWJkMGItMDIyMzVmNDMzZWUwIiwic3ViIjoic
3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.i2728HQZ5NRYdwXpSVNnTXK0UEQJ1va6XFyrJMdJ--2QHvV0xzUfF4uEM0HSMnXUFVoLjRZMarGYJ6_4eptoIW7Px0vKUoWY2--SU-ET-pPXKLFBMQ-c8FSKjuMa1W6-fz1Z4oJyt_4VNRbJrzQbhYcQm8EeiVBs-N0pd4qvoUnJqR2
z_TgdctONPKsQCCy7BadzApoGT00TFGvVEjC-4kOD4ZoWXURItK89VgMAmgxxGmxr3WrD_e-lad9Za_CElA_n90eBPOP5JB3yV0iAxsV6uyo2HZagrvHzbfj7hzmCaHPLlvQc4tApDhnN1QY-RHaJKitZiebQ8TUsmT9DRQ
ca.crt:     1359 bytes
namespace:  11 bytes

登录系统

http://nodeip:50000 访问

8.部署kuboard

kubectl apply -f https://kuboard.cn/install-script/kuboard.yaml

获取token

kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}')

获取nodeport端口

kubectl get svc -n kube-system
kuboard                NodePort    10.0.0.95    <none>        80:32567/TCP             15h

登录系统

http://nodeip:32567 访问

9部署Ingress

服务反向代理
部署Traefik 2.0版本
创建 traefik-crd.yaml 文件 (master-1)

cd /root/ingress

cat >/root/ingress/traefik-crd.yaml<<EOF
## IngressRoute
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:name: ingressroutes.traefik.containo.us
spec:scope: Namespacedgroup: traefik.containo.usversion: v1alpha1names:kind: IngressRouteplural: ingressroutessingular: ingressroute
---
## IngressRouteTCP
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:name: ingressroutetcps.traefik.containo.us
spec:scope: Namespacedgroup: traefik.containo.usversion: v1alpha1names:kind: IngressRouteTCPplural: ingressroutetcpssingular: ingressroutetcp
---
## Middleware
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:name: middlewares.traefik.containo.us
spec:scope: Namespacedgroup: traefik.containo.usversion: v1alpha1names:kind: Middlewareplural: middlewaressingular: middleware
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:name: tlsoptions.traefik.containo.us
spec:scope: Namespacedgroup: traefik.containo.usversion: v1alpha1names:kind: TLSOptionplural: tlsoptionssingular: tlsoption
EOF

创建Traefik CRD资源(master-1)

cd /root/ingress
kubectl create -f traefik-crd.yaml -n kube-system
kubectl get CustomResourceDefinition

创建Traefik RABC文件(master-1)

cat >/root/ingress/traefik-rbac.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:namespace: kube-systemname: traefik-ingress-controller
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:name: traefik-ingress-controller
rules:- apiGroups: [""]resources: ["services","endpoints","secrets"]verbs: ["get","list","watch"]- apiGroups: ["extensions"]resources: ["ingresses"]verbs: ["get","list","watch"]- apiGroups: ["extensions"]resources: ["ingresses/status"]verbs: ["update"]- apiGroups: ["traefik.containo.us"]resources: ["middlewares"]verbs: ["get","list","watch"]- apiGroups: ["traefik.containo.us"]resources: ["ingressroutes"]verbs: ["get","list","watch"]- apiGroups: ["traefik.containo.us"]resources: ["ingressroutetcps"]verbs: ["get","list","watch"]- apiGroups: ["traefik.containo.us"]resources: ["tlsoptions"]verbs: ["get","list","watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:name: traefik-ingress-controller
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: traefik-ingress-controller
subjects:- kind: ServiceAccountname: traefik-ingress-controllernamespace: kube-system
EOF

创建RABC 资源

kubectl create -f traefik-rbac.yaml -n kube-system

创建Traefik ConfigMap (master-1)

cat >/root/ingress/traefik-config.yaml<<EOF
kind: ConfigMap
apiVersion: v1
metadata:name: traefik-config
data:traefik.yaml: |-serversTransport:insecureSkipVerify: trueapi:insecure: truedashboard: truedebug: truemetrics:prometheus: ""entryPoints:web:address: ":80"websecure:address: ":443"providers:kubernetesCRD: ""log:filePath: ""level: errorformat: jsonaccessLog:filePath: ""format: jsonbufferingSize: 0filters:retryAttempts: trueminDuration: 20fields:defaultMode: keepnames:ClientUsername: dropheaders:defaultMode: keepnames:User-Agent: redactAuthorization: dropContent-Type: keep
EOF

创建Traefik ConfigMap资源配置

kubectl apply -f traefik-config.yaml -n kube-system

设置节点标签

设置节点label

kubectl label nodes 172.16.115.238 IngressProxy=true
kubectl label nodes 172.16.115.239 IngressProxy=true

查看节点标签

检查是否成功

kubectl get nodes --show-labels
NAME             STATUS   ROLES    AGE   VERSION    LABELS
172.16.115.238   Ready    <none>   44h   v1.15.10   IngressProxy=true,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.16.115.238,kubernetes.io/os=linux
172.16.115.239   Ready    <none>   44h   v1.15.10   IngressProxy=true,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.16.115.239,kubernetes.io/os=linux

创建 traefik 部署文件

注意每个Node节点的80与443端口不能被占用

netstat -antupl | grep -E "80|443"

cat >/root/ingress/traefik-deploy.yaml<<EOF
apiVersion: v1
kind: Service
metadata:name: traefik
spec:ports:- name: webport: 80- name: websecureport: 443- name: adminport: 8080selector:app: traefik
---
apiVersion: apps/v1
kind: DaemonSet
metadata:name: traefik-ingress-controllerlabels:app: traefik
spec:selector:matchLabels:app: traefiktemplate:metadata:name: traefiklabels:app: traefikspec:serviceAccountName: traefik-ingress-controllerterminationGracePeriodSeconds: 1containers:- image: traefik:v2.0.5name: traefik-ingress-lbports:- name: webcontainerPort: 80hostPort: 80           #hostPort方式，将端口暴露到集群节点- name: websecurecontainerPort: 443hostPort: 443          #hostPort方式，将端口暴露到集群节点- name: admincontainerPort: 8080resources:limits:cpu: 300mmemory: 500Mirequests:cpu: 100mmemory: 102MisecurityContext:capabilities:drop:- ALLadd:- NET_BIND_SERVICEargs:- --configfile=/config/traefik.yamlvolumeMounts:- mountPath: "/config"name: "config"volumes:- name: configconfigMap:name: traefik-config tolerations:              #设置容忍所有污点，防止节点被设置污点- operator: "Exists"nodeSelector:             #设置node筛选器，在特定label的节点上启动IngressProxy: "true"EOF

部署 Traefik 资源

kubectl apply -f traefik-deploy.yaml -n kube-system

kubectl get DaemonSet -A
NAMESPACE     NAME                         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR       AGE
kube-system   traefik-ingress-controller   2         2         0       2            0           IngressProxy=true   6s

Traefik 路由配置
配置Traefik Dashboard

cat >/root/ingress/traefik-dashboard-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:name: traefik-dashboard-route
spec:entryPoints:- webroutes:- match: Host(`traefik.lateautumn4lin.dashboard`)kind: Ruleservices:- name: traefikport: 8080
EOF

创建Ingress (traefik)

kubectl apply -f traefik-dashboard-route.yaml -n kube-system

客户端访问Traefik Dashboard

绑定物理主机Hosts文件或者域名解析
/etc/hosts
172.16.115.239 ingress.hostscc.com

访问web

ingress.hostscc.com

部署访问服务(http)

创建https服务
代理dashboard https 服务
创建自签名证书

cd /root/ingress
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=cloud.lateautumn4lin.dashboard"

将证书存储到 Kubernetes Secret中

kubectl create secret generic cloud-mydlq-tls --from-file=tls.crt --from-file=tls.key -n kube-system

查看系统secret

kubectl get secret -n kube-system
dashboard-tls                            kubernetes.io/tls                     2      41s

创建路由文件

先查询kuberbentes dashboard 的命名空间

cat >/root/ingress/kubernetes-dashboard-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:name: kubernetes-dashboard-route
spec:entryPoints:- websecuretls:secretName: cloud-mydlq-tlsroutes:- match: Host(`cloud.lateautumn4lin.dashboard`) kind: Ruleservices:- name: kubernetes-dashboardport: 443
EOF

创建 Kubernetes Dashboard 路由规则对象

kubectl apply  -f kubernetes-dashboard-route.yaml -n kube-system

查看创建的路由

kubectl get IngressRoute -A

绑定hosts 访问

172.16.115.239 cloud.lateautumn4lin.dashboard

获取token

kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}')

9部署监控系统

监控方案

Prometheus 与 Grafana

安装NFS服务端master节点安装nfs

yum -y install nfs-utils

创建nfs目录

mkdir -p /ifs/kubernetes

修改权限

 chmod -R 777 /ifs/kubernetes

编辑export文件

cat >/etc/exports<<EOF
/ifs/kubernetes *(rw,no_root_squash,sync)
EOF

修改配置启动文件
#修改配置文件

 cat >/etc/systemd/system/sockets.target.wants/rpcbind.socket<<EOFL
[Unit]
Description=RPCbind Server Activation Socket
[Socket]
ListenStream=/var/run/rpcbind.sock
ListenStream=0.0.0.0:111
ListenDatagram=0.0.0.0:111
[Install]
WantedBy=sockets.target
EOFL

启动rpcbind、nfs服务

systemctl restart rpcbind
systemctl enable rpcbind
systemctl restart nfs
systemctl enable nfs

配置生效

 exportfs -f

showmount测试(master-1)

showmount -e k8s-master
Export list for k8s-master:
/ifs/kubernetes *

所有node节点安装客户端

yum -y install nfs-utils

所有的Node检查

showmount -e k8s-master
Export list for k8s-master:
/ifs/kubernetes *

部署PVC
Nfs服务端地址需要修改

mkdir /root/nfs

cat >/root/nfs/nfs-deployment.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:name: nfs-client-provisioner
---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:name: nfs-client-provisioner
spec:replicas: 1strategy:type: Recreatetemplate:metadata:labels:app: nfs-client-provisionerspec:serviceAccountName: nfs-client-provisionercontainers:- name: nfs-client-provisionerimagePullPolicy: IfNotPresentimage: quay.io/external_storage/nfs-client-provisioner:latestvolumeMounts:- name: nfs-client-rootmountPath: /persistentvolumesenv:- name: PROVISIONER_NAMEvalue: fuseim.pri/ifs- name: NFS_SERVERvalue: 172.16.115.237- name: NFS_PATHvalue: /ifs/kubernetesvolumes:- name: nfs-client-rootnfs:server: 172.16.115.237path: /ifs/kubernetes
EOF

cat >/root/nfs/nfs-class.yaml<<EOF
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:name: managed-nfs-storage
provisioner: fuseim.pri/ifs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:archiveOnDelete: "true"
EOF

cat >/root/nfs/nfs-rabc.yaml<<EOF
kind: ServiceAccount
apiVersion: v1
metadata:name: nfs-client-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: nfs-client-provisioner-runner
rules:- apiGroups: [""]resources: ["persistentvolumes"]verbs: ["get", "list", "watch", "create", "delete"]- apiGroups: [""]resources: ["persistentvolumeclaims"]verbs: ["get", "list", "watch", "update"]- apiGroups: ["storage.k8s.io"]resources: ["storageclasses"]verbs: ["get", "list", "watch"]- apiGroups: [""]resources: ["events"]verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: run-nfs-client-provisioner
subjects:- kind: ServiceAccountname: nfs-client-provisionernamespace: default
roleRef:kind: ClusterRolename: nfs-client-provisioner-runnerapiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: leader-locking-nfs-client-provisioner
rules:- apiGroups: [""]resources: ["endpoints"]verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:name: leader-locking-nfs-client-provisioner
subjects:- kind: ServiceAccountname: nfs-client-provisioner# replace with namespace where provisioner is deployednamespace: default
roleRef:kind: Rolename: leader-locking-nfs-client-provisionerapiGroup: rbac.authorization.k8s.io
EOF

kubectl apply  -f  nfs-class.yaml
kubectl apply  -f  nfs-deployment.yaml
kubectl apply  -f  nfs-rabc.yaml

查看nfs pod状态

kubectl get pods
NAME                                     READY   STATUS    RESTARTS   AGE
nfs-client-provisioner-9bf85b8fc-fcj7k   1/1     Running   0          23s

查看是否部署成功

kubectl get StorageClass
NAME                  PROVISIONER      AGE
managed-nfs-storage   fuseim.pri/ifs   61s

部署监控系统

注意需要修改的配置文件
配置configmap文件

cat >prometheus-configmap.yaml<<EOF
apiVersion: v1
kind: ConfigMap
metadata:name: prometheus-confignamespace: kube-system
data:prometheus.yml: |global:scrape_interval:     15sevaluation_interval: 15srule_files:- /etc/prometheus/rules.ymlalerting:alertmanagers:- static_configs:- targets: ["alertmanager:9093"]scrape_configs:- job_name: 'kubernetes-apiservers'kubernetes_sd_configs:- role: endpointsscheme: httpstls_config:ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crtbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/tokenrelabel_configs:- source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]action: keepregex: default;kubernetes;https- job_name: 'kubernetes-cadvisor'kubernetes_sd_configs:- role: nodescheme: httpstls_config:ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crtbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/tokenrelabel_configs:- action: labelmapregex: __meta_kubernetes_node_label_(.+)- target_label: __address__replacement: kubernetes.default.svc:443- source_labels: [__meta_kubernetes_node_name]regex: (.+)target_label: __metrics_path__replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor- job_name: 'kubernetes-service-endpoints'kubernetes_sd_configs:- role: endpointsrelabel_configs:- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]action: keepregex: true- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]action: replacetarget_label: __scheme__regex: (https?)- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]action: replacetarget_label: __metrics_path__regex: (.+)- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]action: replacetarget_label: __address__regex: ([^:]+)(?::\d+)?;(\d+)replacement: $1:$2- action: labelmapregex: __meta_kubernetes_service_label_(.+)- source_labels: [__meta_kubernetes_namespace]action: replacetarget_label: kubernetes_namespace- source_labels: [__meta_kubernetes_service_name]action: replacetarget_label: kubernetes_name- job_name: 'kubernetes-services'kubernetes_sd_configs:- role: servicemetrics_path: /probeparams:module: [http_2xx]relabel_configs:- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]action: keepregex: true- source_labels: [__address__]target_label: __param_target- target_label: __address__replacement: blackbox-exporter.example.com:9115- source_labels: [__param_target]target_label: instance- action: labelmapregex: __meta_kubernetes_service_label_(.+)- source_labels: [__meta_kubernetes_namespace]target_label: kubernetes_namespace- source_labels: [__meta_kubernetes_service_name]target_label: kubernetes_name- job_name: 'kubernetes-ingresses'kubernetes_sd_configs:- role: ingressrelabel_configs:- source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]action: keepregex: trueregex: (.+)- source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]action: replaceregex: ([^:]+)(?::\d+)?;(\d+)replacement: $1:$2target_label: __address__- action: labelmapregex: __meta_kubernetes_pod_label_(.+)- source_labels: [__meta_kubernetes_namespace]action: replacetarget_label: kubernetes_namespace- source_labels: [__meta_kubernetes_pod_name]action: replacetarget_label: kubernetes_pod_name- job_name: 'kubernetes_node'tls_config:ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crtbearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/tokenkubernetes_sd_configs:# 基于endpoint的服务发现，不再经过service代理层面- role: endpointsrelabel_configs:- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_endpoint_port_name]regex: true;prometheus-node-exporteraction: keep- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]action: replacetarget_label: __scheme__regex: (https?)- source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]action: replacetarget_label: __metrics_path__regex: (.+)- source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]action: replacetarget_label: __address__regex: (.+)(?::\d+);(\d+)replacement: $1:$2# 去掉label name中的前缀__meta_kubernetes_service_label_- action: labelmapregex: __meta_kubernetes_service_label_(.+)# 为了区分所属node,把instance 从node-exporter ep的实例，替换成ep所在node的ip- source_labels: [__meta_kubernetes_pod_host_ip]regex: '(.*)'replacement: '${1}'target_label: instance
EOF

部署

kubectl apply -f prometheus-configmap.yaml

部署prometheus工作主程序，注意挂载上面的configmap：

cat >prometheus.deploy.yml<<EOF
apiVersion: apps/v1beta2
kind: Deployment
metadata:labels:name: prometheus-deploymentname: prometheusnamespace: kube-system
spec:replicas: 1selector:matchLabels:app: prometheustemplate:metadata:labels:app: prometheusspec:containers:- image: prom/prometheus:v2.0.0name: prometheuscommand:- "/bin/prometheus"args:- "--config.file=/etc/prometheus/prometheus.yml"- "--storage.tsdb.path=/prometheus"- "--storage.tsdb.retention=24h"ports:- containerPort: 9090protocol: TCPvolumeMounts:- mountPath: "/prometheus"name: data- mountPath: "/etc/prometheus"name: config-volumeresources:requests:cpu: 100mmemory: 100Milimits:cpu: 500mmemory: 2500MiserviceAccountName: prometheus    volumes:- name: dataemptyDir: {}- name: config-volumeconfigMap:name: prometheus-config  EOF

部署

kubectl apply -f prometheus.deploy.yml

部署svc、ingress、rbac授权。
注意：在本地是使用traefik做对外服务代理的，因此修改了默认的NodePort的svc.type为ClusterIP的方式，Ingress添加后，可以以域名方式直接访问。若不做代理，可以无需部署ingress，svc.type使用默认的NodePort，然后通过node ip+port的形式访问

cat >prometheus.svc.yaml<<EOF
kind: Service
apiVersion: v1
metadata:labels:app: prometheusname: prometheusnamespace: kube-system
spec:type: ClusterIPports:- port: 80protocol: TCPtargetPort: 9090selector:app: prometheus
EOF

部署

kubectl apply -f prometheus.svc.yaml

prometheus-route.yaml文件

cat >prometheus-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:name: prometheusnamespace: kube-system
spec:entryPoints:- webroutes:- match: Host(`prometheus.ygbx.com`)kind: Ruleservices:- name: prometheusport: 80
EOF

部署

kubectl apply -f prometheus-route.yaml

rbac-setup.yam文件

cat >rbac-setup.yaml<<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:name: prometheus
rules:
- apiGroups: [""]resources:- nodes- nodes/proxy- services- endpoints- podsverbs: ["get", "list", "watch"]
- apiGroups:- extensionsresources:- ingressesverbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:name: prometheusnamespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:name: prometheus
roleRef:apiGroup: rbac.authorization.k8s.iokind: ClusterRolename: prometheus
subjects:
- kind: ServiceAccountname: prometheusnamespace: kube-systemEOF

部署

kubectl apply -f prometheus-route.yaml

配置好dns记录

访问http://prometheus.ygbx.com/随便选取一个metric，点击execute，查看是否能正常获取结果输出。点击status—target，可以看到metrics的数据来源，即各exporter，点击相应exporter上的链接可查看这个exporter提供的metrics明细。

为了更好的展示图形效果，需要部署grafana，因此前已经部署有grafana，这里不再部署，贴一个all-in-one.yaml部署文件。

cat >grafana-all-in-one.yaml<<EOF
apiVersion: extensions/v1beta1
kind: Deployment
metadata:name: grafana-corenamespace: kube-systemlabels:app: grafanacomponent: core
spec:replicas: 1template:metadata:labels:app: grafanacomponent: corespec:containers:- image: grafana/grafana:4.2.0name: grafana-coreimagePullPolicy: IfNotPresent# env:resources:# keep request = limit to keep this container in guaranteed classlimits:cpu: 100mmemory: 100Mirequests:cpu: 100mmemory: 100Mienv:# The following env variables set up basic auth twith the default admin user and admin password.- name: GF_AUTH_BASIC_ENABLEDvalue: "true"- name: GF_AUTH_ANONYMOUS_ENABLEDvalue: "false"# - name: GF_AUTH_ANONYMOUS_ORG_ROLE#   value: Admin# does not really work, because of template variables in exported dashboards:# - name: GF_DASHBOARDS_JSON_ENABLED#   value: "true"readinessProbe:httpGet:path: /loginport: 3000# initialDelaySeconds: 30# timeoutSeconds: 1volumeMounts:- name: grafana-persistent-storagemountPath: /varvolumes:- name: grafana-persistent-storageemptyDir: {}---
apiVersion: v1
kind: Service
metadata:name: grafananamespace: kube-systemlabels:app: grafanacomponent: core
spec:type: NodePortports:- port: 3000selector:app: grafanacomponent: coreEOF

创建ingress访问

cat >grafana-route.ymal<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:name: grafananamespace: kube-system
spec:entryPoints:- webroutes:- match: Host(`grafana.ygbx.com`)kind: Ruleservices:- name: grafanaport: 3000
EOF

配置好dns记录

访问http://grafana.ygbx.com/

默认管理账号密码为admin admin

选择资源类型，填入prometheus的服务地址及端口号，点击保存

导入展示模板：
点击dashboard，点击import dashboard，在弹出框内填写数字315，会自动加载官方提供的315号模板，然后选择数据源为刚添加的数据源，模板就创建好了.

基本部署到这里就结束了

10容器日志收集方案

把log-agent打包至业务镜像
日志落地至物理节点
每个物理节点启动日志容器

本例中在每个node 节点部署一个pod 收集日志

安装日志组件

设置serviceAccount

kubectl create serviceaccount admin -n kube-system

配置权限

mkdir es

cat >es-rbac.yaml<<EOF
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:name: es-rbac
subjects:- kind: ServiceAccountname: admin namespace: kube-system
roleRef:kind: ClusterRolename: cluster-adminapiGroup: rbac.authorization.k8s.io
EOF

创建权限

kubectl apply -f  es-rbac.yaml

安装Elasticsearch

docker pull registry.cn-hangzhou.aliyuncs.com/cqz/elasticsearch:5.5.1

wget https://acs-logging.oss-cn-hangzhou.aliyuncs.com/elasticsearch.yml