OpenShift——openshift 3.11 集群安装（亲测版，你懂的）

OpenShift 是RedHat的当家产品，在centos7中安装具有重要的意义，赶巧考红帽DO280的时机，我整理了这次安装过程，希望能够给同仁们一点有益的帮助。完全脱机安装，排错，整理，试验，历时一周时间。完全可以做的更好，但苦于DO280考试在即，不得不放手，不过还行，能用。DO280考试做试验估计没问题。希望多提宝贵意见，贴下真知见解，也不枉我一片心血了，不胜感激！
安装环境：在VMWare下启动了6台虚拟机，事先积累了脱机版的镜像源。centos7没有升级。

1.节点规划

序号	ip	主机名
1	192.168.0.71	master.blueicex.com
2	192.168.0.72	node1.blueicex.com
3	192.168.0.73	node2.blueicex.com
4	192.168.0.74	node3.blueicex.com
5	192.168.0.75	etcd.blueicex.com
6	192.168.0.77	resource.blueicex.com

4core CPU，4G memory，40G system disk，20G docker vg，double netface

2. 主机基础搭建

2.1 修改ifc-ens33

操作节点：master nodes etcd resource
被操作节点：master nodes etcd resource

[root@localhost ~]# vi /etc/sysconfig/network-scripts/ifcfg-ens33
ONBOOT="yes"
IPADDR="192.168.0.7*"
GATEWAY="192.168.0.1"
NM_CONTROLLED=yes
[root@localhost ~]# systemctl restart network NetworkManager

★配置验证

[root@localhost ~]# cat etc/sysconfig/network-scripts/ifcfg-ens33
[root@localhost ~]# ip a s  ens33

2.2 root卷扩展

操作节点：master nodes etcd resource
被操作节点：master nodes etcd resource
在生成虚拟主机前，sda卷，保留了10G的预留空间未分配

[root@localhost ~]# fdisk /dev/sda
[root@localhost ~]# partprobe
[root@localhost ~]# lsblk
NAME            MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
sda               8:0    0    20G  0 disk
├─sda1            8:1    0   190M  0 part /boot
├─sda2            8:2    0  10.2G  0 part
│ ├─centos-root 253:0    0    10G  0 lvm  /
│ └─centos-home 253:1    0   192M  0 lvm  /home
└─sda3            8:3    0   9.6G  0 part
..........
[root@localhost ~]# mkfs.xfs /dev/sda3
[root@localhost ~]# vgextend centos /dev/sda3
[root@localhost ~]# vgsVG     #PV #LV #SN Attr   VSize   VFreecentos   2   2   0 wz--n- <19.81g 9.62g
[root@localhost ~]# lvextend -L +9.6G /dev/centos/root
[root@localhost ~]# xfs_growfs /dev/centos/root

★配置验证

[root@localhost ~]# lsblk /dev/sda
NAME            MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
sda               8:0    0   20G  0 disk
├─sda2            8:2    0 10.2G  0 part
│ ├─centos-home 253:1    0  192M  0 lvm  /home
│ └─centos-root 253:0    0 19.6G  0 lvm  /
├─sda3            8:3    0  9.6G  0 part
│ └─centos-root 253:0    0 19.6G  0 lvm  /
└─sda1            8:1    0  190M  0 part /boot

3. 资源主机服务构建

操作节点：resource
被操作节点：resource

3.1 环境准备

[root@localhost ~]# vi /etc/selinux/config
SELINUX=disabled
[root@localhost ~]# systemctl stop firewalld && systemctl disable firewalld
[root@localhost ~]# reboot
[root@localhost ~]# mkdir /blueicex/{soft,temp,image} /mnt/{cdrom,usb} -pv
[root@localhost ~]# lsblk
NAME            MAJ:MIN RM   SIZE RO TYPE MOUNTPOINT
sda               8:0    0    20G  0 disk
├─sda1            8:1    0   190M  0 part /boot
├─sda2            8:2    0  10.2G  0 part
│ ├─centos-root 253:0    0  19.6G  0 lvm  /
│ └─centos-home 253:1    0   192M  0 lvm  /home
└─sda3            8:3    0   9.6G  0 part └─centos-root 253:0    0  19.6G  0 lvm  /
sdb               8:16   0    20G  0 disk
sdc               8:32   0    20G  0 disk
sdd               8:48   1 233.3G  0 disk
├─sdd1            8:49   1 233.3G  0 part
└─sdd2            8:50   1  31.5K  0 part
sr0              11:0    1   4.3G  0 rom
[root@localhost ~]# echo "/dev/cdrom /mnt/cdrom iso9660 defaults 0 0" >> /etc/fstab
[root@localhost ~]# echo "/dev/sdd1 /mnt/usb vfat defaults 0 0" >> /etc/fstab
[root@localhost ~]# mount -a
[root@localhost ~]# rm /etc/yum.repos.d/* -rf
[root@localhost ~]# cp /mnt/usb/config/repos/source.repo /etc/yum.repos.d/
[root@localhost ~]# yum makecache
[root@localhost ~]# yum install -y wget git net-tools bind-utils iptables-services bridge-utils bash-completion kexec-tools sos psacct bash-completion.noarch bash-completion-extras.noarch python-passlib NetworkManager vim lrzsz psmisc tree ntp httpd-tools

★配置验证

[root@localhost ~]# getenforce
[root@localhost ~]# systemctl status firewalld
[root@localhost ~]# mount | egrep 'usb|cdrom'
/dev/sr0 on /mnt/cdrom type iso9660 (ro,relatime)
/dev/sdd1 on /mnt/usb type vfat (rw,relatime,fmask=0022,dmask=0022,codepage=437,iocharset=ascii,shortname=mixed,errors=remount-ro)

3.2 DNS服务构建

[root@localhost ~]# yum install bind -y
[root@localhost ~]# vim /etc/named.conf
options {listen-on port 53 { 127.0.0.1;any; };       allow-query     { localhost;any; };dnssec-enable no;dnssec-validation no;
[root@localhost ~]# vim /etc/named.rfc1912.zones      zone "blueicex.com" IN{type master;file "blueicex.com.zone";allow-update { none; };
};
[root@localhost ~]# cp /var/named/{named.empty,blueicex.com.zone}
[root@localhost ~]# vim /var/named/blueicex.com.zone
$TTL 3H
@       IN SOA  blueicex.com. blueice1980.126.com. (0       ; serial1D      ; refresh1H      ; retry1W      ; expire3H )    ; minimum
@       NS      dns.blueicex.com.
dns             A       192.168.0.77
master          A       192.168.0.71
node1           A       192.168.0.72
node2           A       192.168.0.73
node3           A       192.168.0.74
etcd            A       192.168.0.75
resource        A       192.168.0.77
registry        A       192.168.0.77
[root@localhost ~]# cd /var/named/
[root@localhost named]# chown root:named blueicex.com.zone
[root@localhost named]# named-checkconf /etc/named.conf
[root@localhoste named]# named-checkzone blueicex.com.zone blueicex.com.zone
zone blueicex.com.zone/IN: loaded serial 0
OK
[root@localhost named]# systemctl restart named && systemctl enable named

★配置验证

[root@localhost named]# dig @192.168.0.77 master.blueicex.com | grep master.blueicex.com.
;master.blueicex.com.       IN  A
master.blueicex.com.    10800   IN  A   192.168.0.71

3.3 httpd服务构建

★★★★构建单独的yum镜像源，镜像源文件提前下载到u盘中，整个openshift可脱机安装

[root@localhost ~]# yum -y install httpd
[root@localhost ~]# vim /etc/httpd/conf/httpd.conf
DocumentRoot "/mnt/usb"
[root@localhost ~]# cd /etc/httpd/conf.d/
[root@localhost conf.d]# vim yum.conf
<VirtualHost *:80>ServerAdmin blueice1980@126.comDocumentRoot  "/mnt/usb"# ServerName register.blueicex.com reource.blueicex.com ErrorLog "/var/log/httpd/yum-error_log"CustomLog "/var/log/httpd/yum-access_log" common
</VirtualHost>
<Directory /mnt/usb>Options AllAllowOverride NoneRequire all granted
</Directory>
[root@localhost conf.d]# mv welcome.conf welcome.conf.bak
[root@localhost conf.d]# systemctl start httpd && systemctl enable httpd

★配置验证

[root@localhost conf.d]# systemctl status httpd
[root@localhost conf.d]# curl http://192.168.0.77

3.4 ntp服务器构建

[root@localhost ~]# echo 'restrict 192.168.0.0 mask 255.255.255.0  nomodify notrap' >> /etc/ntp.conf
[root@localhost ~]# echo 'server 127.127.1.0' >> /etc/ntp.conf
[root@localhost ~]# echo 'fudge 127.127.1.0 stratum 10'>> /etc/ntp.conf
[root@localhost ~]# systemctl start ntpd && systemctl enable ntpd

★配置验证
和 “4.11 节点时间同步”共同验证

3.5 docker私有仓库构建

[root@localhost ~]# cp /mnt/usb/config/repos/yum.repo /etc/yum.repos.d/
[root@localhost ~]# yum install -y docker-distribution && systemctl start docker-distribution && systemctl enable docker-distribution

★配置验证

[root@localhost ~]# curl 192.168.0.77:5000/v2/_catalog
{"repositories":[]}

4. 主机基础配置

4.1 修改主机名

操作节点：master
被操作节点：master nodes etcd resource
需要输入主机密码

[root@localhost ~]# vi change-hostname.sh
ssh 192.168.0.71 hostnamectl set-hostname master.blueicex.com
ssh 192.168.0.72 hostnamectl set-hostname node1.blueicex.com
ssh 192.168.0.73 hostnamectl set-hostname node2.blueicex.com
ssh 192.168.0.74 hostnamectl set-hostname node3.blueicex.com
ssh 192.168.0.75 hostnamectl set-hostname etcd.blueicex.com
ssh 192.168.0.77 hostnamectl set-hostname resource.blueicex.com
[root@localhost ~]# bash change-hostname.sh

★配置验证

[root@localhost ~]# hostname
master.blueicex.com

4.2 配置DNS

操作节点：master
被操作节点：master nodes etcd resource
需要输入主机密码

[root@master ~]# vi config-dns.sh
for i in {2,3,4,5,7};
do ssh 192.168.0.7$i 'echo -e "DNS1=192.168.0.77\nDNS2=8.8.8.8\nNM_CONTROLLED=yes" >>/etc/sysconfig/network-scripts/ifcfg-ens33 && systemctl restart network'
done;
[root@master ~]# bash config-dns.sh
#ssh 本身主机不成功，需要从其他主机配置DNS
[root@resource ~]# ssh 192.168.0.71 'echo -e "DNS1=192.168.0.77\nDNS2=8.8.8.8\nNM_CONTROLLED=yes" >>/etc/sysconfig/network-scripts/ifcfg-ens33 && systemctl restart network'

★配置验证

[root@master ~]# cat /etc/resolv.conf | grep 192.168.0.77
nameserver 192.168.0.77

4.3 主机ssh互信

操作节点：master
被操作节点：master nodes etcd resource
需要输入主机密码

[root@master ~]# vi ssh-confidence.sh
rm -rf /root/.ssh/*
ssh-keygen -f /root/.ssh/id_rsa -N ''
ssh-copy-id -i ~/.ssh/id_rsa.pub master.blueicex.com
scp -r /root/.ssh node1.blueicex.com:/root/
scp -r /root/.ssh node2.blueicex.com:/root/
scp -r /root/.ssh node3.blueicex.com:/root/
scp -r /root/.ssh etcd.blueicex.com:/root/
scp -r /root/.ssh resource.blueicex.com:/root/
[root@master ~]# bash ssh-confidence.sh

★配置验证
★★★★★这6步一定要做，否则会报没有权限的错误，进入后，exit命令退出，for循环会进入下一个主机

[root@master ~]# for i in {master,node1,node2,node3,etcd,resource};do ssh $i ;done;
[root@node1 ~]# for i in {master,node1,node2,node3,etcd,resource};do ssh $i ;done;
[root@node2 ~]# for i in {master,node1,node2,node3,etcd,resource};do ssh $i ;done;
[root@node3 ~]# for i in {master,node1,node2,node3,etcd,resource};do ssh $i ;done;
[root@etcd ~]# for i in {master,node1,node2,node3,etcd,resource};do ssh $i ;done;
[root@resource ~]# for i in {master,node1,node2,node3,etcd,resource};do ssh $i ;done;

4.4 主节点配置yum源

操作节点：master
被操作节点：master

[root@master ~]# vi master-config-yum.sh
mkdir /mnt/{cdrom,usb} /blueicex/{soft,image,temp} -pv
rm -rf /etc/yum.repos.d/Ce*.*
scp resource.blueicex.com:/mnt/usb/config/repos/* /etc/yum.repos.d/
echo "/dev/cdrom /mnt/cdrom iso9660 defaults 0 0" >> /etc/fstab
mount -a
yum makecache
[root@master ~]# bash master-config-yum.sh

★★此处做快照镜像

4.5 主节点安装 ansible-2.6.5

操作节点：master
被操作节点：master
安装openshift-ansible-3.11.37后发现，仅仅支持Ansible >= 2.6.5
★★★★★这是个坑

[root@master ~]# cat /usr/share/doc/openshift-ansible-3.11.37/README.md
- Ansible >= 2.6.5, Ansible 2.7 is not yet supported and known to fail

[root@master ~]# yum install -y ansible-2.6.5
[root@master ~]# vi /etc/ansible/hosts
[alls]
master.blueicex.com
node1.blueicex.com
node2.blueicex.com
node3.blueicex.com
etcd.blueicex.com
[nodes]
node1.blueicex.com
node2.blueicex.com
node3.blueicex.com
[master]
master.blueicex.com
[etcd]
etcd.blueicex.com
[resource]
resource.blueicex.com

★配置验证

[root@master ~]# ansible all -m shell -a 'touch /root/test'
[root@master ~]# ansible all -m shell -a 'ls /root/' | grep test | wc -l
6

4.6 其他节点配置yum源

操作节点：master
被操作节点：nodes etcd resource

[root@master ~]# vi othernodes-config-yum.sh
ansible nodes,resource,etcd -m shell -a 'mkdir /mnt/{cdrom,usb} /blueicex/{soft,image,temp} -pv'
ansible nodes,resource,etcd -m shell -a 'rm -rf /etc/yum.repos.d/Ce*.*'
ansible nodes,resource,etcd -m shell -a 'echo "/dev/cdrom /mnt/cdrom iso9660 defaults 0 0">>/etc/fstab'
ansible nodes,resource,etcd -m shell -a 'mount -a'
ansible nodes,resource,etcd -m copy -a 'src=/etc/yum.repos.d/ dest=/etc/yum.repos.d/'
[root@master ~]# bash othernodes-config-yum.sh

★配置验证

#共6台主机
[root@master ~]# ansible all -m shell -a 'ls /etc/yum.repos.d' | grep yum.repo | wc -l
6

4.7 关闭防火墙

操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible all -m shell -a 'systemctl stop firewalld && systemctl disable firewalld'

★配置验证

#共6台主机
[root@master ~]# ansible all -m shell -a 'systemctl status firewalld' | grep inactive | wc -l
6

4.8 开启selinux

操作节点：master
被操作节点：master nodes etcd
默认是开启的，可不手动开启

[root@master ~]# ansible alls -m shell -a 'sed -i s/^SELINUX=.*/SELINUX=enforcing/g /etc/selinux/config'
[root@master ~]# ansible  all -m shell -a 'touch /.autorelabel '
[root@master ~]# ansible nodes,etcd -m shell -a 'reboot'
[root@master ~]# ansible master -m shell -a 'reboot'

★配置验证

#共6台主机,resource主机没有enforcing
[root@master ~]# ansible all -m shell -a 'getenforce' | grep Enforcing | wc -l
5

4.9 配置sshd

操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible all -m shell -a 'echo UseDNS=no >>/etc/ssh/sshd_config'
[root@master ~]# ansible all -m shell -a 'systemctl restart sshd'

★配置验证

#共6台主机
[root@master ~]# ansible all -m shell -a 'grep  -w UseDNS=no /etc/ssh/sshd_config' | wc -l
12

4.10 基础安装

操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible all -m shell -a 'yum install -y wget git net-tools bind-utils iptables-services bridge-utils bash-completion kexec-tools sos psacct bash-completion.noarch bash-completion-extras.noarch python-passlib NetworkManager vim lrzsz psmisc tree ntp httpd-tools'

4.11 节点时间同步

操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible resource -m shell -a 'systemctl stop ntpd && ntpdate ntp.ntsc.ac.cn && systemctl start ntpd'
[root@master ~]# ansible alls -m shell -a 'ntpdate resource.blueicex.com'

★配置验证

[root@master ~]# ansible alls -m shell -a 'date'

4.12 设置ip地址转发及iptables放行

操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible alls -m shell -a ' echo "1" > /proc/sys/net/ipv4/ip_forward'
[root@master ~]# ansible alls -m shell -a 'iptables -P FORWARD ACCEPT'
[root@master ~]# ansible alls -m shell -a '/sbin/iptables -I INPUT -p tcp --dport 8443 -j ACCEPT && service iptables save'

★★★此处做镜像快照

5. docker 部署

5.1 yum安装 docker

操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible all -m shell -a 'yum install docker -y '

5.2 docker 配置

5.2.1 修改镜像仓库地址
操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# vim /etc/containers/registries.conf
[registries.search]
registries = ['registry.docker-cn.com','hub-mirror.c.163.com','docker.mirrors.ustc.edu.cn','resource.blueicex.com:5000','2savhou3.mirror.aliyuncs.com']
[registries.insecure]
registries = ['resource.blueicex.com:5000']
[registries.block]
registries = []
[root@master ~]# ansible nodes,etcd,resource -m copy -a 'src=/etc/containers/registries.conf dest=/etc/containers/'

5.2.2 配置 daemon.json
操作节点：master
被操作节点：master nodes etcd resource
"https://2savhou3.mirror.aliyuncs.com"为阿里云镜像加速器

[root@master ~]# vim /etc/docker/daemon.json
{"registry-mirrors": ["https://registry.docker-cn.com","http://hub-mirror.c.163.com","https://docker.mirrors.ustc.edu.cn","https://2savhou3.mirror.aliyuncs.com"],"insecure-registries":["resource.blueicex.com:5000"]
}
[root@master ~]# ansible nodes,etcd,resource  -m copy -a 'src=/etc/docker/daemon.json  dest=/etc/docker/daemon.json'

5.2.3 配置执行 docker-storage-setup
操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# find / -iname container-storage-setup
/run/container-storage-setup
/usr/bin/container-storage-setup
/usr/share/container-storage-setup
[root@master ~]# vim config-docker-storage-setup
cp /usr/share/container-storage-setup /etc/sysconfig/config-docker-storage-setup
echo DEVS=/dev/sdb >> /etc/sysconfig/config-docker-storage-setup
echo VG=dockervg  >> /etc/sysconfig/config-docker-storage-setup
ansible nodes -m copy -a 'src=/etc/sysconfig/docker-storage-setup dest=/etc/sysconfig/'
ansible all -m shell -a 'bash docker-storage-setup'
[root@master ~]# bash config-docker-storage-setup.sh

5.2.5 启动 docker
修改docker为允许selinux

[root@master ~]#vim /etc/sysconfig/docker
# Modify these options if you want to change the way the docker daemon runs
OPTIONS='--selinux-enabled=true --log-driver=journald --signature-verification=false'
[root@master ~]# ansible nodes -m copy -a 'src=/etc/sysconfig/docker  dest=/etc/sysconfig/'

5.2.6 启动 docker
操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible all -m shell -a 'systemctl start docker  && systemctl enable docker'

5.2.7 敏捷部署 docker
操作节点：master
被操作节点：master nodes etcd resource
以上可以浓缩为 initdocker.sh，docker-storage-setup、daemon.json、registries.conf事先已定义完毕

[root@master ~]# vim initdocker.sh
ansible all -m shell -a 'yum remove docker* -y && yum install docker docker-distribution -y '
ansible all -m get_url -a 'url=http://resource.blueicex.com/config/docker/daemon.json dest=/etc/docker/'
ansible all -m get_url -a 'url=http://resource.blueicex.com/config/docker/docker dest=/etc/sysconfig/'
ansible all -m get_url -a 'url=http://resource.blueicex.com/config/docker/docker-storage-setup dest=/etc/sysconfig/'
ansible all -m get_url -a 'url=http://resource.blueicex.com/config/docker/registries.conf dest=/etc/containers/'
ansible all -m shell -a 'docker-storage-setup && systemctl start docker docker-distribution  && systemctl enable docker docker-distribution '
ansible all -m shell -a 'lsblk | grep docker'
[root@master ~]# bash initdocker.sh

★配置验证

[root@master ~]# ansible all -m shell -a 'docker info'
[root@master ~]# curl resource.blueicex.com:5000/v2/_catalog

6. 装载镜像

操作节点：master
被操作节点：master nodes etcd resource
master 节点下载装载镜像,镜像已在资源主机192.168.0.77下载完毕

[root@master ~]# cd /blueicex/image/
[root@master image]# rm * -rf
[root@master image]# vim openshift-image.txt
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-node-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-control-plane-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-haproxy-router-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-deployer-v3.11.0.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-pod-v3.11.0.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/cluster-monitoring-operator-v0.1.1.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-docker-registry-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-console-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/prometheus-config-reloader-v0.23.2.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/prometheus-operator-v0.23.2.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-web-console-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-metrics-server-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-metrics-heapster-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-metrics-hawkular-metrics-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-metrics-schema-installer-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/origin-metrics-cassandra-v3.11.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/kubernetes-latest.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/prometheus-alertmanager-v0.15.2.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/prometheus-node-exporter-v0.16.0.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/prometheus-v2.3.2.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/grafana-5.2.1.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/kube-state-metrics-v1.3.1.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/oauth-proxy-v1.1.0.tar
http://192.168.0.77/freesoft/openshift-image/openshift3.11/configmap-reload-v0.0.1.tar
[root@master image]# wget -c -i openshift-image.txt
[root@master image]# for i in `ls *.tar` ; do docker image load -i $i;done;
[root@master image]# docker images | awk -F" " '{print $1":"$2}' | awk -F"/" '{print "docker image tag "$1"/"$2"/"$3" resource.blueicex.com:5000/"$2"/"$3}' > temp
[root@master image]# sed '/REPOSITORY/d' temp > marktag.sh
[root@master image]# bash marktag.sh
[root@master image]# awk -F" " '{print "docker push "$5}' marktag.sh >pushimage.sh
[root@master image]# bash pushimage.sh
[root@node1 ~]# watch curl resource.blueicex.com:5000/v2/_catalog
{"repositories":["cockpit/kubernetes","coreos/cluster-monitoring-operator","coreos/configmap-reload","coreos/kube-state-metrics","coreos/prometheus-config-reloader","coreos/prometheus-operator","grafana/grafana","openshift/oauth-proxy","openshift/origin-console","openshift/origin-control-plane","openshift/origin-deployer","openshift/origin-docker-registry","openshift/origin-haproxy-router","openshift/origin-metrics-cassandra","openshift/origin-metrics-hawkular-metrics","openshift/origin-metrics-heapster","openshift/origin-metrics-schema-installer","openshift/origin-metrics-server","openshift/origin-node","openshift/origin-pod","openshift/origin-web-console","openshift/prometheus","openshift/prometheus-alertmanager","openshift/prometheus-node-exporter"]}
#nodes etcd 节点下载装载镜像
[root@master image]# ansible nodes,etcd -m copy -a 'src=/blueicex/image/openshift-image.txt dest=/blueicex/image/'
[root@master image]# ansible nodes,etcd -m shell -a 'cd /blueicex/image && wget -c -i openshift-image.txt'
[root@master image]# ansible nodes,etcd -m shell -a 'for i in `ls /blueicex/image/*.tar` ; do docker image load -i $i;done;'

★配置验证

[root@master ~]# ansible master,nodes,etcd -m shell -a 'docker images | wc -l'

★★★此处做镜像快照

7. openshift-ansible-3.11 安装

操作节点：master
被操作节点：master

#太慢，不建议使用此种方式
[root@master ~]# git clone https://github.com/openshift/openshift-ansible
#推荐yum安装
[root@master ~]# yum list openshift-ansible --showduplica|grep 3.11
Bad id for repo: centos-paas-openshift-origin , byte =   28
openshift-ansible.noarch 3.11.37-1.git.0.3b8b341.el7       centos-paas-openshift-origin311
[root@master ~]# yum install -y openshift-ansible-3.11.37
[root@master ~]# rpm -ql openshift-ansible
/usr/share/ansible/openshift-ansible
/usr/share/doc/openshift-ansible-3.11.37
/usr/share/doc/openshift-ansible-3.11.37/README.md

★配置验证
ansible 要保持在2.6.5版本

[root@master ~]# rpm -ql openshift-ansible
[root@master ~]# ansible --version
ansible 2.6.5

8. 补充更新

操作节点：master
被操作节点：master nodes etcd resource

[root@master ~]# ansible all -m shell -a 'yum install -y  pyOpenSSL python-cryptography python-lxml java-1.8.0-openjdk-headless patch'
[root@master ~]# ansible all -m shell -a 'yum update -y'

9. 配置 openshift-ansible

操作节点：master
被操作节点：master

[root@master ~]# cd /etc/ansible/
[root@master ansible]# vim inventory-3.11
#Create an OSEv3 group that contains the masters and nodes groups
[OSEv3:children]
masters
nodes
etcd
lb
#Set variables common for all OSEv3 hosts
[OSEv3:vars]
#SSH user, this user should allow ssh based auth without requiring a password
ansible_ssh_user=root#If ansible_ssh_user is not root, ansible_become must be set to true
#ansible_become=true
openshift_deployment_type=origin
openshift_release=3.11
openshift_image_tag=v3.11
openshift_pkg_version=-3.11.0
openshift_use_openshift_sdn=true
#uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]openshift_master_default_subdomain=blueicex.com
openshift_disable_check=docker_storage,memory_availability,disk_availability,docker_image_availability
openshift_master_cluster_method=native
openshift_master_cluster_hostname=master.blueicex.com
openshift_master_cluster_public_hostname=master.blueicex.com
#false
openshift_metrics_install_metrics=false
ansible_service_broker_install=false
openshift_enable_service_catalog=false
template_service_broker_install=false
openshift_logging_install_logging=false
enable_excluders=false#registry passwd
#oreg_url=192.168.0.77:5000/openshift3/ose-component:{version}
oreg_url=resource.blueicex.com:5000/openshift/origin-${component}:${version}
openshift_examples_modify_imagestreams=true# Enable cockpit
#osm_use_cockpit=true
#
# Set cockpit plugins
#osm_cockpit_plugins=['cockpit-kubernetes']#docker config
openshift_docker_additional_registries=resource.blueicex.com
openshift_docker_insecure_registries=resource.blueicex.com
#openshift_docker_blocked_registriesopenshift_docker_options="–log-driver json-file --log-opt max-size=1M --log-opt max-file=3"
#--insecure-registry 192.168.0.0/16# OpenShift Router Options
# Router selector (optional)
# Router will only be created if nodes matching this label are present.
# Default value: 'node-role.kubernetes.io/infra=true'
#openshift_hosted_router_selector='node-role.kubernetes.io/infra=true'
#
# Router replicas (optional)
# Unless specified, openshift-ansible will calculate the replica count
# based on the number of nodes matching the openshift router selector.
#openshift_hosted_router_replicas=2# Openshift Registry Options
# Registry selector (optional)
# Registry will only be created if nodes matching this label are present.
# Default value: 'node-role.kubernetes.io/infra=true'
#openshift_hosted_registry_selector='node-role.kubernetes.io/infra=true'
#
# Registry replicas (optional)
# Unless specified, openshift-ansible will calculate the replica count
# based on the number of nodes matching the openshift registry selector.
#openshift_hosted_registry_replicas=2openshift_cluster_monitoring_operator_install=true
# openshift_metrics_install_metrics=true
# openshift_enable_unsupported_configurations=True
# openshift_logging_es_nodeselector='node-role.kubernetes.io/infra: "true"'
# openshift_logging_kibana_nodeselector='node-role.kubernetes.io/infra: "true"'[masters]
master.blueicex.com [lb]
node1.blueicex.com  [etcd]
master.blueicex.com [nodes]
master.blueicex.com openshift_schedulable=True openshift_node_group_name='node-config-master'
node1.blueicex.com openshift_node_group_name='node-config-compute'
node2.blueicex.com openshift_node_group_name='node-config-compute'
etcd.blueicex.com openshift_node_group_name='node-config-compute'
node3.blueicex.com openshift_node_group_name='node-config-infra'
#master.blueicex.com openshift_node_group_name='node-config-master'
#node1.blueicex.com openshift_node_group_name='node-config-compute' openshift_ip=192.168.0.72
#master.blueicex.com openshift_schedulable=True openshift_node_group_name='node-config-all-in-one'
#node2.blueicex.com openshift_node_group_name='node-config-compute' openshift_ip=192.168.0.73
#node3.blueicex.com openshift_node_group_name='node-config-compute' openshift_ip=192.168.0.74
#node2.blueicex.com openshift_node_group_name='node-config-infra'   openshift_ip=192.168.0.73

10. 安装

操作节点：master
被操作节点：master nodes etcd

[root@master ~]# cd /usr/share/ansible/openshift-ansible/roles/openshift_repos/templates
[root@master templates]# cp CentOS-OpenShift-Origin.repo.j2 CentOS-OpenShift-Origin.repo.j2.bak
[root@master templates]# /bin/cp /etc/yum.repos.d/yum.repo CentOS-OpenShift-Origin.repo.j2

★★★★★★建议做快照备份

[root@master ~]# cd /usr/share/ansible/openshift-ansible#安装前检查此处报错，docker不能启动，我略过了安装检查
[root@master openshift-ansible]# ansible-playbook -i /etc/ansible/inventory-3.11 playbooks/prerequisites.yml
#安装
[root@master openshift-ansible]# ansible-playbook -i /etc/ansible/inventory-3.11 playbooks/deploy_cluster.yml#如需重新安装，先卸载
[root@master openshift-ansible]# ansible-playbook -i /etc/ansible/inventory-3.11 playbooks/adhoc/uninstall.yml

安装过程中会报错，docker容器起不来，执行"5.2.5 敏捷部署 docker"。

#开启selinux后，有可能进入不到系统，需要重新打标
touch /.autorelabel

————Blueicex 2020/05/19 23:35 blueice1980@126.com