环境描述
- 介绍
两个节点，一个master节点，另一个当做compute和infra节点，使用的操作系统为rhel 7.4，没有安装EFK、service broker、service catalog、metric，promethues在3.11正式GA，默认就会安装。因为本人有红帽的订阅账号，所以可以从红帽的源进行yum安装，需要提醒的是，从3.11开始，红帽官方的镜像仓库从registry.access.redhat.com变为registry.redhat.io，且拉取镜像也需要红帽的订阅账号了。- 主机信息，两个节点，infra和node复用同一台，openshift-cluster-test2.com为集群的入口
172.16.37.11 master.test2.com
172.16.37.12 node.test2.com
172.16.37.12 infra.test2.com
172.16.37.11 openshift-cluster.test2.com - 操作系统rhel 7.4，ocp 3.11；
yum 源搭建(172.16.37.12)- 本环境没有外部的DNS server作为域名解析，所以使用/etc/hosts进行主机的解析，在两台主机/etc/hosts添加如下解析，环境清单及/etc/hosts文件，操作系统为7.4
172.16.37.11 master.test2.com
172.16.37.12 node.test2.com
172.16.37.12 infra.test2.com
172.16.37.11 openshift-cluster.test2.com- 配置selinux为enforcing：
[root@node ~]# cat /etc/selinux/config
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
# enforcing - SELinux security policy is enforced.
# permissive - SELinux prints warnings instead of enforcing.
# disabled - No SELinux policy is loaded.
SELINUX=enforcing
# SELINUXTYPE= can take one of three two values:
# targeted - Targeted processes are protected,
# minimum - Modification of targeted policy. Only selected processes are protected
# mls - Multi Level Security protection.
SELINUXTYPE=targeted- 上传openshift RPM包至节点，并搭建为yum源
[root@node ~]# tar -zxvf ocp311rpms.tar.gz -C /opt/repos/[root@node ~]# ll /opt/repos/总用量 0drwxr-xr-x. 4 root root 55 Oct 19 23:50 rhel-7-server-ansible-2.6-rpms
drwxr-xr-x. 4 root root 55 Oct 19 23:50 rhel-7-server-extras-rpms
drwxr-xr-x. 4 root root 55 Oct 20 11:04 rhel-7-server-ose-3.11-rpms
drwxr-xr-x. 4 root root 55 Oct 19 23:48 rhel-7-server-rpms- 创建本机yum源：
[root@node ~]# cat /etc/yum.repos.d/ocp-local.repo[rhel-7-server-rpms]
name=rhel-7-server-rpms
baseurl=file:///opt/repos/rhel-7-server-rpms
enabled=1gpgcheck=0[rhel-7-server-extras-rpms]
name=rhel-7-server-extras-rpms
baseurl=file:///opt/repos/rhel-7-server-extras-rpms
enabled=1gpgcheck=0[rhel-7-server-ose-3.11-rpms]
name=rhel-7-server-ose-3.11-rpms
baseurl=file:///opt/repos/rhel-7-server-ose-3.11-rpms
enabled=1gpgcheck=0[rhel-7-server-ansible-2.6-rpms]
name=rhel-7-server-ansible-2.6-rpms
baseurl=file:///opt/repos/rhel-7-server-ansible-2.6-rpms
enabled=1gpgcheck=0- 查看yum源是否正常
[root@node ~]# yum repolistrepolist: 5625- 检查yum源中openshift rpm内容是否正确
[root@node ~]# yum list|grep atomic-openshift | wc -l20- 将本机源设置为yum服务器，安装httpd
yum -y install httpd- 配置httpd：/etc/httpd/conf.d/yum.conf
Alias /repos "/opt/repos"
<Directory "/opt/repos">
Options Indexes FollowSymLinks
Require all granted
</Directory>
<Location /repos>
SetHandler None
</Location>- 修改httpd的服务为81端口，防止与router的80端口冲突
[root@node ~]# vi /etc/httpd/conf/httpd.confListen 81- 启动httpd
systemctl enable httpd;systemctl restart httpd- 检查httpd是否配置正确，正确的话能读取到rpm的内容
[root@node ~]# curl 172.16.37.12:81/repos/- 配置yum文件ocp.repo，将原先的ocp-local.repo修改为ocp-local.repo.bak1
[root@node ~]# mv ocp-local.repo ocp-local.repo.bak1
[root@node ~]# cat /etc/yum.repos.d/ocp.repo
[rhel-7-server-rpms]
name=rhel-7-server-rpms
baseurl=http://172.16.37.12:81/repos/rhel-7-server-rpms
enabled=1
gpgcheck=0
[rhel-7-server-extras-rpms]
name=rhel-7-server-extras-rpms
baseurl=http://172.16.37.12:81/repos/rhel-7-server-extras-rpms
enabled=1
gpgcheck=0
[rhel-7-server-ose-3.11-rpms]
name=rhel-7-server-ose-3.11-rpms
baseurl=http://172.16.37.12:81/repos/rhel-7-server-ose-3.11-rpms
enabled=1
gpgcheck=0
[rhel-7-server-ansible-2.6-rpms]
name=rhel-7-server-ansible-2.6-rpms
baseurl=http://172.16.37.12:81/repos/rhel-7-server-ansible-2.6-rpms
enabled=1
gpgcheck=0- 检查yum源服务器是否配置完成
[root@node ~]# yum clean all;yum repolist- 创建免密登录并且分发密钥至其他节点
ssh-keygen;for i in master.test2.com node.test2.com infra.test2.com
do
echo $i
ssh-copy-id $i;
done;- 脚本批量修改主机名
for i in master.test2.com node.test2.com
do
echo $i
ssh $i
hostnamectl "set-hostname $i";
done;- 安装iptabes服务
yum install iptables-services -y- 配置iptables
cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date " %Y%m%d%H%M%S");
sed -i '/.*--dport 22 -j ACCEPT.*/a-A INPUT -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a-A INPUT -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a-A INPUT -p tcp -m state --state NEW -m tcp --dport 5000 -j ACCEPT' /etc/sysconfig/iptables;
sed -i '/.*--dport 22 -j ACCEPT.*/a-A INPUT -p tcp -m state --state NEW -m tcp --dport 81 -j ACCEPT' /etc/sysconfig/iptables;
systemctl restart iptables;systemctl enable iptables- 安装openshift-ansible
yum install openshift-ansible -y- 配置ansible hosts文件:/etc/ansible/hosts，注意修改cluster域名地址和主机域名
[root@node ~]# cat /etc/ansible/hosts# Create an OSEv3 group that contains the masters, nodes, and etcd groups[OSEv3:children]
masters
nodes
etcd
# Set variables common for all OSEv3 hosts
[OSEv3:vars]
# SSH user, this user should allow ssh based auth without requiring a password
ansible_ssh_user=root
openshift_deployment_type=openshift-enterprise
openshift_image_tag=v3.11.16
# If ansible_ssh_user is not root, ansible_become must be set to true#ansible_become=true
# default selectors for router and registry services# openshift_router_selector='node-role.kubernetes.io/infra=true'# openshift_registry_selector='node-role.kubernetes.io/infra=true'# uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
openshift_master_default_subdomain=test2.com
openshift_disable_check=memory_availability,disk_availability,docker_image_availability
os_sdn_network_plugin_name=redhat/openshift-ovs-multitenant
openshift_master_cluster_method=native
openshift_master_cluster_hostname=openshift-cluster.test2.com
openshift_master_cluster_public_hostname=openshift-cluster.test2.com# false
ansible_service_broker_install=false
openshift_enable_service_catalog=false
template_service_broker_install=false
openshift_logging_install_logging=false# registry passwd
oreg_url=172.16.37.12:5000/openshift3/ose-${component}:${version}
openshift_examples_modify_imagestreams=true# docker config
openshift_docker_additional_registries=172.16.37.12:5000,172.30.0.0/16
openshift_docker_insecure_registries=172.16.37.12:5000,172.30.0.0/16
#openshift_docker_blocked_registries
openshift_docker_options="--log-driver json-file --log-opt max-size=1M --log-opt max-file=3"# openshift_cluster_monitoring_operator_install=false
# openshift_metrics_install_metrics=true
# openshift_enable_unsupported_configurations=True
#openshift_logging_es_nodeselector='node-role.kubernetes.io/infra: "true"'
#openshift_logging_kibana_nodeselector='node-role.kubernetes.io/infra: "true"'
# host group for masters[masters]
master.test2.com
# host group for etcd
[etcd]
master.test2.com
# host group for nodes, includes region info
[nodes]
master.test2.com openshift_node_group_name='node-config-master'
node.test2.com openshift_node_group_name='node-config-compute'
infra.test2.com openshift_node_group_name='node-config-infra'- 分发/etc/hosts文件至其他节点
ansible all -m copy -a "src=/etc/hosts dest=/etc/hosts "- 分发yum配置到各节点
ansible all -m copy -a "src=/etc/yum.repos.d/ocp.repo dest=/etc/yum.repos.d/ocp.repo"- 查看yum源是否正常
for i in master.test2.com node.test2.com
do
echo $i
ssh $i 'yum repolist'
done;所有节点基础包安装
- 基础包
yum install wget git net-tools bind-utils yum-utils iptables-services bridge-utils bash-completion kexec-tools sos psacct vim lrzsz python-setuptools unzip tree docker-1.13.1 -y- 更新
yum -y update配置docker（172.16.37.12）
- 配置/etc/containers/registries.conf为如下内容
# This is a system-wide configuration file used to# keep track of registries for various container backends.# It adheres to TOML format and does not support recursive# lists of registries.# The default location for this configuration file is /etc/containers/registries.conf.# The only valid categories are: 'registries.search', 'registries.insecure',# and 'registries.block'.[registries.search]
registries = ['172.16.37.12:5000']
# If you need to access insecure registries, add the registry's fully-qualified name.# An insecure registry is one that does not have a valid SSL certificate or only does HTTP.
[registries.insecure]
registries = ['172.16.37.12:5000']
# If you need to block pull access from a registry, uncomment the section below# and add the registries fully-qualified name.## Docker only
[registries.block]
registries = ['registry.access.redhat.com']- 分发/etc/containers/registries.conf文件至其他节点
ansible all -m copy -a "src=/etc/containers/registries.conf dest=/etc/containers/registries.conf "- 启动docker服务
ansible all -a 'systemctl start docker';ansible all -a 'systemctl enable docker'- 配置iptables，放行流量
ansible all -m copy -a "src=/etc/sysconfig/iptables dest=/etc/sysconfig/iptables"ansible all -a "systemctl restart iptables"镜像仓库搭建(172.16.37.12)
- 安装docker-distribution
yum install -y docker-distribution;systemctl start docker-distribution;systemctl enable docker-distribution- 上传镜像包并导入，此过程会占用大量磁盘空间和内存资源
docker load -i docker load -i 3.11-images.tar.bz2- 修改镜像tag
docker images|grep registry.redhat.io|awk '{print $1"/"$2}'|awk -F "/" '{print "docker tag "$1"/"$2"/"$3":"$4" 172.16.37.12:5000/"$2"/"$3":"$4}'|sh- 上传镜像至镜像仓库
docker images|grep 172.16.37.12:5000|awk '{print "docker push " $1":"$2}'|shOpenShift部署节点组件(172.16.37.12)
- 部署前检查操作
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml- 开始部署
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/deploy_cluster.yml- 卸载（如有需要则执行）
ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/adhoc/uninstall.yml部署结束后
- 创建用户（master1）
htpasswd -cb /etc/origin/master/htpasswd admin 123456
oc adm policy add-cluster-role-to-user cluster-admin admin- 登录页面
配置好浏览器所在机子的hosts文件里面域名对应master的IP，然后在浏览器输入地址：
https://openshift-cluster.test2.com:8443，输入用户名密码
