上一篇介绍了关于虚机如何结合pipeline实现部署和回滚，并结合了ansible的playbook实现对集群，或者不同的集群进行部署，下面介绍下如何使用pipeline结合k8s实现部署与回滚容器完成部署
设置参数
同样新建一个pipeline风格的任务，然后需要准备几个参数化构建过程的插件，包括：Active Choice Parameter，Choice parameter，Extended Choice Parameter等插件 设置选择分支
代码语言：javascript复制def gettags = ("git ls-remote -h git@coxxxxxxxxxxxxxxxxxxxxxxicheng00001/lorehouse-base-interface.git").execute()
gettags.text.readLines().collect { it.split()[1].replaceAll('refs/heads/', '') }.unique()设置回滚或构建选项
设置要回滚的Tag
pipeline示例
代码语言：javascript复制node {
def registry = "192.168.66.169"
def ops_git_addr = "git@coxxxxxxxxxxxxom:CICDywzx00001/phoenix-sample.git"
def git_address_base = "git@coxxxxxxxxxxxxom:bszsk_wangxicheng00001/lorehouse-base-interface.git"
def git_address_user = "git@coxxxxxxxxxxxxxxxxxxeng00001/lorehouse-user.git"
def git_address_run = "git@codexxxxxxxxxxxxxxxxxxx00001/knowledge-lib-run.git"
def git_auth = "998dc514-4b0c-4194-80f6-5393da0cb710"
def p1 = "lorehouse-base-interface"
def p2 = "lorehouse-user"
def p3 = "lorehouse-lib-run"
def out_ports = '32000'
def apps_name = 'boshi-test'
if (env.Action == "Deploy") {
stage('create dir') {
if (env.Action == "Deploy") {
deleteDir()
sh "mkdir dir1 dir2 dir3"
}
}
try {
stage('base') {
if (env.Action == "Deploy") {
dir('dir1') {
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address_base}"]]])
script {
pom1 = readMavenPom file: 'pom.xml'
v1 = "${pom1.version}"
e1 = sh(script: "git log -1 --pretty=format:'�'", returnStdout: true).trim()
}
sh "mvn package install sonar:sonar -Dsonar.projectKey=${p1} -Dsonar.projectName=${p1} -Dsonar.projectVersion=${v1} -Dsonar.sourceEncoding=UTF-8 -Dsonar.exclusions=src/test/** -Dsonar.sources=src/ -Dsonar.java.binaries=target/classes -Dsonar.host.url=https://sonar.yunshicloud.com -Dsonar.login=a0994774ef9e11c652e9b7545401394d0bae6a00"
}
}
}
}
catch(all) {
currentBuild.result = 'FAILURE'
}
if(currentBuild.result == 'FAILURE') {
stage('user') {
dir('dir2') {
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address_user}"]]])
script {
pom2 = readMavenPom file: 'pom.xml'
v2 = "${pom2.version}"
e2 = sh(script: "git log -1 --pretty=format:'�'", returnStdout: true).trim()
}
sh "mvn package install sonar:sonar -Dsonar.projectKey=${p2} -Dsonar.projectName=${p2} -Dsonar.projectVersion=${v2} -Dsonar.sourceEncoding=UTF-8 -Dsonar.exclusions=src/test/** -Dsonar.sources=src/ -Dsonar.java.binaries=target/classes -Dsonar.host.url=https://sonar.yunshicloud.com -Dsonar.login=a0994774ef9e11c652e9b7545401394d0bae6a00"
}
}
} else {
stage('user') {
dir('dir2') {
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address_user}"]]])
script {
pom2 = readMavenPom file: 'pom.xml'
v2 = "${pom2.version}"
e2 = sh(script: "git log -1 --pretty=format:'�'", returnStdout: true).trim()
}
sh "mvn package install sonar:sonar -Dsonar.projectKey=${p2} -Dsonar.projectName=${p2} -Dsonar.projectVersion=${v2} -Dsonar.sourceEncoding=UTF-8 -Dsonar.exclusions=src/test/** -Dsonar.sources=src/ -Dsonar.java.binaries=target/classes -Dsonar.host.url=https://sonar.yunshicloud.com -Dsonar.login=a0994774ef9e11c652e9b7545401394d0bae6a00"
}
}
}
try {
retry(1) {
stage('base') {
dir('dir1') {
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address_base}"]]])
script {
pom1 = readMavenPom file: 'pom.xml'
v1 = "${pom1.version}"
e1 = sh(script: "git log -1 --pretty=format:'�'", returnStdout: true).trim()
}
sh "mvn package install sonar:sonar -Dsonar.projectKey=${p1} -Dsonar.projectName=${p1} -Dsonar.projectVersion=${v1} -Dsonar.sourceEncoding=UTF-8 -Dsonar.exclusions=src/test/** -Dsonar.sources=src/ -Dsonar.java.binaries=target/classes -Dsonar.host.url=https://sonar.yunshicloud.com -Dsonar.login=a0994774ef9e11c652e9b7545401394d0bae6a00"
}
}
}
}
catch (all) {
currentBuild.result = 'FAILURE'
}
if(currentBuild.result == 'FAILURE') {
throw 'FAILURE'
} else {
stage('run') {
dir('dir3') {
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address_run}"]]])
script {
pom3 = readMavenPom file: 'pom.xml'
v3 = "${pom1.version}"
e3 = sh(script: "git log -1 --pretty=format:'�'", returnStdout: true).trim()
}
sh "mvn package install sonar:sonar -Dsonar.projectKey=${p3} -Dsonar.projectName=${p3} -Dsonar.projectVersion=${v3} -Dsonar.sourceEncoding=UTF-8 -Dsonar.exclusions=src/test/** -Dsonar.sources=src/ -Dsonar.java.binaries=target/classes -Dsonar.host.url=https://sonar.yunshicloud.com -Dsonar.login=a0994774ef9e11c652e9b7545401394d0bae6a00""
sh "rm -rf jenkins-script && mkdir jenkins-script && cd jenkins-script && git clone ${ops_git_addr} && cp -r ${WORKSPACE}/dir6/jenkins-script/phoenix-sample/deploy-jenkins/* ${WORKSPACE}/dir6/target/"
script {
git_cm_id = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()
date_time = sh(script: "date %Y%m%d%H%M", returnStdout: true).trim()
pom = readMavenPom file: 'pom.xml'
img_group = "testhuawei"
img_name = "${registry}/${img_group}/${pom.artifactId}"
cur_version = "${pom.version}"
whole_img_name = "${img_name}:${date_time}-${cur_version}-${git_cm_id}"
tag_info = "${date_time}-${cur_version}-${git_cm_id}"
jar_file_info = "${pom.artifactId}-${pom.version}.jar"
app_name_info = "${pom.artifactId}"
kuber_yml_dirs = "/root/leishengchan-k8s-yaml/${app_name_info}"
}
}
}
}
stage('build and push') {
dir('dir3/target') {
withCredentials([usernamePassword(credentialsId: 'ed481948-5714-4d16-93b3-57023c4a5fbd', passwordVariable: 'dockerpasswd', usernameVariable: 'dockeruser')]) {
sh "cp -r docker-build/* ."
sh "cp -r rollback/* ."
sh "./rollback.sh ${tag_info}"
sh "./docker.sh --app_name=${app_name_info} --jar_file=${jar_file_info} --img_info=${whole_img_name}"
}
}
}
stage('deploy by k8s') {
dir('dir3/target') {
sh "./k8s.sh --app_name=${apps_name} --replica_number=1 --image_address=${whole_img_name} --mem=100Mi --cpu=100m --max_mem=2048Mi --max_cpu=2 --inner_port=8080 --target_port=8080 --out_port=${out_ports} --kuber_yml_dir=${kuber_yml_dirs}"
}
}
stage('Sonar check and send msg') {
sh "/data/jenkins/workspace/scripts/sonar.py ${p1} ${v1}"
sh "/data/jenkins/workspace/scripts/sonar.py ${p2} ${v2}"
sh "/data/jenkins/workspace/scripts/sonar.py ${p3} ${v3}"
}
}
if (env.Action == "Rollback") {
checkout([$class: 'GitSCM', branches: [[name: '${Branch}']], userRemoteConfigs: [[credentialsId: "${git_auth}", url: "${git_address_run}"]]])
sh "rm -rf jenkins-script && mkdir jenkins-script && cd jenkins-script && git clone ${ops_git_addr} && cp -r ${WORKSPACE}/jenkins-script/phoenix-sample/deploy-jenkins/* ${WORKSPACE}/"
script {
poms = readMavenPom file: 'pom.xml'
img_groups = "testhuawei"
img_names = "${registry}/${img_groups}/${poms.artifactId}"
whole_img_names = "${img_names}:${RollbackFile}"
app_name_infos = "${poms.artifactId}"
kuber_yml_dirss = "/root/leishengchan-k8s-yaml/${app_name_infos}"
}
sh "rm -rf jenkins-script && mkdir jenkins-script && cd jenkins-script && git clone ${ops_git_addr} && cp -r ${WORKSPACE}/jenkins-script/phoenix-sample/deploy-jenkins/* ${WORKSPACE}/"
sh "./k8s.sh --app_name=${apps_name} --replica_number=1 --image_address=${whole_img_names} --mem=100Mi --cpu=100m --max_mem=2048Mi --max_cpu=2 --inner_port=8080 --target_port=8080 --out_port=${out_ports} --kuber_yml_dir=${kuber_yml_dirss}"
}
}简单说明下上面的pipeline
这里要对参数Action做判断，当为Deploy时，则进行正常的拉取代码，构建，然后打包成镜像，执行kubectl，如果是Rollback则进行回滚，不需要打包构建，只需选择提供的Tag，也就是这里的变量RollbackFile
另外这里加了一个捕获异常的配置段：
try {
}
catch() {
}因为这个项目在拉取第一个仓库的代码后，进行打包操作会大概率失败，这个时候jenkins在打包失败时是会跳出的，而如果这个打包失败不影响后面的打包，并且在后面的项目打包完成后，再回过头来打包第一个必然会成功，那么这样的一个场景，使用try catch则非常的方便了。
然后pipeline的关键部分还是通过执行脚本来完成的，因为要考虑很多内容，包括：
- 如何获取打包好的jar包名
- 如何命名镜像的tag
- 如何获取时间
- 如何获取版本号
- 如何通过docker build / push 时，进行认证，即连接私有镜像仓库
- 如何结合k8s来创建pod
- 如何把创建pod的yml文件写成模板
这些都是要解决的问题，针对这些问题，我将依次解答 1）如何获取打包好的jar包名？
代码语言：javascript复制 pom = readMavenPom file: 'pom.xml'
jar_file_info = "${pom.artifactId}-${pom.version}.jar"查找了大量资料，发现pipeline里可以读取pom.xml文件，并且获得artifactId，version这两个信息
这样就一下解决了第一个问题，第四个问题
2）如何命名镜像的tag？
这个问题其实是如何命名更加合理，规范，方便回滚等，我这里的tag是以时间 版本号 commit id来命名
其中时间是执行该任务时的时间，通过pipeline执行一个命令获取，并放到变量中，即：
date_time = sh(script: "date %Y%m%d%H%M", returnStdout: true).trim()commit id如何获取？ 同样的方法，通过pipeline执行命令获取
代码语言：javascript复制 git_cm_id = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()这样我们就组成了一个tag，拿到这个tag后，便可以进行docker build操作了
3）如何在docker build/push 时同harbor仓库进行认证
这里有一个技巧，也是用到了pipeline的一个模块吧
### 这里选择withCredentials: Bind credentials to variables，然后选择Username and password(separated) 然后设置存放用户名的变量dockeruser，存放密码的变量dockerpasswd 当然上面的admin用户要提前设置好，设置方法如下： 最后设置好之后，点击生成Pipeline脚本，内容如下：
代码语言：javascript复制withCredentials([usernamePassword(credentialsId: 'ed481948-5714-4d16-93b3-57023c4a5fbd', passwordVariable: 'dockerpasswd', usernameVariable: 'dockeruser')]) {
要执行的内容
}4）如何结合k8s创建pod，即执行kubectl命令？ 这里我使用的是ansible来实现的，因为测试环境的K8S网络有问题，没有结合kubernetes插件，结合kubernetes deploy插件，这里只得通过ansible来远程管理执行命令 实现方式如下：
- 将Jenkins机器的公钥同步到k8s的master机器上
- 编写playbook脚本，内容包括：将启动pod的模板yaml文件传到k8s的master机器上，根据设置好的变量进行命名替换，执行kubectl创建pod
playbook参考
代码语言：javascript复制入口文件参考：
cat setup-k8s.yml
- hosts: k8s-server
roles:
- role: deploy-server-k8s
vars:
app_pod_name: "$AppPodName"
replica_num: "$ReplicaNum"
harbor_image_address: "$HarborImageAddress"
max_cpu_num: "$MaxCpuNum"
max_mem_size: "$MaxMemSize"
cpu_num: "$CpuNum"
mem_size: "$MemSize"
target_server_port: "$TargetServerPort"
inner_server_port: "$InnerServerPort"
out_server_port: "$OutServerPort"
k8s_dst_dir: "$KuberDstDir"
清单文件参考：
cat host/hosts
[k8s-server]
192.168.30.174
[all:vars]
ansible_ssh_user=root
ansible_ssh_port=56968主要内容参考
代码语言：javascript复制tree roles/deploy-server-k8s
tasks
templates
cat main.yml
- name: Check dst dir is already exists.
stat:
path: "{{ k8s_dst_dir }}"
register: dst_dir
- name: Check yaml file is already exists.
stat:
path: "{{ k8s_dst_dir }}/app-service.yaml"
register: yml_file
- name: Create k8s dst dir
file:
path: "{{ k8s_dst_dir }}"
state: directory
owner: root
group: root
mode: 0755
when: dst_dir.stat.exists == False
- name: Copy local deployment yaml file to k8s server
template:
src: app-deployment.yaml.j2
dest: "{{ k8s_dst_dir }}/app-deployment.yaml"
owner: root
group: root
mode: 0644
- name: Copy local service yaml file to k8s server
template:
src: app-service.yaml.j2
dest: "{{ k8s_dst_dir }}/app-service.yaml"
owner: root
group: root
mode: 0644
- name: Start pod
shell: "kubectl delete -f . || kubectl apply -f ."
args:
chdir: "{{ k8s_dst_dir }}"
# when: yml_file.stat.exists == True
#- name: Create pod
# shell: "kubectl apply -f ."
# args:
# chdir: "{{ k8s_dst_dir }}"
# register: result
# until: result.stdout.find("created") == 2
# retries: 3
# delay: 3模板文件参考
代码语言：javascript复制tree templates/
app-deployment.yaml.j2
app-service.yaml.j2
cat app-deployment.yaml.j2
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: {{ app_pod_name }}
labels:
name: {{ app_pod_name }}
spec:
replicas: {{ replica_num }}
strategy:
rollingUpdate:
maxSurge: 1
maxUnavailable: 1
type: RollingUpdate
selector:
matchLabels:
name: {{ app_pod_name }}
template:
metadata:
labels:
name: {{ app_pod_name }}
spec:
containers:
- name: {{ app_pod_name }}
image: {{ harbor_image_address }}
imagePullPolicy: IfNotPresent
resources:
limits:
cpu: {{ max_cpu_num }}
memory: {{ max_mem_size }}
requests:
cpu: {{ cpu_num }}
memory: {{ mem_size }}
imagePullSecrets:
- name: kit-new40
cat app-service.yaml.j2
apiVersion: v1
kind: Service
metadata:
name: {{ app_pod_name }}
labels:
name: {{ app_pod_name }}
spec:
ports:
- name: {{ app_pod_name }}
protocol: TCP
targetPort: {{ target_server_port }}
port: {{ inner_server_port }}
nodePort: {{ out_server_port }}
selector:
name: {{ app_pod_name }}
sessionAffinity: None
type: NodePort以上差不多完成了大部分的问题，下面就是一些脚本，来辅助我们完成镜像构建，远程创建pod的部分
镜像构建
代码语言：javascript复制分为三个部分，一个是Dockerfile用来打一个包含jar包的基础镜像，一个是entrypoings.sh用来在镜像打包完成后的启动命令，另一个是执行docker build/push的脚本
cat Dockerfile
FROM 192.168.66.169/cicd/base-jdk-img:v1.0.0
MAINTAINER chenfei
COPY entrypoints.sh /
RUN mkdir -p /data/$AppName/logs &&
chmod x /entrypoints.sh
COPY $JarName /data/$AppName/
WORKDIR /data/$AppName
ENTRYPOINT ["/entrypoints.sh"]
cat entrypoints.sh
#!/bin/bash
echo '
nameserver 172.18.1.14
nameserver 10.96.0.10
search docker-new40.svc.cluster.local svc.cluster.local cluster.local openstacklocal
options ndots:5
' > /etc/resolv.conf
java -jar $JarName
cat docker.sh
#!/bin/bash
ARGS=`getopt -a -o n:j:i: --long app_name:,jar_file:,img_info:,--base_img::, -- "$@"`
if [ $? != 0 ];then
echo "Terminating..."
exit 1
fi
eval set -- "${ARGS}"
while :
do
case $1 in
-n|--app_name)
app_name=$2
shift
;;
-j|--jar_file)
jar_file=$2
shift
;;
-i|--img_info)
img_info=$2
shift
;;
-b|--base_img)
base_img=$2
shift
;;
--)
shift
break
;;
*)
echo "Internal error!"
exit 1
;;
esac
shift
done
SedFile() {
#sed -i "s#$BaseImg#$base_img#g" Dockerfile
sed -i "s#$JarName#$jar_file#g" Dockerfile
sed -i "s#$AppName#$app_name#g" Dockerfile
sed -i "s#$JarName#$jar_file#g" entrypoints.sh
}
start() {
docker build -t ${img_info} .
docker push ${img_info}
}
SedFile
start上面的脚本要传入几个参数：jar包的完整名，${pom.artifactId}“项目名，镜像名
创建Pod
创建Pod的操作其实就是执行ansible-playbook的过程，执行过程中，根据设置好的变量进行替换，脚本如下：
代码语言：javascript复制#!/bin/bash
export PATH=$PATH
dst_yaml="setup-k8s.yml"
DATE=$(date '%Y-%m-%d_%H-%M')
BackupDir="${JENKINS_HOME}/workspace/backup/${JOB_NAME}"
ARGS=`getopt -a -o a::r::i:ⓜ️:c::M::C::I::T::O::k:: --long app_name::,replica_number::,image_address::,mem::,cpu::,max_mem::,max_cpu::,inner_port::,target_port::,out_port::,kuber_yml_dir::, -- "$@"`
if [ $? != 0 ];then
echo "Terminating..."
exit 1
fi
eval set -- "${ARGS}"
while :
do
case $1 in
-a|--app_name)
app_name=$2
shift
;;
-r|--replica_number)
replica_number=$2
shift
;;
-i|--image_address)
image_address=$2
shift
;;
-m|--mem)
mem=$2
shift
;;
-c|--cpu)
cpu=$2
shift
;;
-M|--max_mem)
max_mem=$2
shift
;;
-C|--max_cpu)
max_cpu=$2
shift
;;
-I|--inner_port)
inner_port=$2
shift
;;
-T|--target_port)
target_port=$2
shift
;;
-O|--out_port)
out_port=$2
shift
;;
-k|--kuber_yml_dir)
kuber_yml_dir=$2
shift
;;
--)
shift
break
;;
*)
echo "Internal error!"
exit 1
;;
esac
shift
done
SedFile() {
sed -i "s#$AppPodName#$app_name#g" ${dst_yaml}
sed -i "s#$ReplicaNum#$replica_number#g" ${dst_yaml}
sed -i "s#$HarborImageAddress#$image_address#g" ${dst_yaml}
sed -i "s#$MaxCpuNum#$max_cpu#g" ${dst_yaml}
sed -i "s#$MaxMemSize#$max_mem#g" ${dst_yaml}
sed -i "s#$CpuNum#$cpu#g" ${dst_yaml}
sed -i "s#$MemSize#$mem#g" ${dst_yaml}
sed -i "s#$TargetServerPort#$target_port#g" ${dst_yaml}
sed -i "s#$InnerServerPort#$inner_port#g" ${dst_yaml}
sed -i "s#$OutServerPort#$out_port#g" ${dst_yaml}
sed -i "s#$KuberDstDir#$kuber_yml_dir#g" ${dst_yaml}
}
ExcuteAnsible() {
ansible-playbook -i host setup-k8s.yml
}
SedFile
ExcuteAnsible需要传入一些参数：pod的名称，副本的数量，镜像的完整地址，内存，CPU，最大内存，最大CPU，程序的监听端口，pod的端口，nodeport的端口，存放yaml文件的目录
回滚操作
这里使用到了一个插件，Extended Choice Parameter
通过这个插件，我们获取放到jenkins服务器中的存放tag的一个文件，该文件结构为：key=v1,v2,v3.v4
这里的key我们可以用来存放是哪个项目，即项目名称，value存放要选择的tag，插件如何使用可以参照上面截图
我们需要在每次构建时都将tag信息存到RollBackFile.txt文件中，所以写一个专门处理这个文件的脚本，内容如下：
cat rollback.sh
#!/bin/bash
export PATH=$PATH
backup_num="7"
rollbackfile="/data/jenkins/workspace/rollback/RollBackFile.txt"
job_name=${JOB_NAME}
img_tag_info="$1"
job_name_stat=$(grep -c ${job_name} ${rollbackfile})
add_content_info="${img_tag_info},"
tag_num=$(awk "/$job_name/" ${rollbackfile} | awk -F '=' '{print $2}' | awk -F ',' '{print NF}')
if [ ${job_name_stat} -eq 0 ];then
echo "${job_name=${img_tag_info}}" >> ${rollbackfile}
else
if [ $(grep -c "${img_tag_info}" ${rollbackfile}) -eq 0 ];then
if [ "${tag_num}" -lt "${backup_num}" ];then
sed -i "s#^${job_name}=#&${add_content_info}#g" ${rollbackfile}
elif [ "${tag_num}" -ge "${backup_num}" ];then
while [ "${tag_num}" -ge "${backup_num}" ]
do
del_field=$(awk "/$job_name/" ${rollbackfile} | awk -F '=' '{print $2}' | awk -F ',' '{print $NF}')
sed -i "s#$,{del_field}##g" ${rollbackfile}
let tag_num--
done
sed -i "s#^${job_name}#&${add_content_info}#g" ${rollbackfile}
fi
fi
fi执行时，传入一个tag参数即可，执行完成后，会将此次的tag记录到RollBackFile.txt中，我们看下这个文件长啥样
cat RollBackFile.txt
info=选择要回滚的镜像标签
knowledge-k8s-test=201908271530-0.0.1-SNAPSHOT-0773959,201908271406-0.0.1-SNAPSHOT-0773959,201908271357-0.0.1-SNAPSHOT-0773959,201908271350-0.0.1-SNAPSHOT-0773959,201908271344-0.0.1-SNAPSHOT-0773959,201908271338-0.0.1-SNAPSHOT-0773959,=201908262217-0.0.1-SNAPSHOT-0773959,201908262217-0.0.1-SNAPSHOT-0773959注意 该文件最好放在 ${WORKSPACE}/rollback/中
当选择Action为Rollback时，则直接选择指定的tag，通过ansible执行kubectl命令创建pod了
