基于K8s插件版的Jenkins动态节点实践【内含最佳实践】

2022-10-04 14:54:35 浏览数 (1)

手动在Jenkins Server上面创建一个Jenkins Agent节点，选择JNLP 类型。然后获取Jenkins Agent与Master连接所需要的JENKINS_AGENT_NAME 和JENKINS_AGENT_SECRET信息。

静态的AGENT节点，需要编写一个Deployment部署文件并传递上面的连接信息，然后kubectl apply；

动态的AGENT节点，需要使用kubernetes插件， 首先配置CLOUD，添加Kubernetes集群的配置信息。然后编写Pipeline Agent部分的YAML和Stages。

1. 创建节点

Jenkins > 节点管理 > 添加节点

配置节点信息

创建成功（后面需要用到节点名称和secret信息）

2. 静态节点

image.png

代码语言：javascript复制
kind: Deployment
apiVersion: apps/v1
metadata:
  labels:
    k8s-app: jenkinsagent
  name: jenkinsagent
  namespace: devops
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      k8s-app: jenkinsagent
  template:
    metadata:
      labels:
        k8s-app: jenkinsagent
      namespace: devops
      name: jenkinsagent
    spec:
      containers:
        - name: jenkinsagent
          image: jenkins/inbound-agent:4.10-3-jdk8
          securityContext:
            privileged: true
          imagePullPolicy: IfNotPresent
          resources:
            limits:
              cpu: 1000m
              memory: 2Gi
            requests:
              cpu: 500m
              memory: 512Mi
          env:
            - name: JENKINS_URL
              value: http://192.168.1.200:8080
            - name: JENKINS_SECRET
              value: 2f3f27ee602171c5b51bc69c2bda501d703ac65f42de16600b3e3007dfdec422
            - name: JENKINS_AGENT_NAME
              value: buildserver1
            - name: JENKINS_AGENT_WORKDIR
              value: /home/jenkins/workspace

image.png

3. 动态节点

4. 最佳实践

image.png

代码语言：javascript复制
@Library("mylib@feature-k8s") _     //加载共享库


podYaml = """
kind: Pod
apiVersion: v1
metadata:
  labels:
    k8s-app: jenkinsagent
  name: jnlp
  namespace: devops
spec:
  securityContext:
    runAsUser: 0
  containers:
  - name: jnlp
    image: jenkins/inbound-agent:4.10-3-jdk8
    imagePullPolicy: IfNotPresent
  - name: maven
    image: maven:3.8.4-jdk-8
    imagePullPolicy: IfNotPresent
    command:
    - cat
    tty: true
  - name: dind
    image: 'docker:stable-dind'
    command:
    - dockerd
    - --host=unix:///var/run/docker.sock
    - --host=tcp://0.0.0.0:8000
    - --insecure-registry=192.168.1.200:8088
    securityContext:
      privileged: true
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "docker"
    tty: true
    image: docker:19.03.15-git
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "curl"
    tty: true
    image: curlimages/curl:7.84.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  - name: "helm"
    tty: true
    image: dtzar/helm-kubectl:3.9.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  volumes:
    - name: docker-dir
      emptyDir: {}
"""


pipeline{
    agent{
        kubernetes{
            label "test01"
            cloud 'kubernetes'
            yaml podYaml

        }
    }
    
    options {
        skipDefaultCheckout()
    }

    stages{
        stage("Checkout"){
            steps{
                script {
                    println("GetCode")
                    checkout([$class: 'GitSCM', 
                        branches: [[name: "${env.branchName}"]], 
                        extensions: [], 
                        userRemoteConfigs: [[credentialsId: '62a3150f-38f8-4c9d-9334-79072b1d75cc', 
                                            url: "${env.srcUrl}"]]])
                    
                    env.commitID = GetCommitID()
                    env.buName = "${JOB_NAME}".split("-")[0]
                    env.serviceName = "${JOB_NAME}".split("_")[0]
                    currentBuild.description = """ branchName: ${env.branchName} n"""
                    currentBuild.displayName = "${env.commitID}"
                }
            }
        }

        stage("Build"){
            steps{
                script{
                    settingsContent = libraryResource 'config/settings.xml'
                    writeFile file: 'settings.xml', text: settingsContent
                }
                container('maven') {
                    sh "${env.buildShell}  -s settings.xml"
                }
            }
        }

        stage("PushArtifact"){
            steps{
                container("curl"){
                    script{ 
                        //Dir /buName/serviceName/version/serviceName-version.xxx
                        version = "${env.branchName}-${env.commitID}"
                        // 重命名
                        JarName = sh returnStdout: true, script: """ls target | grep -E "jar$" """
                        fileName = JarName -"n"
                        fileType = fileName.split('\.')[-1]
                        env.newFileName = "${env.serviceName}-${version}.${fileType}"

                        sh "cd target ; mv ${fileName}  ${newFileName} "
                        PushArtifact("${env.buName}/${env.serviceName}/${version}", "target", "${newFileName}")
                    }
                }
            }
        }


        stage("Docker"){
            steps{
                script{
                    env.registry = "192.168.1.200:8088"
                    // tag: branch-commid
                    env.version = "${env.branchName}-${env.commitID}"
                    env.imageName = "${env.registry}/${env.buName}/${env.serviceName}:${env.version}"
                }
                container('docker'){
                     withCredentials([usernamePassword(credentialsId: 'fad4e7c7-1f8b-45b2-83be-2b914bf08edf', passwordVariable: 'DOCKER_PASSWD', usernameVariable: 'DOCKER_USER')]) {

                        sh """
                            # 构建镜像
                            docker build -t ${env.imageName} .  --build-arg pkgname=target/${env.newFileName}

                            #登录镜像仓库
                            docker login -u ${DOCKER_USER} -p ${DOCKER_PASSWD} ${env.registry}
                            
                            # 上传镜像
                            docker push ${env.imageName}

                            # 删除镜像
                            sleep 2
                            docker rmi ${env.imageName}
                        """
                    }
                }
            }
        }

        stage("ReleaseFile"){
            steps{
                container("curl"){
                    script{

                        // 下载模板文件
                        projectId = 18
                        fileData = GetRepoFile(projectId,"values.yaml", "master" )
                        //println(fileData)

                        yamlData = readYaml text: fileData  

                        // 替换模板文件内容
                        yamlData.image.tag  = "${env.version}"
                        yamlData.image.repository = "${env.registry}/${env.buName}/${env.serviceName}"
                        
                        sh "rm -fr values.yaml"
                        writeYaml  charset: 'UTF-8', file: 'values.yaml', data: yamlData
                        sh "cat values.yaml"

                         //上传替换后的版本文件（新建文件或者更新文件）
                        newYaml = sh returnStdout: true, script: 'cat  values.yaml'
                        //println(newYaml)
                        //更新gitlab文件内容
                        base64Content = newYaml.bytes.encodeBase64().toString()

                        try {
                            CreateRepoFile(projectId,"values.yaml",base64Content, "master")
                        }catch(e){
                            UpdateRepoFile(projectId,"values.yaml",base64Content, "master")
                        }
                    }
                }
            }
        }
    }
}

//获取CommitID
def GetCommitID(){
    ID = sh returnStdout: true, script:"git rev-parse HEAD"
    ID = ID -"n"
    return ID[0..7] 
}

//上传制品
def PushArtifact(targetDir, filePath, fileName){
    sh """
        curl -X POST "http://192.168.1.200:8081/service/rest/v1/components?repository=devops4-local" 
        -H "accept: application/json" 
        -H "Content-Type: multipart/form-data" 
        -F "raw.directory=${targetDir}" 
        -F "raw.asset1=@${filePath}/${fileName};type=application/java-archive" 
        -F "raw.asset1.filename=${fileName}" 
        -u admin:admin123
    """
}

//更新文件内容
def UpdateRepoFile(projectId,filePath,fileContent, branchName){
    apiUrl = "projects/${projectId}/repository/files/${filePath}"
    reqBody = """{"branch": "${branchName}","encoding":"base64", "content": "${fileContent}", "commit_message": "update a new file"}"""
    response = HttpReqByHttpRequest('PUT',apiUrl,reqBody)
    println(response)

}


//创建文件
def CreateRepoFile(projectId,filePath,fileContent, branchName){
    apiUrl = "projects/${projectId}/repository/files/${filePath}"
    reqBody = """{"branch": "${branchName}","encoding":"base64", "content": "${fileContent}", "commit_message": "update a new file"}"""
    response = HttpReqByHttpRequest('POST',apiUrl,reqBody)
    println(response)

}

//获取文件内容
def GetRepoFile(projectId,filePath, branchName ){
   //GET /projects/:id/repository/files/:file_path/raw
   apiUrl = "/projects/${projectId}/repository/files/${filePath}/raw?ref=${branchName}"
   response = HttpReq('GET', apiUrl)
   return response
}


// 封装HTTP
def HttpReqByHttpRequest(reqType, reqUrl,reqBody ){
    def gitServer = "http://192.168.1.200:8077/api/v4"
    withCredentials([string(credentialsId: '553fcb38-81e1-4cdd-aa60-a8886a2eb323', variable: 'GITLABTOKEN')]) {
        response = httpRequest acceptType: 'APPLICATION_JSON_UTF8', 
                          consoleLogResponseBody: true, 
                          contentType: 'APPLICATION_JSON_UTF8', 
                          customHeaders: [[maskValue: false, name: 'PRIVATE-TOKEN', value: "${GITLABTOKEN}"]], 
                          httpMode: "${reqType}", 
                          url: "${gitServer}/${reqUrl}", 
                          wrapAsMultipart: false,
                          requestBody: "${reqBody}"

    }
    return response
}

def HttpReq(method, apiUrl){
    response = sh  returnStdout: true, 
        script: """ 
            curl --location --request ${method} 
            http://192.168.1.200:8077/api/v4/${apiUrl} 
            --header 'PRIVATE-TOKEN: N9mvJV4hq-z7yCcYEsC-' 
        """
    return response

}
代码语言：javascript复制
@Library("mylib@feature-k8s") _     //加载共享库


podYaml = """
kind: Pod
apiVersion: v1
metadata:
  labels:
    k8s-app: jenkinsagent
  name: jnlp
  namespace: devops
spec:
  securityContext:
    runAsUser: 0
  containers:
  - name: jnlp
    image: jenkins/inbound-agent:4.10-3-jdk8
    imagePullPolicy: IfNotPresent
  - name: maven
    image: maven:3.8.4-jdk-8
    imagePullPolicy: IfNotPresent
    command:
    - cat
    tty: true
  - name: dind
    image: 'docker:stable-dind'
    command:
    - dockerd
    - --host=unix:///var/run/docker.sock
    - --host=tcp://0.0.0.0:8000
    - --insecure-registry=192.168.1.200:8088
    securityContext:
      privileged: true
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "docker"
    tty: true
    image: docker:19.03.15-git
    imagePullPolicy: IfNotPresent
    volumeMounts:
    - mountPath: /var/run
      name: docker-dir
  - name: "curl"
    tty: true
    image: curlimages/curl:7.84.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  - name: "helm"
    tty: true
    image: dtzar/helm-kubectl:3.9.0
    imagePullPolicy: IfNotPresent
    command:
    - cat
  volumes:
    - name: docker-dir
      emptyDir: {}
"""

env.projectName = "${JOB_NAME}".split("_")[0]
env.buName = "${JOB_NAME}".split("-")[0]
env.serviceName = "${JOB_NAME}".split("_")[0]

pipeline{
    agent{
        kubernetes{
            label "test01"
            cloud 'kubernetes'
            yaml podYaml

        }
    }
    
    options {
        skipDefaultCheckout()
    }

    stages{
        stage("Checkout"){
            steps{
                script {            
                    
                    println("GetCode")
                    sh "[ -d ${env.serviceName} ] || mkdir ${env.serviceName}"
                    ws("${WORKSPACE}/${env.serviceName}"){
                        checkout([$class: 'GitSCM', 
                                branches: [[name: "${env.branchName}"]], 
                                extensions: [], 
                                userRemoteConfigs: [[credentialsId: '62a3150f-38f8-4c9d-9334-79072b1d75cc', 
                                                    url: "${env.srcUrl}"]]])
                        env.commitID = GetCommitID()
                        currentBuild.description = """ branchName: ${env.branchName} n"""
                        currentBuild.displayName = "${env.commitID}"
                    }
                    
                }
            }
        }
        

        stage("HelmDeploy"){
            steps{
                container("helm"){
                    script{

                        withCredentials([file(credentialsId: 'cdcb0d0b-7420-4c7a-84dd-a47b57878999', variable: 'KUBE_CONFIG')]) {
                            content = sh returnStdout: true, script: "cat $KUBE_CONFIG"
                            writeFile file: './config', text: content
                        }

                        sh " mkdir -p /root/.kube ; mv config /root/.kube/"
                       
                        env.namespace = "${env.buName}"
                        env.appName = "${env.serviceName}"
                        currentBuild.description = "NS: ${env.namespace}"
                      
                        //Helm 发布
                        sh """
                            helm package "${env.appName}/" 
                            helm upgrade --install --create-namespace  "${env.appName}" ./"${env.appName}"-*.tgz -n ${env.namespace} 
                            helm history "${env.appName}"  -n ${env.namespace} 
                        """ 

                        //获取release的历史版本   
                        env.revision = sh returnStdout: true, script: """helm history ${env.appName} -n ${env.namespace}  | grep -v 'REVISION' | awk '{print $1}' """
                        println("${env.revision}")
                        println("${env.revision.split('n').toString()}")
                        env.REVISION = "${env.revision.split('n').toString()}"
                        println("${env.REVISION}")

                        // 获取应用状态
                        5.times{
                            sh "sleep 2; kubectl -n ${env.namespace} get pod  | grep ${env.appName}"
                        }

                    }
                }
            }
        }

        stage("CheckHealth"){
            steps{
                container("curl"){
                    script{
                        sh """
                           echo "192.168.1.200 devops4.ops.service" >>/etc/hosts
                           """
                        result = sh returnStdout: true, script: """ curl "http://${env.domainName}/health" """
                        if (result == "okn"){
                            println("success!")
                        }
                    }
                }
            }
        }

        stage("RollOut"){
            input {
                message "是否进行回滚"
                ok "提交"
                submitter "zeyang,aa"
                parameters {
                    choice(choices: ['yes', 'no'], name: 'opts')
                }
            }

            steps{

                container("helm"){
                    script{
                        switch("${opts}") {
                            case "yes":
                                def result = input  message: "选择回滚版本?", 
                                parameters: [choice(choices: env.REVISION, name: 'rversion')]

                                println("${result}")
                                sh "helm rollback ${env.appName} ${result} -n ${env.namespace} "
                                break

                            case "no":
                            break
                        }
                    }
                }
            }
        }
    }
}

//获取CommitID
def GetCommitID(){
    ID = sh returnStdout: true, script:"git rev-parse HEAD"
    ID = ID -"n"
    return ID[0..7] 
}
jenkins kubernetes 云推荐引擎

0 人点赞