Jenkins 分布式构建1

2022-10-25 | 阅读：次

新环境部署

开发环境(将旧环境合入)
测试环境(将旧环境合入)

部署一个rancher，统一管理所有环境

CICD阶段

maven 缓存

构建工具

maven
node
kubectl
docker

master 分支流程

【发布】(tag)-【推送仓库】-【镜像扫描】-【同步生产】

发布流程

【更新/回滚】-【apply yaml】-【自动化测试】

模块选择，分中心选择（是所有中心同时都上，还是单个上）

多集群config

https://blog.csdn.net/weixin_38299857/article/details/115384071

自定义全局变量

harbor仓库的【系统管理】-【系统配置】-【全局属性】-【环境变量】

建设过程中的问题

版本控制，master，release，dev，bugfix，featrue
由于是多分支场景，需基于某一主干分支去配置jenkinsfile并签出代码，现有场景是已经有很多分支，所以需等下一次发版时，重新迁出，自动生成流水线

jenkinsfile

dev/release

def ImageTag = null
pipeline{
    agent none
    tools{
        jdk 'jdk11'
        maven 'mvn-3.6.3'
    }
    parameters {
        choice (choices: ['agent-service','kernel-common-service','kernel-event-service','kernel-gate-service','kernel-log-service','kernel-protect-service','kernel-scan-service'], description: 'gitlab项目', name: 'GitlabProjoctName')
        string defaultValue: '1', description: '启动的副本数量', name: 'replicas'
    }
    stages{
        stage('Unit Test') {
            agent {
                label 'jenkins-docker'
            }
            steps{
                echo "Unit Testing"
            }
        }
        stage('SonarQube analysis') {
            agent {
                label 'jenkins-docker'
            }
            when {
                anyOf {
                    branch '**/bugfix-*';
                    branch 'dev';
                    branch '**/feature-*'
                } 
            }
            steps{
                withSonarQubeEnv('SonarQube-Server') {
                    echo "SonarQube analysis ................."
                    sh "cd ${GitlabProjoctName} && mvn sonar:sonar -Dsonar.host.url=http://192.168.10.72:9000 -Dsonar.java.binaries=target/sonar"             
                }
            }
        }
        stage("Quality Gate") { 
            agent {
                label 'jenkins-docker'
            }
            when {
                anyOf {
                    branch '**/bugfix-*';
                    branch 'dev';
                    branch '**/feature-*'
                } 
            }
            steps {
                echo "Quality Gate ...................."
                sh "cd ${GitlabProjoctName}"
                // timeout(time: 20, unit: 'MINUTES') { 
                //     waitForQualityGate abortPipeline: false
                // } 
            }
        }
        stage('Compile'){
            agent {
                label 'jenkins-docker'
            }
            steps{
                echo "Current branch: ${env.BRANCH_NAME} "
                // sh 'mvn clean package -Dmaven.test.skip=true -P ${profile} -pl ${GitlabProjoctName} -am'
                sh 'mvn clean package -Dmaven.test.skip=true -P docker -pl ${GitlabProjoctName} -am'
            }
        }
        stage("Build Images") { 
            agent {
                label 'jenkins-docker'
            }
            steps {
                withCredentials([usernamePassword(credentialsId: 'dockerhub', passwordVariable: 'DOCKER_HUB_PASSWORD', usernameVariable: 'DOCKER_HUB_USER')]) {
                    script{
                        ImageTag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
                        sh """
cd ${GitlabProjoctName}
docker login ${DOCKER_REGISTRY} -u  ${DOCKER_HUB_USER} -p ${DOCKER_HUB_PASSWORD}
docker build -t ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${ImageTag} .
docker push ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${ImageTag} 
docker rmi -f ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${ImageTag} 
"""
                }
                }
            }
        }
        stage('Deploy Develop Env'){
            agent {
                label 'jenkins-docker'
            }
            when {
                anyOf {
                    branch '**/bugfix-*';
                    branch 'dev';
                    branch '**/feature-*'
                } 
            }
            steps{
                configFileProvider([configFile(fileId: '10.114-cdev01', targetLocation: 'cdev.kubeconfig')]) {
                        echo "开发环境部署。。。。。。。。。"
                        echo "${env.BRANCH_NAME}"
                        echo "当前部署modules为： ${GitlabProjoctName}"
                        sh """
sed -i "s#<REPLACAS>#${replicas}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
sed -i "s#<DOCKER_REGISTRY>#${DOCKER_REGISTRY}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
sed -i "s#<IMAGE_TAG>#${ImageTag}#g"  ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
kubectl --kubeconfig cdev.kubeconfig apply -f ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
"""       
                }
            }
        }
        stage('Release Branch Deploy'){
            agent {
                label 'jenkins-docker'
            }
            when {
                branch '**/release-*'
            }
            steps{
                script {
                    def userInput = input(
                        id: 'userInput',
                        message: '请输入您要部署的环境：',
                        parameters: [
                            [
                                $class: 'ChoiceParameterDefinition',
                                choices: "Dev\nQA",
                                name: 'Env'
                            ]
                        ]
                    )
                    if (userInput == "Dev") {
                        configFileProvider([configFile(fileId: '10.114-cdev01', targetLocation: 'cdev.kubeconfig')]) {
                            // script {
                            // ImageTag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
                            echo "开发环境部署。。。。。。。。。"
                            echo "${env.BRANCH_NAME}"
                            echo "当前部署modules为： ${GitlabProjoctName}"
                        sh """
sed -i "s#<REPLACAS>#${replicas}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
sed -i "s#<DOCKER_REGISTRY>#${DOCKER_REGISTRY}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
sed -i "s#<IMAGE_TAG>#${ImageTag}#g"  ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
kubectl --kubeconfig cdev.kubeconfig apply -f ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
"""       
                    // }
                }
                    } else if (userInput == "QA"){
                        configFileProvider([configFile(fileId: '10.121-cqa01', targetLocation: 'cqa.kubeconfig')]) {
                            // ImageTag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
                            echo "测试环境部署。。。。。。。。。"
                            echo "${env.BRANCH_NAME}"
                            echo "当前部署modules为： ${GitlabProjoctName}"
                            sh """
sed -i "s#<REPLACAS>#${replicas}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
sed -i "s#<DOCKER_REGISTRY>#${DOCKER_REGISTRY}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
sed -i "s#<IMAGE_TAG>#${ImageTag}#g"  ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
kubectl --kubeconfig cqa.kubeconfig apply -f ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
"""  
                        }
                    }
                }
            }
        }
    }
    post {
        always {
            emailext body: '''
<!DOCTYPE html>
<html>
<head>
<meta charset="UTF-8">
<title>${ENV, var="JOB_NAME"}-第${BUILD_NUMBER}次构建日志</title>
</head>
 
<body leftmargin="8" marginwidth="0" topmargin="8" marginheight="4"
    offset="0">
    <table width="95%" cellpadding="0" cellspacing="0"
        style="font-size: 11pt; font-family: Tahoma, Arial, Helvetica, sans-serif">
        <tr>
            <td>(本邮件是程序自动下发的，请勿回复！)</td>
        </tr>
        <tr>
            <td><h2>
                    <font color="#0000FF">构建结果 - ${BUILD_STATUS}</font>
                </h2></td>
        </tr>
        <tr>
            <td><br />
            <b><font color="#0B610B">构建信息</font></b>
            <hr size="2" width="100%" align="center" /></td>
        </tr>
        <tr>
            <td>
                <ul>
                    <li>项目名称&nbsp;：&nbsp;${PROJECT_NAME}</li>
                    <li>构建编号&nbsp;：&nbsp;第${BUILD_NUMBER}次构建</li>
                    <li>触发原因：&nbsp;${CAUSE}</li>
                    <li>构建日志：&nbsp;<a href="${BUILD_URL}console">${BUILD_URL}console</a></li>
                    <li>构建&nbsp;&nbsp;Url&nbsp;：&nbsp;<a href="${BUILD_URL}">${BUILD_URL}</a></li>
                    <li>工作目录&nbsp;：&nbsp;<a href="${PROJECT_URL}ws">${PROJECT_URL}ws</a></li>
                    <li>项目&nbsp;&nbsp;Url&nbsp;：&nbsp;<a href="${PROJECT_URL}">${PROJECT_URL}</a></li>
                    <li>代码检测质量报告：&nbsp;<a href="http://192.168.10.72:9000/dashboard?id=com.g01%3A${GitlabProjoctName}">http://192.168.10.72:9000/dashboard?id=com.g01%3A${GitlabProjoctName}</a></li>
                </ul>
            </td>
        </tr>
        <tr>
            <td><b><font color="#0B610B">Changes Since Last
                        Successful Build:</font></b>
            <hr size="2" width="100%" align="center" /></td>
        </tr>
        <tr>
            <td>
                <ul>
                    <li>历史变更记录 : <a href="${PROJECT_URL}changes">${PROJECT_URL}changes</a></li>
                </ul> ${CHANGES_SINCE_LAST_SUCCESS,reverse=true, format="Changes for Build #%n:<br />%c<br />",showPaths=true,changesFormat="<pre>[%a]<br />%m</pre>",pathFormat="&nbsp;&nbsp;&nbsp;&nbsp;%p"}
            </td>
        </tr>
        <tr>
            <td><b>Failed Test Results</b>
            <hr size="2" width="100%" align="center" /></td>
        </tr>
        <tr>
            <td><pre
                    style="font-size: 11pt; font-family: Tahoma, Arial, Helvetica, sans-serif">$FAILED_TESTS</pre>
                <br /></td>
        </tr>
        <tr>
            <td><b><font color="#0B610B">构建日志 (最后 100行):</font></b>
            <hr size="2" width="100%" align="center" /></td>
        </tr>
        <!-- <tr>
            <td>Test Logs (if test has ran): <a
                href="${PROJECT_URL}ws/TestResult/archive_logs/Log-Build-${BUILD_NUMBER}.zip">${PROJECT_URL}/ws/TestResult/archive_logs/Log-Build-${BUILD_NUMBER}.zip</a>
                <br />
            <br />
            </td>
        </tr> -->
        <tr>
            <td><textarea cols="80" rows="30" readonly="readonly"
                    style="font-family: Courier New">${BUILD_LOG, maxLines=100}</textarea>
            </td>
        </tr>
    </table>
</body>
</html> 
''',  recipientProviders: [buildUser(), requestor()], subject: "Jenkins pipeline 构建通知--${GitlabProjoctName}: "
        }    
    }
}

master

def INPUT_PARAMS = null
def DATE = null
pipeline{
    agent none
    tools{
        jdk 'jdk11'
    }
    parameters {
        choice (choices: ['agent-setting-service','asset-service','auth-service','common-service','event-analysis-service','event-chain-service','event-service','fh-push-service','notice-service','opn-event-service','outreach-count-service','protect-service','scan-service','security-service','task-scheduler-service','user-service','zh-accept-service'], description: 'gitlab项目', name: 'GitlabProjoctName')
        string defaultValue: '1', description: '预发布环境启动的副本数量', name: 'replicas' 
    }
    stages{
        stage("Version release") { 
            agent {
                label 'jenkins-docker'
            }
            when {
                branch 'master'
            }
            steps {
                timeout(time:1,unit: 'MINUTES'){
                script {
                    build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
                    DATE = sh(returnStdout: true, script: 'date +%Y%m%d')
                    INPUT_PARAMS = input message: '请输入发布信息.', ok: '继续', id: 'userInput',submitter: 'zhenxiyao,mazhenjie,xiaoguotao',
                        parameters: [
                            choice(choices: ['很好', '一般'], description: '今天天气怎么样？', name: 'weather'), 
                            string(defaultValue: "${build_tag}", description: '输入发布的版本号', name: "releaseTag")
                        ]
                    echo "输入发布的版本号为：${INPUT_PARAMS.releaseTag}"
                }
                }
            }
        }
        stage('Master Compile'){
            agent {
                label 'jenkins-docker'
            }
            when {
                branch 'master'
            }
            steps{
                echo "master branch"
                echo "${INPUT_PARAMS.releaseTag}"
                sh 'mvn clean package -Dmaven.test.skip=true -P docker -pl ${GitlabProjoctName} -am'
            }
        }
        stage('Build Image'){
            agent {
                label 'jenkins-docker'
            }
            when {
                branch 'master'
            }
            steps{
                withCredentials([usernamePassword(credentialsId: 'dockerhub', passwordVariable: 'DOCKER_HUB_PASSWORD', usernameVariable: 'DOCKER_HUB_USER')]) {
                    sh '''
#!/bin/bash
DATE=`date +%Y%m%d`
cd ${GitlabProjoctName}
ssh root@192.168.10.124 "mkdir -p /data/hn/jar/$DATE/${GitlabProjoctName}"
scp target/${GitlabProjoctName}-1.0.0.jar root@192.168.10.124:/data/hn/jar/$DATE/${GitlabProjoctName}
'''     
                    sh """
cd ${GitlabProjoctName}
docker login ${DOCKER_REGISTRY} -u  ${DOCKER_HUB_USER} -p ${DOCKER_HUB_PASSWORD}
docker build -t ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag} .
docker push ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag} 
docker rmi -f ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
"""
                }
            }
        }
        stage('Pred Releasse Deploy'){
            agent {
                label 'jenkins-docker'
            }
            when {
                branch 'master'
            }
            steps{
                configFileProvider([configFile(fileId: '10.6-pred01', targetLocation: 'pred.kubeconfig')]) {
                    echo "总中心预发布环境部署"
                    echo "${env.BRANCH_NAME}"
                    echo "当前部署modules为： ${GitlabProjoctName}"
                    sh """
if [ ${GitlabProjoctName} == "event-service" ] || [ ${GitlabProjoctName} == "fh-push-service" ];then 
    echo "This module general center does not need to be deployed。。。。"
else
    sed -i "s#<REPLACAS>#${replicas}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
    sed -i "s#<DOCKER_REGISTRY>#${DOCKER_REGISTRY}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
    sed -i "s#<IMAGE_TAG>#${INPUT_PARAMS.releaseTag}#g"  ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
    kubectl --kubeconfig pred.kubeconfig apply -f ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
fi
"""
                }
                configFileProvider([configFile(fileId: '10.9-spred01', targetLocation: 'spred.kubeconfig')]) {
                    echo "分中心预发布环境部署"
                    echo "${env.BRANCH_NAME}"
                    echo "当前部署modules为： ${GitlabProjoctName}"
                    sh """
if [ ${GitlabProjoctName} == "event-chain-service" ] || [ ${GitlabProjoctName} == "opn-event-service" ] || [ ${GitlabProjoctName} == "zh-accept-service" ];then 
    echo "This module sub center does not need to be deployed。。。。"
else
    sed -i "s#<REPLACAS>#${replicas}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
    sed -i "s#<DOCKER_REGISTRY>#${DOCKER_REGISTRY}#g" ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
    sed -i "s#<IMAGE_TAG>#${INPUT_PARAMS.releaseTag}#g"  ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
    kubectl --kubeconfig spred.kubeconfig apply -f ${GitlabProjoctName}/${GitlabProjoctName}-deployment.yaml
fi
"""
                }
            }
        }
        stage('Push Prod'){
            parallel{
                stage('Push BeiJing'){
                    agent {
                        label 'jenkins-zong'
                    }
                    steps{
                        withCredentials([usernamePassword(credentialsId: '192.168.1.131-harbor', passwordVariable: 'DOCKER_HUB_PASSWORD', usernameVariable: 'DOCKER_HUB_USER')]) {
                            sh """
echo 'Push Image to 总中心'
docker login  ${DOCKER_REGISTRY_ZONG} -u  ${DOCKER_HUB_USER} -p ${DOCKER_HUB_PASSWORD}
docker pull ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker tag ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}  ${DOCKER_REGISTRY_ZONG}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker push ${DOCKER_REGISTRY_ZONG}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker rmi -f ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker rmi -f ${DOCKER_REGISTRY_ZONG}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
"""
                        }
                    }
                }
                stage('Push WuXi'){
                    agent {
                        label 'jenkins-wuxi'
                    }
                    steps{
                        withCredentials([usernamePassword(credentialsId: '192.168.10.232-harbor', passwordVariable: 'DOCKER_HUB_PASSWORD', usernameVariable: 'DOCKER_HUB_USER')]) {
                            sh """
echo 'Push Image to WuXi'
docker login  ${DOCKER_REGISTRY_WUXI} -u  ${DOCKER_HUB_USER} -p ${DOCKER_HUB_PASSWORD}
docker pull ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker tag ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}  ${DOCKER_REGISTRY_WUXI}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker push ${DOCKER_REGISTRY_WUXI}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker rmi -f ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker rmi -f ${DOCKER_REGISTRY_WUXI}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
"""
                        }
                    }
                }
                stage('Push JiangSu'){
                    agent {
                        label 'jenkins-jiangsu'
                    }
                    steps{
                        withCredentials([usernamePassword(credentialsId: '192.168.20.101-harbor', passwordVariable: 'DOCKER_HUB_PASSWORD', usernameVariable: 'DOCKER_HUB_USER')]) {
                            sh """
echo 'Push Image to Jiangsu'
docker login  ${DOCKER_REGISTRY_JIANGSU} -u  ${DOCKER_HUB_USER} -p ${DOCKER_HUB_PASSWORD}
docker pull ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker tag ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}  ${DOCKER_REGISTRY_JIANGSU}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker push ${DOCKER_REGISTRY_JIANGSU}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker rmi -f ${DOCKER_REGISTRY}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
docker rmi -f ${DOCKER_REGISTRY_JIANGSU}/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}
"""
                        }
                    }
                }
                stage('Push HuNan'){
                    agent {
                        label 'jenkins-zong'
                    }
                    steps{
                        sh '''
set -x
DATE=`date +%Y%m%d`
echo "Push Image to hunan"
# 创建对端目录
expect <<EOF
spawn ssh root@192.168.1.1
expect "password" {send  "password\r"}
expect "#" {send  "mkdir -p /data/buildImage/jar_tmp/${GitlabProjoctName}\r"}
expect "#" {send  "exit\r"}
EOF
# 传包
/data/hn/expect /data/hn/jar/$DATE/${GitlabProjoctName}/${GitlabProjoctName}-1.0.0.jar  /data/buildImage/jar_tmp/${GitlabProjoctName}/
'''
                        sh """
expect <<EOF
set timeout -1
spawn ssh root@192.168.1.1
expect "password" {send  "password\r"}
expect "#" {send  "cd /data/buildImage/\r"}
expect "#" {send  "docker build -t 192.168.123.11:9000/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag} -f /data/buildImage/Dockerfile/Dockerfile.${GitlabProjoctName} jar_tmp/${GitlabProjoctName}\r"}
expect "#" {send  "docker push 192.168.123.11:9000/g01/${GitlabProjoctName}:${INPUT_PARAMS.releaseTag}\r"}
expect "#" {send  "exit\r"}
EOF
"""
                    }
                }
            }
        }
    }
    post {
        always {
            emailext body: '${DEFAULT_CONTENT}',  recipientProviders: [buildUser(), requestor()], subject: "Jenkins pipeline 构建通知--${GitlabProjoctName}: "
        }    
    }
}

交付问题

江苏环境

无锡环境

# 连接vpn后，需添加两条静态路由，120为harbor仓库地址，20为jenkins地址
$ route add -host 192.168.10.120 dev ens160
$ route add -host 192.168.10.20 dev ens160

北京环境

huyouba1

甄能忽悠-Linux-个人博客