Skip to main content

Jenkins: Dynamic Parallel 设置变量(适用于Docker/Node/Shell)

最近看到了一篇文章 Jenkins Pipeline Stages 平行處理的寫法 ,对 Jenkins Parallel 的几种使用场景做了一些介绍,对我帮助很大,在此特别感谢!

本文只是针对我的使用场景,对 Dynamic Parallel 做出简单延伸,并且记录我一步步纠错的过程。

Dynamic Parallel 最小化配置

def browsers = ['Chrome', 'Edge', 'Firefox']

pipeline {
    agent any
    stages {
        stage('Example') {
            steps {
                script {
                    def jobs = [:]
                    for (b in browsers) {
                        def browser = b
                        jobs[browser] = {
                            stage(browser) {
                                stage('Build macOS') {
                                    echo "Build ${browser} for macOS"
                                }
                                stage('Build linux') {
                                    echo "Build ${browser} for linux"
                                }
                            }
                        }
                    }
                    parallel jobs
                }
            }
        }
    }
}

这里可以看到我们在 stage('Build linux') 中使用 echo "Build ${browser} for linux" 可以正确的打印出对应的 job

在Jenkins 中有很多种打印变量的方式,一种是像上边一样使用 echo "${browser}",还有一种比较常见的是在shell 中打印 sh 'echo ${browser}',我们来尝试在shell 中打印

@@ -13,9 +13,11 @@ pipeline {
     stage(browser) {
         stage('Build macOS') {
             echo "Build ${browser} for macOS"
+            sh 'echo Build ${browser} for macOS'
         }
         stage('Build linux') {
             echo "Build ${browser} for linux"
+            sh 'echo Build ${browser} for linux'
         }
     }
 }
def browsers = ['Chrome', 'Edge', 'Firefox']

pipeline {
    agent any
    stages {
        stage('Example') {
            steps {
                script {
                    def jobs = [:]
                    for (b in browsers) {
                        def browser = b
                        jobs[browser] = {
                            stage(browser) {
                                stage('Build macOS') {
                                    echo "Build ${browser} for macOS"
                                    sh 'echo Build ${browser} for macOS'
                                }
                                stage('Build linux') {
                                    echo "Build ${browser} for linux"
                                    sh 'echo Build ${browser} for linux'
                                }
                            }
                        }
                    }
                    parallel jobs
                }
            }
        }
    }
}

此时我们会看到shell 中打印的变量其实是空的,因为shell 中的变量其实是环境变量,我们需要手动将此变量转为环境变量

@@ -11,6 +11,9 @@ pipeline {
  def browser = b
  jobs[browser] = {
      stage(browser) {
+         stage('Setup env') {
+             env.browser = browser
+         }
          stage('Build macOS') {
              echo "Build ${browser} for macOS"
              sh 'echo Build ${browser} for macOS'
def browsers = ['Chrome', 'Edge', 'Firefox']

pipeline {
    agent any
    stages {
        stage('Example') {
            steps {
                script {
                    def jobs = [:]
                    for (b in browsers) {
                        def browser = b
                        jobs[browser] = {
                            stage(browser) {
                                stage('Setup env') {
                                    env.browser = browser
                                }
                                stage('Build macOS') {
                                    echo "Build ${browser} for macOS"
                                    sh 'echo Build ${browser} for macOS'
                                }
                                stage('Build linux') {
                                    echo "Build ${browser} for linux"
                                    sh 'echo Build ${browser} for linux'
                                }
                            }
                        }
                    }
                    parallel jobs
                }
            }
        }
    }
}

此时我们可以看到,虽然shell中环境变量打印出来了,但是并不是对应job的值。这可能是由于在单独的stage 中设置导致的,那我们将其拿出来,放在第一个stage 前,看看是否会对所有对stage 都生效

@@ -11,9 +11,7 @@ pipeline {
 def browser = b
 jobs[browser] = {
     stage(browser) {
-        stage('Setup env') {
-            env.browser = browser
-        }
+        env.browser = browser
         stage('Build macOS') {
             echo "Build ${browser} for macOS"
             sh 'echo Build ${browser} for macOS'
def browsers = ['Chrome', 'Edge', 'Firefox']

pipeline {
    agent any
    stages {
        stage('Example') {
            steps {
                script {
                    def jobs = [:]
                    for (b in browsers) {
                        def browser = b
                        jobs[browser] = {
                            stage(browser) {
                                env.browser = browser
                                stage('Build macOS') {
                                    echo "Build ${browser} for macOS"
                                    sh 'echo Build ${browser} for macOS'
                                }
                                stage('Build linux') {
                                    echo "Build ${browser} for linux"
                                    sh 'echo Build ${browser} for linux'
                                }
                            }
                        }
                    }
                    parallel jobs
                }
            }
        }
    }
}

可以看到,结果还是一样的,shell 中并没有打印出正确的环境变量。那我们在每个stage 中都设置环境变量,看看是否可行

@@ -11,13 +11,14 @@ pipeline {
 def browser = b
 jobs[browser] = {
     stage(browser) {
-        env.browser = browser
         stage('Build macOS') {
             echo "Build ${browser} for macOS"
+            env.browser = browser
             sh 'echo Build ${browser} for macOS'
         }
         stage('Build linux') {
             echo "Build ${browser} for linux"
+            env.browser = browser
             sh 'echo Build ${browser} for linux'
         }
     }
def browsers = ['Chrome', 'Edge', 'Firefox']

pipeline {
    agent any
    stages {
        stage('Example') {
            steps {
                script {
                    def jobs = [:]
                    for (b in browsers) {
                        def browser = b
                        jobs[browser] = {
                            stage(browser) {
                                stage('Build macOS') {
                                    echo "Build ${browser} for macOS"
                                    env.browser = browser
                                    sh 'echo Build ${browser} for macOS'
                                }
                                stage('Build linux') {
                                    echo "Build ${browser} for linux"
                                    env.browser = browser
                                    sh 'echo Build ${browser} for linux'
                                }
                            }
                        }
                    }
                    parallel jobs
                }
            }
        }
    }
}

可以看到,到这一步可以打印出正确的环境变量了。那么我的临时方案就是在每一个会调用这个变量的stage 中,都手动设置一遍环境变量。

接下来,我们添加一个使用docker 的stage,并且设置每个job 都在node 中运行,以便工作区隔离。

@@ -11,15 +11,24 @@ pipeline {
 def browser = b
 jobs[browser] = {
     stage(browser) {
-        stage('Build macOS') {
-            echo "Build ${browser} for macOS"
-            env.browser = browser
-            sh 'echo Build ${browser} for macOS'
-        }
-        stage('Build linux') {
-            echo "Build ${browser} for linux"
-            env.browser = browser
-            sh 'echo Build ${browser} for macOS'
+        node('build-agent') {
+            stage('Build macOS') {
+                echo "Build ${browser} for macOS"
+                env.browser = browser
+                sh 'echo Build ${browser} for macOS'
+            }
+            stage('Build linux') {
+                echo "Build ${browser} for linux"
+                env.browser = browser
+                sh 'echo Build ${browser} for linux'
+            }
+            stage('Build in docker') {
+                docker.image('alpine:latest').inside('--privileged -u root --network=host') {
+                    echo "Build ${browser} in docker"
+                    env.browser = browser
+                    sh 'echo Build ${browser} in docker'
+                }
+            }
         }
     }
 }
def browsers = ['Chrome', 'Edge', 'Firefox']

pipeline {
    agent any
    stages {
        stage('Example') {
            steps {
                script {
                    def jobs = [:]
                    for (b in browsers) {
                        def browser = b
                        jobs[browser] = {
                            stage(browser) {
                                node('build-agent') {
                                    stage('Build macOS') {
                                        echo "Build ${browser} for macOS"
                                        env.browser = browser
                                        sh 'echo Build ${browser} for macOS'
                                    }
                                    stage('Build linux') {
                                        echo "Build ${browser} for linux"
                                        env.browser = browser
                                        sh 'echo Build ${browser} for linux'
                                    }
                                    stage('Build in docker') {
                                        docker.image('alpine:latest').inside('--privileged -u root --network=host') {
                                            echo "Build ${browser} in docker"
                                            env.browser = browser
                                            sh 'echo Build ${browser} in docker'
                                        }
                                    }
                                }
                            }
                        }
                    }
                    parallel jobs
                }
            }
        }
    }
}

这次看到shell 中的结果是正常打印的。但是每个stage 中都手动设置一遍env 也太麻烦了吧,有没有更简单的方法? 当然有,我们可以用 withEnv 这个内建函数来设置整个node 的环境变量

@@ -12,21 +12,20 @@ pipeline {
 jobs[browser] = {
     stage(browser) {
         node('build-agent') {
-            stage('Build macOS') {
-                echo "Build ${browser} for macOS"
-                env.browser = browser
-                sh 'echo Build ${browser} for macOS'
-            }
-            stage('Build linux') {
-                echo "Build ${browser} for linux"
-                env.browser = browser
-                sh 'echo Build ${browser} for linux'
-            }
-            stage('Build in docker') {
-                docker.image('alpine:latest').inside('--privileged -u root --network=host') {
-                    echo "Build ${browser} in docker"
-                    env.browser = browser
-                    sh 'echo Build ${browser} in docker'
+            withEnv(["browser=${browser}"]){
+                stage('Build macOS') {
+                    echo "Build ${browser} for macOS"
+                    sh 'echo Build ${browser} for macOS'
+                }
+                stage('Build linux') {
+                    echo "Build ${browser} for linux"
+                    sh 'echo Build ${browser} for linux'
+                }
+                stage('Build in docker') {
+                    docker.image('alpine:latest').inside('--privileged -u root --network=host') {
+                        echo "Build ${browser} in docker"
+                        sh 'echo Build ${browser} in docker'
+                    }
                 }
             }
         }
def browsers = ['Chrome', 'Edge', 'Firefox']

pipeline {
    agent any
    stages {
        stage('Example') {
            steps {
                script {
                    def jobs = [:]
                    for (b in browsers) {
                        def browser = b
                        jobs[browser] = {
                            stage(browser) {
                                node('build-agent') {
                                    withEnv(["browser=${browser}"]){
                                        stage('Build macOS') {
                                            echo "Build ${browser} for macOS"
                                            sh 'echo Build ${browser} for macOS'
                                        }
                                        stage('Build linux') {
                                            echo "Build ${browser} for linux"
                                            sh 'echo Build ${browser} for linux'
                                        }
                                        stage('Build in docker') {
                                            docker.image('alpine:latest').inside('--privileged -u root --network=host') {
                                                echo "Build ${browser} in docker"
                                                sh 'echo Build ${browser} in docker'
                                            }
                                        }
                                    }
                                }
                            }
                        }
                    }
                    parallel jobs
                }
            }
        }
    }
}

结束!