(re)introduce testing

vars/autojob.groovy

@@ -14,18 +14,18 @@ def call(ArrayList<String> dependencyList, String gitUrl='',
 //                                   'tumbleweed-Release']) {
 
 
-  def (organisation, job_type, project) = env.JOB_NAME.tokenize('/')
+  def (organisation, job_type, project, branch) = env.JOB_NAME.tokenize('/')
   env.ORGANISATION = organisation
   env.JOB_TYPE = job_type
   env.PROJECT = project
+  env.BRANCH = branch
 
   if(job_type == 'fasttrack') {
     env.DISABLE_TEST=true
-    buildTestDeploy(dependencyList, gitUrl, ['focal-Debug'])
+    buildAndDeploy(dependencyList, gitUrl, ['focal-Debug'])
   }
-  else if(job_type == 'fasttrack-test') {
-    env.DISABLE_TEST=true
-    buildTestDeploy(dependencyList, gitUrl, ['focal-Debug'])
+  else if(job_type == 'fasttrack-testing') {
+    testing()
   }
   else {
     echo("Unknown job type: ${job_type}")

vars/buildTestDeploy.groovy → vars/buildAndDeploy.groovy

@@ -6,13 +6,7 @@
 
 // This is the function called from the .jenkinsfile
 // The last optional argument is the list of builds to be run. Format must be "<docker_image_name>-<cmake_build_type>"
-def call(ArrayList<String> dependencyList, String gitUrl='',
-         ArrayList<String> builds=['focal-Debug',
-                                   'focal-Release',
-                                   'focal-tsan',
-                                   'focal-asan']) {
-//                                   'tumbleweed-Debug',
-//                                   'tumbleweed-Release']) {
+def call(ArrayList<String> dependencyList, String gitUrl, ArrayList<String> builds) {
 
   def dependencyJobList = new ArrayList<String>()
 
@@ -70,11 +64,12 @@ def call(ArrayList<String> dependencyList, String gitUrl='',
 
     // setup build trigger
     triggers {
-      pollSCM('H/5 * * * *')
+      pollSCM('H/1 * * * *')
       upstream(upstreamProjects: dependencies, threshold: hudson.model.Result.UNSTABLE)
     }
     options {
       //disableConcurrentBuilds()
+      quietPeriod(0)
       copyArtifactPermission('*')
       buildDiscarder(logRotator(numToKeepStr: '15', artifactNumToKeepStr: '2'))
     }
@@ -127,7 +122,7 @@ def call(ArrayList<String> dependencyList, String gitUrl='',
       always {
         node('Docker') {
           script {
-            helper.doPublishBuildTestDeploy(builds)
+            helper.doPublishBuild(builds)
           }
         }
         script {
@@ -156,7 +151,7 @@ def transformIntoStep(ArrayList<String> dependencyList, String buildName, String
         def dockerArgs = "-u 0 --privileged --device=/dev/mtcadummys0 --device=/dev/mtcadummys1 --device=/dev/mtcadummys2 --device=/dev/mtcadummys3 --device=/dev/llrfdummys4 --device=/dev/noioctldummys5 --device=/dev/pcieunidummys6 -v /var/run/lock/mtcadummy:/var/run/lock/mtcadummy -v /opt/matlab_R2016b:/opt/matlab_R2016b"
         docker.image("builder:${label}").inside(dockerArgs) {
           script {
-            helper.doBuildTestDeploy(dependencyList, label, buildType, gitUrl)
+            helper.doBuildAndDeploy(dependencyList, label, buildType, gitUrl)
           }
         }
       }

vars/buildDoocsLibrary.groovy

@@ -42,6 +42,7 @@ def call(String libraryName, ArrayList<String> dependencyList) {
     }
     options {
       disableConcurrentBuilds()
+      quietPeriod(180)
       copyArtifactPermission('*')
       buildDiscarder(logRotator(numToKeepStr: '30', artifactNumToKeepStr: '10'))
     }

vars/helper.groovy

@@ -9,9 +9,13 @@
 // helper function, convert dependency name as listed in the .jenkinsfile into a Jenkins project name
 def dependencyToJenkinsProject(String dependency) {
   def dependencyProjectName = dependency
-  def (dependencyFolder, dependencyProject) = dependencyProjectName.split('/')
+  def (dependencyFolder, dependencyProject) = dependencyProjectName.split('/',2)
+  dependencyProject = dependencyProject.replace('/','%2F')
+  def jobType = env.JOB_TYPE
+  jobType = jobType.minus("-testing")
+  jobType = jobType.minus("-analysis")
   if(dependencyFolder != "DOOCS") {
-    dependencyProjectName = "${dependencyFolder}/${env.JOB_TYPE}/${dependencyProject}/master"
+    dependencyProjectName = "${dependencyFolder}/${jobType}/${dependencyProject}/master"
   }
   else {
     dependencyProjectName = "${dependencyFolder}/${dependencyProject}"
@@ -39,7 +43,7 @@ def gatherDependenciesDeep(ArrayList<String> dependencyList) {
 
 /**********************************************************************************************************************/
 
-def doBuildTestDeploy(ArrayList<String> dependencyList, String label, String buildType, String gitUrl) {
+def doBuildAndDeploy(ArrayList<String> dependencyList, String label, String buildType, String gitUrl) {
 
   // prepare source directory and dependencies
   doPrepare(true, gitUrl)
@@ -48,19 +52,28 @@ def doBuildTestDeploy(ArrayList<String> dependencyList, String label, String bui
   // add inactivity timeout of 30 minutes (build will be interrupted if 30 minutes no log output has been produced)
   timeout(activity: true, time: 30) {
  
-    // start build and tests, then generate artefact
+    // perform build and generate build artefact
     doBuild(label, buildType)
-    if(buildType != "asan" && buildType != "tsan" && !env.DISABLE_TEST) {
-      // tests for asan and tsan are run in the analysis jobs
-      doTest(label, buildType)
-    }
 
-    // Run cppcheck only for focal-debug
-    //if((!env.DISABLE_CPPCHECK || env.DISABLE_CPPCHECK == '') && buildType == "Debug") {
-    //    doCppcheck(label, buildType)
-    //}
+    // deploy and generate deployment artefact
+    doDeploy(label, buildType)
+
+  }
+}
+
+/**********************************************************************************************************************/
+
+def doTesting(String label, String buildType) {
+
+  // prepare source directory and dependencies
+  doPrepare(false)
+  doBuilddirArtefact(label, buildType)
 
-    doInstall(label, buildType)
+  // add inactivity timeout of 30 minutes (build will be interrupted if 30 minutes no log output has been produced)
+  timeout(activity: true, time: 30) {
+ 
+    // run tests
+    doRunTests(label, buildType)
 
   }
 }
@@ -211,8 +224,9 @@ def doBuilddirArtefact(String label, String buildType) {
   
   // obtain artefacts of dependencies
   script {
-    def parentJob = env.JOBNAME_CLEANED[0..-10]     // remove "-analysis" from the job name, which is 9 chars long
-    copyArtifacts filter: "build-${parentJob}-${label}-${buildType}.tgz", fingerprintArtifacts: true, projectName: "${parentJob}", selector: lastSuccessful(), target: "artefacts"
+    def buildJob = env.BUILD_JOB
+    def buildJob_cleaned = buildJob.replace('/','_')
+    copyArtifacts filter: "build-${buildJob_cleaned}-${label}-${buildType}.tgz", fingerprintArtifacts: true, projectName: "${buildJob}", selector: lastSuccessful(), target: "artefacts"
 
     // Unpack artefact into the Docker system root (should only write files to /scratch, which is writable by msk_jenkins).
     // Then obtain artefacts of dependencies (from /scratch/artefact.list)
@@ -227,7 +241,9 @@ def doBuilddirArtefact(String label, String buildType) {
     myFile = readFile(env.WORKSPACE+"/artefact.list")
     myFile.split("\n").each {
       if( it != "" ) {
-        copyArtifacts filter: "install-${it}-${label}-${buildType}.tgz", fingerprintArtifacts: true, projectName: "${it}", selector: lastSuccessful(), target: "artefacts"
+        def dependency = dependencyToJenkinsProject(it)
+        def dependency_cleaned = dependency.replace('/','_')
+        copyArtifacts filter: "install-${dependency_cleaned}-${label}-${buildType}.tgz", fingerprintArtifacts: true, projectName: "${dependency}", selector: lastSuccessful(), target: "artefacts"
       }
     }
   }
@@ -240,6 +256,11 @@ def doBuilddirArtefact(String label, String buildType) {
       done
     fi
   """
+    
+  // fix ownership
+  sh """
+    chown -R msk_jenkins /scratch
+  """
 
 }
 
@@ -300,19 +321,24 @@ EOF
 
 /**********************************************************************************************************************/
 
-def doTest(String label, String buildType) {
+def doRunTests(String label, String buildType) {
   if (env.SKIP_TESTS) {
     currentBuild.result = 'UNSTABLE'
     return
   }
 
+  def buildJob = env.BUILD_JOB
+  def buildJob_cleaned = buildJob.replace('/','_')
+
   // Run the tests via ctest
   // Prefix test names with label and buildType, so we can distinguish them later
   // Copy test results files to the workspace, otherwise they are not available to the xunit plugin
   sh """
     cat > /scratch/script <<EOF
 #!/bin/bash
-cd /scratch/build-${JOBNAME_CLEANED}
+cd /scratch/build-${buildJob_cleaned}
+cmake . -DBUILD_TESTS=ON
+ninja \${MAKEOPTS}
 if [ -z "\${CTESTOPTS}" ]; then
   CTESTOPTS="\${MAKEOPTS}"
 fi
@@ -322,7 +348,8 @@ done
 ctest --no-compress-output \${CTESTOPTS} -T Test -V || true
 echo sed -i Testing/*/Test.xml -e 's|\\(^[[:space:]]*<Name>\\)\\(.*\\)\\(</Name>\\)\$|\\1${label}.${buildType}.\\2\\3|'
 sed -i Testing/*/Test.xml -e 's|\\(^[[:space:]]*<Name>\\)\\(.*\\)\\(</Name>\\)\$|\\1${label}.${buildType}.\\2\\3|'
-cp -r /scratch/build-${JOBNAME_CLEANED}/Testing "${WORKSPACE}"
+rm -rf "${WORKSPACE}/Testing"
+cp -r /scratch/build-${buildJob_cleaned}/Testing "${WORKSPACE}"
 EOF
     cat /scratch/script
     chmod +x /scratch/script
@@ -462,7 +489,7 @@ EOF
 
 /**********************************************************************************************************************/
 
-def doInstall(String label, String buildType) {
+def doDeploy(String label, String buildType) {
 
   // Install, but redirect files into the install directory (instead of installing into the system)
   // Generate tar ball of install directory - this will be the artefact used by our dependents
@@ -484,15 +511,7 @@ def doInstall(String label, String buildType) {
 
 /**********************************************************************************************************************/
 
-def doPublishBuildTestDeploy(ArrayList<String> builds) {
-
-  // Note: this part runs only once per project, not for each branch!
-
-  // Run cppcheck and publish the result. Since this is a static analysis, we don't have to run it for each label
-  //if(!env.DISABLE_CPPCHECK || env.DISABLE_CPPCHECK == '') {
-  //  unstash "cppcheck.xml"
-  //  publishCppcheck pattern: 'cppcheck.xml'
-  //}
+def doPublishBuild(ArrayList<String> builds) {
 
   // Scan for compiler warnings. This is scanning the entire build logs for all labels and build types  
   recordIssues filters: [excludeMessage('.*-Wstrict-aliasing.*')], qualityGates: [[threshold: 1, type: 'TOTAL', unstable: true]], tools: [gcc()]
@@ -521,65 +540,11 @@ def doPublishAnalysis(ArrayList<String> builds) {
         }
       }
       
-      // get valgrind result (only Debug)
-      // -> disable for now
-      /*if(buildType == "Debug") {
-        try {
-          unstash "valgrind-${it}"
-        }
-        catch(all) {
-          echo("Could not retreive stashed valgrind results for ${it}")
-          currentBuild.result = 'FAILURE'
-        }
-      }*/
-      
     }
   }
-/*  
-  // publish valgrind result
-  publishValgrind (
-    failBuildOnInvalidReports: true,
-    failBuildOnMissingReports: true,
-    failThresholdDefinitelyLost: '',
-    failThresholdInvalidReadWrite: '',
-    failThresholdTotal: '',
-    pattern: '* / *.valgrind',
-    publishResultsForAbortedBuilds: false,
-    publishResultsForFailedBuilds: false,
-    sourceSubstitutionPaths: '',
-    unstableThresholdDefinitelyLost: '',
-    unstableThresholdInvalidReadWrite: '',
-    unstableThresholdTotal: '0'
-  )
-  */
+
   // publish cobertura result
   cobertura autoUpdateHealth: false, autoUpdateStability: false, coberturaReportFile: "*/coverage.xml", conditionalCoverageTargets: '70, 0, 0', failNoReports: false, failUnhealthy: false, failUnstable: false, lineCoverageTargets: '80, 0, 0', maxNumberOfBuilds: 0, methodCoverageTargets: '80, 0, 0', onlyStable: false, sourceEncoding: 'ASCII'
   
 }
 
-/**********************************************************************************************************************/
-
-def doCppcheck(String label, String buildType) {
-  // Generate coverage report as HTML and also convert it into cobertura XML file
-  sh """
-    chown msk_jenkins -R /scratch
-    cat > /scratch/script <<EOF
-#!/bin/bash
-cd /scratch/build-${JOBNAME_CLEANED}-${label}-${buildType}
-for VAR in \${JOB_VARIABLES} \${TEST_VARIABLES}; do
-   export \\`eval echo \\\${VAR}\\`
-done
-if [ -e compile_commands.json ]; then
-    cppcheck --inline-suppr --enable=all --xml --xml-version=2  --project=compile_commands.json 2>cppcheck.xml
-else
-    cppcheck --inline-suppr --enable=all --xml --xml-version=2  -ibuild -Iinclude /scratch/source 2>cppcheck.xml
-fi
-cp cppcheck.xml ${WORKSPACE} || true
-EOF
-    cat /scratch/script
-    chmod +x /scratch/script
-    sudo -H -E -u msk_jenkins /scratch/script
-  """
-
-  stash allowEmpty: true, includes: "cppcheck.xml", name: "cppcheck.xml"
-}

vars/testing.groovy

+/***********************************************************************************************************************
+
+  test() is called from autojob for fasttrack-test jobs after the corresponding fasttrack job has completed
+
+***********************************************************************************************************************/
+
+// This is the function called from the .jenkinsfile
+def call() {
+
+  // name of the job which ran the build
+  env.BUILD_JOB = "${env.ORGANISATION}/fasttrack/${env.PROJECT}/${env.BRANCH}"
+  
+  ArrayList<String> builds
+
+  pipeline {
+    agent none
+
+    // setup build trigger
+    triggers {
+      upstream(upstreamProjects: BUILD_JOB, threshold: hudson.model.Result.UNSTABLE)
+    }
+    options {
+      disableConcurrentBuilds()
+      copyArtifactPermission('*')
+      buildDiscarder(logRotator(numToKeepStr: '15', artifactNumToKeepStr: '2'))
+    }
+
+    stages {
+      stage('prepare') {
+        steps {
+          script {
+            node('Docker') {
+              // fetch list of build types
+              copyArtifacts filter: "builds.txt", fingerprintArtifacts: true, projectName: BUILD_JOB, selector: lastSuccessful(), target: "artefacts"  
+              def myFile = readFile(env.WORKSPACE+"/artefacts/builds.txt")
+              builds = myFile.split("\n")
+            }
+          }
+        }
+      }
+      stage('test') {
+        // Run the build stages for all labels + build types in parallel, each in a separate docker container
+        steps {
+          script {
+            parallel builds.collectEntries { ["${it}" : transformIntoStep(it)] }
+          }
+        }
+      } // stage build
+    } // end stages
+    post {
+      failure {
+        emailext body: '$DEFAULT_CONTENT', recipientProviders: [brokenTestsSuspects(), brokenBuildSuspects(), developers()], subject: '[Jenkins] $DEFAULT_SUBJECT', to: env.MAILTO
+        //mattermostSend channel: env.JOB_NAME, color: "danger", message: "Build of ${env.JOB_NAME} failed."
+        //mattermostSend channel: "Jenkins", color: "danger", message: "Build of ${env.JOB_NAME} failed."
+      }
+      always {
+        script {
+          if (currentBuild?.getPreviousBuild()?.result == 'FAILURE') {
+            if (!currentBuild.resultIsWorseOrEqualTo(currentBuild.getPreviousBuild().result)) {
+              //mattermostSend channel: env.JOB_NAME, color: "good", message: "Build of ${env.JOB_NAME} is good again."
+              //mattermostSend channel: "Jenkins", color: "good", message: "Build of ${env.JOB_NAME} is good again."
+            }
+          }
+        }
+      } // end always
+    } // end post
+  } // end pipeline
+}
+
+/**********************************************************************************************************************/
+
+def transformIntoStep(String buildName) {
+  // split the build name at the '-'
+  def (label, buildType) = buildName.tokenize('-')
+  // we need to return a closure here, which is then passed to parallel() for execution
+  return {
+    stage(buildName) {
+      node('Docker') {
+        // we need root access inside the container and access to the dummy pcie devices of the host
+        def dockerArgs = "-u 0 --privileged --device=/dev/mtcadummys0 --device=/dev/mtcadummys1 --device=/dev/mtcadummys2 --device=/dev/mtcadummys3 --device=/dev/llrfdummys4 --device=/dev/noioctldummys5 --device=/dev/pcieunidummys6 -v /var/run/lock/mtcadummy:/var/run/lock/mtcadummy -v /opt/matlab_R2016b:/opt/matlab_R2016b"
+        docker.image("builder:${label}").inside(dockerArgs) {
+          script {
+            helper.doTesting(label, buildType)
+          }
+        }
+      }
+    }
+  }
+}
+
+/**********************************************************************************************************************/
+
+