Merge remote-tracking branch 'origin/ci-adding-route-checks' into integration_2021_wk45_c

ci-scripts/Jenkinsfile-GitLab-Container

@@ -218,6 +218,26 @@ pipeline {
             }
           }
         }
+        stage ("NSA B200 Sanity Check") {
+          when { expression {doMandatoryTests} }
+          steps {
+            script {
+              triggerSlaveJob ('RAN-NSA-B200-Module-LTEBOX-Container', 'Test-NSA-B200')
+            }
+          }
+          post {
+            always {
+              script {
+                finalizeSlaveJob('RAN-NSA-B200-Module-LTEBOX-Container')
+              }
+            }
+            failure {
+              script {
+                currentBuild.result = 'FAILURE'
+              }
+            }
+          }
+        }
       }
     }
     stage ("Images Push to Registries") {
@@ -227,8 +247,14 @@ pipeline {
           triggerSlaveJob ('RAN-DockerHub-Push', 'Push-to-Docker-Hub')
         }
         post {
+          always {
+            script {
+              echo "Push to Docker-Hub OK"
+            }
+          }
           failure {
             script {
+              echo "Push to Docker-Hub KO"
               currentBuild.result = 'FAILURE'
             }
           }
@@ -281,6 +307,11 @@ pipeline {
 // ----  Slave Job functions
 
 def triggerSlaveJob (jobName, gitlabStatusName) {
+  if ("MERGE".equals(env.gitlabActionType)) {
+    MR_NUMBER = env.gitlabMergeRequestIid
+  } else {
+    MR_NUMBER = 'develop'
+  }
   // Workaround for the "cancelled" GitLab pipeline notification
   // The slave job is triggered with the propagate false so the following commands are executed
   // Its status is now PASS/SUCCESS from a stage pipeline point of view
@@ -290,6 +321,7 @@ def triggerSlaveJob (jobName, gitlabStatusName) {
       string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)),
       string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)),
       string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)),
+      string(name: 'eNB_MR', value: String.valueOf(MR_NUMBER)),
       booleanParam(name: 'eNB_mergeRequest', value: "MERGE".equals(env.gitlabActionType)),
       string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch))
     ], propagate: false
@@ -306,6 +338,11 @@ def triggerSlaveJob (jobName, gitlabStatusName) {
 }
 
 def triggerSlaveJobNoGitLab (jobName) {
+  if ("MERGE".equals(env.gitlabActionType)) {
+    MR_NUMBER = env.gitlabMergeRequestIid
+  } else {
+    MR_NUMBER = 'develop'
+  }
   // Workaround for the "cancelled" GitLab pipeline notification
   // The slave job is triggered with the propagate false so the following commands are executed
   // Its status is now PASS/SUCCESS from a stage pipeline point of view
@@ -315,6 +352,7 @@ def triggerSlaveJobNoGitLab (jobName) {
       string(name: 'eNB_Repository', value: String.valueOf(GIT_URL)),
       string(name: 'eNB_Branch', value: String.valueOf(env.gitlabSourceBranch)),
       string(name: 'eNB_CommitID', value: String.valueOf(env.gitlabMergeRequestLastCommit)),
+      string(name: 'eNB_MR', value: String.valueOf(MR_NUMBER)),
       booleanParam(name: 'eNB_mergeRequest', value: "MERGE".equals(env.gitlabActionType)),
       string(name: 'eNB_TargetBranch', value: String.valueOf(env.gitlabTargetBranch))
     ], propagate: false

ci-scripts/Jenkinsfile-tmp-multi-enb-nsa

@@ -51,270 +51,307 @@ def StatusForDb = ""
 
 
 pipeline {
-    agent {label pythonExecutor}
+  agent {label pythonExecutor}
-    options {
+  options {
-        disableConcurrentBuilds()
+    disableConcurrentBuilds()
-        ansiColor('xterm')
+    ansiColor('xterm')
-        lock(extra: [[resource: ciSmartPhonesResource2]], resource: ciSmartPhonesResource1)
+    lock(extra: [[resource: ciSmartPhonesResource2]], resource: ciSmartPhonesResource1)
+  }
+  stages {
+    stage("Build Init") {
+      steps {
+        // update the build name and description
+        buildName "${params.eNB_MR}"
+        buildDescription "Branch : ${params.eNB_Branch}"
+      }
     }
-    stages {
+    stage ("Verify Parameters") {
-        stage("Build Init") {
+      steps {
-            steps {
+        script {
-                // update the build name and description
+          echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
-                buildName "${params.eNB_MR}"
+          def allParametersPresent = true
-                buildDescription "Branch : ${params.eNB_Branch}"
+
+          // It is already to late to check it
+          if (params.pythonExecutor != null) {
+            echo "eNB CI executor node  :   ${pythonExecutor}"
+          }
+          // If not present picking a default Stage Name
+          if (params.pipelineTestStageName == null) {
+            // picking default
+            testStageName = 'Template Test Stage'
+          }
+
+          if (params.SmartPhonesResource1 == null) {
+            allParametersPresent = false
+          }
+          if (params.SmartPhonesResource2 == null) {
+            allParametersPresent = false
+          }
+          // 1st eNB parameters
+          if (params.eNB_IPAddress == null) {
+            allParametersPresent = false
+          }
+          if (params.eNB_SourceCodePath == null) {
+            allParametersPresent = false
+          }
+          if (params.eNB_Credentials == null) {
+            allParametersPresent = false
+          }
+          // 2nd eNB parameters
+          if (params.eNB1_IPAddress == null) {
+            allParametersPresent = false
+          }
+          if (params.eNB1_SourceCodePath == null) {
+            allParametersPresent = false
+          }
+          if (params.eNB1_Credentials == null) {
+            allParametersPresent = false
+          }
+          // 3rd eNB parameters
+          if (params.eNB2_IPAddress == null) {
+            allParametersPresent = false
+          }
+          if (params.eNB2_SourceCodePath == null) {
+            allParametersPresent = false
+          }
+          if (params.eNB2_Credentials == null) {
+            allParametersPresent = false
+          }
+          // the following 4 parameters should be pushed by the master trigger
+          // if not present, take the job GIT variables (used for developing)
+          if (params.eNB_Repository == null) {
+            eNB_Repository = env.GIT_URL
+          } else {
+            eNB_Repository = params.eNB_Repository
+          }
+          echo "eNB_Repository        :   ${eNB_Repository}"
+          if (params.eNB_Branch == null) {
+            eNB_Branch = env.GIT_BRANCH
+          } else {
+            eNB_Branch = params.eNB_Branch
+          }
+          echo "eNB_Branch            :   ${eNB_Branch}"
+          if (params.eNB_CommitID == null) {
+            eNB_CommitID = env.GIT_COMMIT
+          } else {
+            eNB_CommitID = params.eNB_CommitID
+          }
+          echo "eNB_CommitID          :   ${eNB_CommitID}"
+          if (params.eNB_AllowMergeRequestProcess!= null) {
+            eNB_AllowMergeRequestProcess = params.eNB_AllowMergeRequestProcess
+            if (eNB_AllowMergeRequestProcess) {
+              if (params.eNB_TargetBranch != null) {
+                eNB_TargetBranch = params.eNB_TargetBranch
+              } else {
+                eNB_TargetBranch = 'develop'
+              }
+              echo "eNB_TargetBranch      :   ${eNB_TargetBranch}"
             }
-        }
+          }
-        stage ("Verify Parameters") {
-            steps {
-                script {
-                    echo '\u2705 \u001B[32mVerify Parameters\u001B[0m'
-                    def allParametersPresent = true
 
-                    // It is already to late to check it
+          if (params.EPC_IPAddress == null) {
-                    if (params.pythonExecutor != null) {
+            allParametersPresent = false
-                        echo "eNB CI executor node  :   ${pythonExecutor}"
+          }
-                    }
+          if (params.EPC_Type == null) {
-                    // If not present picking a default Stage Name
+            allParametersPresent = false
-                    if (params.pipelineTestStageName == null) {
+          }
-                        // picking default
+          if (params.EPC_SourceCodePath == null) {
-                        testStageName = 'Template Test Stage'
+            allParametersPresent = false
-                    }
+          }
+          if (params.EPC_Credentials == null) {
+            allParametersPresent = false
+          }
 
-                    if (params.SmartPhonesResource1 == null) {
+          if (params.ADB_IPAddress == null) {
-                        allParametersPresent = false
+            allParametersPresent = false
-                    }
+          }
-                    if (params.SmartPhonesResource2 == null) {
+          if (params.ADB_Credentials == null) {
-                        allParametersPresent = false
+            allParametersPresent = false
-                    }
+          }
-                    // 1st eNB parameters
-                    if (params.eNB_IPAddress == null) {
-                        allParametersPresent = false
-                    }
-                    if (params.eNB_SourceCodePath == null) {
-                        allParametersPresent = false
-                    }
-                    if (params.eNB_Credentials == null) {
-                        allParametersPresent = false
-                    }
-                    // 2nd eNB parameters
-                    if (params.eNB1_IPAddress == null) {
-                        allParametersPresent = false
-                    }
-                    if (params.eNB1_SourceCodePath == null) {
-                        allParametersPresent = false
-                    }
-                    if (params.eNB1_Credentials == null) {
-                        allParametersPresent = false
-                    }
-                    // 3rd eNB parameters
-                    if (params.eNB2_IPAddress == null) {
-                        allParametersPresent = false
-                    }
-                    if (params.eNB2_SourceCodePath == null) {
-                        allParametersPresent = false
-                    }
-                    if (params.eNB2_Credentials == null) {
-                        allParametersPresent = false
-                    }
-                    // the following 4 parameters should be pushed by the master trigger
-                    // if not present, take the job GIT variables (used for developing)
-                    if (params.eNB_Repository == null) {
-                        eNB_Repository = env.GIT_URL
-                    } else {
-                        eNB_Repository = params.eNB_Repository
-                    }
-                    echo "eNB_Repository        :   ${eNB_Repository}"
-                    if (params.eNB_Branch == null) {
-                        eNB_Branch = env.GIT_BRANCH
-                    } else {
-                        eNB_Branch = params.eNB_Branch
-                    }
-                    echo "eNB_Branch            :   ${eNB_Branch}"
-                    if (params.eNB_CommitID == null) {
-                        eNB_CommitID = env.GIT_COMMIT
-                    } else {
-                        eNB_CommitID = params.eNB_CommitID
-                    }
-                    echo "eNB_CommitID          :   ${eNB_CommitID}"
-                    if (params.eNB_AllowMergeRequestProcess!= null) {
-                        eNB_AllowMergeRequestProcess = params.eNB_AllowMergeRequestProcess
-                        if (eNB_AllowMergeRequestProcess) {
-                            if (params.eNB_TargetBranch != null) {
-                                eNB_TargetBranch = params.eNB_TargetBranch
-                            } else {
-                                eNB_TargetBranch = 'develop'
-                            }
-                            echo "eNB_TargetBranch      :   ${eNB_TargetBranch}"
-                        }
-                    }
 
-                    if (params.EPC_IPAddress == null) {
+          if (allParametersPresent) {
-                        allParametersPresent = false
+            echo "All parameters are present"
-                    }
+            if (eNB_AllowMergeRequestProcess) {
-                    if (params.EPC_Type == null) {
+              sh "git fetch"
-                        allParametersPresent = false
+              sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest"
-                    }
+            } else {
-                    if (params.EPC_SourceCodePath == null) {
+              sh "git fetch"
-                        allParametersPresent = false
+              sh "git checkout -f ${eNB_CommitID}"
-                    }
-                    if (params.EPC_Credentials == null) {
-                        allParametersPresent = false
-                    }
-
-                    if (params.ADB_IPAddress == null) {
-                        allParametersPresent = false
-                    }
-                    if (params.ADB_Credentials == null) {
-                        allParametersPresent = false
-                    }
-
-                    if (allParametersPresent) {
-                        echo "All parameters are present"
-                        if (eNB_AllowMergeRequestProcess) {
-                            sh "git fetch"
-                            sh "./ci-scripts/doGitLabMerge.sh --src-branch ${eNB_Branch} --src-commit ${eNB_CommitID} --target-branch ${eNB_TargetBranch} --target-commit latest"
-                        } else {
-                            sh "git fetch"
-                            sh "git checkout -f ${eNB_CommitID}"
-                        }
-                    } else {
-                        echo "Some parameters are missing"
-                        sh "./ci-scripts/fail.sh"
-                    }
-                }
             }
+          } else {
+            echo "Some parameters are missing"
+            sh "./ci-scripts/fail.sh"
+          }
         }
-        stage ("Build and Test") {
+      }
-            steps {
+    }
-                script {
+    stage ("Build and Test") {
-                    dir ('ci-scripts') {
+      steps {
-                        echo "\u2705 \u001B[32m${testStageName}\u001B[0m"
+        script {
-                        // If not present picking a default XML file
+          dir ('ci-scripts') {
-                        if (params.pythonTestXmlFile == null) {
+            echo "\u2705 \u001B[32m${testStageName}\u001B[0m"
-                            // picking default
+            // If not present picking a default XML file
-                            testXMLFile = 'xml_files/enb_usrpB210_band7_50PRB.xml'
+            if (params.pythonTestXmlFile == null) {
-                            echo "Test XML file(default):   ${testXMLFile}"
+              // picking default
-                            mainPythonAllXmlFiles += "--XMLTestFile=" + testXMLFile + " "
+              testXMLFile = 'xml_files/enb_usrpB210_band7_50PRB.xml'
-                        } else {
+              echo "Test XML file(default):   ${testXMLFile}"
-                            String[] myXmlTestSuite = testXMLFile.split("\\r?\\n")
+              mainPythonAllXmlFiles += "--XMLTestFile=" + testXMLFile + " "
-                            for (xmlFile in myXmlTestSuite) {
+            } else {
-                                if (fileExists(xmlFile)) {
+              String[] myXmlTestSuite = testXMLFile.split("\\r?\\n")
-                                    mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " "
+              for (xmlFile in myXmlTestSuite) {
-                                    echo "Test XML file         :   ${xmlFile}"
+                if (fileExists(xmlFile)) {
-                                }
+                  mainPythonAllXmlFiles += "--XMLTestFile=" + xmlFile + " "
-                            }
+                  echo "Test XML file     :   ${xmlFile}"
-                        }
-                        withCredentials([
-                            [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'],
-                            [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB1_Credentials}", usernameVariable: 'eNB1_Username', passwordVariable: 'eNB1_Password'],
-                            [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB2_Credentials}", usernameVariable: 'eNB2_Username', passwordVariable: 'eNB2_Password'],
-                            [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'],
-                            [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password']
-                        ]) {
-                            sh "python3 main.py --mode=InitiateHtml --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}"
-                            String[] myXmlTestSuite = testXMLFile.split("\\r?\\n")
-                            for (xmlFile in myXmlTestSuite) {
-                                if (fileExists(xmlFile)) {
-                                    try {
-                                        sh "python3 main.py --mode=TesteNB --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath} --eNB1IPAddress=${params.eNB1_IPAddress} --eNB1UserName=${eNB1_Username} --eNB1Password=${eNB1_Password} --eNB1SourceCodePath=${params.eNB1_SourceCodePath} --eNB2IPAddress=${params.eNB2_IPAddress} --eNB2UserName=${eNB2_Username} --eNB2Password=${eNB2_Password} --eNB2SourceCodePath=${params.eNB2_SourceCodePath} --EPCIPAddress=${params.EPC_IPAddress} --EPCType=${params.EPC_Type} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}"
-                                    } catch (Exception e) {
-                                        currentBuild.result = 'FAILURE'
-                                        buildStageStatus = false
-                                    }
-                                }
-                            }
-                            sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password}"
-                        }
-                    }
                 }
+              }
             }
-        }
+            withCredentials([
-        stage('Log Collection') {
+              [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password'],
-            parallel {
+              [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB1_Credentials}", usernameVariable: 'eNB1_Username', passwordVariable: 'eNB1_Password'],
-                stage('Log Collection (eNB - Build)') {
+              [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB2_Credentials}", usernameVariable: 'eNB2_Username', passwordVariable: 'eNB2_Password'],
-                    steps {
+              [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password'],
-                        withCredentials([
+              [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.ADB_Credentials}", usernameVariable: 'ADB_Username', passwordVariable: 'ADB_Password']
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
+            ]) {
-                        ]) {
+              sh "python3 main.py --mode=InitiateHtml --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} ${mainPythonAllXmlFiles}"
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
+              String[] myXmlTestSuite = testXMLFile.split("\\r?\\n")
-                            sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
+              for (xmlFile in myXmlTestSuite) {
-
+                if (fileExists(xmlFile)) {
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
+                  try {
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
+                    sh "python3 main.py --mode=TesteNB --ranRepository=${eNB_Repository} --ranBranch=${eNB_Branch} --ranCommitID=${eNB_CommitID} --ranAllowMerge=${eNB_AllowMergeRequestProcess} --ranTargetBranch=${eNB_TargetBranch} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath} --eNB1IPAddress=${params.eNB1_IPAddress} --eNB1UserName=${eNB1_Username} --eNB1Password=${eNB1_Password} --eNB1SourceCodePath=${params.eNB1_SourceCodePath} --eNB2IPAddress=${params.eNB2_IPAddress} --eNB2UserName=${eNB2_Username} --eNB2Password=${eNB2_Password} --eNB2SourceCodePath=${params.eNB2_SourceCodePath} --EPCIPAddress=${params.EPC_IPAddress} --EPCType=${params.EPC_Type} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --ADBIPAddress=${params.ADB_IPAddress} --ADBUserName=${ADB_Username} --ADBPassword=${ADB_Password} --XMLTestFile=${xmlFile}"
-                        }
+                  } catch (Exception e) {
-                        script {
+                    currentBuild.result = 'FAILURE'
-                            if(fileExists("build.log.${env.BUILD_ID}.zip")) {
+                    buildStageStatus = false
-                                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
+                  }
-                            }
-                        }
-                    }
-                }
-                stage('Log Collection (eNB - Run)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Collection (eNB - Run)\u001B[0m'
-                            sh "python3 ci-scripts/main.py --mode=LogCollecteNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
-
-                            echo '\u2705 \u001B[32mLog Transfer (eNB - Run)\u001B[0m'
-                            sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/enb.log.zip ./enb.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("enb.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "enb.log.${env.BUILD_ID}.zip"
-                            }
-                        }
-                    }
-                }
-                stage('Log Collection (CN)') {
-                    steps {
-                        withCredentials([
-                             [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password']
-                        ]) {
-                            echo '\u2705 \u001B[32mLog Transfer (CN)\u001B[0m'
-                            sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/logs/oai-cn5g.log.zip ./oai-cn5g.log.${env.BUILD_ID}.zip || true"
-                        }
-                        script {
-                            if(fileExists("oai-cn5g.log.${env.BUILD_ID}.zip")) {
-                                archiveArtifacts "oai-cn5g.log.${env.BUILD_ID}.zip"
-                            }
-                            if(fileExists("ci-scripts/test_results.html")) {
-                                sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html"
-                                sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html"
-                                archiveArtifacts "test_results-${JOB_NAME}.html"
-                            }
-                        }
-                    }
-                }
-                stage ("SQL Collect"){
-                agent {label DataBaseHost}
-                    steps {
-                        script {
-                            if (currentBuild.result=='FAILURE') {StatusForDb = 'FAIL'} else {StatusForDb = 'PASS'}
-                            sh "python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''"
-                        }
-                    }
                 }
+              }
+              sh "python3 main.py --mode=FinalizeHtml --finalStatus=${buildStageStatus} --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password}"
             }
+          }
         }
-    } 
+      }
+    }
+    stage('Log Collection') {
+      parallel {
+        stage('Log Collection (eNB - Build)') {
+          steps {
+            withCredentials([
+               [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
+            ]) {
+              echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
+              sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
 
-    post {
+              echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
-        always {
+              sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
+            }
             script {
-                if (params.pipelineZipsConsoleLog != null) {
+              if(fileExists("build.log.${env.BUILD_ID}.zip")) {
-                    if (params.pipelineZipsConsoleLog) {
+                archiveArtifacts "build.log.${env.BUILD_ID}.zip"
-                        echo "Archiving Jenkins console log"
+              }
-                        sh "wget --no-check-certificate --no-proxy ${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_ID}/consoleText -O consoleText.log || true"
-                        sh "zip -m consoleText.log.${env.BUILD_ID}.zip consoleText.log || true"
-                        if(fileExists("consoleText.log.${env.BUILD_ID}.zip")) {
-                            archiveArtifacts "consoleText.log.${env.BUILD_ID}.zip"
-                        }
-                    }
-                }
             }
+          }
         }
+        stage('Log Collection (eNB - Run)') {
+          steps {
+            withCredentials([
+               [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
+            ]) {
+              echo '\u2705 \u001B[32mLog Collection (eNB - Run)\u001B[0m'
+              sh "python3 ci-scripts/main.py --mode=LogCollecteNB --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
+
+              echo '\u2705 \u001B[32mLog Transfer (eNB - Run)\u001B[0m'
+              sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/enb.log.zip ./enb.log.${env.BUILD_ID}.zip || true"
+            }
+            script {
+              if(fileExists("enb.log.${env.BUILD_ID}.zip")) {
+                archiveArtifacts "enb.log.${env.BUILD_ID}.zip"
+              }
+            }
+          }
+        }
+        stage('Log Collection (CN)') {
+          // Bypassing this stage if EPC server is not defined
+          when {
+            expression { params.EPC_IPAddress != "none" }
+          }
+          steps {
+            script {
+              withCredentials([
+                   [$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.EPC_Credentials}", usernameVariable: 'EPC_Username', passwordVariable: 'EPC_Password']
+              ]) {
+                echo '\u2705 \u001B[32mLog Collection (HSS)\u001B[0m'
+                sh "python3 ci-scripts/main.py --mode=LogCollectHSS --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}"
+                if (params.EPC_Type != 'OAICN5G') {
+                  sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/hss.log.zip ./hss.log.${env.BUILD_ID}.zip || true"
+                }
+                echo '\u2705 \u001B[32mLog Collection (MME or AMF)\u001B[0m'
+                sh "python3 ci-scripts/main.py --mode=LogCollectMME --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}"
+                if (params.EPC_Type == 'OAICN5G') {
+                  sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/mme.log.zip ./amf.log.${env.BUILD_ID}.zip || true"
+                } else {
+                  sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/mme.log.zip ./mme.log.${env.BUILD_ID}.zip || true"
+                }
+                echo '\u2705 \u001B[32mLog Collection (SPGW or SMF/UPF)\u001B[0m'
+                sh "python3 ci-scripts/main.py --mode=LogCollectSPGW --EPCIPAddress=${params.EPC_IPAddress} --EPCUserName=${EPC_Username} --EPCPassword=${EPC_Password} --EPCSourceCodePath=${params.EPC_SourceCodePath} --EPCType=${params.EPC_Type}"
+                if (params.EPC_Type == 'OAICN5G') {
+                  sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/spgw.log.zip ./smf-upf.log.${env.BUILD_ID}.zip || true"
+                } else {
+                  sh "sshpass -p \'${EPC_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${EPC_Username}@${params.EPC_IPAddress}:${EPC_SourceCodePath}/scripts/spgw.log.zip ./spgw.log.${env.BUILD_ID}.zip || true"
+                }
+              }
+              if(fileExists("hss.log.${env.BUILD_ID}.zip")) {
+                archiveArtifacts "hss.log.${env.BUILD_ID}.zip"
+              }
+              if(fileExists("mme.log.${env.BUILD_ID}.zip")) {
+                archiveArtifacts "mme.log.${env.BUILD_ID}.zip"
+              }
+              if(fileExists("spgw.log.${env.BUILD_ID}.zip")) {
+                archiveArtifacts "spgw.log.${env.BUILD_ID}.zip"
+              }
+              if(fileExists("amf.log.${env.BUILD_ID}.zip")) {
+                archiveArtifacts "amf.log.${env.BUILD_ID}.zip"
+              }
+              if(fileExists("smf-upf.log.${env.BUILD_ID}.zip")) {
+                archiveArtifacts "smf-upf.log.${env.BUILD_ID}.zip"
+              }
+              echo '\u2705 \u001B[32mLog Collection for CoreNetwork Done!\u001B[0m'
+            }
+          }
+        }
+        stage ("SQL Collect"){
+          when {
+            expression { DataBaseHost != "none" }
+          }
+          agent {label DataBaseHost}
+          steps {
+            script {
+              if (currentBuild.result=='FAILURE') {StatusForDb = 'FAIL'} else {StatusForDb = 'PASS'}
+              sh "python3 /home/oaicicd/mysql/sql_connect.py ${JOB_NAME} ${params.eNB_MR} ${params.eNB_Branch} ${env.BUILD_ID} ${env.BUILD_URL} ${StatusForDb} ''"
+            }
+          }
+        }
+      }
     }
+  }
+
+  post {
+    always {
+      script {
+        if(fileExists("ci-scripts/test_results.html")) {
+          sh "mv ci-scripts/test_results.html test_results-${JOB_NAME}.html"
+          sh "sed -i -e 's#TEMPLATE_JOB_NAME#${JOB_NAME}#' -e 's@build #TEMPLATE_BUILD_ID@build #${BUILD_ID}@' -e 's#Build-ID: TEMPLATE_BUILD_ID#Build-ID: <a href=\"${BUILD_URL}\">${BUILD_ID}</a>#' -e 's#TEMPLATE_STAGE_NAME#${testStageName}#' test_results-${JOB_NAME}.html"
+          archiveArtifacts "test_results-${JOB_NAME}.html"
+        }
+        if (params.pipelineZipsConsoleLog != null) {
+          if (params.pipelineZipsConsoleLog) {
+            echo "Archiving Jenkins console log"
+            sh "wget --no-check-certificate --no-proxy ${env.JENKINS_URL}/job/${env.JOB_NAME}/${env.BUILD_ID}/consoleText -O consoleText.log || true"
+            sh "zip -m consoleText.log.${env.BUILD_ID}.zip consoleText.log || true"
+            if(fileExists("consoleText.log.${env.BUILD_ID}.zip")) {
+              archiveArtifacts "consoleText.log.${env.BUILD_ID}.zip"
+            }
+          }
+        }
+      }
+    }
+  }
 }

ci-scripts/cls_containerize.py

@@ -101,10 +101,43 @@ class Containerize():
 		self.cliContName = ''
 		self.cliOptions = ''
 
+		self.imageToCopy = ''
+		self.registrySvrId = ''
+		self.testSvrId = ''
+
 #-----------------------------------------------------------
 # Container management functions
 #-----------------------------------------------------------
 
+	def _createWorkspace(self, sshSession, password, sourcePath):
+		# on RedHat/CentOS .git extension is mandatory
+		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
+		if result is not None:
+			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
+		else:
+			full_ran_repo_name = self.ranRepository + '.git'
+		sshSession.command('mkdir -p ' + sourcePath, '\$', 5)
+		sshSession.command('cd ' + sourcePath, '\$', 5)
+		sshSession.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
+		# Raphael: here add a check if git clone or git fetch went smoothly
+		sshSession.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
+		sshSession.command('git config user.name "OAI Jenkins"', '\$', 5)
+
+		sshSession.command('echo ' + password + ' | sudo -S git clean -x -d -ff', '\$', 30)
+		sshSession.command('mkdir -p cmake_targets/log', '\$', 5)
+		# if the commit ID is provided use it to point to it
+		if self.ranCommitID != '':
+			sshSession.command('git checkout -f ' + self.ranCommitID, '\$', 30)
+		# if the branch is not develop, then it is a merge request and we need to do
+		# the potential merge. Note that merge conflicts should already been checked earlier
+		if (self.ranAllowMerge):
+			if self.ranTargetBranch == '':
+				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
+					sshSession.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
+			else:
+				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
+				sshSession.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
+
 	def BuildImage(self, HTML):
 		if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
 			HELP.GenericHelp(CONST.Version)
@@ -173,44 +206,7 @@ class Containerize():
 	
 		self.testCase_id = HTML.testCase_id
 	
-		# on RedHat/CentOS .git extension is mandatory
+		self._createWorkspace(mySSH, lPassWord, lSourcePath)
-		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
-		if result is not None:
-			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
-		else:
-			full_ran_repo_name = self.ranRepository + '.git'
-		mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
-		mySSH.command('cd ' + lSourcePath, '\$', 5)
-		mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
-		# Raphael: here add a check if git clone or git fetch went smoothly
-		mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
-		mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
-
-		mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
-		mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
-		# if the commit ID is provided use it to point to it
-		if self.ranCommitID != '':
-			mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
-		# if the branch is not develop, then it is a merge request and we need to do 
-		# the potential merge. Note that merge conflicts should already been checked earlier
-		imageTag = 'develop'
-		sharedTag = 'develop'
-		forceSharedImageBuild = False
-		if (self.ranAllowMerge):
-			imageTag = 'ci-temp'
-			if self.ranTargetBranch == '':
-				if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
-					mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
-			else:
-				logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
-				mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
-				mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.ran' + self.dockerfileprefix + ' | grep -i INDEX', '\$', 5)
-				result = re.search('index', mySSH.getBefore())
-				if result is not None:
-					forceSharedImageBuild = True
-					sharedTag = 'ci-temp'
-		else:
-			forceSharedImageBuild = True
 
  		# if asterix, copy the entitlement and subscription manager configurations
 		if self.host == 'Red Hat':
@@ -220,6 +216,18 @@ class Containerize():
 			mySSH.command('sudo cp /etc/pki/entitlement/*.pem tmp/entitlement/', '\$', 5)
 
 		sharedimage = 'ran-build'
+		sharedTag = 'develop'
+		forceSharedImageBuild = False
+		imageTag = 'develop'
+		if (self.ranAllowMerge):
+			imageTag = 'ci-temp'
+			if self.ranTargetBranch == 'develop':
+				mySSH.command('git diff HEAD..origin/develop -- docker/Dockerfile.ran' + self.dockerfileprefix + ' | grep --colour=never -i INDEX', '\$', 5)
+				result = re.search('index', mySSH.getBefore())
+				if result is not None:
+					forceSharedImageBuild = True
+					sharedTag = 'ci-temp'
+
 		# Let's remove any previous run artifacts if still there
 		mySSH.command(self.cli + ' image prune --force', '\$', 30)
 		if forceSharedImageBuild:
@@ -397,6 +405,56 @@ class Containerize():
 			HTML.CreateHtmlTabFooter(False)
 			sys.exit(1)
 
+	def Copy_Image_to_Test_Server(self, HTML):
+		imageTag = 'develop'
+		if (self.ranAllowMerge):
+			imageTag = 'ci-temp'
+
+		lSsh = SSH.SSHConnection()
+		# Going to the Docker Registry server
+		if self.registrySvrId == '0':
+			lIpAddr = self.eNBIPAddress
+			lUserName = self.eNBUserName
+			lPassWord = self.eNBPassword
+		elif self.registrySvrId == '1':
+			lIpAddr = self.eNB1IPAddress
+			lUserName = self.eNB1UserName
+			lPassWord = self.eNB1Password
+		elif self.registrySvrId == '2':
+			lIpAddr = self.eNB2IPAddress
+			lUserName = self.eNB2UserName
+			lPassWord = self.eNB2Password
+		lSsh.open(lIpAddr, lUserName, lPassWord)
+		lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.')
+		lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		lSsh.close()
+
+		# Going to the Test Server
+		if self.testSvrId == '0':
+			lIpAddr = self.eNBIPAddress
+			lUserName = self.eNBUserName
+			lPassWord = self.eNBPassword
+		elif self.testSvrId == '1':
+			lIpAddr = self.eNB1IPAddress
+			lUserName = self.eNB1UserName
+			lPassWord = self.eNB1Password
+		elif self.testSvrId == '2':
+			lIpAddr = self.eNB2IPAddress
+			lUserName = self.eNB2UserName
+			lPassWord = self.eNB2Password
+		lSsh.open(lIpAddr, lUserName, lPassWord)
+		lSsh.copyout(lIpAddr, lUserName, lPassWord, './' + self.imageToCopy + '-' + imageTag + '.tar.gz', '~')
+		lSsh.command('docker rmi ' + self.imageToCopy + ':' + imageTag, '\$', 10)
+		lSsh.command('docker load < ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60)
+		lSsh.close()
+
+		if os.path.isfile('./' + self.imageToCopy + '-' + imageTag + '.tar.gz'):
+			os.remove('./' + self.imageToCopy + '-' + imageTag + '.tar.gz')
+
+		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+
 	def DeployObject(self, HTML, EPC):
 		if self.eNB_serverId[self.eNB_instance] == '0':
 			lIpAddr = self.eNBIPAddress
@@ -417,35 +475,33 @@ class Containerize():
 			HELP.GenericHelp(CONST.Version)
 			sys.exit('Insufficient Parameter')
 		logging.debug('\u001B[1m Deploying OAI Object on server: ' + lIpAddr + '\u001B[0m')
+
 		mySSH = SSH.SSHConnection()
 		mySSH.open(lIpAddr, lUserName, lPassWord)
-		# Putting the CPUs in a good state, we do that only on a few servers
-		mySSH.command('hostname', '\$', 5)
-		result = re.search('obelix|asterix',  mySSH.getBefore())
-		if result is not None:
-			mySSH.command('if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -D 0; fi', '\$', 5)
-			time.sleep(5)
 		
+		self._createWorkspace(mySSH, lPassWord, lSourcePath)
+
 		mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5)
 		mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5)
 		imageTag = 'develop'
 		if (self.ranAllowMerge):
 			imageTag = 'ci-temp'
 		mySSH.command('sed -i -e "s/image: oai-enb:latest/image: oai-enb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2)
+		mySSH.command('sed -i -e "s/image: oai-gnb:latest/image: oai-gnb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2)
 		localMmeIpAddr = EPC.MmeIPAddress
 		mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2)
-		if self.flexranCtrlDeployed:
+#		if self.flexranCtrlDeployed:
-			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: "yes"/\' ci-docker-compose.yml', '\$', 2)
+#			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*/FLEXRAN_ENABLED: \'yes\'/" ci-docker-compose.yml', '\$', 2)
-			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2)
+#			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/' + self.flexranCtrlIpAddress + '/" ci-docker-compose.yml', '\$', 2)
-		else:
+#		else:
-			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \"no\"/" ci-docker-compose.yml', '\$', 2)
+#			mySSH.command('sed -i -e "s/FLEXRAN_ENABLED:.*$/FLEXRAN_ENABLED: \'no\'/" ci-docker-compose.yml', '\$', 2)
-			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2)
+#			mySSH.command('sed -i -e "s/CI_FLEXRAN_CTL_IP_ADDR/127.0.0.1/" ci-docker-compose.yml', '\$', 2)
 		# Currently support only one
-		mySSH.command('docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@"', '\$', 2)
+		mySSH.command('docker-compose --file ci-docker-compose.yml config --services | sed -e "s@^@service=@" 2>&1', '\$', 10)
 		result = re.search('service=(?P<svc_name>[a-zA-Z0-9\_]+)', mySSH.getBefore())
 		if result is not None:
 			svcName = result.group('svc_name')
-			mySSH.command('docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 2)
+			mySSH.command('docker-compose --file ci-docker-compose.yml up -d ' + svcName, '\$', 10)
 
 		# Checking Status
 		mySSH.command('docker-compose --file ci-docker-compose.yml config', '\$', 5)
@@ -459,7 +515,7 @@ class Containerize():
 			time.sleep(5)
 			cnt = 0
 			while (cnt < 3):
-				mySSH.command('docker inspect --format=\'{{.State.Health.Status}}\' ' + containerName, '\$', 5)
+				mySSH.command('docker inspect --format="{{.State.Health.Status}}" ' + containerName, '\$', 5)
 				unhealthyNb = mySSH.getBefore().count('unhealthy')
 				healthyNb = mySSH.getBefore().count('healthy') - unhealthyNb
 				startingNb = mySSH.getBefore().count('starting')
@@ -528,12 +584,9 @@ class Containerize():
 			time.sleep(5)
 			mySSH.command('docker logs ' + containerName + ' > ' + lSourcePath + '/cmake_targets/' + self.eNB_logFile[self.eNB_instance], '\$', 30)
 			mySSH.command('docker rm -f ' + containerName, '\$', 30)
+		# Forcing the down now to remove the networks and any artifacts
+		mySSH.command('docker-compose --file ci-docker-compose.yml down', '\$', 5)
 
-		# Putting the CPUs back in a idle state, we do that only on a few servers
-		mySSH.command('hostname', '\$', 5)
-		result = re.search('obelix|asterix',  mySSH.getBefore())
-		if result is not None:
-			mySSH.command('if command -v cpupower &> /dev/null; then echo ' + lPassWord + ' | sudo -S cpupower idle-set -E; fi', '\$', 5)
 		mySSH.close()
 
 		# Analyzing log file!
@@ -841,3 +894,114 @@ class Containerize():
 		else:
 			self.exitStatus = 1
 			HTML.CreateHtmlTestRowQueue(self.cliOptions, 'KO', 1, html_queue)
+
+	def CheckAndAddRoute(self, svrName, ipAddr, userName, password):
+		logging.debug('Checking IP routing on ' + svrName)
+		mySSH = SSH.SSHConnection()
+		if svrName == 'porcepix':
+			mySSH.open(ipAddr, userName, password)
+			# Check if route to asterix gnb exists
+			mySSH.command('ip route | grep --colour=never "192.168.68.64/26"', '\$', 10)
+			result = re.search('192.168.18.194', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.64/26 via 192.168.18.194 dev eno1', '\$', 10)
+			# Check if route to obelix enb exists
+			mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
+			result = re.search('192.168.18.193', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 192.168.18.193 dev eno1', '\$', 10)
+			# Check if route to nepes gnb exists
+			mySSH.command('ip route | grep --colour=never "192.168.68.192/26"', '\$', 10)
+			result = re.search('192.168.18.209', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.192/26 via 192.168.18.209 dev eno1', '\$', 10)
+			# Check if forwarding is enabled
+			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
+			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
+			# Check if iptables forwarding is accepted
+			mySSH.command('echo ' + password + ' | sudo -S iptables -L', '\$', 10)
+			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
+			mySSH.close()
+		if svrName == 'asterix':
+			mySSH.open(ipAddr, userName, password)
+			# Check if route to porcepix epc exists
+			mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
+			result = re.search('192.168.18.210', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 192.168.18.210 dev em1', '\$', 10)
+			# Check if route to porcepix cn5g exists
+			mySSH.command('ip route | grep --colour=never "192.168.70.128/26"', '\$', 10)
+			result = re.search('192.168.18.210', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.70.128/26 via 192.168.18.210 dev em1', '\$', 10)
+			# Check if X2 route to obelix enb exists
+			mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
+			result = re.search('192.168.18.193', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 192.168.18.193 dev em1', '\$', 10)
+			# Check if forwarding is enabled
+			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
+			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
+			# Check if iptables forwarding is accepted
+			mySSH.command('echo ' + password + ' | sudo -S iptables -L', '\$', 10)
+			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
+			mySSH.close()
+		if svrName == 'obelix':
+			mySSH.open(ipAddr, userName, password)
+			# Check if route to porcepix epc exists
+			mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
+			result = re.search('192.168.18.210', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 192.168.18.210 dev eno1', '\$', 10)
+			# Check if X2 route to asterix gnb exists
+			mySSH.command('ip route | grep --colour=never "192.168.68.64/26"', '\$', 10)
+			result = re.search('192.168.18.194', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.64/26 via 192.168.18.194 dev eno1', '\$', 10)
+			# Check if X2 route to nepes gnb exists
+			mySSH.command('ip route | grep --colour=never "192.168.68.192/26"', '\$', 10)
+			result = re.search('192.168.18.209', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.192/26 via 192.168.18.209 dev eno1', '\$', 10)
+			# Check if forwarding is enabled
+			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
+			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
+			# Check if iptables forwarding is accepted
+			mySSH.command('echo ' + password + ' | sudo -S iptables -L', '\$', 10)
+			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
+			mySSH.close()
+		if svrName == 'nepes':
+			mySSH.open(ipAddr, userName, password)
+			# Check if route to porcepix epc exists
+			mySSH.command('ip route | grep --colour=never "192.168.61.192/26"', '\$', 10)
+			result = re.search('192.168.18.210', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.61.192/26 via 192.168.18.210 dev enp0s31f6', '\$', 10)
+			# Check if X2 route to obelix enb exists
+			mySSH.command('ip route | grep --colour=never "192.168.68.128/26"', '\$', 10)
+			result = re.search('192.168.18.193', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S ip route add 192.168.68.128/26 via 192.168.18.193 dev enp0s31f6', '\$', 10)
+			# Check if forwarding is enabled
+			mySSH.command('sysctl net.ipv4.conf.all.forwarding', '\$', 10)
+			result = re.search('net.ipv4.conf.all.forwarding = 1', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S sysctl net.ipv4.conf.all.forwarding=1', '\$', 10)
+			# Check if iptables forwarding is accepted
+			mySSH.command('echo ' + password + ' | sudo -S iptables -L', '\$', 10)
+			result = re.search('Chain FORWARD .*policy ACCEPT', mySSH.getBefore())
+			if result is None:
+				mySSH.command('echo ' + password + ' | sudo -S iptables -P FORWARD ACCEPT', '\$', 10)
+			mySSH.close()

ci-scripts/cls_module_ue.py

@@ -63,7 +63,7 @@ class Module_UE:
 	#if not it will be started
 	def CheckCMProcess(self,CNType):
 		HOST=self.HostUsername+'@'+self.HostIPAddress
-		COMMAND="ps aux | grep " + self.Process['Name'] + " | grep -v grep "
+		COMMAND="ps aux | grep --colour=never " + self.Process['Name'] + " | grep -v grep "
 		logging.debug(COMMAND)
 		ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 		result = ssh.stdout.readlines()
@@ -81,7 +81,7 @@ class Module_UE:
 			#checking the process
 			time.sleep(5)
 			HOST=self.HostUsername+'@'+self.HostIPAddress
-			COMMAND="ps aux | grep " + self.Process['Name'] + " | grep -v grep "
+			COMMAND="ps aux | grep --colour=never " + self.Process['Name'] + " | grep -v grep "
 			logging.debug(COMMAND)
 			ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 			result = ssh.stdout.readlines()
@@ -108,7 +108,7 @@ class Module_UE:
 		response= []
 		tentative = 3 
 		while (len(response)==0) and (tentative>0):
-			COMMAND="ip a show dev " + self.UENetwork + " | grep inet | grep " + self.UENetwork
+			COMMAND="ip a show dev " + self.UENetwork + " | grep --colour=never inet | grep " + self.UENetwork
 			logging.debug(COMMAND)
 			ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 			response = ssh.stdout.readlines()
@@ -136,7 +136,7 @@ class Module_UE:
 		response= []
 		tentative = 3 
 		while (len(response)==0) and (tentative>0):
-			COMMAND="ip a show dev " + self.UENetwork + " | grep mtu"
+			COMMAND="ip a show dev " + self.UENetwork + " | grep --colour=never mtu"
 			logging.debug(COMMAND)
 			ssh = subprocess.Popen(["ssh", "%s" % HOST, COMMAND],shell=False,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 			response = ssh.stdout.readlines()

ci-scripts/cls_oaicitest.py

@@ -192,16 +192,16 @@ class OaiCiTest():
 			result = re.search('LAST_BUILD_INFO', SSH.getBefore())
 			if result is not None:
 				mismatch = False
-				SSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
+				SSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
 				result = re.search(self.ranCommitID, SSH.getBefore())
 				if result is None:
 					mismatch = True
-				SSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
+				SSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
 				if self.ranAllowMerge:
 					result = re.search('YES', SSH.getBefore())
 					if result is None:
 						mismatch = True
-					SSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
+					SSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
 					if self.ranTargetBranch == '':
 						result = re.search('develop', SSH.getBefore())
 					else:
@@ -660,7 +660,7 @@ class OaiCiTest():
 		SSH.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
 		count = 0
 		while count < 5:
-			SSH.command('echo ' + self.ADBPassword + ' | sudo -S lsof | grep ttyUSB0', '\$', 10)
+			SSH.command('echo ' + self.ADBPassword + ' | sudo -S lsof | grep --colour=never ttyUSB0', '\$', 10)
 			result = re.search('picocom', SSH.getBefore())
 			if result is None:
 				count = 10
@@ -1328,7 +1328,7 @@ class OaiCiTest():
 		SSH = sshconnection.SSHConnection()
 		SSH.open(self.ADBIPAddress, self.ADBUserName, self.ADBPassword)
 		if self.ADBCentralized:
-			SSH.command('lsusb | egrep "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15)
+			SSH.command('lsusb | egrep --colour=never "Future Technology Devices International, Ltd FT2232C" | sed -e "s#:.*##" -e "s# #_#g"', '\$', 15)
 			#self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",SSH.getBefore())
 			self.CatMDevices = re.findall("\\\\r\\\\n([A-Za-z0-9_]+)",SSH.getBefore())
 		else:
@@ -2325,7 +2325,7 @@ class OaiCiTest():
 				server_filename = 'iperf_server_' + self.testCase_id + '_' + self.ue_id + '.log'
 				SSH.command('docker exec -it prod-trf-gen /bin/bash -c "killall --signal SIGKILL iperf"', '\$', 5)
 				iperf_cmd = 'echo $USER; nohup bin/iperf -s -u 2>&1 > ' + server_filename
-				cmd = 'docker exec -it prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' 
+				cmd = 'docker exec -d prod-trf-gen /bin/bash -c \"' + iperf_cmd + '\"' 
 				SSH.command(cmd,'\$',5)
 				SSH.close()
 

ci-scripts/epc.py

@@ -53,7 +53,7 @@ import constants as CONST
 class EPCManagement():
 
 	def __init__(self):
-		
+
 		self.IPAddress = ''
 		self.UserName = ''
 		self.Password = ''
@@ -227,6 +227,7 @@ class EPCManagement():
 			sys.exit('Insufficient EPC Parameters')
 		mySSH = SSH.SSHConnection()
 		mySSH.open(self.IPAddress, self.UserName, self.Password)
+		html_cell = '<pre style="background-color:white">\n'
 		if re.match('ltebox', self.Type, re.IGNORECASE):
 			logging.debug('Using the SABOX simulated HSS')
 			mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5)
@@ -237,16 +238,48 @@ class EPCManagement():
 			logging.debug('Using the sabox')
 			mySSH.command('cd /opt/ltebox/tools', '\$', 5)
 			mySSH.command('echo ' + self.Password + ' | sudo -S ./start_sabox', '\$', 5)
+			html_cell += 'N/A\n'
 		elif re.match('OAICN5G', self.Type, re.IGNORECASE):
 			logging.debug('Starting OAI CN5G')
 			mySSH.command('if [ -d ' + self.SourceCodePath + '/scripts ]; then echo ' + self.Password + ' | sudo -S rm -Rf ' + self.SourceCodePath + '/scripts ; fi', '\$', 5)
 			mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5)
 			mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5)
 			mySSH.command('./core-network.sh start nrf spgwu', '\$', 60)
+			time.sleep(2)
+			mySSH.command('docker-compose -p 5gcn ps -a', '\$', 60)
+			if mySSH.getBefore().count('Up (healthy)') != 6:
+				logging.error('Not all container healthy')
+			else:
+				logging.debug('OK')
+			mySSH.command('docker-compose config | grep --colour=never image', '\$', 10)
+			listOfImages = mySSH.getBefore()
+			for imageLine in listOfImages.split('\\r\\n'):
+				res1 = re.search('image: (?P<name>[a-zA-Z0-9\-]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine))
+				res2 = re.search('mysql', str(imageLine))
+				if res1 is not None and res2 is None:
+					html_cell += res1.group('name') + ':' + res1.group('tag') + ' '
+					nbChars = len(res1.group('name')) + len(res1.group('tag')) + 2
+					while (nbChars < 32):
+						html_cell += ' '
+						nbChars += 1
+					mySSH.command('docker image inspect --format="Size = {{.Size}} bytes" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+					res3 = re.search('Size *= *(?P<size>[0-9\-]*) *bytes', mySSH.getBefore())
+					if res3 is not None:
+						imageSize = int(res3.group('size'))
+						imageSize = int(imageSize/(1024*1024))
+						html_cell += str(imageSize) + ' MBytes '
+					mySSH.command('docker image inspect --format="Date = {{.Created}}" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+					res4 = re.search('Date *= *(?P<date>[0-9\-]*)T', mySSH.getBefore())
+					if res4 is not None:
+						html_cell += '(' + res4.group('date') + ')'
+					html_cell += '\n'
 		else:
 			logging.error('This option should not occur!')
+		html_cell += '</pre>'
 		mySSH.close()
-		HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
+		html_queue = SimpleQueue()
+		html_queue.put(html_cell)
+		HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue)
 
 	def SetAmfIPAddress(self):
 		# Not an error if we don't need an 5GCN
@@ -370,7 +403,7 @@ class EPCManagement():
 		elif re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
 			mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGINT oai_hss || true', '\$', 5)
 			time.sleep(2)
-			mySSH.command('stdbuf -o0  ps -aux | grep hss | grep -v grep', '\$', 5)
+			mySSH.command('stdbuf -o0  ps -aux | grep --colour=never hss | grep -v grep', '\$', 5)
 			result = re.search('oai_hss -j', mySSH.getBefore())
 			if result is not None:
 				mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGKILL oai_hss || true', '\$', 5)
@@ -378,7 +411,7 @@ class EPCManagement():
 		elif re.match('OAI', self.Type, re.IGNORECASE):
 			mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGINT run_hss oai_hss || true', '\$', 5)
 			time.sleep(2)
-			mySSH.command('stdbuf -o0  ps -aux | grep hss | grep -v grep', '\$', 5)
+			mySSH.command('stdbuf -o0  ps -aux | grep --colour=never hss | grep -v grep', '\$', 5)
 			result = re.search('\/bin\/bash .\/run_', mySSH.getBefore())
 			if result is not None:
 				mySSH.command('echo ' + self.Password + ' | sudo -S killall --signal SIGKILL run_hss oai_hss || true', '\$', 5)
@@ -464,6 +497,7 @@ class EPCManagement():
 	def Terminate5GCN(self, HTML):
 		mySSH = SSH.SSHConnection()
 		mySSH.open(self.IPAddress, self.UserName, self.Password)
+		message = ''
 		if re.match('ltebox', self.Type, re.IGNORECASE):
 			logging.debug('Terminating SA BOX')
 			mySSH.command('cd /opt/ltebox/tools', '\$', 5)
@@ -474,15 +508,31 @@ class EPCManagement():
 			time.sleep(1)
 			mySSH.command('echo ' + self.Password + ' | sudo -S screen -S simulated_5g_hss -X quit', '\$', 5)
 		elif re.match('OAICN5G', self.Type, re.IGNORECASE):
-			self.LogCollectOAICN5G()
+			logging.debug('OAI CN5G Collecting Log files to workspace')
+			mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5)
+			mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5)
+			containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf']
+			for c in containers_list:
+				mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5)
+
 			logging.debug('Terminating OAI CN5G')
 			mySSH.command('cd /opt/oai-cn5g-fed/docker-compose', '\$', 5)
-			mySSH.command('docker-compose down', '\$', 5)
 			mySSH.command('./core-network.sh stop nrf spgwu', '\$', 60)
+			time.sleep(2)
+			mySSH.command('tshark -r /tmp/oai-cn5g.pcap | egrep --colour=never "Tracking area update" ','\$', 30)
+			result = re.search('Tracking area update request', mySSH.getBefore())
+			if result is not None:
+				message = 'UE requested ' + mySSH.getBefore().count('Tracking area update request') + 'Tracking area update request(s)'
+			else:
+				message = 'No Tracking area update request'
+			logging.debug(message)
 		else:
 			logging.error('This should not happen!')
 		mySSH.close()
-		HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+		html_queue = SimpleQueue()
+		html_cell = '<pre style="background-color:white">' + message + '</pre>'
+		html_queue.put(html_cell)
+		HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue)
 
 	def DeployEpc(self, HTML):
 		logging.debug('Trying to deploy')
@@ -528,6 +578,10 @@ class EPCManagement():
 			mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/mme_fd.sprint.conf', self.SourceCodePath + '/scripts')
 			mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/redis_extern.conf', self.SourceCodePath + '/scripts')
 			mySSH.command('chmod a+x ' + self.SourceCodePath + '/scripts/entrypoint.sh', '\$', 5)
+		else:
+			mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/entrypoint.sh', self.SourceCodePath + '/scripts')
+			mySSH.copyout(self.IPAddress, self.UserName, self.Password, './' + self.yamlPath + '/mme.conf', self.SourceCodePath + '/scripts')
+			mySSH.command('chmod 775 entrypoint.sh', '\$', 60)
 		mySSH.command('wget --quiet --tries=3 --retry-connrefused https://raw.githubusercontent.com/OPENAIRINTERFACE/openair-hss/develop/src/hss_rel14/db/oai_db.cql', '\$', 30)
 		mySSH.command('docker-compose down', '\$', 60)
 		mySSH.command('docker-compose up -d db_init', '\$', 60)
@@ -571,6 +625,30 @@ class EPCManagement():
 			listOfContainers += ' prod-trf-gen'
 			expectedHealthyContainers += 1
 
+		mySSH.command('docker-compose config | grep --colour=never image', '\$', 10)
+		html_cell = '<pre style="background-color:white">\n'
+		listOfImages = mySSH.getBefore()
+		for imageLine in listOfImages.split('\\r\\n'):
+			res1 = re.search('image: (?P<name>[a-zA-Z0-9\-]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine))
+			res2 = re.search('cassandra|redis', str(imageLine))
+			if res1 is not None and res2 is None:
+				html_cell += res1.group('name') + ':' + res1.group('tag') + ' '
+				nbChars = len(res1.group('name')) + len(res1.group('tag')) + 2
+				while (nbChars < 32):
+					html_cell += ' '
+					nbChars += 1
+				mySSH.command('docker image inspect --format="Size = {{.Size}} bytes" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+				res3 = re.search('Size *= *(?P<size>[0-9\-]*) *bytes', mySSH.getBefore())
+				if res3 is not None:
+					imageSize = int(res3.group('size'))
+					imageSize = int(imageSize/(1024*1024))
+					html_cell += str(imageSize) + ' MBytes '
+				mySSH.command('docker image inspect --format="Date = {{.Created}}" ' + res1.group('name') + ':' + res1.group('tag'), '\$', 10)
+				res4 = re.search('Date *= *(?P<date>[0-9\-]*)T', mySSH.getBefore())
+				if res4 is not None:
+					html_cell += '(' + res4.group('date') + ')'
+				html_cell += '\n'
+		html_cell += '</pre>'
 		# Checking if all are healthy
 		cnt = 0
 		while (cnt < 3):
@@ -586,6 +664,8 @@ class EPCManagement():
 		logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)')
 		logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)')
 		logging.debug(' -- ' + str(startingNb) + ' still starting container(s)')
+		html_queue = SimpleQueue()
+		html_queue.put(html_cell)
 		if healthyNb == expectedHealthyContainers:
 			mySSH.command('docker exec -d prod-oai-hss /bin/bash -c "nohup tshark -i any -f \'port 9042 or port 3868\' -w /tmp/hss_check_run.pcap 2>&1 > /dev/null"', '\$', 5)
 			if self.isMagmaUsed:
@@ -597,11 +677,11 @@ class EPCManagement():
 			mySSH.command('docker exec -d prod-oai-spgwu-tiny /bin/bash -c "nohup tshark -i any -f \'port 8805\'  -w /tmp/spgwu_check_run.pcap 2>&1 > /dev/null"', '\$', 10)
 			mySSH.close()
 			logging.debug('Deployment OK')
-			HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
+			HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue)
 		else:
 			mySSH.close()
 			logging.debug('Deployment went wrong')
-			HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER)
+			HTML.CreateHtmlTestRowQueue(self.Type, 'KO', 1, html_queue)
 
 	def UndeployEpc(self, HTML):
 		logging.debug('Trying to undeploy')
@@ -638,6 +718,13 @@ class EPCManagement():
 			mySSH.command('docker cp prod-magma-mme:/tmp/mme_check_run.pcap mme_' + self.testCase_id + '.pcap', '\$', 60)
 		else:
 			mySSH.command('docker cp prod-oai-mme:/tmp/mme_check_run.pcap mme_' + self.testCase_id + '.pcap', '\$', 60)
+		mySSH.command('tshark -r mme_' + self.testCase_id + '.pcap | egrep --colour=never "Tracking area update"', '\$', 60)
+		result = re.search('Tracking area update request', mySSH.getBefore())
+		if result is not None:
+			message = 'UE requested ' + mySSH.getBefore().count('Tracking area update request') + 'Tracking area update request(s)'
+		else:
+			message = 'No Tracking area update request'
+		logging.debug(message)
 		mySSH.command('docker cp prod-oai-spgwc:/tmp/spgwc_check_run.pcap spgwc_' + self.testCase_id + '.pcap', '\$', 60)
 		mySSH.command('docker cp prod-oai-spgwu-tiny:/tmp/spgwu_check_run.pcap spgwu_' + self.testCase_id + '.pcap', '\$', 60)
 		# Remove all
@@ -664,12 +751,15 @@ class EPCManagement():
 		mySSH.command('docker inspect --format=\'{{.Name}}\' prod-oai-public-net prod-oai-private-net', '\$', 10)
 		noMoreNetworkNb = mySSH.getBefore().count('No such object')
 		mySSH.close()
+		html_queue = SimpleQueue()
+		html_cell = '<pre style="background-color:white">' + message + '</pre>'
+		html_queue.put(html_cell)
 		if noMoreContainerNb == nbContainers and noMoreNetworkNb == 2:
 			logging.debug('Undeployment OK')
-			HTML.CreateHtmlTestRow(self.Type, 'OK', CONST.ALL_PROCESSES_OK)
+			HTML.CreateHtmlTestRowQueue(self.Type, 'OK', 1, html_queue)
 		else:
 			logging.debug('Undeployment went wrong')
-			HTML.CreateHtmlTestRow(self.Type, 'KO', CONST.INVALID_PARAMETER)
+			HTML.CreateHtmlTestRowQueu(self.Type, 'KO', 1, html_queue)
 
 	def LogCollectHSS(self):
 		mySSH = SSH.SSHConnection()
@@ -688,6 +778,8 @@ class EPCManagement():
 				mySSH.command('docker cp ' + self.containerPrefix + '-oai-hss:/openair-hss/hss_check_run.log .', '\$', 60)
 				mySSH.command('docker cp ' + self.containerPrefix + '-oai-hss:/tmp/hss_check_run.pcap .', '\$', 60)
 				mySSH.command('zip hss.log.zip hss_check_run.*', '\$', 60)
+		elif re.match('OAICN5G', self.Type, re.IGNORECASE):
+			logging.debug('LogCollect is bypassed for that variant')
 		elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
 			mySSH.command('zip hss.log.zip hss*.log', '\$', 60)
 			mySSH.command('echo ' + self.Password + ' | sudo -S rm hss*.log', '\$', 5)
@@ -718,6 +810,11 @@ class EPCManagement():
 				mySSH.command('docker cp ' + self.containerPrefix + '-oai-mme:/openair-mme/mme_check_run.log .', '\$', 60)
 				mySSH.command('docker cp ' + self.containerPrefix + '-oai-mme:/tmp/mme_check_run.pcap .', '\$', 60)
 				mySSH.command('zip mme.log.zip mme_check_run.*', '\$', 60)
+		elif re.match('OAICN5G', self.Type, re.IGNORECASE):
+			mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5)
+			mySSH.command('cp -f /tmp/oai-cn5g.pcap .','\$', 30)
+			mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g.pcap','\$', 30)
+			mySSH.command('mv mme.log.zip ' + self.SourceCodePath + '/scripts','\$', 30)
 		elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
 			mySSH.command('zip mme.log.zip mme*.log', '\$', 60)
 			mySSH.command('echo ' + self.Password + ' | sudo -S rm mme*.log', '\$', 5)
@@ -747,6 +844,10 @@ class EPCManagement():
 				mySSH.command('docker cp ' + self.containerPrefix + '-oai-spgwc:/tmp/spgwc_check_run.pcap .', '\$', 60)
 				mySSH.command('docker cp ' + self.containerPrefix + '-oai-spgwu-tiny:/tmp/spgwu_check_run.pcap .', '\$', 60)
 				mySSH.command('zip spgw.log.zip spgw*_check_run.*', '\$', 60)
+		elif re.match('OAICN5G', self.Type, re.IGNORECASE):
+			mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5)
+			mySSH.command('zip spgw.log.zip oai-smf.log oai-spgwu.log','\$', 30)
+			mySSH.command('mv spgw.log.zip ' + self.SourceCodePath + '/scripts','\$', 30)
 		elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
 			mySSH.command('zip spgw.log.zip spgw*.log', '\$', 60)
 			mySSH.command('echo ' + self.Password + ' | sudo -S rm spgw*.log', '\$', 5)
@@ -756,17 +857,3 @@ class EPCManagement():
 		else:
 			logging.error('This option should not occur!')
 		mySSH.close()
-
-	def LogCollectOAICN5G(self):
-		mySSH = SSH.SSHConnection()
-		mySSH.open(self.IPAddress, self.UserName, self.Password)
-		logging.debug('OAI CN5G Collecting Log files to workspace')
-		mySSH.command('echo ' + self.Password + ' | sudo rm -rf ' + self.SourceCodePath + '/logs', '\$', 5)
-		mySSH.command('mkdir ' + self.SourceCodePath + '/logs','\$', 5)	
-		containers_list=['oai-smf','oai-spgwu','oai-amf','oai-nrf']
-		for c in containers_list:
-			mySSH.command('docker logs ' + c + ' > ' + self.SourceCodePath + '/logs/' + c + '.log', '\$', 5)
-		mySSH.command('cd ' + self.SourceCodePath + '/logs', '\$', 5)		
-		mySSH.command('zip oai-cn5g.log.zip *.log', '\$', 60)
-		mySSH.close()
-

ci-scripts/main.py

@@ -410,6 +410,17 @@ def GetParametersFromXML(action):
 		if (string_field is not None):
 			CONTAINERS.cliOptions = string_field
 
+	elif action == 'Copy_Image_to_Test':
+		string_field = test.findtext('image_name')
+		if (string_field is not None):
+			CONTAINERS.imageToCopy = string_field
+		string_field = test.findtext('registry_svr_id')
+		if (string_field is not None):
+			CONTAINERS.registrySvrId = string_field
+		string_field = test.findtext('test_svr_id')
+		if (string_field is not None):
+			CONTAINERS.testSvrId = string_field
+
 	else: # ie action == 'Run_PhySim':
 		ldpc.runargs = test.findtext('physim_run_args')
 		
@@ -734,6 +745,22 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 	HTML.SethtmlUEConnected(len(CiTestObj.UEDevices) + len(CiTestObj.CatMDevices))
 	HTML.CreateHtmlTabHeader()
 
+	# On CI bench w/ containers, we need to validate if IP routes are set
+	if EPC.IPAddress == '192.168.18.210':
+		CONTAINERS.CheckAndAddRoute('porcepix', EPC.IPAddress, EPC.UserName, EPC.Password)
+	if CONTAINERS.eNBIPAddress == '192.168.18.194':
+		CONTAINERS.CheckAndAddRoute('asterix', CONTAINERS.eNBIPAddress, CONTAINERS.eNBUserName, CONTAINERS.eNBPassword)
+	if CONTAINERS.eNB1IPAddress == '192.168.18.194':
+		CONTAINERS.CheckAndAddRoute('asterix', CONTAINERS.eNB1IPAddress, CONTAINERS.eNB1UserName, CONTAINERS.eNB1Password)
+	if CONTAINERS.eNBIPAddress == '192.168.18.193':
+		CONTAINERS.CheckAndAddRoute('obelix', CONTAINERS.eNBIPAddress, CONTAINERS.eNBUserName, CONTAINERS.eNBPassword)
+	if CONTAINERS.eNB1IPAddress == '192.168.18.193':
+		CONTAINERS.CheckAndAddRoute('obelix', CONTAINERS.eNB1IPAddress, CONTAINERS.eNB1UserName, CONTAINERS.eNB1Password)
+	if CONTAINERS.eNBIPAddress == '192.168.18.209':
+		CONTAINERS.CheckAndAddRoute('nepes', CONTAINERS.eNBIPAddress, CONTAINERS.eNBUserName, CONTAINERS.eNBPassword)
+	if CONTAINERS.eNB1IPAddress == '192.168.18.209':
+		CONTAINERS.CheckAndAddRoute('nepes', CONTAINERS.eNB1IPAddress, CONTAINERS.eNB1UserName, CONTAINERS.eNB1Password)
+
 	CiTestObj.FailReportCnt = 0
 	RAN.prematureExit=True
 	HTML.startTime=int(round(time.time() * 1000))
@@ -853,6 +880,8 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
 					HTML=ldpc.Run_PhySim(HTML,CONST,id)
 				elif action == 'Build_Image':
 					CONTAINERS.BuildImage(HTML)
+				elif action == 'Copy_Image_to_Test':
+					CONTAINERS.Copy_Image_to_Test_Server(HTML)
 				elif action == 'Deploy_Object':
 					CONTAINERS.DeployObject(HTML, EPC)
 				elif action == 'Undeploy_Object':

ci-scripts/ran.py

@@ -165,16 +165,16 @@ class RANManagement():
 			result = re.search('LAST_BUILD_INFO', mySSH.getBefore())
 			if result is not None:
 				mismatch = False
-				mySSH.command('grep SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
+				mySSH.command('grep --colour=never SRC_COMMIT LAST_BUILD_INFO.txt', '\$', 2)
 				result = re.search(self.ranCommitID, mySSH.getBefore())
 				if result is None:
 					mismatch = True
-				mySSH.command('grep MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
+				mySSH.command('grep --colour=never MERGED_W_TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
 				if (self.ranAllowMerge):
 					result = re.search('YES', mySSH.getBefore())
 					if result is None:
 						mismatch = True
-					mySSH.command('grep TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
+					mySSH.command('grep --colour=never TGT_BRANCH LAST_BUILD_INFO.txt', '\$', 2)
 					if self.ranTargetBranch == '':
 						result = re.search('develop', mySSH.getBefore())
 					else:
@@ -446,7 +446,7 @@ class RANManagement():
 		else:
 			mySSH.command('sed -i -e \'s/FLEXRAN_ENABLED.*;/FLEXRAN_ENABLED        = "no";/\' ' + ci_full_config_file, '\$', 2);
 		self.eNBmbmsEnables[int(self.eNB_instance)] = False
-		mySSH.command('grep enable_enb_m2 ' + ci_full_config_file, '\$', 2);
+		mySSH.command('grep --colour=never enable_enb_m2 ' + ci_full_config_file, '\$', 2);
 		result = re.search('yes', mySSH.getBefore())
 		if result is not None:
 			self.eNBmbmsEnables[int(self.eNB_instance)] = True
@@ -593,8 +593,12 @@ class RANManagement():
 				lPassWord = self.eNBPassword
 			mySSH = SSH.SSHConnection()
 			mySSH.open(lIpAddr, lUserName, lPassWord)
-			mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + self.air_interface[self.eNB_instance] + ' | grep -v grep', '\$', 5)
+			if self.air_interface[self.eNB_instance] == '':
-			result = re.search(self.air_interface[self.eNB_instance], mySSH.getBefore())
+				pattern = 'softmodem'
+			else:
+				pattern = self.air_interface[self.eNB_instance]
+			mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5)
+			result = re.search(pattern, mySSH.getBefore())
 			if result is None:
 				logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
 				status_queue.put(CONST.ENB_PROCESS_FAILED)

ci-scripts/xml_class_list.yml

@@ -37,6 +37,7 @@
   - IdleSleep
   - Perform_X2_Handover
   - Build_Image
+  - Copy_Image_to_Test
   - Deploy_Object
   - Undeploy_Object
   - Cppcheck_Analysis

ci-scripts/xml_files/container_nsa_b200_quectel.xml

@@ -0,0 +1,153 @@
+<!--
+
+ Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The OpenAirInterface Software Alliance licenses this file to You under
+ the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+
+-->
+<testCaseList>
+	<htmlTabRef>TEST-NSA-FR1-TM1-B200</htmlTabRef>
+	<htmlTabName>NSA SanityCheck with QUECTEL</htmlTabName>
+	<htmlTabIcon>tasks</htmlTabIcon>
+	<repeatCount>1</repeatCount>
+	<TestCaseRequestedList>
+ 000001
+ 010002
+ 030000
+ 030101
+ 000001
+ 030102
+ 000001
+ 010000
+ 000001
+ 050000
+ 050001
+ 070000
+ 070001
+ 010002
+ 000001
+ 030202
+ 030201
+	</TestCaseRequestedList>
+	<TestCaseExclusionList></TestCaseExclusionList>
+
+	<testCase id="010000">
+		<class>Initialize_UE</class>
+		<desc>Initialize Quectel</desc>
+		<id>idefix</id>
+		<UE_Trace>yes</UE_Trace>
+	</testCase>
+
+
+	<testCase id="010002">
+		<class>Terminate_UE</class>
+		<desc>Terminate Quectel</desc>
+		<id>idefix</id>
+	</testCase>
+
+	<testCase id="030000">
+		<class>Copy_Image_to_Test</class>
+		<desc>Copy gNB image to test server</desc>
+		<image_name>oai-gnb</image_name>
+		<registry_svr_id>0</registry_svr_id>
+		<test_svr_id>1</test_svr_id>
+	</testCase>
+
+	<testCase id="030101">
+		<class>Deploy_Object</class>
+		<desc>Deploy eNB (FDD/Band7/5MHz/B200) in a container</desc>
+		<yaml_path>ci-scripts/yaml_files/nsa_b200_enb</yaml_path>
+		<eNB_instance>0</eNB_instance>
+		<eNB_serverId>0</eNB_serverId>
+	</testCase>
+
+	<testCase id="030102">
+		<class>Deploy_Object</class>
+		<desc>Deploy gNB (TDD/Band78/40MHz/B200) in a container</desc>
+		<yaml_path>ci-scripts/yaml_files/nsa_b200_gnb</yaml_path>
+		<eNB_instance>1</eNB_instance>
+		<eNB_serverId>1</eNB_serverId>
+	</testCase>
+
+	<testCase id="000001">
+		<class>IdleSleep</class>
+		<desc>Sleep</desc>
+		<idle_sleep_time_in_sec>5</idle_sleep_time_in_sec>
+	</testCase>
+
+	<testCase id="000002">
+		<class>IdleSleep</class>
+		<desc>Sleep</desc>
+		<idle_sleep_time_in_sec>20</idle_sleep_time_in_sec>
+	</testCase>
+
+
+	<testCase id="050000">
+		<class>Ping</class>
+		<desc>Ping: 20pings in 20sec</desc>
+		<id>idefix</id>
+		<ping_args>-c 20</ping_args>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+	</testCase>
+
+	<testCase id="050001">
+		<class>Ping</class>
+		<desc>Ping: 100pings in 20sec</desc>
+		<id>idefix</id>
+		<ping_args>-c 100 -i 0.2</ping_args>
+		<ping_packetloss_threshold>1</ping_packetloss_threshold>
+	</testCase>
+
+	<testCase id="070000">
+		<class>Iperf</class>
+		<desc>iperf (DL/20Mbps/UDP)(60 sec)(single-ue profile)</desc>
+		<iperf_args>-u -b 20M -t 60</iperf_args>
+		<direction>DL</direction>
+		<id>idefix</id>
+		<iperf_packetloss_threshold>3</iperf_packetloss_threshold>
+		<iperf_profile>single-ue</iperf_profile>
+	</testCase>
+
+	<testCase id="070001">
+		<class>Iperf</class>
+		<desc>iperf (UL/2Mbps/UDP)(60 sec)(single-ue profile)</desc>
+		<iperf_args>-u -b 2M -t 60</iperf_args>
+		<direction>UL</direction>
+		<id>idefix</id>
+		<iperf_packetloss_threshold>1</iperf_packetloss_threshold>
+		<iperf_profile>single-ue</iperf_profile>
+	</testCase>
+
+    <testCase id="030201">
+        <class>Undeploy_Object</class>
+        <desc>Undeploy eNB</desc>
+		<yaml_path>ci-scripts/yaml_files/nsa_b200_enb</yaml_path>
+        <eNB_instance>0</eNB_instance>
+        <eNB_serverId>0</eNB_serverId>
+    </testCase>
+
+    <testCase id="030202">
+        <class>Undeploy_Object</class>
+        <desc>Undeploy eNB</desc>
+		<yaml_path>ci-scripts/yaml_files/nsa_b200_gnb</yaml_path>
+        <eNB_instance>1</eNB_instance>
+        <eNB_serverId>1</eNB_serverId>
+    </testCase>
+
+</testCaseList>
+

ci-scripts/yaml_files/fr1_epc_20897/docker-compose.yml

@@ -111,6 +111,9 @@ services:
             TAC_LB_SGW_TEST_0: '03'
             TAC_HB_SGW_TEST_0: '00'
             SGW_IPV4_ADDRESS_FOR_S11_TEST_0: 0.0.0.0
+        volumes:
+            - ./mme.conf:/openair-mme/etc/mme.conf
+            - ./entrypoint.sh:/openair-mme/bin/entrypoint.sh
         healthcheck:
             test: /bin/bash -c "pgrep oai_mme"
             interval: 10s

ci-scripts/yaml_files/fr1_epc_20897/entrypoint.sh

@@ -0,0 +1,73 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# First see if all interfaces are up
+ifconfig
+
+# S10 might be on loopback --> needs bring-up
+if [[ "$MME_INTERFACE_NAME_FOR_S10" == *"lo:"* ]]
+then
+    ifconfig ${MME_INTERFACE_NAME_FOR_S10} ${MME_IPV4_ADDRESS_FOR_S10} up
+fi
+
+LIST_OF_NETWORKS=`ifconfig -s | egrep -v "^Iface|^lo" | cut -d' ' -f1`
+
+for if_name in $LIST_OF_NETWORKS
+do
+    IF_IP_ADDR=`ifconfig $if_name | grep inet | sed -e "s# *inet#inet#" | cut -d' ' -f2`
+    if [[ "${IF_IP_ADDR}" == "${MME_IPV4_ADDRESS_FOR_S1_MME}" ]]; then
+        echo "S1C is on $if_name"
+	MME_INTERFACE_NAME_FOR_S1_MME=$if_name
+    fi
+    if [[ "${IF_IP_ADDR}" == "${MME_IPV4_ADDRESS_FOR_S10}" ]]; then
+        echo "S10 is on $if_name"
+	MME_INTERFACE_NAME_FOR_S10=$if_name
+    fi
+    if [[ "${IF_IP_ADDR}" == "${MME_IPV4_ADDRESS_FOR_S11}" ]]; then
+        echo "S11 is on $if_name"
+	MME_INTERFACE_NAME_FOR_S11=$if_name
+    fi
+done
+
+CONFIG_DIR="/openair-mme/etc"
+
+for c in ${CONFIG_DIR}/mme_fd.conf; do
+    #echo "entrypoint.sh process config file $c"
+    sed -i -e "s#@TAC-LB#@TAC_LB#" -e "s#TAC-HB_#TAC_HB_#" ${c}
+    # grep variable names (format: ${VAR}) from template to be rendered
+    VARS=$(grep -oP '@[a-zA-Z0-9_]+@' ${c} | sort | uniq | xargs)
+    #echo "entrypoint.sh process vars $VARS"
+
+    # create sed expressions for substituting each occurrence of ${VAR}
+    # with the value of the environment variable "VAR"
+    EXPRESSIONS=""
+    for v in ${VARS}; do
+        #echo "var is $v"
+        NEW_VAR=`echo $v | sed -e "s#@##g"`
+        #echo "NEW_VAR is $NEW_VAR"
+        if [[ "${!NEW_VAR}x" == "x" ]]; then
+            echo "Error: Environment variable '${NEW_VAR}' is not set." \
+                "Config file '$(basename $c)' requires all of $VARS."
+            exit 1
+        fi
+        # Some fields require CIDR format
+        if [[ "${NEW_VAR}" == "MME_IPV4_ADDRESS_FOR_S1_MME" ]] || \
+           [[ "${NEW_VAR}" == "MME_IPV4_ADDRESS_FOR_S11" ]] || \
+           [[ "${NEW_VAR}" == "MME_IPV4_ADDRESS_FOR_S10" ]]; then
+            EXPRESSIONS="${EXPRESSIONS};s|${v}|${!NEW_VAR}/24|g"
+        else
+            EXPRESSIONS="${EXPRESSIONS};s|${v}|${!NEW_VAR}|g"
+        fi
+    done
+    EXPRESSIONS="${EXPRESSIONS#';'}"
+
+    # render template and inline replace config file
+    sed -i "${EXPRESSIONS}" ${c}
+done
+
+pushd /openair-mme/scripts
+./check_mme_s6a_certificate ${PREFIX} ${MME_FQDN}
+popd
+
+exec "$@"

ci-scripts/yaml_files/fr1_epc_20897/mme.conf

@@ -0,0 +1,133 @@
+MME : 
+{
+    REALM                                     = "openairinterface.org";                      # YOUR REALM HERE
+    INSTANCE                                  = 1;                     # 0 is the default
+    PID_DIRECTORY                             = "/var/run";              # /var/run is the default
+    MAX_S1_ENB                                = 64;
+    MAX_UE                                    = 4096;
+    RELATIVE_CAPACITY                         = 10;
+    EMERGENCY_ATTACH_SUPPORTED                     = "no";
+    UNAUTHENTICATED_IMSI_SUPPORTED                 = "no";
+    DUMMY_HANDOVER_FORWARDING_ENABLED              = "yes";
+    EPS_NETWORK_FEATURE_SUPPORT_IMS_VOICE_OVER_PS_SESSION_IN_S1      = "no";    # DO NOT CHANGE
+    EPS_NETWORK_FEATURE_SUPPORT_EMERGENCY_BEARER_SERVICES_IN_S1_MODE = "no";    # DO NOT CHANGE
+    EPS_NETWORK_FEATURE_SUPPORT_LOCATION_SERVICES_VIA_EPC            = "no";    # DO NOT CHANGE
+    EPS_NETWORK_FEATURE_SUPPORT_EXTENDED_SERVICE_REQUEST             = "no";    # DO NOT CHANGE
+	
+    # Display statistics about whole system (expressed in seconds)
+    MME_STATISTIC_TIMER                       = 10;
+    MME_MOBILITY_COMPLETION_TIMER             = 2; # Amount of time in seconds the source MME waits to release resources after HANDOVER/TAU is complete (with or without.
+    MME_S10_HANDOVER_COMPLETION_TIMER         = 2; # Amount of time in soconds the target MME waits to check if a handover/tau process has completed successfully.
+   
+    IP_CAPABILITY = "IPV4V6";
+    INTERTASK_INTERFACE :
+    {
+        ITTI_QUEUE_SIZE            = 2000000;
+    };
+
+    S6A :
+    {
+        S6A_CONF                   = "/openair-mme/etc/mme_fd.conf";
+        HSS_HOSTNAME               = "hss.openairinterface.org";                              # THE HSS FQDN ex: hss.epc.mnc001.mcc001.3gppnetwork.org
+        HSS_REALM                  = "openairinterface.org";                             # THE HSS REALM ex: epc.mnc001.mcc001.3gppnetwork.org
+    };
+
+    SCTP :
+    {
+        SCTP_INSTREAMS  = 8;
+        SCTP_OUTSTREAMS = 8;
+    };
+
+    S1AP : 
+    {
+        S1AP_OUTCOME_TIMER = 10;
+    };
+
+    GUMMEI_LIST = ( 
+         {MCC="208" ; MNC="97"; MME_GID="32768" ; MME_CODE="3"; }                   # YOUR GUMMEI CONFIG HERE
+    );
+
+    TAI_LIST = ( 
+         {MCC="208" ; MNC="97";  TAC = "1"; },                       # YOUR TAI CONFIG HERE
+         {MCC="208" ; MNC="97";  TAC = "2"; },                       # YOUR TAI CONFIG HERE
+         {MCC="208" ; MNC="97";  TAC = "3"; }                        # YOUR TAI CONFIG HERE
+    );
+
+    NAS :
+    {
+        ORDERED_SUPPORTED_INTEGRITY_ALGORITHM_LIST = [ "EIA2" , "EIA1" , "EIA0" ];
+        ORDERED_SUPPORTED_CIPHERING_ALGORITHM_LIST = [ "EEA0" , "EEA1" , "EEA2" ];
+        T3402                                 =  12
+        T3412                                 =  0
+        T3422                                 =  6
+        T3450                                 =  6
+        T3460                                 =  6
+        T3470                                 =  6
+        T3485                                 =  3
+        T3486                                 =  3
+        T3489                                 =  4
+        T3495                                 =  3
+        NAS_FORCE_TAU			      =  0
+        STRICT_FILLER_BITS_CHECK              = "yes";
+    };
+
+    NETWORK_INTERFACES : 
+    {
+        # MME binded interface for S1-C or S1-MME  communication (S1AP), can be ethernet interface, virtual ethernet interface, we don't advise wireless interfaces
+        MME_INTERFACE_NAME_FOR_S1_MME   = "eth0";    # YOUR NETWORK CONFIG HERE
+        MME_IPV4_ADDRESS_FOR_S1_MME     = "192.168.61.195/24";      # CIDR, YOUR NETWORK CONFIG HERE
+#        MME_IPV6_ADDRESS_FOR_S1_MME           = "fd00::191/118";               # YOUR NETWORK CONFIG HERE
+        # MME binded interface for S11 communication (GTPV2-C)
+        MME_INTERFACE_NAME_FOR_S11      = "eth0";       # YOUR NETWORK CONFIG HERE
+        MME_IPV4_ADDRESS_FOR_S11        = "192.168.61.195/24";         # CIDR, YOUR NETWORK CONFIG HERE
+#        MME_IPV6_ADDRESS_FOR_S11          = "fd00:0:0:4::191/64";
+        MME_PORT_FOR_S11                  = 2123;                            # YOUR NETWORK CONFIG HERE
+
+
+        #S10 Interface
+        MME_INTERFACE_NAME_FOR_S10      = "lo";       # YOUR NETWORK CONFIG HERE
+        MME_IPV4_ADDRESS_FOR_S10        = "127.0.0.10/24";         # CIDR, YOUR NETWORK CONFIG HERE
+#        MME_IPV6_ADDRESS_FOR_S10          = "fd00:0:0:4::191/64";
+        MME_PORT_FOR_S10                = 2123;                                 # YOUR NETWORK CONFIG HERE
+    };
+
+    LOGGING :
+    {
+        # OUTPUT choice in { "CONSOLE", `path to file`", "`IPv4@`:`TCP port num`"}
+        # `path to file` must start with '.' or '/'
+        # if TCP stream choice, then you can easily dump the traffic on the remote or local host: nc -l `TCP port num` > received.txt
+        OUTPUT            = "CONSOLE";
+        THREAD_SAFE       = "no";                                               # THREAD_SAFE choice in { "yes", "no" }, safe to let 'no'
+        COLOR             = "yes";                                              # COLOR choice in { "yes", "no" } means use of ANSI styling codes or no
+        # Log level choice in { "EMERGENCY", "ALERT", "CRITICAL", "ERROR", "WARNING", "NOTICE", "INFO", "DEBUG", "TRACE"}
+        SCTP_LOG_LEVEL    = "TRACE";
+        S10_LOG_LEVEL     = "TRACE";
+        S11_LOG_LEVEL     = "TRACE";
+        # NEW LOGS FOR MCE	
+        SM_LOG_LEVEL      = "TRACE";
+        MCE_APP_LOG_LEVEL = "TRACE";
+        M2AP_LOG_LEVEL    = "TRACE";
+        GTPV2C_LOG_LEVEL  = "TRACE";
+        UDP_LOG_LEVEL     = "DEBUG";
+        S1AP_LOG_LEVEL    = "DEBUG";
+        NAS_LOG_LEVEL     = "TRACE";
+        MME_APP_LOG_LEVEL = "TRACE";
+        S6A_LOG_LEVEL     = "TRACE";
+        UTIL_LOG_LEVEL    = "ERROR";
+        MSC_LOG_LEVEL     = "ERROR";
+        ITTI_LOG_LEVEL    = "ERROR";
+        ASN1_VERBOSITY    = "annoying";
+    };
+
+
+#    WRR_LIST_SELECTION = (
+#        {ID="tac-lb03.tac-hb00.tac.epc.mnc001.mcc001.3gppnetwork.org" ;        SGW_IP_ADDRESS_FOR_S11="192.168.61.196";},
+#        {ID="tac-lb01.tac-hb00.tac.epc.mnc097.mcc208.3gppnetwork.org" ; SGW_IP_ADDRESS_FOR_S11="192.168.61.196";},
+#        {ID="tac-lb02.tac-hb00.tac.epc.mnc097.mcc208.3gppnetwork.org" ; MME_IP_ADDRESS_FOR_S10="0.0.0.0";},
+#        {ID="tac-lb03.tac-hb00.tac.epc.mnc097.mcc208.3gppnetwork.org" ; MME_IP_ADDRESS_FOR_S10="0.0.0.0";}
+#    );
+    WRR_LIST_SELECTION = (
+        {ID="tac-lb01.tac-hb00.tac.epc.mnc097.mcc208.3gppnetwork.org" ; SGW_IP_ADDRESS_FOR_S11="192.168.61.196";}
+    );
+};
+

ci-scripts/yaml_files/nsa_b200_enb/docker-compose.yml

@@ -0,0 +1,53 @@
+version: '3.8'
+
+services:
+    enb_mono_fdd:
+        image: oai-enb:latest
+        privileged: true
+        container_name: nsa-b200-enb
+        environment:
+            USE_FDD_MONO: 'yes'
+            USE_B2XX: 'yes'
+            ENB_NAME: eNB-in-docker
+            MCC: '222'
+            MNC: '01'
+            MNC_LENGTH: 2
+            TAC: 1
+            UTRA_BAND_ID: 7
+            DL_FREQUENCY_IN_MHZ: 2680
+            UL_FREQUENCY_OFFSET_IN_MHZ: 120
+            NID_CELL: 0
+            NB_PRB: 25
+            ENABLE_MEASUREMENT_REPORTS: 'yes'
+            ENABLE_X2: 'yes'
+            MME_S1C_IP_ADDRESS: 192.168.18.210
+            ENB_S1C_IF_NAME: eth0
+            ENB_S1C_IP_ADDRESS: 192.168.68.130
+            ENB_S1U_IF_NAME: eth0
+            ENB_S1U_IP_ADDRESS: 192.168.68.130
+            ENB_X2_IP_ADDRESS: 192.168.68.130
+            RRC_INACTIVITY_THRESHOLD: 0
+            FLEXRAN_ENABLED: 'no'
+            FLEXRAN_INTERFACE_NAME: eth0
+            FLEXRAN_IPV4_ADDRESS: 192.168.18.210
+            THREAD_PARALLEL_CONFIG: PARALLEL_RU_L1_TRX_SPLIT
+        volumes:
+            - /dev:/dev
+        networks:
+            public_net:
+                ipv4_address: 192.168.68.130
+        healthcheck:
+            # pgrep does NOT work
+            test: /bin/bash -c "ps aux | grep -v grep | grep -c softmodem"
+            interval: 10s
+            timeout: 5s
+            retries: 5
+
+networks:
+    public_net:
+        name: nsa-b200-enb-net
+        ipam:
+            config:
+                - subnet: 192.168.68.128/26
+        driver_opts:
+            com.docker.network.bridge.name: "nsa-enb-net"

ci-scripts/yaml_files/nsa_b200_gnb/docker-compose.yml

@@ -0,0 +1,50 @@
+version: '3.8'
+
+services:
+    gnb_mono_tdd:
+        image: oai-gnb:latest
+        privileged: true
+        container_name: nsa-b200-gnb
+        environment:
+            USE_NSA_TDD_MONO: 'yes'
+            USE_B2XX: 'yes'
+            GNB_NAME: gNB-in-docker
+            MCC: '222'
+            MNC: '01'
+            MNC_LENGTH: 2
+            TAC: 1
+            ENABLE_X2: 'yes'
+            ENB_X2_IP_ADDRESS: 192.168.68.130
+            MME_S1C_IP_ADDRESS: 192.168.18.210
+            GNB_S1C_IF_NAME: eth0
+            GNB_S1C_IP_ADDRESS: 192.168.68.194
+            GNB_S1U_IF_NAME: eth0
+            GNB_S1U_IP_ADDRESS: 192.168.68.194
+            GNB_X2_IP_ADDRESS: 192.168.68.194
+            RRC_INACTIVITY_THRESHOLD: 0
+            FLEXRAN_ENABLED: 'no'
+            FLEXRAN_INTERFACE_NAME: eth0
+            FLEXRAN_IPV4_ADDRESS: 192.168.18.210
+            THREAD_PARALLEL_CONFIG: PARALLEL_RU_L1_TRX_SPLIT
+            USE_ADDITIONAL_OPTIONS: '-E -q'
+        volumes:
+            - /dev:/dev
+        networks:
+            public_net:
+                ipv4_address: 192.168.68.194
+        #entrypoint: /bin/bash -c "sleep infinity"
+        healthcheck:
+            # pgrep does NOT work
+            test: /bin/bash -c "ps aux | grep -v grep | grep -c softmodem"
+            interval: 10s
+            timeout: 5s
+            retries: 5
+
+networks:
+    public_net:
+        name: nsa-b200-gnb-net
+        ipam:
+            config:
+                - subnet: 192.168.68.192/26
+        driver_opts:
+            com.docker.network.bridge.name: "nsa-gnb-net"