From edebdf86983f4e0e7312f94a406ec9e4e2bff3f5 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Tue, 19 Jun 2018 16:15:53 +0200 Subject: [PATCH 1/8] CI: Introducing parallel builds on Virtual Machines Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-gitlab | 32 +++-- ci-scripts/buildOnVM.sh | 233 ++++++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+), 8 deletions(-) create mode 100755 ci-scripts/buildOnVM.sh diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 442e5d3791..2205e46fe3 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -8,8 +8,7 @@ pipeline { disableConcurrentBuilds() timestamps() gitLabConnection('OAI GitLab') - //gitlabBuilds(builds: ["Build", "Test"]) - gitlabBuilds(builds: ["Local Build"]) + gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim"]) } stages { @@ -22,6 +21,7 @@ pipeline { // GitLab-Jenkins plugin integration is lacking to perform the merge by itself // Doing it manually --> it may have merge conflicts sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}" + sh "zip -r localZip.zip ." // Running astyle options on the list of modified files by the merge request // For the moment, there is no fail criteria. Just a notification of number of files that do not follow @@ -38,6 +38,7 @@ pipeline { echo "Git Branch is ${GIT_BRANCH}" echo "Git Commit is ${GIT_COMMIT}" + sh "zip -r localZip.zip ." // Running astyle options on all C/H files in the repository // For the moment, there is no fail criteria. Just a notification of number of files that do not follow sh "./ci-scripts/checkCodingFormattingRules.sh" @@ -45,17 +46,32 @@ pipeline { } } } - stage ("Local Build") { - steps { - gitlabCommitStatus(name: "Local Build") { - sh "./ci-scripts/buildLocally.sh --workspace $WORKSPACE" + + stage ("Variant Builds") { + parallel { + stage ("Build eNb-USRP") { + steps { + gitlabCommitStatus(name: "Build eNb-USRP") { + sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant enb-usrp" + } + } + } + stage ("Build basic simulator") { + steps { + gitlabCommitStatus(name: "Build basic-sim") { + sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant basic-sim" + } + } } } post { always { script { - if(fileExists('archives/local_build_logs.zip')) { - archiveArtifacts artifacts: 'archives/local_build_logs.zip' + dir ('archives') { + sh "zip -r vm_build_logs.zip basic_sim enb_usrp" + } + if(fileExists('archives/vm_build_logs.zip')) { + archiveArtifacts artifacts: 'archives/vm_build_logs.zip' } if ("MERGE".equals(env.gitlabActionType)) { sh "./ci-scripts/reportBuildLocally.sh --git-url ${GIT_URL} --job-name ${JOB_NAME} --build-id ${BUILD_ID} --trigger merge-request --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}" diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh new file mode 100755 index 0000000000..082ff11f81 --- /dev/null +++ b/ci-scripts/buildOnVM.sh @@ -0,0 +1,233 @@ +#!/bin/bash + +function usage { + echo "OAI VM Build Check script" + echo " Original Author: Raphael Defosseux" + echo " Requirements:" + echo " -- uvtool uvtool-libvirt apt-cacher" + echo " -- xenial image already synced" + echo " Default:" + echo " -- eNB with USRP" + echo "" + echo "Usage:" + echo "------" + echo " buildOnVM.sh [OPTIONS]" + echo "" + echo "Options:" + echo "--------" + echo " --workspace #### OR -ws ####" + echo " Specify the workspace" + echo "" + echo " --variant enb-usrp OR -v1" + echo " --variant basic-sim OR -v2" + echo " Specify the variant to build" + echo "" + echo " --help OR -h" + echo " Print this help message." + echo "" +} + +function variant_usage { + echo "OAI VM Build Check script" + echo " Original Author: Raphael Defosseux" + echo "" + echo " --variant enb-usrp OR -v1" + echo " --variant basic-sim OR -v2" + echo "" +} + +if [ $# -ne 2 ] && [ $# -ne 1 ] && [ $# -ne 4 ] && [ $# -ne 3 ] +then + echo "Syntax Error: not the correct number of arguments" + echo "" + usage + exit 1 +fi + +VM_NAME=ci-enb-usrp +ARCHIVES_LOC=enb_usrp +LOG_PATTERN=.Rel14.txt +NB_PATTERN_FILES=4 +BUILD_OPTIONS="--eNB -w USRP" +BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -h|--help) + shift + usage + exit 0 + ;; + -ws|--workspace) + JENKINS_WKSP="$2" + shift + shift + ;; + -v1) + VM_NAME=ci-enb-usrp + ARCHIVES_LOC=enb_usrp + LOG_PATTERN=.Rel14.txt + NB_PATTERN_FILES=4 + BUILD_OPTIONS="--eNB -w USRP" + BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" + shift + ;; + -v2) + VM_NAME=ci-basic-sim + ARCHIVES_LOC=basic_sim + LOG_PATTERN=basic_simulator + NB_PATTERN_FILES=2 + BUILD_OPTIONS="--basic-simulator" + BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" + shift + ;; + --variant) + variant="$2" + case $variant in + enb-usrp) + VM_NAME=ci-enb-usrp + ARCHIVES_LOC=enb_usrp + LOG_PATTERN=.Rel14.txt + NB_PATTERN_FILES=4 + BUILD_OPTIONS="--eNB -w USRP" + BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" + ;; + basic-sim) + VM_NAME=ci-basic-sim + ARCHIVES_LOC=basic_sim + LOG_PATTERN=basic_simulator + NB_PATTERN_FILES=2 + BUILD_OPTIONS="--basic-simulator" + BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" + ;; + *) + echo "" + echo "Syntax Error: Invalid Variant option -> $variant" + echo "" + variant_usage + exit 1 + esac + shift + shift + ;; + *) + echo "Syntax Error: unknown option: $key" + echo "" + usage + exit 1 +esac +done + +if [ ! -f $JENKINS_WKSP/localZip.zip ] +then + echo "Missing localZip.zip file!" + exit 1 +fi +if [ ! -f /etc/apt/apt.conf.d/01proxy ] +then + echo "Missing /etc/apt/apt.conf.d/01proxy file!" + echo "Is apt-cacher installed and configured?" + exit 1 +fi + +VM_CMDS=${VM_NAME}_cmds.txt +ARCHIVES_LOC=${JENKINS_WKSP}/archives/${ARCHIVES_LOC} + +echo "VM_NAME = $VM_NAME" +echo "VM_CMD_FILE = $VM_CMDS" +echo "JENKINS_WKSP = $JENKINS_WKSP" +echo "ARCHIVES_LOC = $ARCHIVES_LOC" +echo "BUILD_OPTIONS = $BUILD_OPTIONS" +echo "BUILD_EXTRA_OPTIONS = $BUILD_EXTRA_OPTIONS" + +echo "############################################################" +echo "Creating VM ($VM_NAME) on Ubuntu Cloud Image base" +echo "############################################################" +uvt-kvm create $VM_NAME release=xenial --memory 2048 --cpu 2 +echo "Waiting for VM to be started" +uvt-kvm wait $VM_NAME --insecure + +VM_IP_ADDR=`uvt-kvm ip $VM_NAME` +echo "$VM_NAME has for IP addr = $VM_IP_ADDR" + +echo "############################################################" +echo "Copying GIT repo into VM ($VM_NAME)" +echo "############################################################" +scp -o StrictHostKeyChecking=no localZip.zip ubuntu@$VM_IP_ADDR:/home/ubuntu +scp -o StrictHostKeyChecking=no /etc/apt/apt.conf.d/01proxy ubuntu@$VM_IP_ADDR:/home/ubuntu + +echo "############################################################" +echo "Running install and build script on VM ($VM_NAME)" +echo "############################################################" +echo "sudo cp 01proxy /etc/apt/apt.conf.d/" > $VM_CMDS +echo "echo \"sudo apt-get --yes --quiet install zip subversion libboost-dev \"" >> $VM_CMDS +echo "sudo apt-get --yes install zip subversion libboost-dev > zip-install.txt 2>&1" >> $VM_CMDS +echo "mkdir tmp" >> $VM_CMDS +echo "cd tmp" >> $VM_CMDS +echo "echo \"unzip -qq ../localZip.zip\"" >> $VM_CMDS +echo "unzip -qq ../localZip.zip" >> $VM_CMDS +echo "echo \"source oaienv\"" >> $VM_CMDS +echo "source oaienv" >> $VM_CMDS +echo "cd cmake_targets/" >> $VM_CMDS +echo "mkdir log" >> $VM_CMDS +echo "cp /home/ubuntu/zip-install.txt log" >> $VM_CMDS +echo "echo \"./build_oai -I $BUILD_OPTIONS \"" >> $VM_CMDS +echo "./build_oai -I $BUILD_OPTIONS $BUILD_EXTRA_OPTIONS > log/install-build.txt 2>&1" >> $VM_CMDS + +ssh -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR < $VM_CMDS + +echo "############################################################" +echo "Creating a tmp folder to store results and artifacts" +echo "############################################################" +if [ ! -d $JENKINS_WKSP/archives ] +then + mkdir $JENKINS_WKSP/archives +fi + +if [ ! -d $ARCHIVES_LOC ] +then + mkdir $ARCHIVES_LOC +fi + +scp -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/*.txt $ARCHIVES_LOC + +echo "############################################################" +echo "Destroying VM" +echo "############################################################" +uvt-kvm destroy $VM_NAME +ssh-keygen -R $VM_IP_ADDR +rm -f $VM_CMDS + +echo "############################################################" +echo "Checking build status" +echo "############################################################" + +LOG_FILES=`ls $ARCHIVES_LOC/*.txt` +STATUS=0 +NB_FOUND_FILES=0 + +for FULLFILE in $LOG_FILES +do + if [[ $FULLFILE == *"$LOG_PATTERN"* ]] + then + filename=$(basename -- "$FULLFILE") + if [ "$LOG_PATTERN" == ".Rel14.txt" ] + then + PASS_PATTERN=`echo $filename | sed -e "s#$LOG_PATTERN##"` + fi + if [ "$LOG_PATTERN" == "basic_simulator" ] + then + PASS_PATTERN="Built target lte-" + fi + LOCAL_STAT=`egrep -c "Built target $PASS_PATTERN" $FULLFILE` + if [ $LOCAL_STAT -eq 0 ]; then STATUS=-1; fi + NB_FOUND_FILES=$((NB_FOUND_FILES + 1)) + fi +done + +if [ $NB_PATTERN_FILES -ne $NB_FOUND_FILES ]; then STATUS=-1; fi + +exit $STATUS From 5df3d10dd6c6c4d8092369170932c420aceb25ed Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Wed, 20 Jun 2018 10:47:28 +0200 Subject: [PATCH 2/8] CI: Fixes on the VM-based Jenkins scripts -- quieter zip of the repository -- remove clock skew and flexran.proto in the warning count -- increase to 4 CPU on VM -- created a new template to use host cpu properties (resolve AXV2 compilation issue) Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-gitlab | 4 ++-- ci-scripts/buildOnVM.sh | 2 +- ci-scripts/reportBuildLocally.sh | 14 +++++++------- ci-scripts/template-host.xml | 27 +++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 ci-scripts/template-host.xml diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 2205e46fe3..0c3fb6bcb9 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -21,7 +21,7 @@ pipeline { // GitLab-Jenkins plugin integration is lacking to perform the merge by itself // Doing it manually --> it may have merge conflicts sh "./ci-scripts/doGitLabMerge.sh --src-branch ${env.gitlabSourceBranch} --src-commit ${env.gitlabMergeRequestLastCommit} --target-branch ${env.gitlabTargetBranch} --target-commit ${GIT_COMMIT}" - sh "zip -r localZip.zip ." + sh "zip -r -qq localZip.zip ." // Running astyle options on the list of modified files by the merge request // For the moment, there is no fail criteria. Just a notification of number of files that do not follow @@ -38,7 +38,7 @@ pipeline { echo "Git Branch is ${GIT_BRANCH}" echo "Git Commit is ${GIT_COMMIT}" - sh "zip -r localZip.zip ." + sh "zip -r -qq localZip.zip ." // Running astyle options on all C/H files in the repository // For the moment, there is no fail criteria. Just a notification of number of files that do not follow sh "./ci-scripts/checkCodingFormattingRules.sh" diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh index 082ff11f81..ac8b274db9 100755 --- a/ci-scripts/buildOnVM.sh +++ b/ci-scripts/buildOnVM.sh @@ -146,7 +146,7 @@ echo "BUILD_EXTRA_OPTIONS = $BUILD_EXTRA_OPTIONS" echo "############################################################" echo "Creating VM ($VM_NAME) on Ubuntu Cloud Image base" echo "############################################################" -uvt-kvm create $VM_NAME release=xenial --memory 2048 --cpu 2 +uvt-kvm create $VM_NAME release=xenial --memory 2048 --cpu 4 --unsafe-caching --template ci-scripts/template-host.xml echo "Waiting for VM to be started" uvt-kvm wait $VM_NAME --insecure diff --git a/ci-scripts/reportBuildLocally.sh b/ci-scripts/reportBuildLocally.sh index d3dd1f52a0..55e1754593 100755 --- a/ci-scripts/reportBuildLocally.sh +++ b/ci-scripts/reportBuildLocally.sh @@ -315,7 +315,7 @@ then else echo " $STATUS" >> ./build_results.html fi - STATUS=`egrep -c "warning:" ./archives/enb_usrp/lte-softmodem.Rel14.txt` + STATUS=`egrep "warning:" ./archives/enb_usrp/lte-softmodem.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` if [ $STATUS -eq 0 ] then echo " $STATUS" >> ./build_results.html @@ -346,7 +346,7 @@ then else echo " $STATUS" >> ./build_results.html fi - STATUS=`egrep -c "warning:" ./archives/enb_usrp/coding.Rel14.txt` + STATUS=`egrep "warning:" ./archives/enb_usrp/coding.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` if [ $STATUS -eq 0 ] then echo " $STATUS" >> ./build_results.html @@ -377,7 +377,7 @@ then else echo " $STATUS" >> ./build_results.html fi - STATUS=`egrep -c "warning:" ./archives/enb_usrp/oai_usrpdevif.Rel14.txt` + STATUS=`egrep "warning:" ./archives/enb_usrp/oai_usrpdevif.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` if [ $STATUS -eq 0 ] then echo " $STATUS" >> ./build_results.html @@ -408,7 +408,7 @@ then else echo " $STATUS" >> ./build_results.html fi - STATUS=`egrep -c "warning:" ./archives/enb_usrp/params_libconfig.Rel14.txt` + STATUS=`egrep "warning:" ./archives/enb_usrp/params_libconfig.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` if [ $STATUS -eq 0 ] then echo " $STATUS" >> ./build_results.html @@ -452,7 +452,7 @@ then else echo " $STATUS" >> ./build_results.html fi - STATUS=`egrep -c "warning:" ./archives/basic_sim/basic_simulator_enb.txt` + STATUS=`egrep "warning:" ./archives/basic_sim/basic_simulator_enb.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` if [ $STATUS -eq 0 ] then echo " $STATUS" >> ./build_results.html @@ -483,7 +483,7 @@ then else echo " $STATUS" >> ./build_results.html fi - STATUS=`egrep -c "warning:" ./archives/basic_sim/basic_simulator_ue.txt` + STATUS=`egrep "warning:" ./archives/basic_sim/basic_simulator_ue.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` if [ $STATUS -eq 0 ] then echo " $STATUS" >> ./build_results.html @@ -514,7 +514,7 @@ then else echo " $STATUS" >> ./build_results.html fi - STATUS=`egrep -c "warning:" ./archives/basic_sim/conf2uedata.Rel14.txt` + STATUS=`egrep "warning:" ./archives/basic_sim/conf2uedata.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` if [ $STATUS -eq 0 ] then echo " $STATUS" >> ./build_results.html diff --git a/ci-scripts/template-host.xml b/ci-scripts/template-host.xml new file mode 100644 index 0000000000..efd9912c7a --- /dev/null +++ b/ci-scripts/template-host.xml @@ -0,0 +1,27 @@ + + + hvm + + + + + + + + + + + + + + + + + + + + + + + From 29ffb1169355d4c40bc92cca192c4f44331ffd5c Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Wed, 20 Jun 2018 11:13:22 +0200 Subject: [PATCH 3/8] CI: fix on the pass pattern for basic simulator Signed-off-by: Raphael Defosseux --- ci-scripts/buildOnVM.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh index ac8b274db9..dbb946ac1d 100755 --- a/ci-scripts/buildOnVM.sh +++ b/ci-scripts/buildOnVM.sh @@ -220,7 +220,7 @@ do fi if [ "$LOG_PATTERN" == "basic_simulator" ] then - PASS_PATTERN="Built target lte-" + PASS_PATTERN="lte-" fi LOCAL_STAT=`egrep -c "Built target $PASS_PATTERN" $FULLFILE` if [ $LOCAL_STAT -eq 0 ]; then STATUS=-1; fi From 20db2d2f77d28328115de997c173719aaa306135 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Wed, 20 Jun 2018 17:25:09 +0200 Subject: [PATCH 4/8] CI: Improvements: -- Added VM-build support for physical simulators -- More details in the reported HTML files Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-gitlab | 9 +- ci-scripts/buildOnVM.sh | 25 ++ ci-scripts/reportBuildLocally.sh | 376 ++++++++++++------------------- 3 files changed, 172 insertions(+), 238 deletions(-) diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 0c3fb6bcb9..2021e8347f 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -8,7 +8,7 @@ pipeline { disableConcurrentBuilds() timestamps() gitLabConnection('OAI GitLab') - gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim"]) + gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim", "Build phy-sim"]) } stages { @@ -63,6 +63,13 @@ pipeline { } } } + stage ("Build physical simulators") { + steps { + gitlabCommitStatus(name: "Build phy-sim") { + sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant phy-sim" + } + } + } } post { always { diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh index dbb946ac1d..d34fb9a6c4 100755 --- a/ci-scripts/buildOnVM.sh +++ b/ci-scripts/buildOnVM.sh @@ -20,6 +20,7 @@ function usage { echo "" echo " --variant enb-usrp OR -v1" echo " --variant basic-sim OR -v2" + echo " --variant phy-sim OR -v3" echo " Specify the variant to build" echo "" echo " --help OR -h" @@ -33,6 +34,7 @@ function variant_usage { echo "" echo " --variant enb-usrp OR -v1" echo " --variant basic-sim OR -v2" + echo " --variant phy-sim OR -v3" echo "" } @@ -84,6 +86,15 @@ case $key in BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" shift ;; + -v3) + VM_NAME=ci-phy-sim + ARCHIVES_LOC=phy_sim + LOG_PATTERN=.Rel14.txt + NB_PATTERN_FILES=3 + BUILD_OPTIONS="--phy_simulators" + BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" + shift + ;; --variant) variant="$2" case $variant in @@ -103,6 +114,14 @@ case $key in BUILD_OPTIONS="--basic-simulator" BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" ;; + phy-sim) + VM_NAME=ci-phy-sim + ARCHIVES_LOC=phy_sim + LOG_PATTERN=.Rel14.txt + NB_PATTERN_FILES=3 + BUILD_OPTIONS="--phy_simulators" + BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" + ;; *) echo "" echo "Syntax Error: Invalid Variant option -> $variant" @@ -230,4 +249,10 @@ done if [ $NB_PATTERN_FILES -ne $NB_FOUND_FILES ]; then STATUS=-1; fi +if [ $STATUS -eq 0 ] +then + echo "STATUS seems OK" +else + echo "STATUS failed?" +fi exit $STATUS diff --git a/ci-scripts/reportBuildLocally.sh b/ci-scripts/reportBuildLocally.sh index 55e1754593..bfa5b258c2 100755 --- a/ci-scripts/reportBuildLocally.sh +++ b/ci-scripts/reportBuildLocally.sh @@ -68,6 +68,113 @@ function trigger_usage { echo "" } +function details_table { + echo "

$1

" >> $3 + + echo " " >> $3 + echo " " >> $3 + echo " " >> $3 + echo " " >> $3 + echo " " >> $3 + echo " " >> $3 + echo " " >> $3 + + LIST_MESSAGES=`egrep "error:|warning:" $2 | egrep -v "jobserver unavailable|Clock skew detected.|flexran.proto"` + COMPLETE_MESSAGE="start" + for MESSAGE in $LIST_MESSAGES + do + if [[ $MESSAGE == *"/home/ubuntu/tmp"* ]] + then + FILENAME=`echo $MESSAGE | sed -e "s#^/home/ubuntu/tmp/##" | awk -F ":" '{print $1}'` + LINENB=`echo $MESSAGE | awk -F ":" '{print $2}'` + if [ "$COMPLETE_MESSAGE" != "start" ] + then + COMPLETE_MESSAGE=`echo $COMPLETE_MESSAGE | sed -e "s#‘#'#g" -e "s#’#'#g"` + echo " " >> $3 + echo " " >> $3 + fi + echo " " >> $3 + echo " " >> $3 + echo " " >> $3 + else + if [[ $MESSAGE == *"warning:"* ]] || [[ $MESSAGE == *"error:"* ]] + then + MSGTYPE=`echo $MESSAGE | sed -e "s#:##g"` + echo " " >> $3 + COMPLETE_MESSAGE="" + else + COMPLETE_MESSAGE=$COMPLETE_MESSAGE" "$MESSAGE + fi + fi + done + + if [ "$COMPLETE_MESSAGE" != "start" ] + then + COMPLETE_MESSAGE=`echo $COMPLETE_MESSAGE | sed -e "s#‘#'#g" -e "s#’#'#g"` + echo " " >> $3 + echo " " >> $3 + fi + echo "
FileLine NumberStatusMessage
$COMPLETE_MESSAGE
$FILENAME$LINENB$MSGTYPE$COMPLETE_MESSAGE
" >> $3 +} + +function summary_table_header { + echo "

$1

" >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html +} + +function summary_table_row { + echo " " >> ./build_results.html + echo " " >> ./build_results.html +} + +function summary_table_footer { + echo "
ElementStatusNb ErrorsNb Warnings
$1" >> ./build_results.html + if [ -f $2 ] + then + STATUS=`egrep -c "$3" $2` + if [ $STATUS -eq 1 ] + then + echo " OK" >> ./build_results.html + else + echo " KO" >> ./build_results.html + fi + NB_ERRORS=`egrep -c "error:" $2` + if [ $NB_ERRORS -eq 0 ] + then + echo " $NB_ERRORS" >> ./build_results.html + else + echo " $NB_ERRORS" >> ./build_results.html + fi + NB_WARNINGS=`egrep "warning:" $2 | egrep -v "jobserver unavailable|Clock skew detected.|flexran.proto" | egrep -c "warning:"` + if [ $NB_WARNINGS -eq 0 ] + then + echo " $NB_WARNINGS" >> ./build_results.html + else + if [ $NB_WARNINGS -gt 20 ] + then + echo " $NB_WARNINGS" >> ./build_results.html + else + echo " $NB_WARNINGS" >> ./build_results.html + fi + fi + if [ $NB_ERRORS -ne 0 ] || [ $NB_WARNINGS -ne 0 ] + then + details_table "$1" $2 $4 + fi + else + echo " Unknown" >> ./build_results.html + echo " --" >> ./build_results.html + echo " --" >> ./build_results.html + fi + echo "
" >> ./build_results.html +} + jb_checker=0 mr_checker=0 pu_checker=0 @@ -289,245 +396,40 @@ then echo " " >> ./build_results.html fi -echo "

OAI Build eNb -- USRP option

" >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo "
ElementStatusNb ErrorsNb Warnings
LTE SoftModem - Release 14" >> ./build_results.html -if [ -f ./archives/enb_usrp/lte-softmodem.Rel14.txt ] -then - STATUS=`egrep -c "Built target lte-softmodem" ./archives/enb_usrp/lte-softmodem.Rel14.txt` - if [ $STATUS -eq 1 ] - then - echo " OK" >> ./build_results.html - else - echo " KO" >> ./build_results.html - fi - STATUS=`egrep -c "error:" ./archives/enb_usrp/lte-softmodem.Rel14.txt` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi - STATUS=`egrep "warning:" ./archives/enb_usrp/lte-softmodem.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi -else - echo " Unknown" >> ./build_results.html - echo " --" >> ./build_results.html - echo " --" >> ./build_results.html -fi -echo "
Coding - Release 14" >> ./build_results.html -if [ -f ./archives/enb_usrp/coding.Rel14.txt ] -then - STATUS=`egrep -c "Built target coding" ./archives/enb_usrp/coding.Rel14.txt` - if [ $STATUS -eq 1 ] - then - echo " OK" >> ./build_results.html - else - echo " KO" >> ./build_results.html - fi - STATUS=`egrep -c "error:" ./archives/enb_usrp/coding.Rel14.txt` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi - STATUS=`egrep "warning:" ./archives/enb_usrp/coding.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi -else - echo " Unknown" >> ./build_results.html - echo " --" >> ./build_results.html - echo " --" >> ./build_results.html -fi -echo "
OAI USRP device if - Release 14" >> ./build_results.html -if [ -f ./archives/enb_usrp/oai_usrpdevif.Rel14.txt ] -then - STATUS=`egrep -c "Built target oai_usrpdevif" ./archives/enb_usrp/oai_usrpdevif.Rel14.txt` - if [ $STATUS -eq 1 ] - then - echo " OK" >> ./build_results.html - else - echo " KO" >> ./build_results.html - fi - STATUS=`egrep -c "error:" ./archives/enb_usrp/oai_usrpdevif.Rel14.txt` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi - STATUS=`egrep "warning:" ./archives/enb_usrp/oai_usrpdevif.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi -else - echo " Unknown" >> ./build_results.html - echo " --" >> ./build_results.html - echo " --" >> ./build_results.html -fi -echo "
Parameters Lib Config - Release 14" >> ./build_results.html -if [ -f ./archives/enb_usrp/params_libconfig.Rel14.txt ] -then - STATUS=`egrep -c "Built target params_libconfig" ./archives/enb_usrp/params_libconfig.Rel14.txt` - if [ $STATUS -eq 1 ] - then - echo " OK" >> ./build_results.html - else - echo " KO" >> ./build_results.html - fi - STATUS=`egrep -c "error:" ./archives/enb_usrp/params_libconfig.Rel14.txt` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi - STATUS=`egrep "warning:" ./archives/enb_usrp/params_libconfig.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi -else - echo " Unknown" >> ./build_results.html - echo " --" >> ./build_results.html - echo " --" >> ./build_results.html -fi -echo "
" >> ./build_results.html +summary_table_header "OAI Build eNB -- USRP option" +summary_table_row "LTE SoftModem - Release 14" ./archives/enb_usrp/lte-softmodem.Rel14.txt "Built target lte-softmodem" ./enb_usrp_row1.html +summary_table_row "Coding - Release 14" ./archives/enb_usrp/coding.Rel14.txt "Built target coding" ./enb_usrp_row2.html +summary_table_row "OAI USRP device if - Release 14" ./archives/enb_usrp/oai_usrpdevif.Rel14.txt "Built target oai_usrpdevif" ./enb_usrp_row3.html +summary_table_row "Parameters Lib Config - Release 14" ./archives/enb_usrp/params_libconfig.Rel14.txt "Built target params_libconfig" ./enb_usrp_row4.html +summary_table_footer -# conf2uedata.Rel14.txt -# archives/basic_sim +summary_table_header "OAI Build basic simulator option" +summary_table_row "Basic Simulator eNb - Release 14" ./archives/basic_sim/basic_simulator_enb.txt "Built target lte-softmodem" ./basic_sim_row1.html +summary_table_row "Basic Simulator UE - Release 14" ./archives/basic_sim/basic_simulator_ue.txt "Built target lte-uesoftmodem" ./basic_sim_row2.html +summary_table_row "Conf 2 UE data - Release 14" ./archives/basic_sim/conf2uedata.Rel14.txt "Built target conf2uedata" ./basic_sim_row3.html +summary_table_footer -echo "

OAI Build basic simulator option

" >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo " " >> ./build_results.html -echo "
ElementStatusNb ErrorsNb Warnings
Basic Simulator eNb - Release 14" >> ./build_results.html -if [ -f ./archives/basic_sim/basic_simulator_enb.txt ] -then - STATUS=`egrep -c "Built target lte-softmodem" ./archives/basic_sim/basic_simulator_enb.txt` - if [ $STATUS -eq 1 ] - then - echo " OK" >> ./build_results.html - else - echo " KO" >> ./build_results.html - fi - STATUS=`egrep -c "error:" ./archives/basic_sim/basic_simulator_enb.txt` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi - STATUS=`egrep "warning:" ./archives/basic_sim/basic_simulator_enb.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi -else - echo " Unknown" >> ./build_results.html - echo " --" >> ./build_results.html - echo " --" >> ./build_results.html -fi -echo "
Basic Simulator UE - Release 14" >> ./build_results.html -if [ -f ./archives/basic_sim/basic_simulator_ue.txt ] -then - STATUS=`egrep -c "Built target lte-uesoftmodem" ./archives/basic_sim/basic_simulator_ue.txt` - if [ $STATUS -eq 1 ] - then - echo " OK" >> ./build_results.html - else - echo " KO" >> ./build_results.html - fi - STATUS=`egrep -c "error:" ./archives/basic_sim/basic_simulator_ue.txt` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi - STATUS=`egrep "warning:" ./archives/basic_sim/basic_simulator_ue.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi -else - echo " Unknown" >> ./build_results.html - echo " --" >> ./build_results.html - echo " --" >> ./build_results.html -fi -echo "
Conf 2 UE data - Release 14" >> ./build_results.html -if [ -f ./archives/basic_sim/conf2uedata.Rel14.txt ] -then - STATUS=`egrep -c "Built target conf2uedata" ./archives/basic_sim/conf2uedata.Rel14.txt` - if [ $STATUS -eq 1 ] - then - echo " OK" >> ./build_results.html - else - echo " KO" >> ./build_results.html - fi - STATUS=`egrep -c "error:" ./archives/basic_sim/conf2uedata.Rel14.txt` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi - STATUS=`egrep "warning:" ./archives/basic_sim/conf2uedata.Rel14.txt | egrep -v "Clock skew detected.|flexran.proto" | egrep -c "warning:"` - if [ $STATUS -eq 0 ] - then - echo " $STATUS" >> ./build_results.html - else - echo " $STATUS" >> ./build_results.html - fi -else - echo " Unknown" >> ./build_results.html - echo " --" >> ./build_results.html - echo " --" >> ./build_results.html -fi -echo "
" >> ./build_results.html +summary_table_header "OAI Build Physical simulators option" +summary_table_row "DL Simulator - Release 14" ./archives/phy_sim/dlsim.Rel14.txt "Built target dlsim" ./phy_sim_row1.html +summary_table_row "UL Simulator - Release 14" ./archives/phy_sim/ulsim.Rel14.txt "Built target ulsim" ./phy_sim_row2.html +summary_table_row "Coding - Release 14" ./archives/phy_sim/coding.Rel14.txt "Built target coding" ./phy_sim_row3.html +summary_table_footer + +echo "

Details

" >> ./build_results.html + +for DETAILS_TABLE in `ls ./enb_usrp_row*.html` +do + cat $DETAILS_TABLE >> ./build_results.html +done +for DETAILS_TABLE in `ls ./basic_sim_row*.html` +do + cat $DETAILS_TABLE >> ./build_results.html +done +for DETAILS_TABLE in `ls ./phy_sim_row*.html` +do + cat $DETAILS_TABLE >> ./build_results.html +done +rm -f ./enb_usrp_row*.html ./basic_sim_row*.html ./phy_sim_row*.html echo "" >> ./build_results.html echo "" >> ./build_results.html From 854247d2658f9f8a637bc73a4fa99c50f6832084 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 21 Jun 2018 13:46:40 +0200 Subject: [PATCH 5/8] CI: Adding Static Code Analysis stage -- new stage in parallel on a VM -- detailed reporting of CPPCHECK -- Option to keep vm alive after build (not used yet on jenkins pipeline) Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-gitlab | 11 +++- ci-scripts/buildOnVM.sh | 94 +++++++++++++++++++++++++------- ci-scripts/reportBuildLocally.sh | 75 +++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 22 deletions(-) diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 2021e8347f..7e82261ab8 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -8,7 +8,7 @@ pipeline { disableConcurrentBuilds() timestamps() gitLabConnection('OAI GitLab') - gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim", "Build phy-sim"]) + gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim", "Build phy-sim", "Analysis with cppcheck"]) } stages { @@ -49,6 +49,13 @@ pipeline { stage ("Variant Builds") { parallel { + stage ("Analysis with cppcheck") { + steps { + gitlabCommitStatus(name: "Analysis with cppcheck") { + sh "./ci-scripts/buildOnVM.sh --workspace $WORKSPACE --variant cppcheck" + } + } + } stage ("Build eNb-USRP") { steps { gitlabCommitStatus(name: "Build eNb-USRP") { @@ -75,7 +82,7 @@ pipeline { always { script { dir ('archives') { - sh "zip -r vm_build_logs.zip basic_sim enb_usrp" + sh "zip -r vm_build_logs.zip basic_sim enb_usrp cppcheck" } if(fileExists('archives/vm_build_logs.zip')) { archiveArtifacts artifacts: 'archives/vm_build_logs.zip' diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh index d34fb9a6c4..1ea5eaf50e 100755 --- a/ci-scripts/buildOnVM.sh +++ b/ci-scripts/buildOnVM.sh @@ -16,12 +16,16 @@ function usage { echo "Options:" echo "--------" echo " --workspace #### OR -ws ####" - echo " Specify the workspace" + echo " Specify the workspace." echo "" echo " --variant enb-usrp OR -v1" echo " --variant basic-sim OR -v2" echo " --variant phy-sim OR -v3" - echo " Specify the variant to build" + echo " --variant cppcheck OR -v4" + echo " Specify the variant to build." + echo "" + echo " --keep-vm-alive OR -k" + echo " Keep the VM alive after the build." echo "" echo " --help OR -h" echo " Print this help message." @@ -35,10 +39,11 @@ function variant_usage { echo " --variant enb-usrp OR -v1" echo " --variant basic-sim OR -v2" echo " --variant phy-sim OR -v3" + echo " --variant cppcheck OR -v4" echo "" } -if [ $# -ne 2 ] && [ $# -ne 1 ] && [ $# -ne 4 ] && [ $# -ne 3 ] +if [ $# -lt 1 ] || [ $# -gt 5 ] then echo "Syntax Error: not the correct number of arguments" echo "" @@ -52,6 +57,7 @@ LOG_PATTERN=.Rel14.txt NB_PATTERN_FILES=4 BUILD_OPTIONS="--eNB -w USRP" BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" +KEEP_VM_ALIVE=0 while [[ $# -gt 0 ]] do @@ -68,6 +74,10 @@ case $key in shift shift ;; + -k|--keep-vm-alive) + KEEP_VM_ALIVE=1 + shift + ;; -v1) VM_NAME=ci-enb-usrp ARCHIVES_LOC=enb_usrp @@ -95,6 +105,15 @@ case $key in BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" shift ;; + -v4) + VM_NAME=ci-cppcheck + ARCHIVES_LOC=cppcheck + LOG_PATTERN=cppcheck.xml + NB_PATTERN_FILES=1 + BUILD_OPTIONS="--enable=warning --force --xml --xml-version=2" + BUILD_EXTRA_OPTIONS="" + shift + ;; --variant) variant="$2" case $variant in @@ -122,6 +141,14 @@ case $key in BUILD_OPTIONS="--phy_simulators" BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" ;; + cppcheck) + VM_NAME=ci-cppcheck + ARCHIVES_LOC=cppcheck + LOG_PATTERN=cppcheck.xml + NB_PATTERN_FILES=1 + BUILD_OPTIONS="--enable=warning --force --xml --xml-version=2" + BUILD_EXTRA_OPTIONS="" + ;; *) echo "" echo "Syntax Error: Invalid Variant option -> $variant" @@ -182,20 +209,34 @@ echo "############################################################" echo "Running install and build script on VM ($VM_NAME)" echo "############################################################" echo "sudo cp 01proxy /etc/apt/apt.conf.d/" > $VM_CMDS -echo "echo \"sudo apt-get --yes --quiet install zip subversion libboost-dev \"" >> $VM_CMDS -echo "sudo apt-get --yes install zip subversion libboost-dev > zip-install.txt 2>&1" >> $VM_CMDS +if [ "$VM_NAME" == "ci-cppcheck" ] +then + echo "echo \"sudo apt-get --yes --quiet install zip cppcheck \"" >> $VM_CMDS + echo "sudo apt-get update > zip-install.txt 2>&1" >> $VM_CMDS + echo "sudo apt-get --yes install zip cppcheck >> zip-install.txt 2>&1" >> $VM_CMDS +else + echo "echo \"sudo apt-get --yes --quiet install zip subversion libboost-dev \"" >> $VM_CMDS + echo "sudo apt-get --yes install zip subversion libboost-dev > zip-install.txt 2>&1" >> $VM_CMDS +fi echo "mkdir tmp" >> $VM_CMDS echo "cd tmp" >> $VM_CMDS echo "echo \"unzip -qq ../localZip.zip\"" >> $VM_CMDS echo "unzip -qq ../localZip.zip" >> $VM_CMDS -echo "echo \"source oaienv\"" >> $VM_CMDS -echo "source oaienv" >> $VM_CMDS -echo "cd cmake_targets/" >> $VM_CMDS -echo "mkdir log" >> $VM_CMDS -echo "cp /home/ubuntu/zip-install.txt log" >> $VM_CMDS -echo "echo \"./build_oai -I $BUILD_OPTIONS \"" >> $VM_CMDS -echo "./build_oai -I $BUILD_OPTIONS $BUILD_EXTRA_OPTIONS > log/install-build.txt 2>&1" >> $VM_CMDS - +if [ "$VM_NAME" == "ci-cppcheck" ] +then + echo "mkdir cmake_targets/log" >> $VM_CMDS + echo "cp /home/ubuntu/zip-install.txt cmake_targets/log" >> $VM_CMDS + echo "echo \"cppcheck $BUILD_OPTIONS . \"" >> $VM_CMDS + echo "cppcheck $BUILD_OPTIONS . 2> cmake_targets/log/cppcheck.xml 1> cmake_targets/log/cppcheck_build.txt" >> $VM_CMDS +else + echo "echo \"source oaienv\"" >> $VM_CMDS + echo "source oaienv" >> $VM_CMDS + echo "cd cmake_targets/" >> $VM_CMDS + echo "mkdir log" >> $VM_CMDS + echo "cp /home/ubuntu/zip-install.txt log" >> $VM_CMDS + echo "echo \"./build_oai -I $BUILD_OPTIONS \"" >> $VM_CMDS + echo "./build_oai -I $BUILD_OPTIONS $BUILD_EXTRA_OPTIONS > log/install-build.txt 2>&1" >> $VM_CMDS +fi ssh -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR < $VM_CMDS echo "############################################################" @@ -212,19 +253,26 @@ then fi scp -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/*.txt $ARCHIVES_LOC +if [ "$VM_NAME" == "ci-cppcheck" ] +then + scp -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/*.xml $ARCHIVES_LOC +fi -echo "############################################################" -echo "Destroying VM" -echo "############################################################" -uvt-kvm destroy $VM_NAME -ssh-keygen -R $VM_IP_ADDR +if [ $KEEP_VM_ALIVE -eq 0 ] +then + echo "############################################################" + echo "Destroying VM" + echo "############################################################" + uvt-kvm destroy $VM_NAME + ssh-keygen -R $VM_IP_ADDR +fi rm -f $VM_CMDS echo "############################################################" echo "Checking build status" echo "############################################################" -LOG_FILES=`ls $ARCHIVES_LOC/*.txt` +LOG_FILES=`ls $ARCHIVES_LOC/*.txt $ARCHIVES_LOC/*.xml` STATUS=0 NB_FOUND_FILES=0 @@ -241,7 +289,13 @@ do then PASS_PATTERN="lte-" fi - LOCAL_STAT=`egrep -c "Built target $PASS_PATTERN" $FULLFILE` + if [ "$LOG_PATTERN" == "cppcheck.xml" ] + then + PASS_PATTERN="results version" + LOCAL_STAT=`egrep -c "$PASS_PATTERN" $FULLFILE` + else + LOCAL_STAT=`egrep -c "Built target $PASS_PATTERN" $FULLFILE` + fi if [ $LOCAL_STAT -eq 0 ]; then STATUS=-1; fi NB_FOUND_FILES=$((NB_FOUND_FILES + 1)) fi diff --git a/ci-scripts/reportBuildLocally.sh b/ci-scripts/reportBuildLocally.sh index bfa5b258c2..5194b81d4f 100755 --- a/ci-scripts/reportBuildLocally.sh +++ b/ci-scripts/reportBuildLocally.sh @@ -175,6 +175,69 @@ function summary_table_footer { echo " " >> ./build_results.html } +function sca_summary_table_header { + echo "

$1

" >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo "0" > ccp_error_cnt.txt +} + +function sca_summary_table_row { + echo " " >> ./build_results.html + echo " " >> ./build_results.html + if [ -f $1 ] + then + NB_ERRORS=`egrep "severity=\"error\"" $1 | egrep -c "id=\"$3\""` + echo " " >> ./build_results.html + echo " " >> ./build_results.html + if [ -f ccp_error_cnt.txt ] + then + TOTAL_ERRORS=`cat ccp_error_cnt.txt` + TOTAL_ERRORS=$((TOTAL_ERRORS + NB_ERRORS)) + echo $TOTAL_ERRORS > ccp_error_cnt.txt + fi + else + echo " " >> ./build_results.html + echo " " >> ./build_results.html + fi + echo " " >> ./build_results.html +} + +function sca_summary_table_footer { + echo " " >> ./build_results.html + if [ -f $1 ] + then + NB_ERRORS=`egrep -c "severity=\"error\"" $1` + if [ -f ccp_error_cnt.txt ] + then + echo " " >> ./build_results.html + TOTAL_ERRORS=`cat ccp_error_cnt.txt` + TOTAL_ERRORS=$((NB_ERRORS - TOTAL_ERRORS)) + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + rm -f ccp_error_cnt.txt + fi + echo " " >> ./build_results.html + echo " " >> ./build_results.html + NB_WARNINGS=`egrep -c "severity=\"warning\"" $1` + echo " " >> ./build_results.html + else + echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html + fi + echo " " >> ./build_results.html + echo "
Error TypeNb ErrorsNb Warnings
$2$NB_ERRORSN/AUnknownUnknown
Others$TOTAL_ERRORSN/A
Total$NB_ERRORS$NB_WARNINGSTotalUnknownUnknown
" >> ./build_results.html + echo "

Full details in zipped artifact (cppcheck/cppcheck.xml)

" >> ./build_results.html + echo "

Graphical Interface tool : cppcheck-gui -l cppcheck/cppcheck.xml

" >> ./build_results.html +} + jb_checker=0 mr_checker=0 pu_checker=0 @@ -396,6 +459,18 @@ then echo " " >> ./build_results.html fi +sca_summary_table_header "OAI Static Code Analysis with CPPCHECK" +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Uninitialized variable" uninitvar +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Uninitialized struct member" uninitStructMember +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Memory leak" memleak +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Memory is freed twice" doubleFree +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Resource leak" resourceLeak +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Possible null pointer dereference" nullPointer +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Array access out of bounds" arrayIndexOutOfBounds +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Buffer is accessed out of bounds" bufferAccessOutOfBounds +sca_summary_table_row ./archives/cppcheck/cppcheck.xml "Expression depends on order of evaluation of side effects" unknownEvaluationOrder +sca_summary_table_footer ./archives/cppcheck/cppcheck.xml + summary_table_header "OAI Build eNB -- USRP option" summary_table_row "LTE SoftModem - Release 14" ./archives/enb_usrp/lte-softmodem.Rel14.txt "Built target lte-softmodem" ./enb_usrp_row1.html summary_table_row "Coding - Release 14" ./archives/enb_usrp/coding.Rel14.txt "Built target coding" ./enb_usrp_row2.html From 89344c5dabd9b7f1a2609e3f6e1015f04cb6385a Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 21 Jun 2018 15:18:41 +0200 Subject: [PATCH 6/8] CI: improvements -- missing phy simulator logs in artifacts -- enhancement on HTML report Signed-off-by: Raphael Defosseux --- ci-scripts/Jenkinsfile-gitlab | 2 +- ci-scripts/reportBuildLocally.sh | 43 +++++++++++++++++++++----------- 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 7e82261ab8..ad97588d9b 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -82,7 +82,7 @@ pipeline { always { script { dir ('archives') { - sh "zip -r vm_build_logs.zip basic_sim enb_usrp cppcheck" + sh "zip -r vm_build_logs.zip basic_sim enb_usrp phy_sim cppcheck" } if(fileExists('archives/vm_build_logs.zip')) { archiveArtifacts artifacts: 'archives/vm_build_logs.zip' diff --git a/ci-scripts/reportBuildLocally.sh b/ci-scripts/reportBuildLocally.sh index 5194b81d4f..b4f2f324e7 100755 --- a/ci-scripts/reportBuildLocally.sh +++ b/ci-scripts/reportBuildLocally.sh @@ -71,8 +71,8 @@ function trigger_usage { function details_table { echo "

$1

" >> $3 - echo " " >> $3 - echo " " >> $3 + echo "
" >> $3 + echo " " >> $3 echo " " >> $3 echo " " >> $3 echo " " >> $3 @@ -119,8 +119,8 @@ function details_table { function summary_table_header { echo "

$1

" >> ./build_results.html - echo "
FileLine NumberStatus
" >> ./build_results.html - echo " " >> ./build_results.html + echo "
" >> ./build_results.html + echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html @@ -177,9 +177,9 @@ function summary_table_footer { function sca_summary_table_header { echo "

$1

" >> ./build_results.html - echo "
ElementStatusNb Errors
" >> ./build_results.html - echo " " >> ./build_results.html - echo " " >> ./build_results.html + echo "
Error Type
" >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html @@ -188,7 +188,7 @@ function sca_summary_table_header { function sca_summary_table_row { echo " " >> ./build_results.html - echo " " >> ./build_results.html + echo " " >> ./build_results.html if [ -f $1 ] then NB_ERRORS=`egrep "severity=\"error\"" $1 | egrep -c "id=\"$3\""` @@ -208,26 +208,27 @@ function sca_summary_table_row { } function sca_summary_table_footer { - echo " " >> ./build_results.html if [ -f $1 ] then NB_ERRORS=`egrep -c "severity=\"error\"" $1` + NB_WARNINGS=`egrep -c "severity=\"warning\"" $1` if [ -f ccp_error_cnt.txt ] then - echo " " >> ./build_results.html + echo " " >> ./build_results.html + echo " " >> ./build_results.html TOTAL_ERRORS=`cat ccp_error_cnt.txt` TOTAL_ERRORS=$((NB_ERRORS - TOTAL_ERRORS)) echo " " >> ./build_results.html - echo " " >> ./build_results.html + echo " " >> ./build_results.html echo " " >> ./build_results.html - echo " " >> ./build_results.html rm -f ccp_error_cnt.txt fi + echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html - NB_WARNINGS=`egrep -c "severity=\"warning\"" $1` echo " " >> ./build_results.html else + echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html @@ -394,9 +395,23 @@ echo "" > ./build_results.html echo "" >> ./build_results.html echo "" >> ./build_results.html echo " Build Results for $JOB_NAME job build #$BUILD_ID" >> ./build_results.html +echo " " >> ./build_results.html echo "" >> ./build_results.html echo "" >> ./build_results.html -echo "

Job Summary -- Job: $JOB_NAME -- Build-ID: $BUILD_ID

" >> ./build_results.html +echo "
Error / Warning TypeNb ErrorsNb Warnings
$2$2
Others
Others$TOTAL_ERRORSN/A$NB_WARNINGS
Total$NB_ERRORS$NB_WARNINGS
TotalUnknownUnknown
" >> ./build_results.html +echo " " >> ./build_results.html +echo " " >> ./build_results.html +echo " " >> ./build_results.html +echo " " >> ./build_results.html +echo "
" >> ./build_results.html +echo " " >> ./build_results.html +echo " \"\"" >> ./build_results.html +echo " " >> ./build_results.html +echo " " >> ./build_results.html +echo " " >> ./build_results.html +echo " Job Summary -- Job: eNb-CI -- Build-ID: 23" >> ./build_results.html +echo "
" >> ./build_results.html +echo "
" >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html From 2ac3c5832a61960dcc97060c7b71f0b36eaccfba Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 21 Jun 2018 16:42:41 +0200 Subject: [PATCH 7/8] CI: improvements -- remove extra build options (not necessary anymore) -- fix in reporting Signed-off-by: Raphael Defosseux --- ci-scripts/buildOnVM.sh | 12 +----------- ci-scripts/reportBuildLocally.sh | 2 +- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh index 1ea5eaf50e..b5fef44ba6 100755 --- a/ci-scripts/buildOnVM.sh +++ b/ci-scripts/buildOnVM.sh @@ -56,7 +56,6 @@ ARCHIVES_LOC=enb_usrp LOG_PATTERN=.Rel14.txt NB_PATTERN_FILES=4 BUILD_OPTIONS="--eNB -w USRP" -BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" KEEP_VM_ALIVE=0 while [[ $# -gt 0 ]] @@ -84,7 +83,6 @@ case $key in LOG_PATTERN=.Rel14.txt NB_PATTERN_FILES=4 BUILD_OPTIONS="--eNB -w USRP" - BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" shift ;; -v2) @@ -93,7 +91,6 @@ case $key in LOG_PATTERN=basic_simulator NB_PATTERN_FILES=2 BUILD_OPTIONS="--basic-simulator" - BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" shift ;; -v3) @@ -102,7 +99,6 @@ case $key in LOG_PATTERN=.Rel14.txt NB_PATTERN_FILES=3 BUILD_OPTIONS="--phy_simulators" - BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" shift ;; -v4) @@ -111,7 +107,6 @@ case $key in LOG_PATTERN=cppcheck.xml NB_PATTERN_FILES=1 BUILD_OPTIONS="--enable=warning --force --xml --xml-version=2" - BUILD_EXTRA_OPTIONS="" shift ;; --variant) @@ -123,7 +118,6 @@ case $key in LOG_PATTERN=.Rel14.txt NB_PATTERN_FILES=4 BUILD_OPTIONS="--eNB -w USRP" - BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" ;; basic-sim) VM_NAME=ci-basic-sim @@ -131,7 +125,6 @@ case $key in LOG_PATTERN=basic_simulator NB_PATTERN_FILES=2 BUILD_OPTIONS="--basic-simulator" - BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" ;; phy-sim) VM_NAME=ci-phy-sim @@ -139,7 +132,6 @@ case $key in LOG_PATTERN=.Rel14.txt NB_PATTERN_FILES=3 BUILD_OPTIONS="--phy_simulators" - BUILD_EXTRA_OPTIONS="--cflags_processor \"-mssse3 -msse4.1 -mavx2\"" ;; cppcheck) VM_NAME=ci-cppcheck @@ -147,7 +139,6 @@ case $key in LOG_PATTERN=cppcheck.xml NB_PATTERN_FILES=1 BUILD_OPTIONS="--enable=warning --force --xml --xml-version=2" - BUILD_EXTRA_OPTIONS="" ;; *) echo "" @@ -187,7 +178,6 @@ echo "VM_CMD_FILE = $VM_CMDS" echo "JENKINS_WKSP = $JENKINS_WKSP" echo "ARCHIVES_LOC = $ARCHIVES_LOC" echo "BUILD_OPTIONS = $BUILD_OPTIONS" -echo "BUILD_EXTRA_OPTIONS = $BUILD_EXTRA_OPTIONS" echo "############################################################" echo "Creating VM ($VM_NAME) on Ubuntu Cloud Image base" @@ -235,7 +225,7 @@ else echo "mkdir log" >> $VM_CMDS echo "cp /home/ubuntu/zip-install.txt log" >> $VM_CMDS echo "echo \"./build_oai -I $BUILD_OPTIONS \"" >> $VM_CMDS - echo "./build_oai -I $BUILD_OPTIONS $BUILD_EXTRA_OPTIONS > log/install-build.txt 2>&1" >> $VM_CMDS + echo "./build_oai -I $BUILD_OPTIONS > log/install-build.txt 2>&1" >> $VM_CMDS fi ssh -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR < $VM_CMDS diff --git a/ci-scripts/reportBuildLocally.sh b/ci-scripts/reportBuildLocally.sh index b4f2f324e7..3a19bae812 100755 --- a/ci-scripts/reportBuildLocally.sh +++ b/ci-scripts/reportBuildLocally.sh @@ -407,7 +407,7 @@ echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html echo " " >> ./build_results.html echo "
GIT Repository" >> ./build_results.html -echo " Job Summary -- Job: eNb-CI -- Build-ID: 23" >> ./build_results.html +echo " Job Summary -- Job: $JOB_NAME -- Build-ID: $BUILD_ID" >> ./build_results.html echo "
" >> ./build_results.html From 1231aa98a10d505d2520b80b06c6b39c5fb22a2e Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 22 Jun 2018 09:52:25 +0200 Subject: [PATCH 8/8] CI: stability fix -- when unzip, added --DD option: skip restoration of timestamps for all extracted items. should resolve clock skew issues Signed-off-by: Raphael Defosseux --- ci-scripts/buildOnVM.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-scripts/buildOnVM.sh b/ci-scripts/buildOnVM.sh index b5fef44ba6..c5389c2b50 100755 --- a/ci-scripts/buildOnVM.sh +++ b/ci-scripts/buildOnVM.sh @@ -210,8 +210,8 @@ else fi echo "mkdir tmp" >> $VM_CMDS echo "cd tmp" >> $VM_CMDS -echo "echo \"unzip -qq ../localZip.zip\"" >> $VM_CMDS -echo "unzip -qq ../localZip.zip" >> $VM_CMDS +echo "echo \"unzip -qq -DD ../localZip.zip\"" >> $VM_CMDS +echo "unzip -qq -DD ../localZip.zip" >> $VM_CMDS if [ "$VM_NAME" == "ci-cppcheck" ] then echo "mkdir cmake_targets/log" >> $VM_CMDS