diff --git a/ci-scripts/Jenkinsfile-gitlab b/ci-scripts/Jenkinsfile-gitlab index 2a29742c36..e80324e3f1 100644 --- a/ci-scripts/Jenkinsfile-gitlab +++ b/ci-scripts/Jenkinsfile-gitlab @@ -8,7 +8,7 @@ pipeline { disableConcurrentBuilds() timestamps() gitLabConnection('OAI GitLab') - gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim", "Build phy-sim", "Build eNb-ethernet", "Build UE-ethernet", "Analysis with cppcheck"]) + gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim", "Build phy-sim", "Build eNb-ethernet", "Build UE-ethernet", "Analysis with cppcheck", "Test phy-sim"]) } stages { @@ -150,6 +150,37 @@ pipeline { } } + stage ("Variant Tests") { + parallel { + stage ("Test physical simulators") { + steps { + gitlabCommitStatus(name: "Test phy-sim") { + sh "./ci-scripts/runTestOnVM.sh --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive" + } + } + } + stage ("Test basic simulator") { + steps { + //gitlabCommitStatus(name: "Test basic-sim") { + sh "./ci-scripts/runTestOnVM.sh --workspace $WORKSPACE --variant phy-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID} --keep-vm-alive" + //} + } + } + } + post { + always { + script { + dir ('archives') { + sh "zip -r -qq vm_tests_logs.zip */test" + } + if(fileExists('archives/vm_tests_logs.zip')) { + archiveArtifacts artifacts: 'archives/vm_tests_logs.zip' + archiveArtifacts artifacts: 'archives/*/test/results_autotests*.xml' + } + } + } + } + } stage ("Destroy all Virtual Machines") { steps { sh "./ci-scripts/destroyAllRunningVM.sh --job-name ${JOB_NAME} --build-id ${BUILD_ID}" diff --git a/ci-scripts/runTestOnVM.sh b/ci-scripts/runTestOnVM.sh new file mode 100755 index 0000000000..e679d790df --- /dev/null +++ b/ci-scripts/runTestOnVM.sh @@ -0,0 +1,291 @@ +#!/bin/bash + +function usage { + echo "OAI VM Test Run script" + echo " Original Author: Raphael Defosseux" + echo " Default:" + echo " -- eNB with USRP" + echo "" + echo "Usage:" + echo "------" + echo " runTestOnVM.sh [OPTIONS]" + echo "" + echo "Options:" + echo "--------" + echo " --job-name #### OR -jn ####" + echo " Specify the name of the Jenkins job." + echo "" + echo " --build-id #### OR -id ####" + echo " Specify the build ID of the Jenkins job." + echo "" + echo " --workspace #### OR -ws ####" + echo " Specify the workspace." + echo "" + echo " --variant enb-usrp OR -v1" + echo " --variant basic-sim OR -v2" + echo " --variant phy-sim OR -v3" + echo " --variant cppcheck OR -v4" + echo " --variant enb-ethernet OR -v7" + echo " --variant ue-ethernet OR -v8" + echo " Specify the variant to build." + echo "" + echo " --keep-vm-alive OR -k" + echo " Keep the VM alive after the build." + echo "" + echo " --help OR -h" + echo " Print this help message." + echo "" +} + +function variant_usage { + echo "OAI VM Test Run script" + echo " Original Author: Raphael Defosseux" + echo "" + echo " --variant enb-usrp OR -v1" + echo " --variant basic-sim OR -v2" + echo " --variant phy-sim OR -v3" + echo " --variant cppcheck OR -v4" + echo " --variant enb-ethernet OR -v7" + echo " --variant ue-ethernet OR -v8" + echo "" +} + +if [ $# -lt 1 ] || [ $# -gt 9 ] +then + echo "Syntax Error: not the correct number of arguments" + echo "" + usage + exit 1 +fi + +VM_TEMPLATE=ci- +JOB_NAME=XX +BUILD_ID=XX +VM_NAME=ci-enb-usrp +ARCHIVES_LOC=enb_usrp/test +KEEP_VM_ALIVE=0 +RUN_OPTIONS="none" +STATUS=0 + +while [[ $# -gt 0 ]] +do +key="$1" + +case $key in + -h|--help) + shift + usage + exit 0 + ;; + -jn|--job-name) + JOB_NAME="$2" + shift + shift + ;; + -id|--build-id) + BUILD_ID="$2" + shift + shift + ;; + -ws|--workspace) + JENKINS_WKSP="$2" + shift + shift + ;; + -k|--keep-vm-alive) + KEEP_VM_ALIVE=1 + shift + ;; + -v1) + VM_NAME=ci-enb-usrp + shift + ;; + -v2) + VM_NAME=ci-basic-sim + shift + ;; + -v3) + VM_NAME=ci-phy-sim + RUN_OPTIONS="./run_exec_autotests.bash -g \"01510*\" -q -np -b" + ARCHIVES_LOC=phy_sim/test + shift + ;; + -v4) + VM_NAME=ci-cppcheck + shift + ;; + -v7) + VM_NAME=ci-enb-ethernet + shift + ;; + -v8) + VM_NAME=ci-ue-ethernet + shift + ;; + --variant) + variant="$2" + case $variant in + enb-usrp) + VM_NAME=ci-enb-usrp + ;; + basic-sim) + VM_NAME=ci-basic-sim + ;; + phy-sim) + VM_NAME=ci-phy-sim + ;; + cppcheck) + VM_NAME=ci-cppcheck + ;; + enb-ethernet) + VM_NAME=ci-enb-ethernet + ;; + ue-ethernet) + VM_NAME=ci-ue-ethernet + ;; + *) + echo "" + echo "Syntax Error: Invalid Variant option -> $variant" + echo "" + variant_usage + exit 1 + esac + shift + shift + ;; + *) + echo "Syntax Error: unknown option: $key" + echo "" + usage + exit 1 +esac +done + +if [ "$JOB_NAME" == "XX" ] || [ "$BUILD_ID" == "XX" ] +then + VM_TEMPLATE=ci- +else + VM_TEMPLATE=${JOB_NAME}-b${BUILD_ID}- +fi + +VM_NAME=`echo $VM_NAME | sed -e "s#ci-#$VM_TEMPLATE#"` +VM_CMDS=${VM_NAME}_cmds.txt +ARCHIVES_LOC=${JENKINS_WKSP}/archives/${ARCHIVES_LOC} + +echo "VM_NAME = $VM_NAME" +echo "VM_CMD_FILE = $VM_CMDS" +echo "JENKINS_WKSP = $JENKINS_WKSP" +echo "ARCHIVES_LOC = $ARCHIVES_LOC" + +echo "############################################################" +echo "Waiting for VM to be started" +echo "############################################################" +uvt-kvm wait $VM_NAME --insecure + +VM_IP_ADDR=`uvt-kvm ip $VM_NAME` +echo "$VM_NAME has for IP addr = $VM_IP_ADDR" + +if [ "$RUN_OPTIONS" == "none" ] +then + echo "No run on VM testing for this variant currently" +else + + echo "############################################################" + echo "Running test script on VM ($VM_NAME)" + echo "############################################################" + echo "echo \"sudo apt-get --yes --quiet install bc \"" > $VM_CMDS + echo "sudo apt-get update > bc-install.txt 2>&1" >> $VM_CMDS + echo "sudo apt-get --yes install bc >> bc-install.txt 2>&1" >> $VM_CMDS + echo "cd tmp" >> $VM_CMDS + echo "echo \"source oaienv\"" >> $VM_CMDS + echo "source oaienv" >> $VM_CMDS + echo "echo \"cd cmake_targets/autotests\"" >> $VM_CMDS + echo "cd cmake_targets/autotests" >> $VM_CMDS + echo "echo \"rm -Rf log\"" >> $VM_CMDS + echo "rm -Rf log" >> $VM_CMDS + echo "$RUN_OPTIONS" | sed -e 's@"@\\"@g' -e 's@^@echo "@' -e 's@$@"@' >> $VM_CMDS + echo "$RUN_OPTIONS" >> $VM_CMDS + echo "cp /home/ubuntu/bc-install.txt log" >> $VM_CMDS + echo "cd log" >> $VM_CMDS + echo "zip -r -qq tmp.zip *.* 0*" >> $VM_CMDS + + ssh -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR < $VM_CMDS + + echo "############################################################" + echo "Creating a tmp folder to store results and artifacts" + echo "############################################################" + + if [ -d $ARCHIVES_LOC ] + then + rm -Rf $ARCHIVES_LOC + fi + mkdir --parents $ARCHIVES_LOC + + scp -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/autotests/log/tmp.zip $ARCHIVES_LOC + pushd $ARCHIVES_LOC + unzip -qq -DD tmp.zip + rm tmp.zip + if [ -f results_autotests.xml ] + then + FUNCTION=`echo $VM_NAME | sed -e "s@$VM_TEMPLATE@@"` + NEW_NAME=`echo "results_autotests.xml" | sed -e "s@results_autotests@results_autotests-$FUNCTION@"` + mv results_autotests.xml $NEW_NAME + fi + popd + + if [ $KEEP_VM_ALIVE -eq 0 ] + then + echo "############################################################" + echo "Destroying VM" + echo "############################################################" + uvt-kvm destroy $VM_NAME + ssh-keygen -R $VM_IP_ADDR + fi + rm -f $VM_CMDS + + echo "############################################################" + echo "Checking build status" + echo "############################################################" + + LOG_FILES=`ls $ARCHIVES_LOC/results_autotests*.xml` + NB_FOUND_FILES=0 + NB_RUNS=0 + NB_FAILURES=0 + + for FULLFILE in $LOG_FILES + do + TESTSUITES=`egrep "testsuite errors" $FULLFILE` + for TESTSUITE in $TESTSUITES + do + if [[ "$TESTSUITE" == *"tests="* ]] + then + RUNS=`echo $TESTSUITE | awk 'BEGIN{FS="="}{print $2}END{}' | sed -e "s@'@@g" ` + NB_RUNS=$((NB_RUNS + RUNS)) + fi + if [[ "$TESTSUITE" == *"failures="* ]] + then + FAILS=`echo $TESTSUITE | awk 'BEGIN{FS="="}{print $2}END{}' | sed -e "s@'@@g" ` + NB_FAILURES=$((NB_FAILURES + FAILS)) + fi + done + NB_FOUND_FILES=$((NB_FOUND_FILES + 1)) + done + + echo "NB_FOUND_FILES = $NB_FOUND_FILES" + echo "NB_RUNS = $NB_RUNS" + echo "NB_FAILURES = $NB_FAILURES" + + if [ $NB_FOUND_FILES -eq 0 ]; then STATUS=-1; fi + if [ $NB_RUNS -eq 0 ]; then STATUS=-1; fi + if [ $NB_FAILURES -ne 0 ]; then STATUS=-1; fi + + if [ $STATUS -eq 0 ] + then + echo "STATUS seems OK" + else + echo "STATUS failed?" + fi + +fi + +exit $STATUS diff --git a/ci-scripts/template.xsl b/ci-scripts/template.xsl new file mode 100644 index 0000000000..64fb23e4cd --- /dev/null +++ b/ci-scripts/template.xsl @@ -0,0 +1,61 @@ + + + + + + + +

TEMPLATE Results Summary

+ + + + + + + + + + + + + + + + + +
HostnameNb TestsFailuresTimestamp
+ + + + + + + +
+

Details

+ + + + + + + + + + + + + + + +
Test NameResultTime
+ + + + + +
+ + + + diff --git a/cmake_targets/autotests/run_exec_autotests.bash b/cmake_targets/autotests/run_exec_autotests.bash index ff38ae9d3c..6c5b9193b6 100755 --- a/cmake_targets/autotests/run_exec_autotests.bash +++ b/cmake_targets/autotests/run_exec_autotests.bash @@ -200,6 +200,7 @@ function test_compile() { #\param $14 -> tags to help identify the test case for readability in output xml file #\param $15 => password for the user to run certain commands as sudo #\param $16 => test config file params to be modified +#\param $17 => bypass flag if main_exec if available function test_compile_and_run() { xUnit_start @@ -221,6 +222,7 @@ function test_compile_and_run() { tags=${14} mypassword=${15} test_config_file=${16} + bypass_compile=${17} build_dir=$tdir/$1/build #exec_file=$build_dir/$6 @@ -231,8 +233,6 @@ function test_compile_and_run() { rm -fr $log_dir mkdir -p $log_dir - rm -fr $OPENAIR_DIR/cmake_targets/log - echo "" > $temp_exec_log echo "" > $log_file #echo "log_dir = $log_dir" @@ -257,7 +257,13 @@ function test_compile_and_run() { main_exec_args_array=() readarray -t main_exec_args_array <<< "$exec_args" - + REAL_MAIN_EXEC=`eval "echo $main_exec"` + if [ "$bypass_compile" == "1" ] && [ -f $REAL_MAIN_EXEC ] + then + echo "Bypassing compilation for $main_exec" + else + rm -fr $OPENAIR_DIR/cmake_targets/log + #for search_expr in "${compile_prog_array[@]}" #do echo "Compiling test case $test_case_name Log file = $log_file" @@ -283,6 +289,7 @@ function test_compile_and_run() { }>> $log_file 2>&1 echo "" >> $log_file 2>&1 #done + fi #process the test case if it is that of execution if [ "$class" == "execution" ]; then @@ -393,10 +400,18 @@ Options Run test cases in a group. For example, ./run_exec_autotests "0101* 010102" -p Use password for logging +-np | --no-password + No need for a password +-q | --quiet + Quiet mode; eliminate informational messages and comment prompts. +-b | --bypass-compile + Bypass compilation of main-exec if already present ' } function main () { +QUIET=0 +BYPASS_COMPILE=0 RUN_GROUP=0 SET_PASSWORD=0 passwd="" @@ -419,6 +434,16 @@ until [ -z "$1" ] SET_PASSWORD=1 passwd=$2 shift 2;; + -np|--no-password) + SET_PASSWORD=1 + shift ;; + -q|--quiet) + QUIET=1 + shift ;; + -b|--bypass-compile) + BYPASS_COMPILE=1 + echo "bypass option ON" + shift ;; -h | --help) print_help exit 1;; @@ -449,15 +474,15 @@ xml_conf="$OPENAIR_DIR/cmake_targets/autotests/test_case_list.xml" test_case_list=`xmlstarlet sel -T -t -m /testCaseList/testCase -s A:N:- "@id" -v "@id" -n $xml_conf` test_case_excl_list=`xmlstarlet sel -t -v "/testCaseList/TestCaseExclusionList" $xml_conf` -echo "Test Case Exclusion List = $test_case_excl_list " +if [ $QUIET -eq 0 ]; then echo "Test Case Exclusion List = $test_case_excl_list "; fi test_case_excl_list=`sed "s/\+/\*/g" <<< "$test_case_excl_list" ` # Replace + with * for bash string substituion read -a test_case_excl_array <<< "$test_case_excl_list" -echo "test_case_list = $test_case_list" +if [ $QUIET -eq 0 ]; then echo "test_case_list = $test_case_list"; fi -echo "Test Case Exclusion List = $test_case_excl_list \n" +if [ $QUIET -eq 0 ]; then echo "Test Case Exclusion List = $test_case_excl_list \n"; fi readarray -t test_case_array <<<"$test_case_list" @@ -484,7 +509,7 @@ for search_expr in "${test_case_array[@]}" do if [[ $search_expr == $search_excl ]];then flag_run_test_case=0 - echo_info "Test case $search_expr match found in test case excl group. Will skip the test case for execution..." + if [ $QUIET -eq 0 ]; then echo_info "Test case $search_expr match found in test case excl group. Will skip the test case for execution..."; fi break fi done @@ -533,8 +558,8 @@ for search_expr in "${test_case_array[@]}" search_array_true=() - IFS=\" #set the shell's field separator - set -f #don't try to glob + IFS=\" #set the shell field separator + set -f #dont try to glob #set -- $search_expr_true #split on $IFS for i in $search_expr_true do echo "i = $i" @@ -551,7 +576,7 @@ for search_expr in "${test_case_array[@]}" test_compile "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" elif [ "$class" == "execution" ]; then echo \'passwd\' | $SUDO killall -q oaisim_nos1 - test_compile_and_run "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" "$mypassword" "$test_config_file" + test_compile_and_run "$name" "$compile_prog" "$compile_prog_args" "$pre_exec" "$pre_exec_args" "$main_exec" "$main_exec_args" "search_array_true[@]" "$search_expr_false" "$nruns" "$pre_compile_prog" "$class" "$compile_prog_out" "$tags" "$mypassword" "$test_config_file" "$BYPASS_COMPILE" else echo "Unexpected class of test case...Skipping the test case $name ...." fi diff --git a/cmake_targets/autotests/test_case_list.xml b/cmake_targets/autotests/test_case_list.xml index d9ed38a4ec..27f43f6f2f 100644 --- a/cmake_targets/autotests/test_case_list.xml +++ b/cmake_targets/autotests/test_case_list.xml @@ -955,7 +955,7 @@ --phy_simulators -c $OPENAIR_DIR/cmake_targets/autotests/tools/free_mem.bash - $OPENAIR_DIR/cmake_targets/lte-simulators/build/dlsim + $OPENAIR_DIR/targets/bin/dlsim.Rel14 -m5 -gF -s-1 -w1.0 -f.2 -n500 -B50 -c2 -z2 -O70 -m4 -gF -s0 -w1.0 -f.2 -n500 -B6 -c4 -z2 -O70 -m15 -gF -s6.7 -w1.0 -f.2 -n500 -B50 -c2 -z2 -O70 @@ -989,7 +989,7 @@ --phy_simulators -c $OPENAIR_DIR/cmake_targets/autotests/tools/free_mem.bash - $OPENAIR_DIR/cmake_targets/lte-simulators/build/ulsim + $OPENAIR_DIR/targets/bin/ulsim.Rel14 -B25 -m5 -y1 -gN -x1 -s6 -w1.0 -e.1 -P -n500 -O70 -L -B25 -m16 -y1 -gN -x1 -s12 -w1.0 -e.1 -P -n500 -O70 -L -B50 -m5 -y1 -gN -x1 -s6 -w1.0 -e.1 -P -n500 -O70 -L