Merge branch 'develop' into dlsch_scheduler_opti

2019-01-28 08:36:21 +01:00
parent b5fb496882 5e35d90974
commit ed1401ac00
4 changed files with 212 additions and 11 deletions
@@ -41,7 +41,7 @@ pipeline {
        disableConcurrentBuilds()
        timestamps()
        gitLabConnection('OAI GitLab')
-        gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim", "Build phy-sim", "Build eNb-ethernet", "Build UE-ethernet", "Analysis with cppcheck", "Test phy-sim", "Test basic-sim", "Test-FDD-Band7", "Test-TDD-Band40", "Test-IF4p5-FDD-Band7", "Test-IF4p5-TDD-Band40"])
+        gitlabBuilds(builds: ["Build eNb-USRP", "Build basic-sim", "Build phy-sim", "Build eNb-ethernet", "Build UE-ethernet", "Analysis with cppcheck", "Test phy-sim", "Test basic-sim", "Test L2-sim", "Test-FDD-Band7", "Test-TDD-Band40", "Test-IF4p5-FDD-Band7", "Test-IF4p5-TDD-Band40"])
        ansiColor('xterm')
    }

@@ -332,11 +332,11 @@ pipeline {
                }
                stage ("Test L2 simulator") {
                    steps {
-                        //gitlabCommitStatus(name: "Test basic-sim") {
+                        gitlabCommitStatus(name: "Test L2-sim") {
                            timeout (time: 30, unit: 'MINUTES') {
                                sh "./ci-scripts/oai-ci-vm-tool test --workspace $WORKSPACE --variant l2-sim --job-name ${JOB_NAME} --build-id ${BUILD_ID}"
                            }
-                        //}
+                        }
                    }
                }
                stage ("Build Flexran Controller") {
@@ -534,7 +534,7 @@ pipeline {
                always {
                    script {
                        dir ('archives') {
-                            sh "if [ -d basic_sim/test ] || [ -d phy_sim/test ]; then zip -r -qq vm_tests_logs.zip */test ; fi"
+                            sh "if [ -d basic_sim/test ] || [ -d phy_sim/test ] || [ -d l2_sim/test ]; then zip -r -qq vm_tests_logs.zip */test ; fi"
                        }
                        if(fileExists('archives/vm_tests_logs.zip')) {
                            archiveArtifacts artifacts: 'archives/vm_tests_logs.zip'
@@ -417,6 +417,130 @@ function report_test {
        echo "   </div>" >> ./test_simulator_results.html
    fi

+    ARCHIVES_LOC=archives/l2_sim/test
+    if [ -d $ARCHIVES_LOC ]
+    then
+        echo "   <h3>L2-NFAPI Simulator Check</h3>" >> ./test_simulator_results.html
+
+        if [ -f $ARCHIVES_LOC/test_final_status.log ]
+        then
+            if [ `grep -c TEST_OK $ARCHIVES_LOC/test_final_status.log` -eq 1 ]
+            then
+                echo "   <div class=\"alert alert-success\">" >> ./test_simulator_results.html
+                echo "      <strong>TEST was SUCCESSFUL <span class=\"glyphicon glyphicon-ok-circle\"></span></strong>" >> ./test_simulator_results.html
+                echo "   </div>" >> ./test_simulator_results.html
+            else
+                echo "   <div class=\"alert alert-danger\">" >> ./test_simulator_results.html
+                echo "      <strong>TEST was a FAILURE! <span class=\"glyphicon glyphicon-ban-circle\"></span></strong>" >> ./test_simulator_results.html
+                echo "   </div>" >> ./test_simulator_results.html
+            fi
+        else
+            echo "   <div class=\"alert alert-danger\">" >> ./test_simulator_results.html
+            echo "      <strong>COULD NOT DETERMINE TEST FINAL STATUS! <span class=\"glyphicon glyphicon-ban-circle\"></span></strong>" >> ./test_simulator_results.html
+            echo "   </div>" >> ./test_simulator_results.html
+        fi
+
+        echo "   <button data-toggle=\"collapse\" data-target=\"#oai-l2-sim-test-details\">More details on L2-NFAPI Simulator test results</button>" >> ./test_simulator_results.html
+        echo "   <div id=\"oai-l2-sim-test-details\" class=\"collapse\">" >> ./test_simulator_results.html
+        echo "   <table border = \"1\">" >> ./test_simulator_results.html
+        echo "      <tr bgcolor = \"#33CCFF\" >" >> ./test_simulator_results.html
+        echo "        <th>Log File Name</th>" >> ./test_simulator_results.html
+        echo "        <th>Command</th>" >> ./test_simulator_results.html
+        echo "        <th>Status</th>" >> ./test_simulator_results.html
+        echo "        <th>Statistics</th>" >> ./test_simulator_results.html
+        echo "      </tr>" >> ./test_simulator_results.html
+
+        TRANS_MODES=("fdd")
+        BW_CASES=(05)
+        for TMODE in ${TRANS_MODES[@]}
+        do
+            for BW in ${BW_CASES[@]}
+            do
+                ENB_LOG=$ARCHIVES_LOC/${TMODE}_${BW}MHz_enb.log
+                UE_LOG=`echo $ENB_LOG | sed -e "s#enb#ue#"`
+                if [ -f $ENB_LOG ] && [ -f $UE_LOG ]
+                then
+                    NAME_ENB=`echo $ENB_LOG | sed -e "s#$ARCHIVES_LOC/##"`
+                    NAME_UE=`echo $UE_LOG | sed -e "s#$ARCHIVES_LOC/##"`
+                    echo "      <tr>" >> ./test_simulator_results.html
+                    echo "        <td>$NAME_ENB --- $NAME_UE</td>" >> ./test_simulator_results.html
+                    echo "        <td>N/A</td>" >> ./test_simulator_results.html
+                    NB_ENB_GOT_SYNC=`egrep -c "got sync" $ENB_LOG`
+                    NB_UE_GOT_SYNC=`egrep -c "got sync" $UE_LOG`
+                    NB_ENB_SYNCED_WITH_UE=`egrep -c "Sending NFAPI_START_RESPONSE" $UE_LOG`
+                    if [ $NB_ENB_GOT_SYNC -eq 1 ] && [ $NB_UE_GOT_SYNC -eq 3 ] && [ $NB_ENB_SYNCED_WITH_UE -eq 1 ]
+                    then
+                        echo "        <td bgcolor = \"green\" >OK</td>" >> ./test_simulator_results.html
+                    else
+                        echo "        <td bgcolor = \"red\" >KO</td>" >> ./test_simulator_results.html
+                    fi
+                    echo "        <td><pre>" >> ./test_simulator_results.html
+                    if [ $NB_ENB_GOT_SYNC -eq 1 ]
+                    then
+                        echo "<font color = \"blue\">- eNB --> got sync</font>" >> ./test_simulator_results.html
+                    else
+                        echo "<font color = \"red\"><b>- eNB NEVER got sync</b></font>" >> ./test_simulator_results.html
+                    fi
+                    if [ $NB_UE_GOT_SYNC -eq 3 ]
+                    then
+                        echo "<font color = \"blue\">- UE --> got sync</font>" >> ./test_simulator_results.html
+                    else
+                        echo "<font color = \"red\"><b>- UE NEVER got sync</b></font>" >> ./test_simulator_results.html
+                    fi
+                    if [ $NB_ENB_SYNCED_WITH_UE -eq 1 ]
+                    then
+                        echo "<font color = \"blue\">- UE attached to eNB</font>" >> ./test_simulator_results.html
+                    else
+                        echo "<font color = \"red\"><b>- UE NEVER attached to eNB</b></font>" >> ./test_simulator_results.html
+                    fi
+                    echo "        </pre></td>" >> ./test_simulator_results.html
+                    echo "      </tr>" >> ./test_simulator_results.html
+                fi
+                PING_CASE=$ARCHIVES_LOC/${TMODE}_${BW}MHz_ping_epc.txt
+                if [ -f $PING_CASE ]
+                then
+                    echo "      <tr>" >> ./test_simulator_results.html
+                    NAME=`echo $PING_CASE | sed -e "s#$ARCHIVES_LOC/##"`
+                    echo "        <td>$NAME</td>" >> ./test_simulator_results.html
+                    CMD=`egrep "COMMAND IS" $PING_CASE | sed -e "s#COMMAND IS: ##"`
+                    echo "        <td>$CMD</td>" >> ./test_simulator_results.html
+                    FILE_COMPLETE=`egrep -c "ping statistics" $PING_CASE`
+                    if [ $FILE_COMPLETE -eq 0 ]
+                    then
+                        echo "        <td bgcolor = \"red\" >KO</td>" >> ./test_simulator_results.html
+                        echo "        <td>N/A</td>" >> ./test_simulator_results.html
+                    else
+                        NB_TR_PACKETS=`egrep "packets transmitted" $PING_CASE | sed -e "s# packets transmitted.*##"`
+                        NB_RC_PACKETS=`egrep "packets transmitted" $PING_CASE | sed -e "s#^.*packets transmitted, ##" -e "s# received,.*##"`
+                        if [ $NB_TR_PACKETS -eq $NB_RC_PACKETS ]
+                        then
+                            echo "        <td bgcolor = \"green\" >OK</td>" >> ./test_simulator_results.html
+                        else
+                            echo "        <td bgcolor = \"red\" >KO</td>" >> ./test_simulator_results.html
+                        fi
+                        echo "        <td>" >> ./test_simulator_results.html
+                        echo "            <pre>" >> ./test_simulator_results.html
+                        STATS=`egrep "packets transmitted" $PING_CASE | sed -e "s#^.*received, ##" -e "s#, time.*##" -e "s# packet loss##"`
+                        echo "Packet Loss : $STATS" >> ./test_simulator_results.html
+                        RTTMIN=`egrep "rtt min" $PING_CASE | awk '{split($4,a,"/"); print a[1] " " $5}'`
+                        echo "RTT Minimal : $RTTMIN" >> ./test_simulator_results.html
+                        RTTAVG=`egrep "rtt min" $PING_CASE | awk '{split($4,a,"/"); print a[2] " " $5}'`
+                        echo "RTT Average : $RTTAVG" >> ./test_simulator_results.html
+                        RTTMAX=`egrep "rtt min" $PING_CASE | awk '{split($4,a,"/"); print a[3] " " $5}'`
+                        echo "RTT Maximal : $RTTMAX" >> ./test_simulator_results.html
+                        echo "            </pre>" >> ./test_simulator_results.html
+                        echo "        </td>" >> ./test_simulator_results.html
+                    fi
+                    echo "      </tr>" >> ./test_simulator_results.html
+                fi
+
+            done
+        done
+
+        echo "   </table>" >> ./test_simulator_results.html
+        echo "   </div>" >> ./test_simulator_results.html
+    fi
+
    ARCHIVES_LOC=archives/phy_sim/test
    if [ -d $ARCHIVES_LOC ]
    then
@@ -148,6 +148,15 @@ function ping_ue_ip_addr {
    rm -f $1
 }

+function ping_epc_ip_addr {
+    echo "echo \"COMMAND IS: ping -I oip1 -c 20 $3\" > $4" > $1
+    echo "rm -f $4" >> $1
+    echo "ping -I oip1 -c 20 $3 | tee -a $4" >> $1
+    cat $1
+    ssh -o StrictHostKeyChecking=no ubuntu@$2 < $1
+    rm -f $1
+}
+
 function check_ping_result {
    local LOC_PING_FILE=$1
    local LOC_NB_PINGS=$2
@@ -157,16 +166,19 @@ function check_ping_result {
        if [ $FILE_COMPLETE -eq 0 ]
        then
            PING_STATUS=-1
+            echo "ping file incomplete"
        else
            local ALL_PACKET_RECEIVED=`egrep -c "$LOC_NB_PINGS received" $LOC_PING_FILE`
            if [ $ALL_PACKET_RECEIVED -eq 1 ]
            then
                echo "got all ping packets"
            else
+                echo "got NOT all ping packets"
                PING_STATUS=-1
            fi
        fi
    else
+        echo "ping file not present"
        PING_STATUS=-1
    fi
 }
@@ -533,8 +545,30 @@ function start_l2_sim_enb {
    echo "sudo -E daemon --inherit --unsafe --name=enb_daemon --chdir=/home/ubuntu/tmp/cmake_targets/lte_build_oai/build/ -o /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE ./my-lte-softmodem-run.sh" >> $1

    ssh -o StrictHostKeyChecking=no ubuntu@$LOC_VM_IP_ADDR < $1
-    sleep 10
    rm $1
+
+    local i="0"
+    echo "egrep -c \"Waiting for PHY_config_req\" /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE" > $1
+    while [ $i -lt 10 ]
+    do
+        sleep 5
+        CONNECTED=`ssh -o StrictHostKeyChecking=no ubuntu@$LOC_VM_IP_ADDR < $1`
+        if [ $CONNECTED -ne 0 ]
+        then
+            i="100"
+        else
+            i=$[$i+1]
+        fi
+    done
+    rm $1
+    if [ $i -lt 50 ]
+    then
+        ENB_SYNC=0
+        echo "L2-SIM eNB is NOT sync'ed: process still alive?"
+    else
+        ENB_SYNC=1
+        echo "L2-SIM eNB is sync'ed: waiting for UE(s) to connect"
+    fi
 }

 function start_l2_sim_ue {
@@ -551,8 +585,31 @@ function start_l2_sim_ue {
    echo "sudo -E daemon --inherit --unsafe --name=ue_daemon --chdir=/home/ubuntu/tmp-ue/cmake_targets/lte_build_oai/build/ -o /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE ./my-lte-softmodem-run.sh" >> $1

    ssh -o StrictHostKeyChecking=no ubuntu@$LOC_VM_IP_ADDR < $1
-    sleep 10
    rm $1
+
+    local i="0"
+    echo "egrep -c \"Received NFAPI_START_REQ\" /home/ubuntu/tmp/cmake_targets/log/$LOC_LOG_FILE" > $1
+    while [ $i -lt 10 ]
+    do
+        sleep 5
+        CONNECTED=`ssh -o StrictHostKeyChecking=no ubuntu@$LOC_VM_IP_ADDR < $1`
+        if [ $CONNECTED -eq 1 ]
+        then
+            i="100"
+        else
+            i=$[$i+1]
+        fi
+    done
+    rm $1
+    if [ $i -lt 50 ]
+    then
+        UE_SYNC=0
+        echo "L2-SIM UE is NOT sync'ed w/eNB"
+    else
+        UE_SYNC=1
+        echo "L2-SIM UE is sync'ed w/eNB"
+    fi
+    sleep 10
 }

 function run_test_on_vm {
@@ -1197,10 +1254,24 @@ function run_test_on_vm {
        echo "############################################################"
        CURRENT_UE_LOG_FILE=fdd_05MHz_ue.log
        start_l2_sim_ue $VM_CMDS $VM_IP_ADDR $CURRENT_UE_LOG_FILE ue.nfapi.conf
+        if [ $UE_SYNC -eq 0 ]
+        then
+            echo "Problem w/ eNB and UE not syncing"
+            terminate_enb_ue_basic_sim $VM_CMDS $VM_IP_ADDR
+            scp -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/$CURRENT_ENB_LOG_FILE $ARCHIVES_LOC
+            scp -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/tmp/cmake_targets/log/$CURRENT_UE_LOG_FILE $ARCHIVES_LOC
+            terminate_epc $EPC_VM_CMDS $EPC_VM_IP_ADDR
+            echo "TEST_KO" > $ARCHIVES_LOC/test_final_status.log
+            exit -1
+        fi

-        sleep 30
-        echo "ping -I oip1 -c 20 $REAL_EPC_IP_ADDR" > $VM_CMDS
-        ssh -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR < $VM_CMDS
+        echo "############################################################"
+        echo "Pinging the EPC from UE"
+        echo "############################################################"
+        PING_LOG_FILE=fdd_05MHz_ping_epc.txt
+        ping_epc_ip_addr $VM_CMDS $VM_IP_ADDR $REAL_EPC_IP_ADDR $PING_LOG_FILE
+        scp -o StrictHostKeyChecking=no ubuntu@$VM_IP_ADDR:/home/ubuntu/$PING_LOG_FILE $ARCHIVES_LOC
+        check_ping_result $ARCHIVES_LOC/$PING_LOG_FILE 20

        echo "############################################################"
        echo "Terminate enb/ue simulators"
@@ -1226,6 +1297,11 @@ function run_test_on_vm {
            ssh-keygen -R $EPC_VM_IP_ADDR
        fi

+        echo "############################################################"
+        echo "Checking run status"
+        echo "############################################################"
+
+        if [ $PING_STATUS -ne 0 ]; then STATUS=-1; fi
        if [ $STATUS -eq 0 ]
        then
            echo "TEST_OK" > $ARCHIVES_LOC/test_final_status.log
@@ -1237,11 +1237,11 @@ int pnf_connect(pnf_t* pnf)
 	}

 	struct addrinfo *p = servinfo;
-	int i = 0;
 	int connected = 0;

 	while(p != NULL && connected == 0)
 	{
+#ifdef NFAPI_TRACE_ENABLED
 		char* family = "Unknown";
 		char* address = "Unknown";
 		char _addr[128];
@@ -1260,7 +1260,8 @@ int pnf_connect(pnf_t* pnf)
 			address = inet_ntoa(addr->sin_addr);
 		}

-		//NFAPI_TRACE(NFAPI_TRACE_NOTE, "Host address info  %d Family:%s Address:%s\n", i++, family, address);
+		NFAPI_TRACE(NFAPI_TRACE_NOTE, "Host address info  %d Family:%s Address:%s\n", i++, family, address);
+#endif

 		if (pnf->sctp)
 		{