From 8393744c3f26c85f6549a590339216c560e28c1b Mon Sep 17 00:00:00 2001 From: Cedric Roux Date: Wed, 14 Apr 2021 15:34:26 +0200 Subject: [PATCH 01/16] nr: bugfix: bad check 38.323 5.2.2.1 says: "if t-Reordering is running, and if RX_DELIV >= RX_REORD" Let's do as is said. --- openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c index 1822239171..83c005ac55 100644 --- a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c +++ b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c @@ -138,7 +138,7 @@ static void nr_pdcp_entity_recv_pdu(nr_pdcp_entity_t *entity, entity->rx_deliv = count; } - if (entity->t_reordering_start != 0 && entity->rx_deliv > entity->rx_reord) { + if (entity->t_reordering_start != 0 && entity->rx_deliv >= entity->rx_reord) { /* stop and reset t-Reordering */ entity->t_reordering_start = 0; } From 913f40228025b82b92a6071c1830ef3b2a3cc234 Mon Sep 17 00:00:00 2001 From: Cedric Roux Date: Wed, 14 Apr 2021 15:38:26 +0200 Subject: [PATCH 02/16] nr: pdcp: accept "infinity" for t-reordering "infinity" is -1 in the code. --- openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c | 4 ++++ openair2/LAYER2/nr_pdcp/nr_pdcp_entity.h | 4 ++-- openair2/LAYER2/nr_pdcp/nr_pdcp_oai_api.c | 8 +++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c index 83c005ac55..f8c4d70130 100644 --- a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c +++ b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c @@ -217,6 +217,10 @@ static void check_t_reordering(nr_pdcp_entity_t *entity) { uint32_t count; + /* if t_reordering is set to "infinity" (seen as -1) then do nothing */ + if (entity->t_reordering == -1) + return; + if (entity->t_reordering_start == 0 || entity->t_current <= entity->t_reordering_start + entity->t_reordering) return; diff --git a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.h b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.h index 4ae7a45abb..2d391eb3e0 100644 --- a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.h +++ b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.h @@ -55,8 +55,8 @@ typedef struct nr_pdcp_entity_t { int rb_id; int sn_size; /* SN size, in bits */ - int t_reordering; /* unit: ms */ - int discard_timer; /* unit: ms */ + int t_reordering; /* unit: ms, -1 for infinity */ + int discard_timer; /* unit: ms, -1 for infinity */ int sn_max; /* (2^SN_size) - 1 */ int window_size; /* 2^(SN_size - 1) */ diff --git a/openair2/LAYER2/nr_pdcp/nr_pdcp_oai_api.c b/openair2/LAYER2/nr_pdcp/nr_pdcp_oai_api.c index 39b770508e..c106d8e4fd 100644 --- a/openair2/LAYER2/nr_pdcp/nr_pdcp_oai_api.c +++ b/openair2/LAYER2/nr_pdcp/nr_pdcp_oai_api.c @@ -610,11 +610,17 @@ static void add_drb_am(int is_gnb, int rnti, struct NR_DRB_ToAddMod *s, nr_pdcp_ue_t *ue; int drb_id = s->drb_Identity; - int t_reordering = decode_t_reordering(*s->pdcp_Config->t_Reordering); + int t_reordering; int sn_size_ul = decode_sn_size_ul(*s->pdcp_Config->drb->pdcp_SN_SizeUL); int sn_size_dl = decode_sn_size_dl(*s->pdcp_Config->drb->pdcp_SN_SizeDL); int discard_timer = decode_discard_timer(*s->pdcp_Config->drb->discardTimer); + /* if pdcp_Config->t_Reordering is not present, it means infinity (-1) */ + if (s->pdcp_Config->t_Reordering != NULL) + t_reordering = decode_t_reordering(*s->pdcp_Config->t_Reordering); + else + t_reordering = -1; + /* TODO(?): accept different UL and DL SN sizes? */ if (sn_size_ul != sn_size_dl) { LOG_E(PDCP, "%s:%d:%s: fatal, bad SN sizes, must be same. ul=%d, dl=%d\n", From e257f76dd0b326e4415f6f2c6f29f1290283107a Mon Sep 17 00:00:00 2001 From: Cedric Roux Date: Wed, 14 Apr 2021 15:47:08 +0200 Subject: [PATCH 03/16] nr: rlc: minor: set log level to error for max retx reached Just in case there is a problem, better to see it. max retx reached means "radio link failure". We should do something about it at some point. For the moment, let's inform the user that there is a problem. We can set back to LOG_D later in time when we deal with it properly. --- openair2/LAYER2/nr_rlc/nr_rlc_oai_api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openair2/LAYER2/nr_rlc/nr_rlc_oai_api.c b/openair2/LAYER2/nr_rlc/nr_rlc_oai_api.c index 66553e813b..7cd1a87784 100644 --- a/openair2/LAYER2/nr_rlc/nr_rlc_oai_api.c +++ b/openair2/LAYER2/nr_rlc/nr_rlc_oai_api.c @@ -561,7 +561,7 @@ static void max_retx_reached(void *_ue, nr_rlc_entity_t *entity) exit(1); rb_found: - LOG_D(RLC, "max RETX reached on %s %d\n", + LOG_E(RLC, "max RETX reached on %s %d\n", is_srb ? "SRB" : "DRB", rb_id); From e26c1f6f6d726f49c593195b663e3f7e3f90a2b3 Mon Sep 17 00:00:00 2001 From: Cedric Roux Date: Sat, 22 May 2021 12:47:53 +0200 Subject: [PATCH 04/16] nr pdcp: bugfix: bad HFN computation at PDU reception --- openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c index 1822239171..91564f18b4 100644 --- a/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c +++ b/openair2/LAYER2/nr_pdcp/nr_pdcp_entity.c @@ -77,7 +77,7 @@ static void nr_pdcp_entity_recv_pdu(nr_pdcp_entity_t *entity, } rx_deliv_sn = entity->rx_deliv & entity->sn_max; - rx_deliv_hfn = (entity->rx_deliv >> entity->sn_size) & ~entity->sn_max; + rx_deliv_hfn = entity->rx_deliv >> entity->sn_size; if (rcvd_sn < rx_deliv_sn - entity->window_size) { rcvd_hfn = rx_deliv_hfn + 1; From 02cedc0ca16c9baceae368aacbb16cb2a5b70f7d Mon Sep 17 00:00:00 2001 From: Cedric Roux Date: Sat, 22 May 2021 12:49:05 +0200 Subject: [PATCH 05/16] nr pdcp: bugfix: better nettle version detection This fix may fail to work. we should stop supporting old versions of nettle at some point. --- openair2/LAYER2/nr_pdcp/nr_pdcp_security_nea2.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openair2/LAYER2/nr_pdcp/nr_pdcp_security_nea2.c b/openair2/LAYER2/nr_pdcp/nr_pdcp_security_nea2.c index fc07f661dd..52afe95673 100644 --- a/openair2/LAYER2/nr_pdcp/nr_pdcp_security_nea2.c +++ b/openair2/LAYER2/nr_pdcp/nr_pdcp_security_nea2.c @@ -27,12 +27,21 @@ #include #include +#ifndef NETTLE_VERSION_MAJOR +/* hack: include bignum.h, not version.h because version.h does not exist + * in old versions and bignum.h includes version.h (as of today). + * May completely fail to work... maybe we should skip support of old + * versions of nettle. + */ +#include +#endif + void *nr_pdcp_security_nea2_init(unsigned char *ciphering_key) { void *ctx = calloc(1, nettle_aes128.context_size); if (ctx == NULL) exit(1); -#if NETTLE_VERSION_MAJOR < 3 +#if !defined(NETTLE_VERSION_MAJOR) || NETTLE_VERSION_MAJOR < 3 nettle_aes128.set_encrypt_key(ctx, 16, ciphering_key); #else nettle_aes128.set_encrypt_key(ctx, ciphering_key); From 773f2a58e4c66362cf4ddfb36640cdd46d2e3563 Mon Sep 17 00:00:00 2001 From: Remi Hardy Date: Mon, 7 Jun 2021 22:52:33 +0200 Subject: [PATCH 06/16] add QLog traces and fix autoterminate for module --- ci-scripts/ci_ueinfra.yaml | 2 +- ci-scripts/cls_module_ue.py | 15 ++++- ci-scripts/cls_oaicitest.py | 83 +++++++++++++----------- ci-scripts/main.py | 14 ++-- ci-scripts/ran.py | 2 +- ci-scripts/xml_files/fr1_nsa_quectel.xml | 6 +- 6 files changed, 71 insertions(+), 51 deletions(-) diff --git a/ci-scripts/ci_ueinfra.yaml b/ci-scripts/ci_ueinfra.yaml index c778b1cad3..67331fdfb2 100644 --- a/ci-scripts/ci_ueinfra.yaml +++ b/ci-scripts/ci_ueinfra.yaml @@ -8,7 +8,7 @@ idefix: WakeupScript : ci_ctl_qtel.py /dev/ttyUSB2 wup DetachScript : ci_ctl_qtel.py /dev/ttyUSB2 detach PLMN : 22201 - UENetwork : enxb22a941dbf00 + UENetwork : wwan0 HostIPAddress : 192.168.18.188 HostUsername : oaicicd HostPassword : oaicicd diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index abfa8c1106..3130dbfa7d 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -126,7 +126,18 @@ class Module_UE: logging.debug('\u001B[1;37;41m Module IP Address Not Found! \u001B[0m') return -1 + def EnableTrace(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + #delete old artifacts + mySSH.command('echo ' + self.HostPassword + ' | sudo -S rm -rf ci_qlog','\$',5) + #start Trace + mySSH.command('echo $USER; nohup sudo -E QLog/QLog -s ci_qlog -f NR5G.cfg &','\$', 5) + mySSH.close() - - + def DisableTrace(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + mySSH.command('echo ' + self.HostPassword + ' | sudo -S killall --signal=SIGINT *QLog*', '\$',5) + mySSH.close() diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 7202e7c881..a9507b9c5d 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -389,17 +389,21 @@ class OaiCiTest(): is_module=Module_UE.CheckCMProcess() if is_module: Module_UE.Command("wup") + logging.debug("Waiting for IP address to be assigned") + time.sleep(20) + logging.debug("Retrieve IP address") status=Module_UE.GetModuleIPAddress() + Module_UE.EnableTrace() if status==0: HTML.CreateHtmlTestRow(Module_UE.UEIPAddress, 'OK', CONST.ALL_PROCESSES_OK) logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress) else: #status==-1 failed to retrieve IP address HTML.CreateHtmlTestRow('N/A', 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return - def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE): + def InitializeOAIUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '' or self.UESourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -623,7 +627,7 @@ class OaiCiTest(): HTML.htmlUEFailureMsg='nr-uesoftmodem did NOT synced' HTML.CreateHtmlTestRow(self.air_interface + ' ' + self.Initialize_OAI_UE_args, 'KO', CONST.OAI_UE_PROCESS_COULD_NOT_SYNC, 'OAI UE') logging.error('\033[91mInitialize OAI UE Failed! \033[0m') - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def checkDevTTYisUnlocked(self): SSH = sshconnection.SSHConnection() @@ -701,7 +705,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) self.checkDevTTYisUnlocked() - def AttachCatM(self,HTML,RAN,COTS_UE,EPC): + def AttachCatM(self,HTML,RAN,COTS_UE,EPC,InfraUE): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -774,9 +778,9 @@ class OaiCiTest(): html_cell = '
CAT-M module Attachment Failed
' html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue('N/A', 'KO', 1, html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) - def PingCatM(self,HTML,RAN,EPC,COTS_UE): + def PingCatM(self,HTML,RAN,EPC,COTS_UE,InfraUE): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -785,7 +789,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return try: statusQueue = SimpleQueue() @@ -806,7 +810,7 @@ class OaiCiTest(): moduleIPAddr = result.group('ipaddr') else: HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return ping_time = re.findall("-c (\d+)",str(self.ping_args)) device_id = 'catm' @@ -870,7 +874,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', 1, statusQueue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', 1, statusQueue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) except: os.kill(os.getppid(),signal.SIGUSR1) @@ -962,7 +966,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def AttachUE(self,HTML,RAN,EPC,COTS_UE): + def AttachUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.ue_id=='':#no ID specified, then it is a COTS controlled by ADB if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -972,7 +976,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return multi_jobs = [] status_queue = SimpleQueue() @@ -991,7 +995,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return else: attach_status = True @@ -1020,7 +1024,7 @@ class OaiCiTest(): time.sleep(5) else: HTML.CreateHtmlTestRowQueue('N/A', 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def DetachUE_common(self, device_id, idx,COTS_UE): try: @@ -1054,7 +1058,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow('N/A', 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return multi_jobs = [] cnt = 0 @@ -1079,6 +1083,7 @@ class OaiCiTest(): else:#if an ID is specified, it is a module from the yaml infrastructure file Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) Module_UE.Command("detach") + Module_UE.DisableTrace() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) @@ -1329,7 +1334,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE): + def CheckStatusUE(self,HTML,RAN,EPC,COTS_UE,InfraUE): if self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') @@ -1376,7 +1381,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(htmlOptions, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: check_status = True html_queue = SimpleQueue() @@ -1392,7 +1397,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(htmlOptions, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(htmlOptions, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def GetAllUEIPAddresses(self): SSH = sshconnection.SSHConnection() @@ -1612,14 +1617,14 @@ class OaiCiTest(): html_queue.put(html_cell) HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - def PingNoS1(self,HTML,RAN,EPC,COTS_UE): + def PingNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE): SSH=sshconnection.SSHConnection() check_eNB = True check_OAI_UE = True pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return ping_from_eNB = re.search('oaitun_enb1', str(self.ping_args)) if ping_from_eNB is not None: @@ -1708,7 +1713,7 @@ class OaiCiTest(): def Ping(self,HTML,RAN,EPC,COTS_UE, InfraUE): result = re.search('noS1', str(RAN.Initialize_eNB_args)) if result is not None: - self.PingNoS1(HTML,RAN,EPC,COTS_UE) + self.PingNoS1(HTML,RAN,EPC,COTS_UE,InfraUE) return if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -1721,7 +1726,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return if self.ue_id=="": @@ -1729,7 +1734,7 @@ class OaiCiTest(): ueIpStatus = self.GetAllUEIPAddresses() if (ueIpStatus < 0): HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return else: self.UEIPAddresses=[] @@ -1756,7 +1761,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(self.ping_args, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: ping_status = True html_queue = SimpleQueue() @@ -1773,7 +1778,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.ping_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def Iperf_ComputeTime(self): result = re.search('-t (?P\d+)', str(self.iperf_args)) @@ -2467,7 +2472,7 @@ class OaiCiTest(): except: os.kill(os.getppid(),signal.SIGUSR1) - def IperfNoS1(self,HTML,RAN,EPC,COTS_UE): + def IperfNoS1(self,HTML,RAN,EPC,COTS_UE,InfraUE): SSH = sshconnection.SSHConnection() if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or self.UEIPAddress == '' or self.UEUserName == '' or self.UEPassword == '': HELP.GenericHelp(CONST.Version) @@ -2477,7 +2482,7 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return server_on_enb = re.search('-R', str(self.iperf_args)) if server_on_enb is not None: @@ -2576,12 +2581,12 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def Iperf(self,HTML,RAN,EPC,COTS_UE, InfraUE): result = re.search('noS1', str(RAN.Initialize_eNB_args)) if result is not None: - self.IperfNoS1(HTML,RAN,EPC,COTS_UE) + self.IperfNoS1(HTML,RAN,EPC,COTS_UE,InfraUE) return if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.SourceCodePath == '' or self.ADBIPAddress == '' or self.ADBUserName == '' or self.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -2594,14 +2599,14 @@ class OaiCiTest(): pStatus = self.CheckProcessExist(check_eNB, check_OAI_UE,RAN,EPC) if (pStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', pStatus) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return if self.ue_id=="":#is not a module, follow legacy code ueIpStatus = self.GetAllUEIPAddresses() if (ueIpStatus < 0): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.UE_IP_ADDRESS_ISSUE) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) return else: #is a module self.UEIPAddresses=[] @@ -2646,7 +2651,7 @@ class OaiCiTest(): if (status_queue.empty()): HTML.CreateHtmlTestRow(self.iperf_args, 'KO', CONST.ALL_PROCESSES_OK) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfaUE) else: iperf_status = True iperf_noperf = False @@ -2668,7 +2673,7 @@ class OaiCiTest(): HTML.CreateHtmlTestRowQueue(self.iperf_args, 'OK', len(self.UEDevices), html_queue) else: HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', len(self.UEDevices), html_queue) - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) def CheckProcessExist(self, check_eNB, check_OAI_UE,RAN,EPC): multi_jobs = [] @@ -3088,7 +3093,7 @@ class OaiCiTest(): job.join() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC): + def TerminateOAIUE(self,HTML,RAN,COTS_UE,EPC, InfraUE): SSH = sshconnection.SSHConnection() SSH.open(self.UEIPAddress, self.UEUserName, self.UEPassword) SSH.command('cd ' + self.UESourceCodePath + '/cmake_targets', '\$', 5) @@ -3129,11 +3134,11 @@ class OaiCiTest(): # Not an error then if (logStatus != CONST.OAI_UE_PROCESS_COULD_NOT_SYNC) or (ueAction != 'Sniffing'): self.Initialize_OAI_UE_args = '' - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: if (logStatus == CONST.OAI_UE_PROCESS_COULD_NOT_SYNC): self.Initialize_OAI_UE_args = '' - self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC) + self.AutoTerminateUEandeNB(HTML,RAN,COTS_UE,EPC,InfraUE) else: logging.debug('\u001B[1m' + ueAction + ' Completed \u001B[0m') HTML.htmlUEFailureMsg='' + ueAction + ' Completed\n' + HTML.htmlUEFailureMsg @@ -3142,7 +3147,7 @@ class OaiCiTest(): else: HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC): + def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC,InfraUE): if (self.ADBIPAddress != 'none'): self.testCase_id = 'AUTO-KILL-UE' HTML.testCase_id=self.testCase_id @@ -3156,7 +3161,7 @@ class OaiCiTest(): self.desc = 'Automatic Termination of OAI-UE' HTML.desc='Automatic Termination of OAI-UE' self.ShowTestID() - self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + self.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) if (RAN.Initialize_eNB_args != ''): self.testCase_id = 'AUTO-KILL-RAN' HTML.testCase_id=self.testCase_id @@ -3176,6 +3181,10 @@ class OaiCiTest(): HTML.desc='Automatic Termination of FlexRan CTL' self.ShowTestID() self.TerminateFlexranCtrl(HTML,RAN,EPC) + if self.ue_id!='': + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.Command("detach") + Module_UE.DisableTrace() RAN.prematureExit=True def IdleSleep(self,HTML): diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 3f5b57e016..f99229053d 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -474,7 +474,7 @@ elif re.match('^TerminateOAIUE$', mode, re.IGNORECASE): HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') signal.signal(signal.SIGUSR1, receive_signal) - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) elif re.match('^TerminateHSS$', mode, re.IGNORECASE): if EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '': HELP.GenericHelp(CONST.Version) @@ -717,7 +717,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'Terminate_UE': CiTestObj.TerminateUE(HTML,COTS_UE) elif action == 'Attach_UE': - CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.AttachUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Detach_UE': CiTestObj.DetachUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'DataDisable_UE': @@ -725,23 +725,23 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re elif action == 'DataEnable_UE': CiTestObj.DataEnableUE(HTML) elif action == 'CheckStatusUE': - CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.CheckStatusUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Build_OAI_UE': CiTestObj.BuildOAIUE(HTML) elif action == 'Initialize_OAI_UE': - CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE) + CiTestObj.InitializeOAIUE(HTML,RAN,EPC,COTS_UE,InfraUE) elif action == 'Terminate_OAI_UE': - CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC) + CiTestObj.TerminateOAIUE(HTML,RAN,COTS_UE,EPC,InfraUE) elif action == 'Initialize_CatM_module': CiTestObj.InitializeCatM(HTML) elif action == 'Terminate_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Attach_CatM_module': - CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC) + CiTestObj.AttachCatM(HTML,RAN,COTS_UE,EPC,InfraUE) elif action == 'Detach_CatM_module': CiTestObj.TerminateCatM(HTML) elif action == 'Ping_CatM_module': - CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC) + CiTestObj.PingCatM(HTML,RAN,EPC,COTS_UE,EPC,InfraUE) elif action == 'Ping': CiTestObj.Ping(HTML,RAN,EPC,COTS_UE, InfraUE) elif action == 'Iperf': diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 83945b3c17..66c8392662 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -388,7 +388,7 @@ class RANManagement(): # do not reset board twice in IF4.5 case result = re.search('^rru|^enb|^du.band', str(config_file)) if result is not None: - mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) + mySSH.command2('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) result = re.search('type: b200', mySSH.getBefore()) if result is not None: logging.debug('Found a B2xx device --> resetting it') diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml index 6c2bd6bd26..8992221608 100644 --- a/ci-scripts/xml_files/fr1_nsa_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml @@ -24,7 +24,7 @@ TEST-NSA-FR1-TM1 NSA Ping DL UL with QUECTEL tasks - 3 + 1 030000 040000 @@ -61,7 +61,7 @@ Initialize_eNB Initialize eNB - -O ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf + -O ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf --T_stdout 0 0 0 lte @@ -71,7 +71,7 @@ Initialize_eNB Initialize gNB - -O ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpn310.conf -q + -O ci-scripts/conf_files/gnb.band78.tm1.fr1.106PRB.usrpb210.conf -E -q 1 1 nr From 97d097347687d0caf1f6ccc2386a3a01f5a85af6 Mon Sep 17 00:00:00 2001 From: Remi Hardy Date: Tue, 8 Jun 2021 17:38:40 +0200 Subject: [PATCH 07/16] update module mgt and log collect for nsa test --- ci-scripts/cls_module_ue.py | 10 +++++++++- ci-scripts/cls_oaicitest.py | 14 +++++++++----- ci-scripts/ran.py | 20 ++++++++++++++++++-- ci-scripts/xml_files/fr1_nsa_quectel.xml | 8 ++------ 4 files changed, 38 insertions(+), 14 deletions(-) diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index 3130dbfa7d..e3ea863b21 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -101,7 +101,7 @@ class Module_UE: def GetModuleIPAddress(self): HOST=self.HostIPAddress response= [] - tentative = 10 + tentative = 3 while (len(response)==0) and (tentative>0): COMMAND="ip a show dev " + self.UENetwork + " | grep inet | grep " + self.UENetwork logging.debug(COMMAND) @@ -141,3 +141,11 @@ class Module_UE: mySSH.command('echo ' + self.HostPassword + ' | sudo -S killall --signal=SIGINT *QLog*', '\$',5) mySSH.close() + + def DisableCM(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + mySSH.command('echo ' + self.HostPassword + ' | sudo -S killall --signal SIGKILL *'+self.Process['Name']+'*', '\$', 5) + mySSH.close() + + diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index a9507b9c5d..31230a3df3 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -388,12 +388,13 @@ class OaiCiTest(): Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) is_module=Module_UE.CheckCMProcess() if is_module: + Module_UE.EnableTrace() + time.sleep(5) Module_UE.Command("wup") logging.debug("Waiting for IP address to be assigned") time.sleep(20) logging.debug("Retrieve IP address") status=Module_UE.GetModuleIPAddress() - Module_UE.EnableTrace() if status==0: HTML.CreateHtmlTestRow(Module_UE.UEIPAddress, 'OK', CONST.ALL_PROCESSES_OK) logging.debug('UE IP addresss : '+ Module_UE.UEIPAddress) @@ -1084,6 +1085,7 @@ class OaiCiTest(): Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) Module_UE.Command("detach") Module_UE.DisableTrace() + Module_UE.DisableCM() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) @@ -3148,6 +3150,12 @@ class OaiCiTest(): HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) def AutoTerminateUEandeNB(self,HTML,RAN,COTS_UE,EPC,InfraUE): + if self.ue_id!='': + Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) + Module_UE.Command("detach") + Module_UE.DisableTrace() + Module_UE.DisableCM() + if (self.ADBIPAddress != 'none'): self.testCase_id = 'AUTO-KILL-UE' HTML.testCase_id=self.testCase_id @@ -3181,10 +3189,6 @@ class OaiCiTest(): HTML.desc='Automatic Termination of FlexRan CTL' self.ShowTestID() self.TerminateFlexranCtrl(HTML,RAN,EPC) - if self.ue_id!='': - Module_UE = cls_module_ue.Module_UE(InfraUE.ci_ue_infra[self.ue_id]) - Module_UE.Command("detach") - Module_UE.DisableTrace() RAN.prematureExit=True def IdleSleep(self,HTML): diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 66c8392662..b7675cfdac 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -344,6 +344,21 @@ class RANManagement(): HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus) HTML.CreateHtmlTabFooter(False) sys.exit(1) + #Get pcap on S1 and X2 interface always + if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): + mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5) + result = re.search('interfaceToUse=(?P[a-zA-Z0-9\-\_]+)done', mySSH.getBefore()) + if result is not None: + eth_interface = result.group('eth_interface') + logging.debug('\u001B[1m Launching tshark on interface ' + eth_interface + '\u001B[0m') + pcapfile = 'enb_' + self.testCase_id + '_s1x2log.pcap' + mySSH.command('echo ' + lPassWord + ' | sudo -S rm -f /tmp/' + pcapfile , '\$', 5) + mySSH.command('echo $USER; nohup sudo -E tshark -i ' + eth_interface + ' -w /tmp/' + pcapfile + ' 2>&1 &','\$', 5) + mySSH.close() + + + # If tracer options is on, running tshark on EPC side and capture traffic b/ EPC and eNB result = re.search('T_stdout', str(self.Initialize_eNB_args)) if (result is not None): @@ -650,9 +665,10 @@ class RANManagement(): mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) + mySSH.command('cp /tmp/enb_*_s1x2log.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log', '\$', 60) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt', '\$', 5) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 60) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) mySSH.close() def AnalyzeLogFile_eNB(self, eNBlogFile, HTML): diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml index 8992221608..7ac7b990db 100644 --- a/ci-scripts/xml_files/fr1_nsa_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml @@ -30,15 +30,11 @@ 040000 000002 010000 + 000001 050000 050001 000001 - 070000 - 000001 - 070000 - 000001 010002 - 000001 080001 080000 @@ -61,7 +57,7 @@ Initialize_eNB Initialize eNB - -O ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf --T_stdout 0 + -O ci-scripts/conf_files/enb.band7.tm1.fr1.25PRB.usrpb210.conf 0 0 lte From 19aadc07bbd0107a51304c70e4a94fb048c89401 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 9 Jun 2021 09:04:59 +0200 Subject: [PATCH 08/16] no longer need 3 resources reservation --- ci-scripts/Jenkinsfile-tmp-multi-enb-nsa | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa index 66d12eed33..1e4933838b 100644 --- a/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa +++ b/ci-scripts/Jenkinsfile-tmp-multi-enb-nsa @@ -33,10 +33,9 @@ def buildStageStatus = true // Name of the test stage def testStageName = params.pipelineTestStageName -// Name of the phone resource +// Name of the phone/server resource def ciSmartPhonesResource1 = params.SmartPhonesResource1 def ciSmartPhonesResource2 = params.SmartPhonesResource2 -def ciSmartPhonesResource3 = params.SmartPhonesResource3 // Global Parameters. Normally they should be populated when the master job // triggers the slave job with parameters @@ -53,7 +52,7 @@ pipeline { options { disableConcurrentBuilds() ansiColor('xterm') - lock(extra: [[resource: ciSmartPhonesResource2],[resource: ciSmartPhonesResource3]], resource: ciSmartPhonesResource1) + lock(extra: [[resource: ciSmartPhonesResource2]], resource: ciSmartPhonesResource1) } stages { stage("Build Init") { @@ -85,9 +84,6 @@ pipeline { if (params.SmartPhonesResource2 == null) { allParametersPresent = false } - if (params.SmartPhonesResource3== null) { - allParametersPresent = false - } // 1st eNB parameters if (params.eNB_IPAddress == null) { allParametersPresent = false From 4e238b87004c4c1932aa0bae6c88f637c7778c4a Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 9 Jun 2021 10:25:29 +0200 Subject: [PATCH 09/16] fixing copy of pcap artifacts --- ci-scripts/ran.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index b7675cfdac..2108fb7c8b 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -665,7 +665,7 @@ class RANManagement(): mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) - mySSH.command('cp /tmp/enb_*_s1x2log.pcap .','\$',20) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S cp /tmp/enb_*_s1x2log.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 60) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) From 19f5c7baa20ebc91a01aa152a2ec645a5061e481 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 9 Jun 2021 12:23:05 +0200 Subject: [PATCH 10/16] nsa test for usrp update --- ci-scripts/xml_files/fr1_nsa_quectel.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml index 7ac7b990db..1deb81dd27 100644 --- a/ci-scripts/xml_files/fr1_nsa_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml @@ -34,6 +34,10 @@ 050000 050001 000001 + 070000 + 000001 + 070001 + 000001 010002 080001 080000 From 688cb7ae4e3529cb1c3230e0b3ca82b48bb4e03b Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 9 Jun 2021 14:18:36 +0200 Subject: [PATCH 11/16] Qlog archive mgt --- ci-scripts/cls_module_ue.py | 13 ++++++++++++- ci-scripts/cls_oaicitest.py | 6 ++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index e3ea863b21..bc79b87c66 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -37,7 +37,7 @@ import time import re import subprocess - +from datetime import datetime class Module_UE: @@ -149,3 +149,14 @@ class Module_UE: mySSH.close() + def LogCollect(self): + mySSH = sshconnection.SSHConnection() + mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) + #archive qlog to /opt/ci_qlogs with datetime suffix + now=datetime.now() + now_string = now.strftime("%Y/%m/%d-%H:%M") + source='ci_qlog' + destination='/opt/ci_qlogs/ci_qlog_'+now_string + mySSH.command('echo ' + self.HostPassword + ' | sudo -S cp -r '+source+' '+destination,'\$', 20) + mySSH.close() + return destination diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index 31230a3df3..809af4ebec 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -1086,7 +1086,8 @@ class OaiCiTest(): Module_UE.Command("detach") Module_UE.DisableTrace() Module_UE.DisableCM() - HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + archive_destination=Module_UE.LogCollect() + HTML.CreateHtmlTestRow('QLog at : '+archive_destination, 'OK', CONST.ALL_PROCESSES_OK) @@ -3155,7 +3156,8 @@ class OaiCiTest(): Module_UE.Command("detach") Module_UE.DisableTrace() Module_UE.DisableCM() - + archive_destination=Module_UE.LogCollect() + HTML.CreateHtmlTestRow('QLog at : '+archive_destination, 'OK', CONST.ALL_PROCESSES_OK) if (self.ADBIPAddress != 'none'): self.testCase_id = 'AUTO-KILL-UE' HTML.testCase_id=self.testCase_id From 2c2d8bbf62673c4518fea5edc174e29b862838be Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 9 Jun 2021 15:51:07 +0200 Subject: [PATCH 12/16] fixing tshark termination ; zip of QLog file --- ci-scripts/cls_module_ue.py | 6 +++--- ci-scripts/ran.py | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_module_ue.py b/ci-scripts/cls_module_ue.py index bc79b87c66..f93cb9f036 100644 --- a/ci-scripts/cls_module_ue.py +++ b/ci-scripts/cls_module_ue.py @@ -154,9 +154,9 @@ class Module_UE: mySSH.open(self.HostIPAddress, self.HostUsername, self.HostPassword) #archive qlog to /opt/ci_qlogs with datetime suffix now=datetime.now() - now_string = now.strftime("%Y/%m/%d-%H:%M") + now_string = now.strftime("%Y%m%d-%H%M") source='ci_qlog' - destination='/opt/ci_qlogs/ci_qlog_'+now_string - mySSH.command('echo ' + self.HostPassword + ' | sudo -S cp -r '+source+' '+destination,'\$', 20) + destination='/opt/ci_qlogs/ci_qlog_'+now_string+'.zip' + mySSH.command('echo $USER; echo ' + self.HostPassword + ' | nohup sudo -S zip -r '+destination+' '+source+' &','\$', 10) mySSH.close() return destination diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 2108fb7c8b..5fb7dd39e8 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -594,6 +594,10 @@ class RANManagement(): mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5) time.sleep(5) mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) + #stopping tshark on eNB side + logging.debug('\u001B[1m Stopping tshark \u001B[0m') + mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) + time.sleep(1) mySSH.close() # If tracer options is on, stopping tshark on EPC side result = re.search('T_stdout', str(self.Initialize_eNB_args)) From 2f84dc6bd0dc6bdfbd46a1b9594cf5cab412bc85 Mon Sep 17 00:00:00 2001 From: hardy Date: Wed, 9 Jun 2021 21:45:50 +0200 Subject: [PATCH 13/16] making enb pcap generation optional in xml --- ci-scripts/main.py | 1 + ci-scripts/ran.py | 10 ++++++---- ci-scripts/xml_files/fr1_nsa_quectel.xml | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci-scripts/main.py b/ci-scripts/main.py index f99229053d..cbd323366b 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -152,6 +152,7 @@ def GetParametersFromXML(action): RAN.eNB_serverId[RAN.eNB_instance]=eNB_serverId elif action == 'Initialize_eNB': + RAN.eNB_Trace=test.findtext('eNB_Trace') RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args') eNB_instance=test.findtext('eNB_instance') if (eNB_instance is None): diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 5fb7dd39e8..4f12dee5e5 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -93,6 +93,7 @@ class RANManagement(): self.epcPcapFile = '' self.runtime_stats= '' self.datalog_rt_stats={} + self.eNB_Trace = '' #if 'yes', Tshark will be launched at initialization @@ -344,8 +345,9 @@ class RANManagement(): HTML.CreateHtmlTestRow(self.air_interface[self.eNB_instance] + ' ' + self.Initialize_eNB_args, 'KO', self.pStatus) HTML.CreateHtmlTabFooter(False) sys.exit(1) - #Get pcap on S1 and X2 interface always - if (self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb'): + #Get pcap on S1 and X2 eNB interface, if enabled in the xml + #will not work for gNB at this stage + if ((self.air_interface[self.eNB_instance] == 'lte-softmodem') or (self.air_interface[self.eNB_instance] == 'ocp-enb')) and self.eNB_Trace=='yes': mySSH.open(lIpAddr, lUserName, lPassWord) mySSH.command('ip addr show | awk -f /tmp/active_net_interfaces.awk | egrep -v "lo|tun"', '\$', 5) result = re.search('interfaceToUse=(?P[a-zA-Z0-9\-\_]+)done', mySSH.getBefore()) @@ -594,7 +596,7 @@ class RANManagement(): mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL -r .*-softmodem ocp-enb || true', '\$', 5) time.sleep(5) mySSH.command('rm -f my-lte-softmodem-run' + str(self.eNB_instance) + '.sh', '\$', 5) - #stopping tshark on eNB side + #stopping tshark (valid if eNB and enabled in xml, will not harm otherwise) logging.debug('\u001B[1m Stopping tshark \u001B[0m') mySSH.command('echo ' + lPassWord + ' | sudo -S killall --signal SIGKILL tshark', '\$', 5) time.sleep(1) @@ -669,7 +671,7 @@ class RANManagement(): mySSH.open(self.eNBIPAddress, self.eNBUserName, self.eNBPassword) mySSH.command('cd ' + self.eNBSourceCodePath, '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) - mySSH.command('echo ' + self.eNBPassword + ' | sudo -S cp /tmp/enb_*_s1x2log.pcap .','\$',20) + mySSH.command('echo ' + self.eNBPassword + ' | sudo -S mv /tmp/enb_*_s1x2log.pcap .','\$',20) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm -f enb.log.zip', '\$', 5) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S zip enb.log.zip enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 60) mySSH.command('echo ' + self.eNBPassword + ' | sudo -S rm enb*.log core* enb_*record.raw enb_*.pcap enb_*txt physim_*.log *stats.log', '\$', 5) diff --git a/ci-scripts/xml_files/fr1_nsa_quectel.xml b/ci-scripts/xml_files/fr1_nsa_quectel.xml index 1deb81dd27..fbaa32aea7 100644 --- a/ci-scripts/xml_files/fr1_nsa_quectel.xml +++ b/ci-scripts/xml_files/fr1_nsa_quectel.xml @@ -65,6 +65,7 @@ 0 0 lte + yes From cd25915cd029fdb545ade12dee05a50c3bf0b687 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 10 Jun 2021 13:18:35 +0200 Subject: [PATCH 14/16] new html log to detect errors (removing ue) --- ci-scripts/ran.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 4f12dee5e5..20a8867922 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -729,6 +729,8 @@ class RANManagement(): real_time_stats = {} #count "problem receiving samples" msg pb_receiving_samples_cnt = 0 + #count "removing UE" msg + removing_ue = 0 #NSA specific log markers nsa_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': []} @@ -925,6 +927,10 @@ class RANManagement(): result = re.search('\[PHY\]\s+problem receiving samples', str(line)) if result is not None: pb_receiving_samples_cnt += 1 + #count "Removing UE" msg + result = re.search('\[MAC\]\s+Removing UE', str(line)) + if result is not None: + removing_ue += 1 #nsa markers logging for k in nsa_markers: @@ -1014,6 +1020,11 @@ class RANManagement(): htmleNBFailureMsg += statMsg else: + #Removing UE log + statMsg = '[MAC] Removing UE msg count = '+str(removing_ue) + htmlMsg = statMsg+'\n' + logging.debug(statMsg) + htmleNBFailureMsg += htmlMsg #nsa markers statMsg = 'logfile line count = ' + str(line_cnt) htmlMsg = statMsg+'\n' From dafa4487900abf905df96aef3948264942d9c474 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Thu, 10 Jun 2021 16:37:56 +0200 Subject: [PATCH 15/16] CI: fix command2, killing record process and building w/ ninja Signed-off-by: Raphael Defosseux --- ci-scripts/ran.py | 3 ++- ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml | 2 +- ci-scripts/xml_files/enb_usrp210_band13_build.xml | 2 +- ci-scripts/xml_files/enb_usrp210_band40_build.xml | 2 +- ci-scripts/xml_files/enb_usrp210_band7_build.xml | 2 +- ci-scripts/xml_files/if4p5_usrp210_band40_build.xml | 2 +- ci-scripts/xml_files/if4p5_usrp210_band7_build.xml | 2 +- ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml | 2 +- 8 files changed, 9 insertions(+), 8 deletions(-) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 20a8867922..470572f2ff 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -405,7 +405,7 @@ class RANManagement(): # do not reset board twice in IF4.5 case result = re.search('^rru|^enb|^du.band', str(config_file)) if result is not None: - mySSH.command2('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) + mySSH.command('echo ' + lPassWord + ' | sudo -S uhd_find_devices', '\$', 90) result = re.search('type: b200', mySSH.getBefore()) if result is not None: logging.debug('Found a B2xx device --> resetting it') @@ -618,6 +618,7 @@ class RANManagement(): mySSH.close() logging.debug('\u001B[1m Replaying RAW record file\u001B[0m') mySSH.open(lIpAddr, lUserName, lPassWord) + mySSH.command('killall --signal SIGKILL record', '\$', 5) mySSH.command('cd ' + lSourcePath + '/common/utils/T/tracer/', '\$', 5) enbLogFile = self.eNBLogFiles[int(self.eNB_instance)] raw_record_file = enbLogFile.replace('.log', '_record.raw') diff --git a/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml b/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml index 64c3517166..bfaaceaa2d 100644 --- a/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml +++ b/ci-scripts/xml_files/enb_ue_usrp210_band7_build.xml @@ -32,7 +32,7 @@ Build_eNB Build eNB (USRP) - -w USRP -c --eNB + -w USRP -c --eNB --ninja diff --git a/ci-scripts/xml_files/enb_usrp210_band13_build.xml b/ci-scripts/xml_files/enb_usrp210_band13_build.xml index 888a1230ae..53066573c8 100644 --- a/ci-scripts/xml_files/enb_usrp210_band13_build.xml +++ b/ci-scripts/xml_files/enb_usrp210_band13_build.xml @@ -33,7 +33,7 @@ Build_eNB Build eNB (USRP) - -w USRP -c --eNB + -w USRP -c --eNB --ninja diff --git a/ci-scripts/xml_files/enb_usrp210_band40_build.xml b/ci-scripts/xml_files/enb_usrp210_band40_build.xml index 64c3517166..bfaaceaa2d 100644 --- a/ci-scripts/xml_files/enb_usrp210_band40_build.xml +++ b/ci-scripts/xml_files/enb_usrp210_band40_build.xml @@ -32,7 +32,7 @@ Build_eNB Build eNB (USRP) - -w USRP -c --eNB + -w USRP -c --eNB --ninja diff --git a/ci-scripts/xml_files/enb_usrp210_band7_build.xml b/ci-scripts/xml_files/enb_usrp210_band7_build.xml index 64c3517166..bfaaceaa2d 100644 --- a/ci-scripts/xml_files/enb_usrp210_band7_build.xml +++ b/ci-scripts/xml_files/enb_usrp210_band7_build.xml @@ -32,7 +32,7 @@ Build_eNB Build eNB (USRP) - -w USRP -c --eNB + -w USRP -c --eNB --ninja diff --git a/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml b/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml index ab221c918d..d383dbb61b 100644 --- a/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml +++ b/ci-scripts/xml_files/if4p5_usrp210_band40_build.xml @@ -33,7 +33,7 @@ Build_eNB Build eNB (USRP -- Ethernet Fronthaul) - -w USRP -c --eNB + -w USRP -c --eNB --ninja diff --git a/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml b/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml index ab221c918d..d383dbb61b 100644 --- a/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml +++ b/ci-scripts/xml_files/if4p5_usrp210_band7_build.xml @@ -33,7 +33,7 @@ Build_eNB Build eNB (USRP -- Ethernet Fronthaul) - -w USRP -c --eNB + -w USRP -c --eNB --ninja diff --git a/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml b/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml index c5a912a33d..0ac6bfcc0d 100644 --- a/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml +++ b/ci-scripts/xml_files/x2ho_enb_usrp210_band13_build.xml @@ -42,7 +42,7 @@ Build_eNB Build Slave eNB (USRP) - -w USRP -c --eNB + -w USRP -c --eNB --ninja 1 1 From 328664b4d607ee341e16a73de23f4698046cf6f1 Mon Sep 17 00:00:00 2001 From: hardy Date: Thu, 10 Jun 2021 21:55:28 +0200 Subject: [PATCH 16/16] remove benetel from nsa trigger (temporarily) --- ci-scripts/Jenkinsfile-trig-nsa | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ci-scripts/Jenkinsfile-trig-nsa b/ci-scripts/Jenkinsfile-trig-nsa index b9172225ca..8f623ffb05 100644 --- a/ci-scripts/Jenkinsfile-trig-nsa +++ b/ci-scripts/Jenkinsfile-trig-nsa @@ -36,7 +36,7 @@ pipeline { stage ("Launcher") { steps { script { - //retrieve MR that are opened nd with tag READY_TO_BE_MERGED + //retrieve MR that are opened nd with tag NSA MR_LIST= sh returnStdout: true, script: 'curl --silent "https://gitlab.eurecom.fr/api/v4/projects/oai%2Fopenairinterface5g/merge_requests?state=opened&per_page=100&labels=NSA" | jq ".[].iid" || true ' echo "List of selected MR:\n${MR_LIST}" def MR_ARRAY = MR_LIST.split('\n') @@ -48,7 +48,7 @@ pipeline { COMMIT_ID=COMMIT_ID.trim() echo "Testing NSA on : ${MR} ${SRC_BRANCH} ${COMMIT_ID}" //calling NSA sub job - build job: "RAN-CI-NSA-B210-N310-ModuleUE", wait : false, propagate : false, parameters: [ + build job: "RAN-NSA-Mini-Module", wait : false, propagate : false, parameters: [ string(name: 'eNB_MR', value: String.valueOf(MR)), string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)), string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)), @@ -56,13 +56,13 @@ pipeline { booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE)) ] //calling Benetel sub job - build job: "RAN-CI-BENETEL", wait : false, propagate : false, parameters: [ - string(name: 'eNB_MR', value: String.valueOf(MR)), - string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)), - string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)), - string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)), - booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE)) - ] + //build job: "RAN-CI-BENETEL", wait : false, propagate : false, parameters: [ + // string(name: 'eNB_MR', value: String.valueOf(MR)), + // string(name: 'eNB_Branch', value: String.valueOf(SRC_BRANCH)), + // string(name: 'eNB_CommitID', value: String.valueOf(COMMIT_ID)), + // string(name: 'eNB_TargetBranch', value: String.valueOf(TARGET_BRANCH)), + // booleanParam(name: 'eNB_AllowMergeRequestProcess', value: Boolean.valueOf(ALLOW_MERGE)) + //] } } }