From 6da8869ed4f0c85da281610fbe8434921e47fe82 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 14 Oct 2022 11:46:12 +0200 Subject: [PATCH 01/12] feat(ci): adding push / pull images operations to/from a local registry Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 143 ++++++++++++++++++ ci-scripts/cls_log_mgt.py | 5 - ci-scripts/cls_oai_html.py | 6 +- ci-scripts/cls_oaicitest.py | 12 +- ci-scripts/main.py | 77 +++++++--- ci-scripts/sshconnection.py | 52 +++---- ci-scripts/xml_class_list.yml | 3 + .../xml_files/container_image_build.xml | 7 + 8 files changed, 241 insertions(+), 64 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 5cf6b76b38..1b756b84c3 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -208,6 +208,7 @@ class Containerize(): self.imageToCopy = '' self.registrySvrId = '' self.testSvrId = '' + self.imageToPull = [] #checkers from xml self.ran_checkers={} @@ -616,6 +617,148 @@ class Containerize(): HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True + def ImageTagToUse(self, imageName): + shortCommit = self.ranCommitID[0:8] + if self.ranAllowMerge: + tagToUse = f'{self.ranBranch}-{shortCommit}' + else: + tagToUse = f'develop-{shortCommit}' + fullTag = f'porcepix.sboai.cs.eurecom.fr/{imageName}:{tagToUse}' + return fullTag + + def Push_Image_to_Local_Registry(self, HTML): + if self.registrySvrId == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.registrySvrId == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.registrySvrId == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Pushing images from server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + + mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Login Succeeded', mySSH.getBefore()) is None: + logging.error('Could not log into local registry') + return False + + orgTag = 'develop' + if self.ranAllowMerge: + orgTag = 'ci-temp' + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + mySSH.command(f'docker image tag {image}:{orgTag} {tagToUse}', '\$', 5) + mySSH.command(f'docker push {tagToUse}', '\$', 120) + if re.search(': digest:', mySSH.getBefore()) is None: + logging.debug(mySSH.getBefore()) + logging.error(f'Could not push {image} to local registry : {tagToUse}') + return False + mySSH.command(f'docker rmi {tagToUse}', '\$', 5) + + mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Removing login credentials', mySSH.getBefore()) is None: + logging.error('Could not log off from local registry') + return False + + mySSH.close() + return True + + def Pull_Image_from_Local_Registry(self, HTML): + if self.testSvrId == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.testSvrId == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.testSvrId == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Pulling images onto server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + + mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Login Succeeded', mySSH.getBefore()) is None: + logging.error('Could not log into local registry') + return False + + orgTag = 'develop' + if self.ranAllowMerge: + orgTag = 'ci-temp' + + for image in self.imageToPull: + tagToUse = self.ImageTagToUse(image) + mySSH.command(f'docker pull {tagToUse}', '\$', 120) + if re.search('Status: Downloaded newer image for |Status: Image is up to date for', mySSH.getBefore()) is None: + logging.debug(mySSH.getBefore()) + logging.error(f'Could not pull {image} from local registry : {tagToUse}') + return False + + mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Removing login credentials', mySSH.getBefore()) is None: + logging.error('Could not log off from local registry') + return False + + mySSH.close() + return True + + def Clean_Test_Server_Images(self, HTML): + if self.testSvrId == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.testSvrId == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.testSvrId == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Removing test images from server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + + orgTag = 'develop' + if self.ranAllowMerge: + orgTag = 'ci-temp' + tagToUse = self.ImageTagToUse(self.imageToPull) + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + mySSH.command(f'docker rmi {tagToUse} || true', '\$', 5) + + mySSH.close() + return True + def DeployObject(self, HTML, EPC): if self.eNB_serverId[self.eNB_instance] == '0': lIpAddr = self.eNBIPAddress diff --git a/ci-scripts/cls_log_mgt.py b/ci-scripts/cls_log_mgt.py index f1dbf6b8ec..a78eee5a33 100644 --- a/ci-scripts/cls_log_mgt.py +++ b/ci-scripts/cls_log_mgt.py @@ -35,11 +35,6 @@ import re import subprocess import sshconnection -logging.basicConfig( - level=logging.DEBUG, - format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s" -) - class Log_Mgt: def __init__(self,Username, IPAddress,Password,Path): diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py index 536f2ff49e..b7514172dc 100644 --- a/ci-scripts/cls_oai_html.py +++ b/ci-scripts/cls_oai_html.py @@ -102,9 +102,9 @@ class HTMLManagement(): def CreateHtmlHeader(self, ADBIPAddress): if (not self.htmlHeaderCreated): - logging.debug('\u001B[1m----------------------------------------\u001B[0m') - logging.debug('\u001B[1m Creating HTML header \u001B[0m') - logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m Creating HTML header \u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') self.htmlFile = open('test_results.html', 'w') self.htmlFile.write('\n') self.htmlFile.write('\n') diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index d3f4191c02..f16df2cb97 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -44,10 +44,6 @@ import datetime import signal import statistics as stat from multiprocessing import Process, Lock, SimpleQueue -logging.basicConfig( - level=logging.DEBUG, - format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s" -) #import our libs import helpreadme as HELP @@ -3937,7 +3933,7 @@ class OaiCiTest(): sys.exit(1) def ShowTestID(self): - logging.debug('\u001B[1m----------------------------------------\u001B[0m') - logging.debug('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m') - logging.debug('\u001B[1m' + self.desc + '\u001B[0m') - logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1mTest ID:' + self.testCase_id + '\u001B[0m') + logging.info('\u001B[1m' + self.desc + '\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') diff --git a/ci-scripts/main.py b/ci-scripts/main.py index eb9e60a83e..9c7bab4efd 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -70,7 +70,7 @@ import subprocess from multiprocessing import Process, Lock, SimpleQueue logging.basicConfig( level=logging.DEBUG, - format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s" + format="[%(asctime)s] %(levelname)8s: %(message)s" ) @@ -84,7 +84,7 @@ logging.basicConfig( def CheckClassValidity(xml_class_list,action,id): if action not in xml_class_list: - logging.debug('ERROR: test-case ' + id + ' has unlisted class ' + action + ' ##CHECK xml_class_list.yml') + logging.error('test-case ' + id + ' has unlisted class ' + action + ' ##CHECK xml_class_list.yml') resp=False else: resp=True @@ -328,14 +328,14 @@ def GetParametersFromXML(action): CiTestObj.iperf_profile = 'balanced' else: if CiTestObj.iperf_profile != 'balanced' and CiTestObj.iperf_profile != 'unbalanced' and CiTestObj.iperf_profile != 'single-ue': - logging.debug('ERROR: test-case has wrong profile ' + CiTestObj.iperf_profile) + logging.error('test-case has wrong profile ' + CiTestObj.iperf_profile) CiTestObj.iperf_profile = 'balanced' CiTestObj.iperf_options = test.findtext('iperf_options') if (CiTestObj.iperf_options is None): CiTestObj.iperf_options = 'check' else: if CiTestObj.iperf_options != 'check' and CiTestObj.iperf_options != 'sink': - logging.debug('ERROR: test-case has wrong option ' + CiTestObj.iperf_options) + logging.error('test-case has wrong option ' + CiTestObj.iperf_options) CiTestObj.iperf_options = 'check' elif action == 'IdleSleep': @@ -464,8 +464,27 @@ def GetParametersFromXML(action): elif action == 'Cppcheck_Analysis': pass + elif action == 'Push_Local_Registry': + string_field = test.findtext('registry_svr_id') + if (string_field is not None): + CONTAINERS.registrySvrId = string_field + + elif action == 'Pull_Local_Registry': + string_field = test.findtext('test_svr_id') + if (string_field is not None): + CONTAINERS.testSvrId = string_field + for image in test.findall('image_to_pull'): + string_field = image.findtext('image_name') + if (string_field is not None): + CONTAINERS.imageToPull.append(string_field) + + elif action == 'Clean_Test_Server_Images': + string_field = test.findtext('test_svr_id') + if (string_field is not None): + CONTAINERS.testSvrId = string_field + else: - logging.error(f"unknown action {action}") + logging.warning(f"unknown action {action} from option-parsing point-of-view") #check if given test is in list @@ -617,7 +636,7 @@ elif re.match('^LogCollecteNB$', mode, re.IGNORECASE): sys.exit('Insufficient Parameter') if RAN.eNBIPAddress == 'none': cmd = 'zip -r enb.log.' + RAN.BuildId + '.zip cmake_targets/log' - logging.debug(cmd) + logging.info(cmd) zipStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=60) sys.exit(0) RAN.LogCollecteNB() @@ -683,17 +702,17 @@ elif re.match('^InitiateHtml$', mode, re.IGNORECASE): HTML.htmlNb_CATM_Modules=len(CiTestObj.CatMDevices) HTML.CreateHtmlHeader(CiTestObj.ADBIPAddress) elif re.match('^FinalizeHtml$', mode, re.IGNORECASE): - logging.debug('\u001B[1m----------------------------------------\u001B[0m') - logging.debug('\u001B[1m Creating HTML footer \u001B[0m') - logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m Creating HTML footer \u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') CiTestObj.RetrieveSystemVersion('eNB',HTML,RAN) CiTestObj.RetrieveSystemVersion('UE',HTML,RAN) HTML.CreateHtmlFooter(CiTestObj.finalStatus) elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re.IGNORECASE): - logging.debug('\u001B[1m----------------------------------------\u001B[0m') - logging.debug('\u001B[1m Starting Scenario: ' + CiTestObj.testXMLfiles[0] + '\u001B[0m') - logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m Starting Scenario: ' + CiTestObj.testXMLfiles[0] + '\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') if re.match('^TesteNB$', mode, re.IGNORECASE): if RAN.eNBIPAddress == '' or RAN.ranRepository == '' or RAN.ranBranch == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '' or EPC.IPAddress == '' or EPC.UserName == '' or EPC.Password == '' or EPC.Type == '' or EPC.SourceCodePath == '' or CiTestObj.ADBIPAddress == '' or CiTestObj.ADBUserName == '' or CiTestObj.ADBPassword == '': HELP.GenericHelp(CONST.Version) @@ -736,7 +755,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML.testUnstable = True CiTestObj.testMinStableId = xmlRoot.findtext('TestMinId',default='999999') HTML.testMinStableId = CiTestObj.testMinStableId - logging.debug('Test is tagged as Unstable -- starting from TestID ' + str(CiTestObj.testMinStableId)) + logging.warning('Test is tagged as Unstable -- starting from TestID ' + str(CiTestObj.testMinStableId)) all_tests=xmlRoot.findall('testCase') exclusion_tests=exclusion_tests.split() @@ -747,10 +766,10 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re for test in exclusion_tests: if (not re.match('^[0-9]{6}$', test) and not re.match('^[0-9]{1,5}\+$', test)): - logging.debug('ERROR: exclusion test is invalidly formatted: ' + test) + logging.error('exclusion test is invalidly formatted: ' + test) sys.exit(1) else: - logging.debug(test) + logging.info(test) #check that requested tests are well formatted #(6 digits or less than 6 digits followed by +) @@ -758,9 +777,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re for test in requested_tests: if (re.match('^[0-9]{6}$', test) or re.match('^[0-9]{1,5}\+$', test)): - logging.debug('INFO: test group/case requested: ' + test) + logging.info('test group/case requested: ' + test) else: - logging.debug('ERROR: requested test is invalidly formatted: ' + test) + logging.error('requested test is invalidly formatted: ' + test) sys.exit(1) if (EPC.IPAddress != '') and (EPC.IPAddress != 'none'): CiTestObj.CheckFlexranCtrlInstallation(RAN,EPC,CONTAINERS) @@ -771,9 +790,9 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re todo_tests=[] for test in requested_tests: if (test_in_list(test, exclusion_tests)): - logging.debug('INFO: test will be skipped: ' + test) + logging.info('test will be skipped: ' + test) else: - #logging.debug('INFO: test will be run: ' + test) + #logging.info('test will be run: ' + test) todo_tests.append(test) signal.signal(signal.SIGUSR1, receive_signal) @@ -941,6 +960,18 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re success = CONTAINERS.Copy_Image_to_Test_Server(HTML) if not success: RAN.prematureExit = True + elif action == 'Push_Local_Registry': + success = CONTAINERS.Push_Image_to_Local_Registry(HTML) + if not success: + RAN.prematureExit = True + elif action == 'Pull_Local_Registry': + success = CONTAINERS.Pull_Image_from_Local_Registry(HTML) + if not success: + RAN.prematureExit = True + elif action == 'Clean_Test_Server_Images': + success = CONTAINERS.Clean_Test_Server_Images(HTML) + if not success: + RAN.prematureExit = True elif action == 'Deploy_Object': CONTAINERS.DeployObject(HTML, EPC) if CONTAINERS.exitStatus==1: @@ -973,19 +1004,21 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re RAN.prematureExit = True elif action == 'StatsFromGenObject': CONTAINERS.StatsFromGenObject(HTML) + elif action == 'Push_Images_To_Test_Servers': + logging.debug('To be implemented') else: sys.exit('Invalid class (action) from xml') if RAN.prematureExit: if CiTestObj.testCase_id == CiTestObj.testMinStableId: - logging.debug('Scenario has reached minimal stability point') + logging.warning('Scenario has reached minimal stability point') CiTestObj.testStabilityPointReached = True HTML.testStabilityPointReached = True CiTestObj.FailReportCnt += 1 if CiTestObj.FailReportCnt == CiTestObj.repeatCounts[0] and RAN.prematureExit: - logging.debug('Scenario failed ' + str(CiTestObj.FailReportCnt) + ' time(s)') + logging.error('Scenario failed ' + str(CiTestObj.FailReportCnt) + ' time(s)') HTML.CreateHtmlTabFooter(False) if CiTestObj.testUnstable and (CiTestObj.testStabilityPointReached or CiTestObj.testMinStableId == '999999'): - logging.debug('Scenario has reached minimal stability point -- Not a Failure') + logging.warning('Scenario has reached minimal stability point -- Not a Failure') else: sys.exit('Failed Scenario') else: diff --git a/ci-scripts/sshconnection.py b/ci-scripts/sshconnection.py index 0946ce0f34..7c9991119e 100644 --- a/ci-scripts/sshconnection.py +++ b/ci-scripts/sshconnection.py @@ -74,7 +74,7 @@ class SSHConnection(): count = 10 connect_status = True else: - logging.debug('self.sshresponse = ' + str(self.sshresponse)) + logging.warning('self.sshresponse = ' + str(self.sshresponse)) elif self.sshresponse == 1: self.ssh.sendline(password) self.sshresponse = self.ssh.expect([prompt, 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT]) @@ -82,7 +82,7 @@ class SSHConnection(): count = 10 connect_status = True else: - logging.debug('self.sshresponse = ' + str(self.sshresponse)) + logging.warning('self.sshresponse = ' + str(self.sshresponse)) elif self.sshresponse == 2: # We directly ended up on the remote server because of pubkey auth count = 10 @@ -91,8 +91,8 @@ class SSHConnection(): self.sshresponse = self.ssh.expect([prompt]) else: # debug output - logging.debug(str(self.ssh.before)) - logging.debug('self.sshresponse = ' + str(self.sshresponse)) + logging.warning(str(self.ssh.before)) + logging.warning('self.sshresponse = ' + str(self.sshresponse)) # adding a tempo when failure if not connect_status: time.sleep(1) @@ -108,7 +108,7 @@ class SSHConnection(): def cde_check_value(self, commandline, expected, timeout): - logging.debug(commandline) + logging.info(commandline) self.ssh.timeout = timeout self.ssh.sendline(commandline) expected.append(pexpect.EOF) @@ -118,7 +118,7 @@ class SSHConnection(): def command(self, commandline, expectedline, timeout, silent=False, resync=False): if not silent: - logging.debug(commandline) + logging.info(commandline) self.ssh.timeout = timeout # Nasty patch when pexpect output is out of sync. # Much pronounced when running back-to-back-back oc commands @@ -133,27 +133,27 @@ class SSHConnection(): if self.sshresponse == 0: return 0 elif self.sshresponse == 1: - logging.debug('\u001B[1;37;41m Unexpected EOF \u001B[0m') - logging.debug('Expected Line : ' + expectedline) - logging.debug(str(self.ssh.before)) + logging.error('\u001B[1;37;41m Unexpected EOF \u001B[0m') + logging.error('Expected Line : ' + expectedline) + logging.error(str(self.ssh.before)) sys.exit(self.sshresponse) elif self.sshresponse == 2: - logging.debug('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m') - logging.debug('Expected Line : ' + expectedline) + logging.error('\u001B[1;37;41m Unexpected TIMEOUT \u001B[0m') + logging.error('Expected Line : ' + expectedline) result = re.search('ping |iperf |picocom', str(commandline)) if result is None: - logging.debug(str(self.ssh.before)) + logging.warning(str(self.ssh.before)) sys.exit(self.sshresponse) else: return -1 else: - logging.debug('\u001B[1;37;41m Unexpected Others \u001B[0m') - logging.debug('Expected Line : ' + expectedline) + logging.error('\u001B[1;37;41m Unexpected Others \u001B[0m') + logging.error('Expected Line : ' + expectedline) sys.exit(self.sshresponse) def command2(self, commandline, timeout, silent=False): if not silent: - logging.debug(commandline) + logging.info(commandline) self.cmd2Results = '' noHistoryCmd = 'unset HISTFILE; ' + commandline myHost = self.username + '@' + self.ipaddress @@ -165,7 +165,7 @@ class SSHConnection(): def command3(self, commandline, timeout, silent=False): if not silent: - logging.debug(commandline) + logging.info(commandline) self.cmd2Results = '' noHistoryCmd = 'unset HISTFILE; ' + commandline myHost = self.username + '@' + self.ipaddress @@ -186,14 +186,14 @@ class SSHConnection(): pass elif self.sshresponse == 1: if not self.picocom_closure: - logging.debug('\u001B[1;37;41m Unexpected TIMEOUT during closing\u001B[0m') + logging.warning('\u001B[1;37;41m Unexpected TIMEOUT during closing\u001B[0m') else: - logging.debug('\u001B[1;37;41m Unexpected Others during closing\u001B[0m') + logging.warning('\u001B[1;37;41m Unexpected Others during closing\u001B[0m') def copyin(self, ipaddress, username, password, source, destination): count = 0 copy_status = False - logging.debug('scp -r '+ username + '@' + ipaddress + ':' + source + ' ' + destination) + logging.info('scp -r '+ username + '@' + ipaddress + ':' + source + ' ' + destination) while count < 10: scp_spawn = pexpect.spawn('scp -r '+ username + '@' + ipaddress + ':' + source + ' ' + destination, timeout = 100) scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT]) @@ -206,7 +206,7 @@ class SSHConnection(): count = 10 copy_status = True else: - logging.debug('1 - scp_response = ' + str(scp_response)) + logging.warning('1 - scp_response = ' + str(scp_response)) elif scp_response == 1: scp_spawn.sendline(password) scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT]) @@ -214,12 +214,12 @@ class SSHConnection(): count = 10 copy_status = True else: - logging.debug('2 - scp_response = ' + str(scp_response)) + logging.warning('2 - scp_response = ' + str(scp_response)) elif scp_response == 2: count = 10 copy_status = True else: - logging.debug('3 - scp_response = ' + str(scp_response)) + logging.warning('3 - scp_response = ' + str(scp_response)) # adding a tempo when failure if not copy_status: time.sleep(1) @@ -232,7 +232,7 @@ class SSHConnection(): def copyout(self, ipaddress, username, password, source, destination): count = 0 copy_status = False - logging.debug('scp -r ' + source + ' ' + username + '@' + ipaddress + ':' + destination) + logging.info('scp -r ' + source + ' ' + username + '@' + ipaddress + ':' + destination) while count < 4: scp_spawn = pexpect.spawn('scp -r ' + source + ' ' + username + '@' + ipaddress + ':' + destination, timeout = 100) scp_response = scp_spawn.expect(['Are you sure you want to continue connecting (yes/no)?', 'password:', pexpect.EOF, pexpect.TIMEOUT]) @@ -245,7 +245,7 @@ class SSHConnection(): count = 10 copy_status = True else: - logging.debug('1 - scp_response = ' + str(scp_response)) + logging.warning('1 - scp_response = ' + str(scp_response)) elif scp_response == 1: scp_spawn.sendline(password) scp_response = scp_spawn.expect(['\$', 'Permission denied', 'password:', pexpect.EOF, pexpect.TIMEOUT]) @@ -253,12 +253,12 @@ class SSHConnection(): count = 10 copy_status = True else: - logging.debug('2 - scp_response = ' + str(scp_response)) + logging.warning('2 - scp_response = ' + str(scp_response)) elif scp_response == 2: count = 10 copy_status = True else: - logging.debug('3 - scp_response = ' + str(scp_response)) + logging.warning('3 - scp_response = ' + str(scp_response)) # adding a tempo when failure if not copy_status: time.sleep(1) diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index 6497fd3020..d77683f9c9 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -51,3 +51,6 @@ - IperfFromContainer - StatsFromGenObject - LicenceAndFormattingCheck + - Push_Local_Registry + - Pull_Local_Registry + - Clean_Test_Server_Images diff --git a/ci-scripts/xml_files/container_image_build.xml b/ci-scripts/xml_files/container_image_build.xml index e7eabc0794..7684e0f997 100644 --- a/ci-scripts/xml_files/container_image_build.xml +++ b/ci-scripts/xml_files/container_image_build.xml @@ -26,6 +26,7 @@ wrench 000001 + 000010 @@ -38,4 +39,10 @@ True + + Push_Local_Registry + Push Images to Local Registry + 0 + + From 4f46622383994e2092042624d2dad31b7456b275 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Fri, 14 Oct 2022 14:31:39 +0200 Subject: [PATCH 02/12] fix(ci): forgot to add HTML rows to test-report Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 15 +++++++++++++++ ci-scripts/main.py | 6 +++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 1b756b84c3..4692574709 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -652,6 +652,8 @@ class Containerize(): mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Login Succeeded', mySSH.getBefore()) is None: logging.error('Could not log into local registry') + mySSH.close() + HTML.CreateHtmlTestRow('Could not log into local registry', 'KO', CONST.ALL_PROCESSES_OK) return False orgTag = 'develop' @@ -665,15 +667,20 @@ class Containerize(): if re.search(': digest:', mySSH.getBefore()) is None: logging.debug(mySSH.getBefore()) logging.error(f'Could not push {image} to local registry : {tagToUse}') + mySSH.close() + HTML.CreateHtmlTestRow(f'Could not push {image} to local registry : {tagToUse}', 'KO', CONST.ALL_PROCESSES_OK) return False mySSH.command(f'docker rmi {tagToUse}', '\$', 5) mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Removing login credentials', mySSH.getBefore()) is None: logging.error('Could not log off from local registry') + mySSH.close() + HTML.CreateHtmlTestRow('Could not log off from local registry', 'KO', CONST.ALL_PROCESSES_OK) return False mySSH.close() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True def Pull_Image_from_Local_Registry(self, HTML): @@ -702,6 +709,8 @@ class Containerize(): mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Login Succeeded', mySSH.getBefore()) is None: logging.error('Could not log into local registry') + mySSH.close() + HTML.CreateHtmlTestRow('Could not log into local registry', 'KO', CONST.ALL_PROCESSES_OK) return False orgTag = 'develop' @@ -714,14 +723,19 @@ class Containerize(): if re.search('Status: Downloaded newer image for |Status: Image is up to date for', mySSH.getBefore()) is None: logging.debug(mySSH.getBefore()) logging.error(f'Could not pull {image} from local registry : {tagToUse}') + mySSH.close() + HTML.CreateHtmlTestRow(f'Could not pull {image} from local registry : {tagToUse}', 'KO', CONST.ALL_PROCESSES_OK) return False mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Removing login credentials', mySSH.getBefore()) is None: logging.error('Could not log off from local registry') + mySSH.close() + HTML.CreateHtmlTestRow('Could not log off from local registry', 'KO', CONST.ALL_PROCESSES_OK) return False mySSH.close() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True def Clean_Test_Server_Images(self, HTML): @@ -757,6 +771,7 @@ class Containerize(): mySSH.command(f'docker rmi {tagToUse} || true', '\$', 5) mySSH.close() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True def DeployObject(self, HTML, EPC): diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 9c7bab4efd..67957b7fba 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -1015,14 +1015,14 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re HTML.testStabilityPointReached = True CiTestObj.FailReportCnt += 1 if CiTestObj.FailReportCnt == CiTestObj.repeatCounts[0] and RAN.prematureExit: - logging.error('Scenario failed ' + str(CiTestObj.FailReportCnt) + ' time(s)') + logging.error('\u001B[1;37;41mScenario failed ' + str(CiTestObj.FailReportCnt) + ' time(s)\u001B[0m') HTML.CreateHtmlTabFooter(False) if CiTestObj.testUnstable and (CiTestObj.testStabilityPointReached or CiTestObj.testMinStableId == '999999'): - logging.warning('Scenario has reached minimal stability point -- Not a Failure') + logging.warning('\u001B[1;30;43mScenario has reached minimal stability point -- Not a Failure\u001B[0m') else: sys.exit('Failed Scenario') else: - logging.info('Scenario passed after ' + str(CiTestObj.FailReportCnt) + ' time(s)') + logging.info('\u001B[1;37;42mScenario passed after ' + str(CiTestObj.FailReportCnt) + ' time(s)\u001B[0m') HTML.CreateHtmlTabFooter(True) elif re.match('^LoadParams$', mode, re.IGNORECASE): pass From 5480e983d69219ee733b8517990ce9539ff91d13 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Mon, 17 Oct 2022 10:25:03 +0200 Subject: [PATCH 03/12] feat(ci): the simulation scenarios are now pulling / using images from local registry Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 198 +++++++++++------- .../xml_files/container_4g_l2sim_tdd.xml | 13 ++ .../xml_files/container_4g_l2sim_tdd_down.xml | 7 + .../container_4g_rfsim_fdd_05MHz.xml | 17 +- .../container_4g_rfsim_fdd_05MHz_down.xml | 7 + .../container_4g_rfsim_fdd_05MHz_noS1.xml | 17 +- ...container_4g_rfsim_fdd_05MHz_noS1_down.xml | 7 + .../container_4g_rfsim_fdd_10MHz.xml | 17 +- .../container_4g_rfsim_fdd_10MHz_down.xml | 7 + .../container_4g_rfsim_fdd_20MHz.xml | 17 +- .../container_4g_rfsim_fdd_20MHz_down.xml | 7 + .../container_4g_rfsim_tdd_05MHz.xml | 19 +- .../container_4g_rfsim_tdd_05MHz_down.xml | 7 + .../xml_files/container_5g_f1_rfsim.xml | 15 +- .../xml_files/container_5g_f1_rfsim_down.xml | 7 + .../xml_files/container_5g_fdd_rfsim.xml | 15 +- .../xml_files/container_5g_fdd_rfsim_down.xml | 7 + .../xml_files/container_5g_l2sim_tdd.xml | 13 ++ .../xml_files/container_5g_l2sim_tdd_down.xml | 7 + ci-scripts/xml_files/container_5g_rfsim.xml | 17 +- .../xml_files/container_5g_rfsim_down.xml | 7 + 21 files changed, 336 insertions(+), 92 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 4692574709..3ad8fafd21 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -651,9 +651,10 @@ class Containerize(): mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Login Succeeded', mySSH.getBefore()) is None: - logging.error('Could not log into local registry') + msg = 'Could not log into local registry' + logging.error(msg) mySSH.close() - HTML.CreateHtmlTestRow('Could not log into local registry', 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) return False orgTag = 'develop' @@ -666,17 +667,21 @@ class Containerize(): mySSH.command(f'docker push {tagToUse}', '\$', 120) if re.search(': digest:', mySSH.getBefore()) is None: logging.debug(mySSH.getBefore()) - logging.error(f'Could not push {image} to local registry : {tagToUse}') + msg = f'Could not push {image} to local registry : {tagToUse}' + logging.error(msg) mySSH.close() - HTML.CreateHtmlTestRow(f'Could not push {image} to local registry : {tagToUse}', 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) return False + # TODO: once we are done migrating all types of tests, rmi also {image}:{orgTag} + # so when we pull on builder as a test server, it will really pull mySSH.command(f'docker rmi {tagToUse}', '\$', 5) mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Removing login credentials', mySSH.getBefore()) is None: - logging.error('Could not log off from local registry') + msg = 'Could not log off from local registry' + logging.error(msg) mySSH.close() - HTML.CreateHtmlTestRow('Could not log off from local registry', 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) return False mySSH.close() @@ -684,6 +689,8 @@ class Containerize(): return True def Pull_Image_from_Local_Registry(self, HTML): + # This method can be called either onto a remote server (different from python executor) + # or directly on the python executor (ie lIpAddr == 'none') if self.testSvrId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName @@ -702,43 +709,68 @@ class Containerize(): if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') - logging.debug('Pulling images onto server: ' + lIpAddr) - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) + if lIpAddr != 'none': + logging.debug('Pulling images onto server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + else: + logging.debug('Pulling images locally') - mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) - if re.search('Login Succeeded', mySSH.getBefore()) is None: - logging.error('Could not log into local registry') - mySSH.close() - HTML.CreateHtmlTestRow('Could not log into local registry', 'KO', CONST.ALL_PROCESSES_OK) + cmd = 'echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True, stderr=subprocess.STDOUT) + if re.search('Login Succeeded', response) is None: + msg = 'Could not log into local registry' + logging.error(msg) + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) return False - orgTag = 'develop' - if self.ranAllowMerge: - orgTag = 'ci-temp' - for image in self.imageToPull: tagToUse = self.ImageTagToUse(image) - mySSH.command(f'docker pull {tagToUse}', '\$', 120) - if re.search('Status: Downloaded newer image for |Status: Image is up to date for', mySSH.getBefore()) is None: - logging.debug(mySSH.getBefore()) - logging.error(f'Could not pull {image} from local registry : {tagToUse}') - mySSH.close() - HTML.CreateHtmlTestRow(f'Could not pull {image} from local registry : {tagToUse}', 'KO', CONST.ALL_PROCESSES_OK) + cmd = f'docker pull {tagToUse}' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 120) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True) + if re.search('Status: Downloaded newer image for |Status: Image is up to date for', response) is None: + logging.debug(response) + msg = f'Could not pull {image} from local registry : {tagToUse}' + logging.error(msg) + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow('msg', 'KO', CONST.ALL_PROCESSES_OK) return False - mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) - if re.search('Removing login credentials', mySSH.getBefore()) is None: - logging.error('Could not log off from local registry') + cmd = 'docker logout porcepix.sboai.cs.eurecom.fr' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True) + if re.search('Removing login credentials', response) is None: + msg = 'Could not log off from local registry' + logging.error(msg) mySSH.close() - HTML.CreateHtmlTestRow('Could not log off from local registry', 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) return False - mySSH.close() + if lIpAddr != 'none': + mySSH.close() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True def Clean_Test_Server_Images(self, HTML): + # This method can be called either onto a remote server (different from python executor) + # or directly on the python executor (ie lIpAddr == 'none') if self.testSvrId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName @@ -757,20 +789,24 @@ class Containerize(): if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': HELP.GenericHelp(CONST.Version) sys.exit('Insufficient Parameter') - logging.debug('Removing test images from server: ' + lIpAddr) - mySSH = SSH.SSHConnection() - mySSH.open(lIpAddr, lUserName, lPassWord) + if lIpAddr != 'none': + logging.debug('Removing test images from server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + else: + logging.debug('Removing test images locally') - orgTag = 'develop' - if self.ranAllowMerge: - orgTag = 'ci-temp' - tagToUse = self.ImageTagToUse(self.imageToPull) imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] for image in imageNames: - tagToUse = self.ImageTagToUse(image) - mySSH.command(f'docker rmi {tagToUse} || true', '\$', 5) + cmd = f'docker rmi {self.ImageTagToUse(image)} || true' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + else: + logging.info(cmd) + subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - mySSH.close() + if lIpAddr != 'none': + mySSH.close() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True @@ -958,33 +994,40 @@ class Containerize(): def DeployGenObject(self, HTML, RAN, UE): self.exitStatus = 0 - logging.info('\u001B[1m Checking Services to deploy\u001B[0m') + logging.debug('\u001B[1m Checking Services to deploy\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services' - logging.debug(cmd) + logging.info(cmd) try: listServices = subprocess.check_output(cmd, shell=True, universal_newlines=True) except Exception as e: self.exitStatus = 1 HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) return + displayUsedTag = False for reqSvc in self.services[0].split(' '): res = re.search(reqSvc, listServices) if res is None: logging.error(reqSvc + ' not found in specified docker-compose') self.exitStatus = 1 + res = re.search('oai-gnb|oai-nr-ue|oai-cu|oai-du|oai_enb|oai_ue', reqSvc) + if res is not None: + displayUsedTag = True if (self.exitStatus == 1): HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) return - if (self.ranAllowMerge): - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@ci-temp@" docker-compose.y*ml > docker-compose-ci.yml' - else: - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@develop@" docker-compose.y*ml > docker-compose-ci.yml' - logging.debug(cmd) - subprocess.run(cmd, shell=True) + cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml' + subprocess.run(cmd, shell=True) + if displayUsedTag: + tagToUse = self.ImageTagToUse('oai-xxx') + logging.info(f'\u001B[1m Using Image Tag: {tagToUse}\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml up -d ' + self.services[0] - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -993,7 +1036,7 @@ class Containerize(): HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK) return - logging.info('\u001B[1m Checking if all deployed healthy\u001B[0m') + logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a' count = 0 healthy = 0 @@ -1088,7 +1131,7 @@ class Containerize(): cmd += ' -w /tmp/capture_' ymlPath = self.yamlPath[0].split('/') cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &' - logging.debug(cmd) + logging.info(cmd) networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) self.tsharkStarted = True @@ -1097,22 +1140,23 @@ class Containerize(): ymlPath = self.yamlPath[0].split('/') logPath = '../cmake_targets/log/' + ymlPath[1] - if (self.ranAllowMerge): - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@ci-temp@" docker-compose.y*ml > docker-compose-ci.yml' - else: - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@develop@" docker-compose.y*ml > docker-compose-ci.yml' - logging.debug(cmd) + cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' subprocess.run(cmd, shell=True) + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml' + subprocess.run(cmd, shell=True) # check which containers are running for log recovery later cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' - logging.debug(cmd) + logging.info(cmd) deployStatusLogs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) # Stop the containers to shut down objects logging.debug('\u001B[1m Stopping containers \u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml stop' - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -1134,16 +1178,16 @@ class Containerize(): anyLogs = True cName = res.group('container_name') cmd = 'cd ' + self.yamlPath[0] + ' && docker logs ' + cName + ' > ' + cName + '.log 2>&1' - logging.debug(cmd) + logging.info(cmd) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) if re.search('magma-mme', cName) is not None: cmd = 'cd ' + self.yamlPath[0] + ' && docker cp -L ' + cName + ':/var/log/mme.log ' + cName + '-full.log' - logging.debug(cmd) + logging.info(cmd) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) fullStatus = True if anyLogs: cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) # Analyzing log file(s)! @@ -1193,25 +1237,25 @@ class Containerize(): HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK) cmd = 'rm ' + self.yamlPath[0] + '/*.log' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) if self.tsharkStarted: self.tsharkStarted = True ymlPath = self.yamlPath[0].split('/') cmd = 'sudo chmod 666 /tmp/capture_' + ymlPath[1] + '.pcap' - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) cmd = 'cp /tmp/capture_' + ymlPath[1] + '.pcap ' + logPath - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) cmd = 'sudo rm /tmp/capture_' + ymlPath[1] + '.pcap' - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) self.tsharkStarted = False logging.debug('\u001B[1m Undeploying \u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml down' - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -1224,7 +1268,7 @@ class Containerize(): self.deployedContainers = [] # Cleaning any created tmp volume cmd = 'docker volume prune --force || true' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) if fullStatus: @@ -1232,7 +1276,7 @@ class Containerize(): logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m') else: HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK) - logging.info('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m') + logging.error('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m') def StatsFromGenObject(self, HTML): self.exitStatus = 0 @@ -1241,7 +1285,7 @@ class Containerize(): # if the containers are running, recover the logs! cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) cmd = 'docker stats --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" ' anyLogs = False @@ -1257,7 +1301,7 @@ class Containerize(): cmd += res.group('container_name') + ' ' message = '' if anyLogs: - logging.debug(cmd) + logging.info(cmd) stats = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) for statLine in stats.split('\n'): logging.debug(statLine) @@ -1277,7 +1321,7 @@ class Containerize(): cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ' + logPath + '/ping_' + HTML.testCase_id + '.log || true' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', deployStatus) @@ -1334,9 +1378,9 @@ class Containerize(): logging.error('\u001B[1;37;41m ping test FAIL -- ' + message + ' \u001B[0m') HTML.CreateHtmlTestRowQueue(self.pingOptions, 'KO', 1, html_queue) # Automatic undeployment - logging.debug('----------------------------------------') - logging.debug('\u001B[1m Starting Automatic undeployment \u001B[0m') - logging.debug('----------------------------------------') + logging.warning('----------------------------------------') + logging.warning('\u001B[1m Starting Automatic undeployment \u001B[0m') + logging.warning('----------------------------------------') HTML.testCase_id = 'AUTO-UNDEPLOY' HTML.desc = 'Automatic Un-Deployment' self.UndeployGenObject(HTML, RAN, UE) @@ -1352,23 +1396,23 @@ class Containerize(): # Start the server process cmd = 'docker exec -d ' + self.svrContName + ' /bin/bash -c "nohup iperf ' + self.svrOptions + ' > /tmp/iperf_server.log 2>&1" || true' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) time.sleep(5) # Start the client process cmd = 'docker exec ' + self.cliContName + ' /bin/bash -c "iperf ' + self.cliOptions + '" 2>&1 | tee '+ logPath + '/iperf_client_' + HTML.testCase_id + '.log || true' - logging.debug(cmd) + logging.info(cmd) clientStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) # Stop the server process cmd = 'docker exec ' + self.svrContName + ' /bin/bash -c "pkill iperf" || true' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) time.sleep(5) cmd = 'docker cp ' + self.svrContName + ':/tmp/iperf_server.log '+ logPath + '/iperf_server_' + HTML.testCase_id + '.log' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) # Analyze client output diff --git a/ci-scripts/xml_files/container_4g_l2sim_tdd.xml b/ci-scripts/xml_files/container_4g_l2sim_tdd.xml index 3ac32e1989..3c6361459c 100644 --- a/ci-scripts/xml_files/container_4g_l2sim_tdd.xml +++ b/ci-scripts/xml_files/container_4g_l2sim_tdd.xml @@ -26,6 +26,7 @@ wrench 1 + 111111 100001 000000 000001 @@ -42,6 +43,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-enb + + + oai-lte-ue + + + DeployGenObject Deploy Cassandra Database diff --git a/ci-scripts/xml_files/container_4g_l2sim_tdd_down.xml b/ci-scripts/xml_files/container_4g_l2sim_tdd_down.xml index ec0f2523e2..011c3dbf6a 100644 --- a/ci-scripts/xml_files/container_4g_l2sim_tdd_down.xml +++ b/ci-scripts/xml_files/container_4g_l2sim_tdd_down.xml @@ -26,6 +26,7 @@ trash 100002 + 222222 @@ -35,4 +36,10 @@ yaml_files/4g_l2sim_fdd + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml index a598b2bdc1..4e927d3243 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml @@ -25,6 +25,7 @@ Monolithic eNB - FDD 05MHz wrench + 111111 100011 000011 000001 @@ -42,6 +43,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-enb + + + oai-lte-ue + + + DeployGenObject Deploy Cassandra Database @@ -74,7 +87,7 @@ DeployGenObject Deploy OAI 4G eNB RF sim (FDD 05MHz) yaml_files/4g_rfsimulator_fdd_05MHz - enb + oai_enb0 8 @@ -97,7 +110,7 @@ PingFromContainer Ping LTE-UE from Traffic-Gen - rfsim4g-trf-gen + rfsim4g-trf-gen -c 20 12.0.0.2 5 diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_down.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_down.xml index 1c3b1d5a7c..068a4063f4 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_down.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_down.xml @@ -26,6 +26,7 @@ trash 100011 + 222222 @@ -35,4 +36,10 @@ yaml_files/4g_rfsimulator_fdd_05MHz + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml index 07ea358ad5..a2c56125ec 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml @@ -25,6 +25,7 @@ Monolithic eNB - FDD 05MHz - noS1 wrench + 111111 100011 000013 000001 @@ -38,6 +39,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-enb + + + oai-lte-ue + + + IdleSleep Sleep @@ -54,7 +67,7 @@ DeployGenObject Deploy OAI 4G eNB RF sim (FDD 05MHz) yaml_files/4g_rfsimulator_fdd_05MHz_noS1 - enb + oai_enb0 1 @@ -77,7 +90,7 @@ PingFromContainer Ping LTE-UE from eNB - rfsim4g-oai-enb + rfsim4g-oai-enb -I oaitun_enb1 -c 20 10.0.1.2 5 diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1_down.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1_down.xml index 6db1dfd9de..f528ae0c21 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1_down.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1_down.xml @@ -26,6 +26,7 @@ trash 100011 + 222222 @@ -35,4 +36,10 @@ yaml_files/4g_rfsimulator_fdd_05MHz_noS1 + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml index 29acfa3b09..0d8dfa1161 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml @@ -25,6 +25,7 @@ Monolithic eNB - FDD 10MHz wrench + 111111 100011 000011 000001 @@ -42,6 +43,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-enb + + + oai-lte-ue + + + DeployGenObject Deploy Cassandra Database @@ -74,7 +87,7 @@ DeployGenObject Deploy OAI 4G eNB RF sim (FDD 10MHz) yaml_files/4g_rfsimulator_fdd_10MHz - enb + oai_enb0 8 @@ -97,7 +110,7 @@ PingFromContainer Ping LTE-UE from Traffic-Gen - rfsim4g-trf-gen + rfsim4g-trf-gen -c 20 12.0.0.2 5 diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz_down.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz_down.xml index 7d03d56ff4..37295911a5 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz_down.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz_down.xml @@ -26,6 +26,7 @@ trash 100011 + 222222 @@ -35,4 +36,10 @@ yaml_files/4g_rfsimulator_fdd_10MHz + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml index 74f712ebda..c666128e39 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml @@ -25,6 +25,7 @@ Monolithic eNB - FDD 20MHz wrench + 111111 100011 000011 000001 @@ -42,6 +43,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-enb + + + oai-lte-ue + + + DeployGenObject Deploy Cassandra Database @@ -74,7 +87,7 @@ DeployGenObject Deploy OAI 4G eNB RF sim (FDD 20MHz) yaml_files/4g_rfsimulator_fdd_20MHz - enb + oai_enb0 8 @@ -97,7 +110,7 @@ PingFromContainer Ping LTE-UE from Traffic-Gen - rfsim4g-trf-gen + rfsim4g-trf-gen -c 20 12.0.0.2 5 diff --git a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz_down.xml b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz_down.xml index 0a721c5091..2017aec636 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz_down.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz_down.xml @@ -26,6 +26,7 @@ trash 100011 + 222222 @@ -35,4 +36,10 @@ yaml_files/4g_rfsimulator_fdd_20MHz + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml index 7a29e50f39..0a638df278 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml @@ -24,8 +24,9 @@ rfsim-4glte-tdd05mhz Monolithic eNB - TDD 05MHz wrench - 1 + 1 + 111111 100011 000011 000001 @@ -43,6 +44,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-enb + + + oai-lte-ue + + + DeployGenObject Deploy Cassandra Database @@ -75,7 +88,7 @@ DeployGenObject Deploy OAI 4G eNB RF sim (TDD 05MHz) yaml_files/4g_rfsimulator_tdd_05MHz - enb + oai_enb0 7 @@ -98,7 +111,7 @@ PingFromContainer Ping LTE-UE from Traffic-Gen - rfsim4g-trf-gen + rfsim4g-trf-gen -c 20 12.0.0.2 5 diff --git a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz_down.xml b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz_down.xml index 7e27bac73c..7e526d4f48 100644 --- a/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz_down.xml +++ b/ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz_down.xml @@ -26,6 +26,7 @@ trash 100011 + 222222 @@ -35,4 +36,10 @@ yaml_files/4g_rfsimulator_tdd_05MHz + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_5g_f1_rfsim.xml b/ci-scripts/xml_files/container_5g_f1_rfsim.xml index 61157f72df..295d544122 100644 --- a/ci-scripts/xml_files/container_5g_f1_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_f1_rfsim.xml @@ -25,6 +25,7 @@ CU-DU F1 split wrench + 111111 100021 000020 000021 @@ -38,6 +39,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-gnb + + + oai-nr-ue + + + DeployGenObject Deploy MySql Database @@ -89,7 +102,7 @@ PingFromContainer Ping NR-UE from ext-dn - rfsim5g-oai-ext-dn + rfsim5g-oai-ext-dn -c 20 12.1.1.2 5 diff --git a/ci-scripts/xml_files/container_5g_f1_rfsim_down.xml b/ci-scripts/xml_files/container_5g_f1_rfsim_down.xml index bb640381cf..e083286a09 100644 --- a/ci-scripts/xml_files/container_5g_f1_rfsim_down.xml +++ b/ci-scripts/xml_files/container_5g_f1_rfsim_down.xml @@ -26,6 +26,7 @@ trash 100022 + 222222 @@ -35,4 +36,10 @@ yaml_files/5g_f1_rfsimulator + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml index 578d5f221f..7de6007b01 100644 --- a/ci-scripts/xml_files/container_5g_fdd_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_fdd_rfsim.xml @@ -26,6 +26,7 @@ wrench 1 + 111111 100011 000010 000011 @@ -39,6 +40,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-gnb + + + oai-nr-ue + + + DeployGenObject Deploy MySql Database @@ -82,7 +95,7 @@ PingFromContainer Ping NR-UE from ext-dn - rfsim5g-oai-ext-dn + rfsim5g-oai-ext-dn -c 20 12.1.1.2 5 diff --git a/ci-scripts/xml_files/container_5g_fdd_rfsim_down.xml b/ci-scripts/xml_files/container_5g_fdd_rfsim_down.xml index 12648ca750..a2cae33f4c 100644 --- a/ci-scripts/xml_files/container_5g_fdd_rfsim_down.xml +++ b/ci-scripts/xml_files/container_5g_fdd_rfsim_down.xml @@ -26,6 +26,7 @@ trash 100012 + 222222 @@ -35,4 +36,10 @@ yaml_files/5g_fdd_rfsimulator + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_5g_l2sim_tdd.xml b/ci-scripts/xml_files/container_5g_l2sim_tdd.xml index cc3c1ae391..9aa7be0b20 100644 --- a/ci-scripts/xml_files/container_5g_l2sim_tdd.xml +++ b/ci-scripts/xml_files/container_5g_l2sim_tdd.xml @@ -26,6 +26,7 @@ wrench 1 + 111111 100001 000000 000001 @@ -40,6 +41,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-gnb + + + oai-nr-ue + + + DeployGenObject Deploy MySql Database diff --git a/ci-scripts/xml_files/container_5g_l2sim_tdd_down.xml b/ci-scripts/xml_files/container_5g_l2sim_tdd_down.xml index d3630cd905..906ce7aa81 100644 --- a/ci-scripts/xml_files/container_5g_l2sim_tdd_down.xml +++ b/ci-scripts/xml_files/container_5g_l2sim_tdd_down.xml @@ -26,6 +26,7 @@ trash 100002 + 222222 @@ -35,4 +36,10 @@ yaml_files/5g_l2sim_tdd + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + diff --git a/ci-scripts/xml_files/container_5g_rfsim.xml b/ci-scripts/xml_files/container_5g_rfsim.xml index 5ad400ecc1..09ec6e07d7 100644 --- a/ci-scripts/xml_files/container_5g_rfsim.xml +++ b/ci-scripts/xml_files/container_5g_rfsim.xml @@ -26,6 +26,7 @@ wrench 1 + 111111 100001 000000 000001 @@ -42,6 +43,18 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-gnb + + + oai-nr-ue + + + DeployGenObject Deploy MySql Database @@ -93,7 +106,7 @@ PingFromContainer Ping NR-UE from ext-dn - rfsim5g-oai-ext-dn + rfsim5g-oai-ext-dn -c 20 12.1.1.2 5 @@ -109,7 +122,7 @@ PingFromContainer Ping Second NR-UE from ext-dn - rfsim5g-oai-ext-dn + rfsim5g-oai-ext-dn -c 20 12.1.1.3 5 diff --git a/ci-scripts/xml_files/container_5g_rfsim_down.xml b/ci-scripts/xml_files/container_5g_rfsim_down.xml index 028ff61864..a2f048a7f4 100644 --- a/ci-scripts/xml_files/container_5g_rfsim_down.xml +++ b/ci-scripts/xml_files/container_5g_rfsim_down.xml @@ -26,6 +26,7 @@ trash 100002 + 222222 @@ -35,4 +36,10 @@ yaml_files/5g_rfsimulator + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + From 612956d354872bc6bc0653e6980971e362a988ee Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Mon, 17 Oct 2022 16:25:37 +0200 Subject: [PATCH 04/12] feat(ci): adding pull strategy to b200 nsa/sa scenarios * also removing the Copy_Image_to_Test method, useless now Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 78 +------------------ ci-scripts/main.py | 16 +--- ci-scripts/xml_class_list.yml | 1 - .../xml_files/container_nsa_b200_quectel.xml | 29 ++++--- .../container_nsa_b200_terminate.xml | 14 ++++ .../xml_files/container_sa_b200_quectel.xml | 19 ++--- .../xml_files/container_sa_b200_terminate.xml | 7 ++ 7 files changed, 56 insertions(+), 108 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 3ad8fafd21..0ee4b3f00f 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -548,75 +548,6 @@ class Containerize(): HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) - def Copy_Image_to_Test_Server(self, HTML): - imageTag = 'develop' - if (self.ranAllowMerge): - imageTag = 'ci-temp' - - lSsh = SSH.SSHConnection() - # Going to the Docker Registry server - if self.registrySvrId == '0': - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - elif self.registrySvrId == '1': - lIpAddr = self.eNB1IPAddress - lUserName = self.eNB1UserName - lPassWord = self.eNB1Password - elif self.registrySvrId == '2': - lIpAddr = self.eNB2IPAddress - lUserName = self.eNB2UserName - lPassWord = self.eNB2Password - lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip --fast > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - ret = lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') - if ret != 0: - HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('cannot remove'): - HTML.CreateHtmlTestRow('file not created by docker save', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.close() - - # Going to the Test Server - if self.testSvrId == '0': - lIpAddr = self.eNBIPAddress - lUserName = self.eNBUserName - lPassWord = self.eNBPassword - elif self.testSvrId == '1': - lIpAddr = self.eNB1IPAddress - lUserName = self.eNB1UserName - lPassWord = self.eNB1Password - elif self.testSvrId == '2': - lIpAddr = self.eNB2IPAddress - lUserName = self.eNB2UserName - lPassWord = self.eNB2Password - lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.copyout(lIpAddr, lUserName, lPassWord, './' + self.imageToCopy + '-' + imageTag + '.tar.gz', '~') - # copyout has no return code and will quit if something fails - lSsh.command('docker rmi ' + self.imageToCopy + ':' + imageTag, '\$', 10) - lSsh.command('docker load < ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('o such file') or lSsh.getBefore().count('invalid tar header'): - logging.debug(lSsh.getBefore()) - HTML.CreateHtmlTestRow('problem during docker load', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('cannot remove'): - HTML.CreateHtmlTestRow('file not copied during scp?', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.close() - - if os.path.isfile('./' + self.imageToCopy + '-' + imageTag + '.tar.gz'): - os.remove('./' + self.imageToCopy + '-' + imageTag + '.tar.gz') - - HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) - return True - def ImageTagToUse(self, imageName): shortCommit = self.ranCommitID[0:8] if self.ranAllowMerge: @@ -838,11 +769,10 @@ class Containerize(): mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5) - imageTag = 'develop' - if (self.ranAllowMerge): - imageTag = 'ci-temp' - mySSH.command('sed -i -e "s/image: oai-enb:latest/image: oai-enb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) - mySSH.command('sed -i -e "s/image: oai-gnb:latest/image: oai-gnb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) + imagesList = ['oai-enb', 'oai-gnb'] + for image in imagesList: + imageToUse = self.ImageTagToUse(image) + mySSH.command(f'sed -i -e "s#image: {image}:latest#image: {imageToUse}#" ci-docker-compose.yml', '\$', 2) localMmeIpAddr = EPC.MmeIPAddress mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2) # if self.flexranCtrlDeployed: diff --git a/ci-scripts/main.py b/ci-scripts/main.py index 67957b7fba..1a8f7d97fe 100644 --- a/ci-scripts/main.py +++ b/ci-scripts/main.py @@ -444,17 +444,6 @@ def GetParametersFromXML(action): if (string_field is not None): CONTAINERS.cliOptions = string_field - elif action == 'Copy_Image_to_Test': - string_field = test.findtext('image_name') - if (string_field is not None): - CONTAINERS.imageToCopy = string_field - string_field = test.findtext('registry_svr_id') - if (string_field is not None): - CONTAINERS.registrySvrId = string_field - string_field = test.findtext('test_svr_id') - if (string_field is not None): - CONTAINERS.testSvrId = string_field - elif action == 'Run_LDPCTest' or action == 'Run_NRulsimTest': ldpc.runargs = test.findtext('physim_run_args') @@ -473,6 +462,7 @@ def GetParametersFromXML(action): string_field = test.findtext('test_svr_id') if (string_field is not None): CONTAINERS.testSvrId = string_field + CONTAINERS.imageToPull.clear() for image in test.findall('image_to_pull'): string_field = image.findtext('image_name') if (string_field is not None): @@ -956,10 +946,6 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re CONTAINERS.BuildImage(HTML) elif action == 'Build_Proxy': CONTAINERS.BuildProxy(HTML) - elif action == 'Copy_Image_to_Test': - success = CONTAINERS.Copy_Image_to_Test_Server(HTML) - if not success: - RAN.prematureExit = True elif action == 'Push_Local_Registry': success = CONTAINERS.Push_Image_to_Local_Registry(HTML) if not success: diff --git a/ci-scripts/xml_class_list.yml b/ci-scripts/xml_class_list.yml index d77683f9c9..a191bd3332 100755 --- a/ci-scripts/xml_class_list.yml +++ b/ci-scripts/xml_class_list.yml @@ -40,7 +40,6 @@ - IdleSleep - Perform_X2_Handover - Build_Image - - Copy_Image_to_Test - Deploy_Object - Undeploy_Object - Cppcheck_Analysis diff --git a/ci-scripts/xml_files/container_nsa_b200_quectel.xml b/ci-scripts/xml_files/container_nsa_b200_quectel.xml index 51e6668cf0..0bed99fdb2 100644 --- a/ci-scripts/xml_files/container_nsa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_nsa_b200_quectel.xml @@ -26,9 +26,10 @@ tasks 1 + 111110 + 111111 000001 010002 - 030000 030101 000001 030102 @@ -49,6 +50,24 @@ + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-enb + + + + + Pull_Local_Registry + Pull Images from Local Registry + 1 + + oai-gnb + + + Initialize_UE Initialize Quectel @@ -62,14 +81,6 @@ idefix - - Copy_Image_to_Test - Copy gNB image to test server - oai-gnb - 0 - 1 - - Deploy_Object Deploy eNB (FDD/Band7/5MHz/B200) in a container diff --git a/ci-scripts/xml_files/container_nsa_b200_terminate.xml b/ci-scripts/xml_files/container_nsa_b200_terminate.xml index 4b446988cd..a766fb7bcb 100644 --- a/ci-scripts/xml_files/container_nsa_b200_terminate.xml +++ b/ci-scripts/xml_files/container_nsa_b200_terminate.xml @@ -28,6 +28,8 @@ 030202 030201 + 222220 + 222221 @@ -47,5 +49,17 @@ 1 + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + + + Clean_Test_Server_Images + Clean Test Images on Test Server + 1 + + diff --git a/ci-scripts/xml_files/container_sa_b200_quectel.xml b/ci-scripts/xml_files/container_sa_b200_quectel.xml index 7f0475629d..127900343a 100644 --- a/ci-scripts/xml_files/container_sa_b200_quectel.xml +++ b/ci-scripts/xml_files/container_sa_b200_quectel.xml @@ -26,8 +26,8 @@ tasks 1 + 111111 010002 - 030000 030101 000001 010000 @@ -47,6 +47,15 @@ --> + + Pull_Local_Registry + Pull Images from Local Registry + 0 + + oai-gnb + + + Initialize_UE Initialize Quectel @@ -60,14 +69,6 @@ idefix - - Copy_Image_to_Test - Copy gNB image to test server - oai-gnb - 1 - 0 - - Deploy_Object Deploy gNB (TDD/Band78/40MHz/B200) in a container diff --git a/ci-scripts/xml_files/container_sa_b200_terminate.xml b/ci-scripts/xml_files/container_sa_b200_terminate.xml index 5a2b6bd454..8e7e0eff44 100644 --- a/ci-scripts/xml_files/container_sa_b200_terminate.xml +++ b/ci-scripts/xml_files/container_sa_b200_terminate.xml @@ -27,6 +27,7 @@ 1 030201 + 222222 @@ -39,4 +40,10 @@ sa-test + + Clean_Test_Server_Images + Clean Test Images on Test Server + 0 + + From 73b1130fefa9fbb28f3042b0d8440304b359c69f Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Tue, 18 Oct 2022 08:50:18 +0200 Subject: [PATCH 05/12] fix(ci): longer timeouts for merge operations Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 2 +- ci-scripts/cls_oaicitest.py | 4 ++-- ci-scripts/cls_physim.py | 4 ++-- ci-scripts/cls_physim1.py | 4 ++-- ci-scripts/cls_static_code_analysis.py | 8 ++++---- ci-scripts/ran.py | 4 ++-- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 0ee4b3f00f..94c0ac8113 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -81,7 +81,7 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge if ranTargetBranch == '': ranTargetBranch = 'develop' logging.debug(f'Merging with the target branch: {ranTargetBranch}') - sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 5) + sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 30) def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw): sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5) diff --git a/ci-scripts/cls_oaicitest.py b/ci-scripts/cls_oaicitest.py index f16df2cb97..92d5f5a720 100644 --- a/ci-scripts/cls_oaicitest.py +++ b/ci-scripts/cls_oaicitest.py @@ -259,10 +259,10 @@ class OaiCiTest(): if self.ranAllowMerge: if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - SSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + SSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) - SSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + SSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30) SSH.command('source oaienv', '\$', 5) SSH.command('cd cmake_targets', '\$', 5) SSH.command('mkdir -p log', '\$', 5) diff --git a/ci-scripts/cls_physim.py b/ci-scripts/cls_physim.py index 14abd9116e..5efbc25aae 100644 --- a/ci-scripts/cls_physim.py +++ b/ci-scripts/cls_physim.py @@ -180,10 +180,10 @@ class PhySim: if (self.ranAllowMerge): if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) else: logging.info('Merging with the target branch: ' + self.ranTargetBranch) - mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30) #build mySSH.command('source oaienv', '\$', 5) diff --git a/ci-scripts/cls_physim1.py b/ci-scripts/cls_physim1.py index e664353fa4..20d2481c56 100644 --- a/ci-scripts/cls_physim1.py +++ b/ci-scripts/cls_physim1.py @@ -108,10 +108,10 @@ class PhySim: imageTag = "ci-temp" if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) - mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30) else: imageTag = "develop" diff --git a/ci-scripts/cls_static_code_analysis.py b/ci-scripts/cls_static_code_analysis.py index 131f2d72bc..ee3f1bbebf 100644 --- a/ci-scripts/cls_static_code_analysis.py +++ b/ci-scripts/cls_static_code_analysis.py @@ -123,10 +123,10 @@ class StaticCodeAnalysis(): if (self.ranAllowMerge): if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) - mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30) mySSH.command('docker image rm oai-cppcheck:bionic oai-cppcheck:focal || true', '\$', 60) mySSH.command('sed -e "s@xenial@bionic@" ci-scripts/docker/Dockerfile.cppcheck.xenial > ci-scripts/docker/Dockerfile.cppcheck.bionic', '\$', 6) @@ -281,11 +281,11 @@ class StaticCodeAnalysis(): argToPass = '--build-arg MERGE_REQUEST=true --build-arg SRC_BRANCH=' + self.ranBranch if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) argToPass += ' --build-arg TARGET_BRANCH=develop ' else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) - mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30) argToPass += ' --build-arg TARGET_BRANCH=' + self.ranTargetBranch + ' ' mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60) diff --git a/ci-scripts/ran.py b/ci-scripts/ran.py index 1d81862647..1de609cc83 100644 --- a/ci-scripts/ran.py +++ b/ci-scripts/ran.py @@ -202,10 +202,10 @@ class RANManagement(): if (self.ranAllowMerge): if self.ranTargetBranch == '': if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'): - mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 30) else: logging.debug('Merging with the target branch: ' + self.ranTargetBranch) - mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5) + mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 30) logging.debug(mySSH.getBefore()) # print what git said when merging/checking out mySSH.command('source oaienv', '\$', 5) mySSH.command('cd cmake_targets', '\$', 5) From 7b3dcc8c498b22e55843d7868f80dee4752c46d8 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Tue, 18 Oct 2022 08:51:57 +0200 Subject: [PATCH 06/12] fix(ci): proper service names for enb Signed-off-by: Raphael Defosseux --- ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/README.md | 2 +- .../yaml_files/4g_rfsimulator_fdd_05MHz/docker-compose.yml | 2 +- .../yaml_files/4g_rfsimulator_fdd_05MHz_noS1/docker-compose.yml | 2 +- .../yaml_files/4g_rfsimulator_fdd_10MHz/docker-compose.yml | 2 +- .../yaml_files/4g_rfsimulator_fdd_20MHz/docker-compose.yml | 2 +- .../yaml_files/4g_rfsimulator_tdd_05MHz/docker-compose.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/README.md b/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/README.md index d146c8abec..d134ffc824 100644 --- a/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/README.md +++ b/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/README.md @@ -146,7 +146,7 @@ rfsim4g-trf-gen /bin/bash -c ip route add ... Up (healthy) ## 2.3. Deploy OAI eNB in RF simulator mode ## ```bash -$ docker-compose up -d enb +$ docker-compose up -d oai_enb0 Creating rfsim4g-oai-enb ... done ``` diff --git a/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/docker-compose.yml b/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/docker-compose.yml index a260857a34..b44e09e454 100644 --- a/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz/docker-compose.yml @@ -179,7 +179,7 @@ services: timeout: 5s retries: 5 - enb: + oai_enb0: image: oai-enb:develop privileged: true container_name: rfsim4g-oai-enb diff --git a/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz_noS1/docker-compose.yml b/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz_noS1/docker-compose.yml index 6e3916b8ca..e0a059222f 100644 --- a/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz_noS1/docker-compose.yml +++ b/ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz_noS1/docker-compose.yml @@ -1,7 +1,7 @@ version: '3.8' services: - enb: + oai_enb0: image: oai-enb:develop privileged: true container_name: rfsim4g-oai-enb diff --git a/ci-scripts/yaml_files/4g_rfsimulator_fdd_10MHz/docker-compose.yml b/ci-scripts/yaml_files/4g_rfsimulator_fdd_10MHz/docker-compose.yml index 3f63953b0e..730cedd720 100644 --- a/ci-scripts/yaml_files/4g_rfsimulator_fdd_10MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/4g_rfsimulator_fdd_10MHz/docker-compose.yml @@ -179,7 +179,7 @@ services: timeout: 5s retries: 5 - enb: + oai_enb0: image: oai-enb:develop privileged: true container_name: rfsim4g-oai-enb diff --git a/ci-scripts/yaml_files/4g_rfsimulator_fdd_20MHz/docker-compose.yml b/ci-scripts/yaml_files/4g_rfsimulator_fdd_20MHz/docker-compose.yml index 1165400273..c1ea936f58 100644 --- a/ci-scripts/yaml_files/4g_rfsimulator_fdd_20MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/4g_rfsimulator_fdd_20MHz/docker-compose.yml @@ -179,7 +179,7 @@ services: timeout: 5s retries: 5 - enb: + oai_enb0: image: oai-enb:develop privileged: true container_name: rfsim4g-oai-enb diff --git a/ci-scripts/yaml_files/4g_rfsimulator_tdd_05MHz/docker-compose.yml b/ci-scripts/yaml_files/4g_rfsimulator_tdd_05MHz/docker-compose.yml index 86f6c80b59..f497709221 100644 --- a/ci-scripts/yaml_files/4g_rfsimulator_tdd_05MHz/docker-compose.yml +++ b/ci-scripts/yaml_files/4g_rfsimulator_tdd_05MHz/docker-compose.yml @@ -179,7 +179,7 @@ services: timeout: 5s retries: 5 - enb: + oai_enb0: image: oai-enb:develop privileged: true container_name: rfsim4g-oai-enb From a1f1d65429dac452d267a99decf56aca804e0eff Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Tue, 18 Oct 2022 09:32:23 +0200 Subject: [PATCH 07/12] feat(ci): adding used image info to HTML report Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 94c0ac8113..6ddb60c359 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -603,9 +603,7 @@ class Containerize(): mySSH.close() HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) return False - # TODO: once we are done migrating all types of tests, rmi also {image}:{orgTag} - # so when we pull on builder as a test server, it will really pull - mySSH.command(f'docker rmi {tagToUse}', '\$', 5) + mySSH.command(f'docker rmi {tagToUse} {image}:{orgTag}', '\$', 30) mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Removing login credentials', mySSH.getBefore()) is None: @@ -796,6 +794,8 @@ class Containerize(): healthyNb = 0 startingNb = 0 containerName = '' + usedImage = '' + imageInfo = '' if result is not None: containerName = result.group('container_name') time.sleep(5) @@ -810,6 +810,10 @@ class Containerize(): else: time.sleep(10) cnt += 1 + mySSH.command(f'docker inspect -f "{{.Config.Image}}" {containerName}', '\$', 5) + usedImage = str(mySSH.getBefore()).strip() + mySSH.command(f'docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" {usedImage}', '\$', 5) + imageInfo = str(mySSH.getBefore()) logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)') logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') @@ -839,11 +843,24 @@ class Containerize(): mySSH.copyin(lIpAddr, lUserName, lPassWord, logfilename, '.') mySSH.close() + html_queue = SimpleQueue() + html_cell = '
\n'
+		if usedImage != '':
+			html_cell += f'Used Image = {usedImage}:\n'
+			html_cell += imageInfo
+		else:
+			html_cell += 'Could not retrieve used image info!\n'
 		if status:
-			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+			html_cell += '\nHealthy deployment!\n'
+		else:
+			html_cell += '\nUnhealthy deployment! -- Check logs for reason!\n'
+		html_cell += '
' + html_queue.put(html_cell) + if status: + HTML.CreateHtmlTestRowQueue('N/A', 'OK', CONST.ENB_PROCESS_OK, html_queue) else: self.exitStatus = 1 - HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRowQueue('N/A', 'KO', CONST.ENB_PROCESS_OK, html_queue) def UndeployObject(self, HTML, RAN): From 28fb8fbdd46c7b2a1b39dec8304b7f8239175865 Mon Sep 17 00:00:00 2001 From: Raphael Defosseux Date: Tue, 18 Oct 2022 15:32:41 +0200 Subject: [PATCH 08/12] fix(ci): post-review changes and fix on the HTML reporting Signed-off-by: Raphael Defosseux --- ci-scripts/cls_containerize.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 6ddb60c359..e3893b8179 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -580,7 +580,7 @@ class Containerize(): mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) - mySSH.command('echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) + mySSH.command('docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) if re.search('Login Succeeded', mySSH.getBefore()) is None: msg = 'Could not log into local registry' logging.error(msg) @@ -645,7 +645,7 @@ class Containerize(): else: logging.debug('Pulling images locally') - cmd = 'echo oaicicd | docker login --password-stdin -u oaicicd porcepix.sboai.cs.eurecom.fr' + cmd = 'docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr' if lIpAddr != 'none': mySSH.command(cmd, '\$', 5) response = mySSH.getBefore() @@ -762,7 +762,7 @@ class Containerize(): mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) - + CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) @@ -810,10 +810,20 @@ class Containerize(): else: time.sleep(10) cnt += 1 - mySSH.command(f'docker inspect -f "{{.Config.Image}}" {containerName}', '\$', 5) - usedImage = str(mySSH.getBefore()).strip() - mySSH.command(f'docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" {usedImage}', '\$', 5) - imageInfo = str(mySSH.getBefore()) + mySSH.command('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName, '\$', 5) + for stdoutLine in mySSH.getBefore().split('\n'): + if stdoutLine.count('ImageUsed: porcepix'): + usedImage = stdoutLine.replace('ImageUsed: porcepix', 'porcepix').strip() + logging.debug('Used image is ' + usedImage) + if usedImage != '': + mySSH.command('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage, '\$', 5, silent=True) + for stdoutLine in mySSH.getBefore().split('\n'): + if re.search('Size = [0-9]', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' + if re.search('Creation = [0-9]', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' + if re.search('Id = sha256', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)') logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') @@ -846,7 +856,7 @@ class Containerize(): html_queue = SimpleQueue() html_cell = '
\n'
 		if usedImage != '':
-			html_cell += f'Used Image = {usedImage}:\n'
+			html_cell += f'Used Image = {usedImage} :\n'
 			html_cell += imageInfo
 		else:
 			html_cell += 'Could not retrieve used image info!\n'

From a42716d5fb5198983f2e0e2b87ef517d6bd61fee Mon Sep 17 00:00:00 2001
From: Raphael Defosseux 
Date: Tue, 18 Oct 2022 15:33:13 +0200
Subject: [PATCH 09/12] feat(ci): the first tab in HTML report is now open

Signed-off-by: Raphael Defosseux 
---
 ci-scripts/cls_oai_html.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py
index b7514172dc..48123f52ad 100644
--- a/ci-scripts/cls_oai_html.py
+++ b/ci-scripts/cls_oai_html.py
@@ -33,6 +33,7 @@
 #-----------------------------------------------------------
 import sys              # arg
 import re               # reg
+import fileinput
 import logging
 import os
 import time
@@ -195,7 +196,10 @@ class HTMLManagement():
 			self.htmlFile.write('