diff --git a/ci-scripts/cls_containerize.py b/ci-scripts/cls_containerize.py index 5cf6b76b38..e3893b8179 100644 --- a/ci-scripts/cls_containerize.py +++ b/ci-scripts/cls_containerize.py @@ -81,7 +81,7 @@ def CreateWorkspace(sshSession, sourcePath, ranRepository, ranCommitID, ranTarge if ranTargetBranch == '': ranTargetBranch = 'develop' logging.debug(f'Merging with the target branch: {ranTargetBranch}') - sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 5) + sshSession.command(f'git merge --ff origin/{ranTargetBranch} -m "Temporary merge for CI"', '\$', 30) def CopyLogsToExecutor(sshSession, sourcePath, log_name, scpIp, scpUser, scpPw): sshSession.command(f'cd {sourcePath}/cmake_targets', '\$', 5) @@ -208,6 +208,7 @@ class Containerize(): self.imageToCopy = '' self.registrySvrId = '' self.testSvrId = '' + self.imageToPull = [] #checkers from xml self.ran_checkers={} @@ -547,72 +548,194 @@ class Containerize(): HTML.CreateHtmlTestRow('commit ' + tag, 'OK', CONST.ALL_PROCESSES_OK) HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, allImagesSize) - def Copy_Image_to_Test_Server(self, HTML): - imageTag = 'develop' - if (self.ranAllowMerge): - imageTag = 'ci-temp' + def ImageTagToUse(self, imageName): + shortCommit = self.ranCommitID[0:8] + if self.ranAllowMerge: + tagToUse = f'{self.ranBranch}-{shortCommit}' + else: + tagToUse = f'develop-{shortCommit}' + fullTag = f'porcepix.sboai.cs.eurecom.fr/{imageName}:{tagToUse}' + return fullTag - lSsh = SSH.SSHConnection() - # Going to the Docker Registry server + def Push_Image_to_Local_Registry(self, HTML): if self.registrySvrId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath elif self.registrySvrId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath elif self.registrySvrId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password - lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.command('docker save ' + self.imageToCopy + ':' + imageTag + ' | gzip --fast > ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - ret = lSsh.copyin(lIpAddr, lUserName, lPassWord, '~/' + self.imageToCopy + '-' + imageTag + '.tar.gz', '.') - if ret != 0: - HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('cannot remove'): - HTML.CreateHtmlTestRow('file not created by docker save', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.close() + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + logging.debug('Pushing images from server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) - # Going to the Test Server + mySSH.command('docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Login Succeeded', mySSH.getBefore()) is None: + msg = 'Could not log into local registry' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + + orgTag = 'develop' + if self.ranAllowMerge: + orgTag = 'ci-temp' + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + mySSH.command(f'docker image tag {image}:{orgTag} {tagToUse}', '\$', 5) + mySSH.command(f'docker push {tagToUse}', '\$', 120) + if re.search(': digest:', mySSH.getBefore()) is None: + logging.debug(mySSH.getBefore()) + msg = f'Could not push {image} to local registry : {tagToUse}' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + mySSH.command(f'docker rmi {tagToUse} {image}:{orgTag}', '\$', 30) + + mySSH.command('docker logout porcepix.sboai.cs.eurecom.fr', '\$', 5) + if re.search('Removing login credentials', mySSH.getBefore()) is None: + msg = 'Could not log off from local registry' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + + mySSH.close() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + return True + + def Pull_Image_from_Local_Registry(self, HTML): + # This method can be called either onto a remote server (different from python executor) + # or directly on the python executor (ie lIpAddr == 'none') if self.testSvrId == '0': lIpAddr = self.eNBIPAddress lUserName = self.eNBUserName lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath elif self.testSvrId == '1': lIpAddr = self.eNB1IPAddress lUserName = self.eNB1UserName lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath elif self.testSvrId == '2': lIpAddr = self.eNB2IPAddress lUserName = self.eNB2UserName lPassWord = self.eNB2Password - lSsh.open(lIpAddr, lUserName, lPassWord) - lSsh.copyout(lIpAddr, lUserName, lPassWord, './' + self.imageToCopy + '-' + imageTag + '.tar.gz', '~') - # copyout has no return code and will quit if something fails - lSsh.command('docker rmi ' + self.imageToCopy + ':' + imageTag, '\$', 10) - lSsh.command('docker load < ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('o such file') or lSsh.getBefore().count('invalid tar header'): - logging.debug(lSsh.getBefore()) - HTML.CreateHtmlTestRow('problem during docker load', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.command('rm ' + self.imageToCopy + '-' + imageTag + '.tar.gz', '\$', 60) - if lSsh.getBefore().count('cannot remove'): - HTML.CreateHtmlTestRow('file not copied during scp?', 'KO', CONST.ALL_PROCESSES_OK) - self.exitStatus = 1 - return False - lSsh.close() + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + if lIpAddr != 'none': + logging.debug('Pulling images onto server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + else: + logging.debug('Pulling images locally') - if os.path.isfile('./' + self.imageToCopy + '-' + imageTag + '.tar.gz'): - os.remove('./' + self.imageToCopy + '-' + imageTag + '.tar.gz') + cmd = 'docker login -u oaicicd -p oaicicd porcepix.sboai.cs.eurecom.fr' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True, stderr=subprocess.STDOUT) + if re.search('Login Succeeded', response) is None: + msg = 'Could not log into local registry' + logging.error(msg) + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + for image in self.imageToPull: + tagToUse = self.ImageTagToUse(image) + cmd = f'docker pull {tagToUse}' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 120) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True) + if re.search('Status: Downloaded newer image for |Status: Image is up to date for', response) is None: + logging.debug(response) + msg = f'Could not pull {image} from local registry : {tagToUse}' + logging.error(msg) + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow('msg', 'KO', CONST.ALL_PROCESSES_OK) + return False + + cmd = 'docker logout porcepix.sboai.cs.eurecom.fr' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + response = mySSH.getBefore() + else: + logging.info(cmd) + response = subprocess.check_output(cmd, shell=True, universal_newlines=True) + if re.search('Removing login credentials', response) is None: + msg = 'Could not log off from local registry' + logging.error(msg) + mySSH.close() + HTML.CreateHtmlTestRow(msg, 'KO', CONST.ALL_PROCESSES_OK) + return False + + if lIpAddr != 'none': + mySSH.close() + HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) + return True + + def Clean_Test_Server_Images(self, HTML): + # This method can be called either onto a remote server (different from python executor) + # or directly on the python executor (ie lIpAddr == 'none') + if self.testSvrId == '0': + lIpAddr = self.eNBIPAddress + lUserName = self.eNBUserName + lPassWord = self.eNBPassword + lSourcePath = self.eNBSourceCodePath + elif self.testSvrId == '1': + lIpAddr = self.eNB1IPAddress + lUserName = self.eNB1UserName + lPassWord = self.eNB1Password + lSourcePath = self.eNB1SourceCodePath + elif self.testSvrId == '2': + lIpAddr = self.eNB2IPAddress + lUserName = self.eNB2UserName + lPassWord = self.eNB2Password + lSourcePath = self.eNB2SourceCodePath + if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '': + HELP.GenericHelp(CONST.Version) + sys.exit('Insufficient Parameter') + if lIpAddr != 'none': + logging.debug('Removing test images from server: ' + lIpAddr) + mySSH = SSH.SSHConnection() + mySSH.open(lIpAddr, lUserName, lPassWord) + else: + logging.debug('Removing test images locally') + + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + cmd = f'docker rmi {self.ImageTagToUse(image)} || true' + if lIpAddr != 'none': + mySSH.command(cmd, '\$', 5) + else: + logging.info(cmd) + subprocess.run(cmd, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + if lIpAddr != 'none': + mySSH.close() HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK) return True @@ -639,16 +762,15 @@ class Containerize(): mySSH = SSH.SSHConnection() mySSH.open(lIpAddr, lUserName, lPassWord) - + CreateWorkspace(mySSH, lSourcePath, self.ranRepository, self.ranCommitID, self.ranTargetBranch, self.ranAllowMerge) mySSH.command('cd ' + lSourcePath + '/' + self.yamlPath[self.eNB_instance], '\$', 5) mySSH.command('cp docker-compose.yml ci-docker-compose.yml', '\$', 5) - imageTag = 'develop' - if (self.ranAllowMerge): - imageTag = 'ci-temp' - mySSH.command('sed -i -e "s/image: oai-enb:latest/image: oai-enb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) - mySSH.command('sed -i -e "s/image: oai-gnb:latest/image: oai-gnb:' + imageTag + '/" ci-docker-compose.yml', '\$', 2) + imagesList = ['oai-enb', 'oai-gnb'] + for image in imagesList: + imageToUse = self.ImageTagToUse(image) + mySSH.command(f'sed -i -e "s#image: {image}:latest#image: {imageToUse}#" ci-docker-compose.yml', '\$', 2) localMmeIpAddr = EPC.MmeIPAddress mySSH.command('sed -i -e "s/CI_MME_IP_ADDR/' + localMmeIpAddr + '/" ci-docker-compose.yml', '\$', 2) # if self.flexranCtrlDeployed: @@ -672,6 +794,8 @@ class Containerize(): healthyNb = 0 startingNb = 0 containerName = '' + usedImage = '' + imageInfo = '' if result is not None: containerName = result.group('container_name') time.sleep(5) @@ -686,6 +810,20 @@ class Containerize(): else: time.sleep(10) cnt += 1 + mySSH.command('docker inspect --format="ImageUsed: {{.Config.Image}}" ' + containerName, '\$', 5) + for stdoutLine in mySSH.getBefore().split('\n'): + if stdoutLine.count('ImageUsed: porcepix'): + usedImage = stdoutLine.replace('ImageUsed: porcepix', 'porcepix').strip() + logging.debug('Used image is ' + usedImage) + if usedImage != '': + mySSH.command('docker image inspect --format "* Size = {{.Size}} bytes\n* Creation = {{.Created}}\n* Id = {{.Id}}" ' + usedImage, '\$', 5, silent=True) + for stdoutLine in mySSH.getBefore().split('\n'): + if re.search('Size = [0-9]', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' + if re.search('Creation = [0-9]', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' + if re.search('Id = sha256', stdoutLine) is not None: + imageInfo += stdoutLine.strip() + '\n' logging.debug(' -- ' + str(healthyNb) + ' healthy container(s)') logging.debug(' -- ' + str(unhealthyNb) + ' unhealthy container(s)') logging.debug(' -- ' + str(startingNb) + ' still starting container(s)') @@ -715,11 +853,24 @@ class Containerize(): mySSH.copyin(lIpAddr, lUserName, lPassWord, logfilename, '.') mySSH.close() + html_queue = SimpleQueue() + html_cell = '
\n'
+		if usedImage != '':
+			html_cell += f'Used Image = {usedImage} :\n'
+			html_cell += imageInfo
+		else:
+			html_cell += 'Could not retrieve used image info!\n'
 		if status:
-			HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
+			html_cell += '\nHealthy deployment!\n'
+		else:
+			html_cell += '\nUnhealthy deployment! -- Check logs for reason!\n'
+		html_cell += '
' + html_queue.put(html_cell) + if status: + HTML.CreateHtmlTestRowQueue('N/A', 'OK', CONST.ENB_PROCESS_OK, html_queue) else: self.exitStatus = 1 - HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ALL_PROCESSES_OK) + HTML.CreateHtmlTestRowQueue('N/A', 'KO', CONST.ENB_PROCESS_OK, html_queue) def UndeployObject(self, HTML, RAN): @@ -800,33 +951,40 @@ class Containerize(): def DeployGenObject(self, HTML, RAN, UE): self.exitStatus = 0 - logging.info('\u001B[1m Checking Services to deploy\u001B[0m') + logging.debug('\u001B[1m Checking Services to deploy\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose config --services' - logging.debug(cmd) + logging.info(cmd) try: listServices = subprocess.check_output(cmd, shell=True, universal_newlines=True) except Exception as e: self.exitStatus = 1 HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) return + displayUsedTag = False for reqSvc in self.services[0].split(' '): res = re.search(reqSvc, listServices) if res is None: logging.error(reqSvc + ' not found in specified docker-compose') self.exitStatus = 1 + res = re.search('oai-gnb|oai-nr-ue|oai-cu|oai-du|oai_enb|oai_ue', reqSvc) + if res is not None: + displayUsedTag = True if (self.exitStatus == 1): HTML.CreateHtmlTestRow('SVC not Found', 'KO', CONST.ALL_PROCESSES_OK) return - if (self.ranAllowMerge): - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@ci-temp@" docker-compose.y*ml > docker-compose-ci.yml' - else: - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@develop@" docker-compose.y*ml > docker-compose-ci.yml' - logging.debug(cmd) - subprocess.run(cmd, shell=True) + cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml' + subprocess.run(cmd, shell=True) + if displayUsedTag: + tagToUse = self.ImageTagToUse('oai-xxx') + logging.info(f'\u001B[1m Using Image Tag: {tagToUse}\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml up -d ' + self.services[0] - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -835,7 +993,7 @@ class Containerize(): HTML.CreateHtmlTestRow('Could not deploy', 'KO', CONST.ALL_PROCESSES_OK) return - logging.info('\u001B[1m Checking if all deployed healthy\u001B[0m') + logging.debug('\u001B[1m Checking if all deployed healthy\u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps -a' count = 0 healthy = 0 @@ -930,7 +1088,7 @@ class Containerize(): cmd += ' -w /tmp/capture_' ymlPath = self.yamlPath[0].split('/') cmd += ymlPath[1] + '.pcap > /tmp/tshark.log 2>&1 &' - logging.debug(cmd) + logging.info(cmd) networkNames = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) self.tsharkStarted = True @@ -939,22 +1097,23 @@ class Containerize(): ymlPath = self.yamlPath[0].split('/') logPath = '../cmake_targets/log/' + ymlPath[1] - if (self.ranAllowMerge): - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@ci-temp@" docker-compose.y*ml > docker-compose-ci.yml' - else: - cmd = 'cd ' + self.yamlPath[0] + ' && sed -e "s@develop@develop@" docker-compose.y*ml > docker-compose-ci.yml' - logging.debug(cmd) + cmd = 'cd ' + self.yamlPath[0] + ' && cp docker-compose.y*ml docker-compose-ci.yml' subprocess.run(cmd, shell=True) + imageNames = ['oai-enb', 'oai-gnb', 'oai-lte-ue', 'oai-nr-ue', 'oai-lte-ru'] + for image in imageNames: + tagToUse = self.ImageTagToUse(image) + cmd = f'cd {self.yamlPath[0]} && sed -i -e "s@{image}:develop@{tagToUse}@" docker-compose-ci.yml' + subprocess.run(cmd, shell=True) # check which containers are running for log recovery later cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' - logging.debug(cmd) + logging.info(cmd) deployStatusLogs = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) # Stop the containers to shut down objects logging.debug('\u001B[1m Stopping containers \u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml stop' - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -976,16 +1135,16 @@ class Containerize(): anyLogs = True cName = res.group('container_name') cmd = 'cd ' + self.yamlPath[0] + ' && docker logs ' + cName + ' > ' + cName + '.log 2>&1' - logging.debug(cmd) + logging.info(cmd) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) if re.search('magma-mme', cName) is not None: cmd = 'cd ' + self.yamlPath[0] + ' && docker cp -L ' + cName + ':/var/log/mme.log ' + cName + '-full.log' - logging.debug(cmd) + logging.info(cmd) subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) fullStatus = True if anyLogs: cmd = 'mkdir -p '+ logPath + ' && cp ' + self.yamlPath[0] + '/*.log ' + logPath - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) # Analyzing log file(s)! @@ -1035,25 +1194,25 @@ class Containerize(): HTML.CreateHtmlTestRow('UE log Analysis', 'OK', CONST.ALL_PROCESSES_OK) cmd = 'rm ' + self.yamlPath[0] + '/*.log' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) if self.tsharkStarted: self.tsharkStarted = True ymlPath = self.yamlPath[0].split('/') cmd = 'sudo chmod 666 /tmp/capture_' + ymlPath[1] + '.pcap' - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) cmd = 'cp /tmp/capture_' + ymlPath[1] + '.pcap ' + logPath - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) cmd = 'sudo rm /tmp/capture_' + ymlPath[1] + '.pcap' - logging.debug(cmd) + logging.info(cmd) copyStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) self.tsharkStarted = False logging.debug('\u001B[1m Undeploying \u001B[0m') cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml down' - logging.debug(cmd) + logging.info(cmd) try: deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) except Exception as e: @@ -1066,7 +1225,7 @@ class Containerize(): self.deployedContainers = [] # Cleaning any created tmp volume cmd = 'docker volume prune --force || true' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) if fullStatus: @@ -1074,7 +1233,7 @@ class Containerize(): logging.info('\u001B[1m Undeploying OAI Object(s) PASS\u001B[0m') else: HTML.CreateHtmlTestRow('n/a', 'KO', CONST.ALL_PROCESSES_OK) - logging.info('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m') + logging.error('\u001B[1m Undeploying OAI Object(s) FAIL\u001B[0m') def StatsFromGenObject(self, HTML): self.exitStatus = 0 @@ -1083,7 +1242,7 @@ class Containerize(): # if the containers are running, recover the logs! cmd = 'cd ' + self.yamlPath[0] + ' && docker-compose -f docker-compose-ci.yml ps --all' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) cmd = 'docker stats --no-stream --format "table {{.Container}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}" ' anyLogs = False @@ -1099,7 +1258,7 @@ class Containerize(): cmd += res.group('container_name') + ' ' message = '' if anyLogs: - logging.debug(cmd) + logging.info(cmd) stats = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) for statLine in stats.split('\n'): logging.debug(statLine) @@ -1119,7 +1278,7 @@ class Containerize(): cmd = 'docker exec ' + self.pingContName + ' /bin/bash -c "ping ' + self.pingOptions + '" 2>&1 | tee ' + logPath + '/ping_' + HTML.testCase_id + '.log || true' - logging.debug(cmd) + logging.info(cmd) deployStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) result = re.search(', (?P[0-9\.]+)% packet loss, time [0-9\.]+ms', deployStatus) @@ -1176,9 +1335,9 @@ class Containerize(): logging.error('\u001B[1;37;41m ping test FAIL -- ' + message + ' \u001B[0m') HTML.CreateHtmlTestRowQueue(self.pingOptions, 'KO', 1, html_queue) # Automatic undeployment - logging.debug('----------------------------------------') - logging.debug('\u001B[1m Starting Automatic undeployment \u001B[0m') - logging.debug('----------------------------------------') + logging.warning('----------------------------------------') + logging.warning('\u001B[1m Starting Automatic undeployment \u001B[0m') + logging.warning('----------------------------------------') HTML.testCase_id = 'AUTO-UNDEPLOY' HTML.desc = 'Automatic Un-Deployment' self.UndeployGenObject(HTML, RAN, UE) @@ -1194,23 +1353,23 @@ class Containerize(): # Start the server process cmd = 'docker exec -d ' + self.svrContName + ' /bin/bash -c "nohup iperf ' + self.svrOptions + ' > /tmp/iperf_server.log 2>&1" || true' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) time.sleep(5) # Start the client process cmd = 'docker exec ' + self.cliContName + ' /bin/bash -c "iperf ' + self.cliOptions + '" 2>&1 | tee '+ logPath + '/iperf_client_' + HTML.testCase_id + '.log || true' - logging.debug(cmd) + logging.info(cmd) clientStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=100) # Stop the server process cmd = 'docker exec ' + self.svrContName + ' /bin/bash -c "pkill iperf" || true' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=10) time.sleep(5) cmd = 'docker cp ' + self.svrContName + ':/tmp/iperf_server.log '+ logPath + '/iperf_server_' + HTML.testCase_id + '.log' - logging.debug(cmd) + logging.info(cmd) serverStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=30) # Analyze client output diff --git a/ci-scripts/cls_log_mgt.py b/ci-scripts/cls_log_mgt.py index f1dbf6b8ec..a78eee5a33 100644 --- a/ci-scripts/cls_log_mgt.py +++ b/ci-scripts/cls_log_mgt.py @@ -35,11 +35,6 @@ import re import subprocess import sshconnection -logging.basicConfig( - level=logging.DEBUG, - format="[%(asctime)s] %(name)s:%(levelname)s: %(message)s" -) - class Log_Mgt: def __init__(self,Username, IPAddress,Password,Path): diff --git a/ci-scripts/cls_oai_html.py b/ci-scripts/cls_oai_html.py index 536f2ff49e..48123f52ad 100644 --- a/ci-scripts/cls_oai_html.py +++ b/ci-scripts/cls_oai_html.py @@ -33,6 +33,7 @@ #----------------------------------------------------------- import sys # arg import re # reg +import fileinput import logging import os import time @@ -102,9 +103,9 @@ class HTMLManagement(): def CreateHtmlHeader(self, ADBIPAddress): if (not self.htmlHeaderCreated): - logging.debug('\u001B[1m----------------------------------------\u001B[0m') - logging.debug('\u001B[1m Creating HTML header \u001B[0m') - logging.debug('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') + logging.info('\u001B[1m Creating HTML header \u001B[0m') + logging.info('\u001B[1m----------------------------------------\u001B[0m') self.htmlFile = open('test_results.html', 'w') self.htmlFile.write('\n') self.htmlFile.write('\n') @@ -195,7 +196,10 @@ class HTMLManagement(): self.htmlFile.write('