Merge remote-tracking branch 'origin/ci-fix-cn5g-deploy' into integration_2022_wk19
This commit is contained in:
Robert Schmidt
@@ -59,7 +59,7 @@ if [ $# -eq 0 ]
|
||||
then
|
||||
echo " ---- Checking the whole repository ----"
|
||||
echo ""
|
||||
NB_FILES_TO_FORMAT=`astyle --dry-run --options=ci-scripts/astyle-options.txt --recursive *.c *.h | grep -c Formatted || true`
|
||||
NB_FILES_TO_FORMAT=`astyle --dry-run --options=ci-scripts/astyle-options.txt --recursive --exclude=ci-scripts --exclude=cmake_targets *.c *.h | grep -c Formatted || true`
|
||||
echo "Nb Files that do NOT follow OAI rules: $NB_FILES_TO_FORMAT"
|
||||
echo $NB_FILES_TO_FORMAT > ./oai_rules_result.txt
|
||||
|
||||
@@ -136,7 +136,7 @@ fi
|
||||
# Merge request scenario
|
||||
|
||||
MERGE_COMMMIT=`git log -n1 --pretty=format:%H`
|
||||
TARGET_INIT_COMMIT=`cat .git/refs/remotes/origin/$TARGET_BRANCH`
|
||||
TARGET_INIT_COMMIT=`git log -n1 --pretty=format:%H origin/$TARGET_BRANCH`
|
||||
|
||||
echo " ---- Checking the modified files by the merge request ----"
|
||||
echo ""
|
||||
|
||||
@@ -37,6 +37,7 @@ import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
from multiprocessing import Process, Lock, SimpleQueue
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# OAI Testing modules
|
||||
@@ -237,3 +238,205 @@ class StaticCodeAnalysis():
|
||||
|
||||
return 0
|
||||
|
||||
def LicenceAndFormattingCheck(self, HTML):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter')
|
||||
lIpAddr = self.eNBIPAddress
|
||||
lUserName = self.eNBUserName
|
||||
lPassWord = self.eNBPassword
|
||||
lSourcePath = self.eNBSourceCodePath
|
||||
|
||||
if lIpAddr == '' or lUserName == '' or lPassWord == '' or lSourcePath == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter')
|
||||
logging.debug('Building on server: ' + lIpAddr)
|
||||
mySSH = SSH.SSHConnection()
|
||||
mySSH.open(lIpAddr, lUserName, lPassWord)
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
|
||||
# on RedHat/CentOS .git extension is mandatory
|
||||
result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
|
||||
if result is not None:
|
||||
full_ran_repo_name = self.ranRepository.replace('git/', 'git')
|
||||
else:
|
||||
full_ran_repo_name = self.ranRepository + '.git'
|
||||
mySSH.command('mkdir -p ' + lSourcePath, '\$', 5)
|
||||
mySSH.command('cd ' + lSourcePath, '\$', 5)
|
||||
mySSH.command('if [ ! -e .git ]; then stdbuf -o0 git clone ' + full_ran_repo_name + ' .; else stdbuf -o0 git fetch --prune; fi', '\$', 600)
|
||||
# Raphael: here add a check if git clone or git fetch went smoothly
|
||||
mySSH.command('git config user.email "jenkins@openairinterface.org"', '\$', 5)
|
||||
mySSH.command('git config user.name "OAI Jenkins"', '\$', 5)
|
||||
|
||||
mySSH.command('echo ' + lPassWord + ' | sudo -S git clean -x -d -ff', '\$', 30)
|
||||
mySSH.command('mkdir -p cmake_targets/log', '\$', 5)
|
||||
# if the commit ID is provided use it to point to it
|
||||
if self.ranCommitID != '':
|
||||
mySSH.command('git checkout -f ' + self.ranCommitID, '\$', 30)
|
||||
# if the branch is not develop, then it is a merge request and we need to do
|
||||
# the potential merge. Note that merge conflicts should already been checked earlier
|
||||
argToPass = ''
|
||||
if (self.ranAllowMerge):
|
||||
argToPass = '--build-arg MERGE_REQUEST=true --build-arg SRC_BRANCH=' + self.ranBranch
|
||||
if self.ranTargetBranch == '':
|
||||
if (self.ranBranch != 'develop') and (self.ranBranch != 'origin/develop'):
|
||||
mySSH.command('git merge --ff origin/develop -m "Temporary merge for CI"', '\$', 5)
|
||||
argToPass += ' --build-arg TARGET_BRANCH=develop '
|
||||
else:
|
||||
logging.debug('Merging with the target branch: ' + self.ranTargetBranch)
|
||||
mySSH.command('git merge --ff origin/' + self.ranTargetBranch + ' -m "Temporary merge for CI"', '\$', 5)
|
||||
argToPass += ' --build-arg TARGET_BRANCH=' + self.ranTargetBranch + ' '
|
||||
|
||||
mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60)
|
||||
mySSH.command('docker build --target oai-formatting-check --tag oai-formatting-check:latest ' + argToPass + '--file ci-scripts/docker/Dockerfile.formatting.bionic . > cmake_targets/log/oai-formatting-check.txt 2>&1', '\$', 600)
|
||||
|
||||
mySSH.command('docker image rm oai-formatting-check:latest || true', '\$', 60)
|
||||
mySSH.command('docker image prune --force', '\$', 60)
|
||||
mySSH.command('docker volume prune --force', '\$', 60)
|
||||
|
||||
# Analyzing the logs
|
||||
mySSH.command('cd ' + lSourcePath + '/cmake_targets', '\$', 5)
|
||||
mySSH.command('mkdir -p build_log_' + self.testCase_id, '\$', 5)
|
||||
mySSH.command('mv log/* ' + 'build_log_' + self.testCase_id, '\$', 5)
|
||||
mySSH.close()
|
||||
|
||||
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/build_log_' + self.testCase_id + '/*', '.')
|
||||
|
||||
finalStatus = 0
|
||||
if (os.path.isfile('./oai-formatting-check.txt')):
|
||||
analyzed = False
|
||||
nbFilesNotFormatted = 0
|
||||
listFiles = False
|
||||
listFilesNotFormatted = []
|
||||
circularHeaderDependency = False
|
||||
circularHeaderDependencyFiles = []
|
||||
gnuGplLicence = False
|
||||
gnuGplLicenceFiles = []
|
||||
suspectLicence = False
|
||||
suspectLicenceFiles = []
|
||||
with open('./oai-formatting-check.txt', 'r') as logfile:
|
||||
for line in logfile:
|
||||
ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line))
|
||||
if ret is not None:
|
||||
analyzed = True
|
||||
if analyzed:
|
||||
ret = re.search('Nb Files that do NOT follow OAI rules: (?P<nb_errors>[0-9\.]+)', str(line))
|
||||
if ret is not None:
|
||||
nbFilesNotFormatted = int(ret.group('nb_errors'))
|
||||
|
||||
if re.search('=== Files not properly formatted ===', str(line)) is not None:
|
||||
listFiles = True
|
||||
if listFiles:
|
||||
if re.search('Removing intermediate container', str(line)) is not None:
|
||||
listFiles = False
|
||||
elif re.search('Running in|Files not properly formatted', str(line)) is not None:
|
||||
pass
|
||||
else:
|
||||
listFilesNotFormatted.append(str(line).strip())
|
||||
|
||||
if re.search('=== Files with incorrect define protection ===', str(line)) is not None:
|
||||
circularHeaderDependency = True
|
||||
if circularHeaderDependency:
|
||||
if re.search('Removing intermediate container', str(line)) is not None:
|
||||
circularHeaderDependency = False
|
||||
elif re.search('Running in|Files with incorrect define protection', str(line)) is not None:
|
||||
pass
|
||||
else:
|
||||
circularHeaderDependencyFiles.append(str(line).strip())
|
||||
|
||||
if re.search('=== Files with a GNU GPL licence Banner ===', str(line)) is not None:
|
||||
gnuGplLicence = True
|
||||
if gnuGplLicence:
|
||||
if re.search('Removing intermediate container', str(line)) is not None:
|
||||
gnuGplLicence = False
|
||||
elif re.search('Running in|Files with a GNU GPL licence Banner', str(line)) is not None:
|
||||
pass
|
||||
else:
|
||||
gnuGplLicenceFiles.append(str(line).strip())
|
||||
|
||||
if re.search('=== Files with a suspect Banner ===', str(line)) is not None:
|
||||
suspectLicence = True
|
||||
if suspectLicence:
|
||||
if re.search('Removing intermediate container', str(line)) is not None:
|
||||
suspectLicence = False
|
||||
elif re.search('Running in|Files with a suspect Banner', str(line)) is not None:
|
||||
pass
|
||||
else:
|
||||
suspectLicenceFiles.append(str(line).strip())
|
||||
|
||||
logfile.close()
|
||||
if analyzed:
|
||||
logging.debug('files not formatted properly: ' + str(nbFilesNotFormatted))
|
||||
if nbFilesNotFormatted == 0:
|
||||
HTML.CreateHtmlTestRow('File(s) Format', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
else:
|
||||
html_queue = SimpleQueue()
|
||||
html_cell = '<pre style="background-color:white">\n'
|
||||
html_cell += 'Number of files not following OAI Rules: ' + str(nbFilesNotFormatted) + '\n'
|
||||
for nFile in listFilesNotFormatted:
|
||||
html_cell += str(nFile).strip() + '\n'
|
||||
html_cell += '</pre>'
|
||||
html_queue.put(html_cell)
|
||||
HTML.CreateHtmlTestRowQueue('File(s) Format', 'KO', 1, html_queue)
|
||||
del(html_cell)
|
||||
del(html_queue)
|
||||
|
||||
logging.debug('header files not respecting the circular dependency protection: ' + str(len(circularHeaderDependencyFiles)))
|
||||
if len(circularHeaderDependencyFiles) == 0:
|
||||
HTML.CreateHtmlTestRow('Header Circular Dependency', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
else:
|
||||
html_queue = SimpleQueue()
|
||||
html_cell = '<pre style="background-color:white">\n'
|
||||
html_cell += 'Number of files not respecting: ' + str(len(circularHeaderDependencyFiles)) + '\n'
|
||||
for nFile in circularHeaderDependencyFiles:
|
||||
html_cell += str(nFile).strip() + '\n'
|
||||
html_cell += '</pre>'
|
||||
html_queue.put(html_cell)
|
||||
HTML.CreateHtmlTestRowQueue('Header Circular Dependency', 'KO', 1, html_queue)
|
||||
del(html_cell)
|
||||
del(html_queue)
|
||||
finalStatus = -1
|
||||
|
||||
logging.debug('files with a GNU GPL license: ' + str(len(gnuGplLicenceFiles)))
|
||||
if len(gnuGplLicenceFiles) == 0:
|
||||
HTML.CreateHtmlTestRow('Files w/ GNU GPL License', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
else:
|
||||
html_queue = SimpleQueue()
|
||||
html_cell = '<pre style="background-color:white">\n'
|
||||
html_cell += 'Number of files not respecting: ' + str(len(gnuGplLicenceFiles)) + '\n'
|
||||
for nFile in gnuGplLicenceFiles:
|
||||
html_cell += str(nFile).strip() + '\n'
|
||||
html_cell += '</pre>'
|
||||
html_queue.put(html_cell)
|
||||
HTML.CreateHtmlTestRowQueue('Files w/ GNU GPL License', 'KO', 1, html_queue)
|
||||
del(html_cell)
|
||||
del(html_queue)
|
||||
finalStatus = -1
|
||||
|
||||
logging.debug('files with a suspect license: ' + str(len(suspectLicenceFiles)))
|
||||
if len(suspectLicenceFiles) == 0:
|
||||
HTML.CreateHtmlTestRow('Files with suspect license', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
else:
|
||||
html_queue = SimpleQueue()
|
||||
html_cell = '<pre style="background-color:white">\n'
|
||||
html_cell += 'Number of files not respecting: ' + str(len(suspectLicenceFiles)) + '\n'
|
||||
for nFile in suspectLicenceFiles:
|
||||
html_cell += str(nFile).strip() + '\n'
|
||||
html_cell += '</pre>'
|
||||
html_queue.put(html_cell)
|
||||
HTML.CreateHtmlTestRowQueue('Files with suspect license', 'KO', 1, html_queue)
|
||||
del(html_cell)
|
||||
del(html_queue)
|
||||
finalStatus = -1
|
||||
|
||||
else:
|
||||
finalStatus = -1
|
||||
HTML.htmleNBFailureMsg = 'Could not fully analyze oai-formatting-check.txt file'
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
|
||||
else:
|
||||
finalStatus = -1
|
||||
HTML.htmleNBFailureMsg = 'Could not access oai-formatting-check.txt file'
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
|
||||
|
||||
return finalStatus
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
FROM ubuntu:bionic AS oai-formatting-check
|
||||
|
||||
ARG MERGE_REQUEST
|
||||
ARG SRC_BRANCH
|
||||
ARG TARGET_BRANCH
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
RUN apt-get update && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get upgrade --yes && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
astyle \
|
||||
gawk \
|
||||
git
|
||||
|
||||
WORKDIR /oai-ran
|
||||
COPY . .
|
||||
|
||||
RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then echo 'Source Branch = $SRC_BRANCH'; echo 'Target Branch = $TARGET_BRANCH'; else echo 'Push to develop'; fi"
|
||||
RUN /bin/bash -c "if [[ -v MERGE_REQUEST ]]; then ./ci-scripts/checkCodingFormattingRules.sh --src-branch $SRC_BRANCH --target-branch $TARGET_BRANCH; else ./ci-scripts/checkCodingFormattingRules.sh; fi"
|
||||
|
||||
RUN echo "=== Files not properly formatted ===" && \
|
||||
/bin/bash -c "if [[ -f oai_rules_result_list.txt ]]; then cat oai_rules_result_list.txt; fi"
|
||||
|
||||
RUN echo "=== Files with incorrect define protection ===" && \
|
||||
/bin/bash -c "if [[ -f header-files-w-incorrect-define.txt ]]; then cat header-files-w-incorrect-define.txt; fi"
|
||||
|
||||
RUN echo "=== Files with a GNU GPL licence Banner ===" && \
|
||||
/bin/bash -c "if [[ -f files-w-gnu-gpl-license-banner.txt ]]; then cat files-w-gnu-gpl-license-banner.txt; fi"
|
||||
|
||||
RUN echo "=== Files with a suspect Banner ===" && \
|
||||
/bin/bash -c "if [[ -f files-w-suspect-banner.txt ]]; then cat files-w-suspect-banner.txt; fi"
|
||||
+11
-8
@@ -255,16 +255,19 @@ class EPCManagement():
|
||||
mySSH.command('mkdir -p ' + self.SourceCodePath + '/scripts', '\$', 5)
|
||||
mySSH.command('cd /opt/oai-cn5g-fed-v1.3/docker-compose', '\$', 5)
|
||||
mySSH.command('python3 ./core-network.py '+self.cfgDeploy, '\$', 60)
|
||||
time.sleep(2)
|
||||
mySSH.command('docker-compose -p 5gcn ps -a', '\$', 60)
|
||||
if re.search('start-mini-as-ue', self.cfgDeploy):
|
||||
dFile = 'docker-compose-mini-nrf-asue.yaml'
|
||||
else:
|
||||
dFile = 'docker-compose-mini-nrf.yaml'
|
||||
mySSH.command('docker-compose -p 5gcn -f ' + dFile + ' ps -a', '\$', 60)
|
||||
if mySSH.getBefore().count('Up (healthy)') != 6:
|
||||
logging.error('Not all container healthy')
|
||||
else:
|
||||
logging.debug('OK')
|
||||
mySSH.command('docker-compose config | grep --colour=never image', '\$', 10)
|
||||
logging.debug('OK --> all containers are healthy')
|
||||
mySSH.command('docker-compose -p 5gcn -f ' + dFile + ' config | grep --colour=never image', '\$', 10)
|
||||
listOfImages = mySSH.getBefore()
|
||||
for imageLine in listOfImages.split('\\r\\n'):
|
||||
res1 = re.search('image: (?P<name>[a-zA-Z0-9\-]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine))
|
||||
res1 = re.search('image: (?P<name>[a-zA-Z0-9\-/]+):(?P<tag>[a-zA-Z0-9\-]+)', str(imageLine))
|
||||
res2 = re.search('mysql', str(imageLine))
|
||||
if res1 is not None and res2 is None:
|
||||
html_cell += res1.group('name') + ':' + res1.group('tag') + ' '
|
||||
@@ -536,7 +539,7 @@ class EPCManagement():
|
||||
mySSH.command('python3 ./core-network.py '+self.cfgUnDeploy, '\$', 60)
|
||||
mySSH.command('docker volume prune --force || true', '\$', 60)
|
||||
time.sleep(2)
|
||||
mySSH.command('tshark -r /tmp/oai-cn5g.pcap | egrep --colour=never "Tracking area update" ','\$', 30)
|
||||
mySSH.command('tshark -r /tmp/oai-cn5g-v1.3.pcap | egrep --colour=never "Tracking area update" ','\$', 30)
|
||||
result = re.search('Tracking area update request', mySSH.getBefore())
|
||||
if result is not None:
|
||||
message = 'UE requested ' + str(mySSH.getBefore().count('Tracking area update request')) + 'Tracking area update request(s)'
|
||||
@@ -830,8 +833,8 @@ class EPCManagement():
|
||||
mySSH.command('zip mme.log.zip mme_check_run.*', '\$', 60)
|
||||
elif re.match('OAICN5G', self.Type, re.IGNORECASE):
|
||||
mySSH.command('cd ' + self.SourceCodePath + '/logs','\$', 5)
|
||||
mySSH.command('cp -f /tmp/oai-cn5g.pcap .','\$', 30)
|
||||
mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g.pcap','\$', 30)
|
||||
mySSH.command('cp -f /tmp/oai-cn5g-v1.3.pcap .','\$', 30)
|
||||
mySSH.command('zip mme.log.zip oai-amf.log oai-nrf.log oai-cn5g*.pcap','\$', 30)
|
||||
mySSH.command('mv mme.log.zip ' + self.SourceCodePath + '/scripts','\$', 30)
|
||||
elif re.match('OAI', self.Type, re.IGNORECASE) or re.match('OAI-Rel14-CUPS', self.Type, re.IGNORECASE):
|
||||
mySSH.command('zip mme.log.zip mme*.log', '\$', 60)
|
||||
|
||||
+5
-1
@@ -799,7 +799,7 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
|
||||
HTML.startTime=int(round(time.time() * 1000))
|
||||
while CiTestObj.FailReportCnt < CiTestObj.repeatCounts[0] and RAN.prematureExit:
|
||||
RAN.prematureExit=False
|
||||
# At every iteratin of the retry loop, a separator will be added
|
||||
# At every iteration of the retry loop, a separator will be added
|
||||
# pass CiTestObj.FailReportCnt as parameter of HTML.CreateHtmlRetrySeparator
|
||||
HTML.CreateHtmlRetrySeparator(CiTestObj.FailReportCnt)
|
||||
for test_case_id in todo_tests:
|
||||
@@ -923,6 +923,10 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
|
||||
CONTAINERS.UndeployObject(HTML, RAN)
|
||||
elif action == 'Cppcheck_Analysis':
|
||||
SCA.CppCheckAnalysis(HTML)
|
||||
elif action == 'LicenceAndFormattingCheck':
|
||||
ret = SCA.LicenceAndFormattingCheck(HTML)
|
||||
if ret != 0:
|
||||
RAN.prematureExit = True
|
||||
elif action == 'Deploy_Run_PhySim':
|
||||
PHYSIM.Deploy_PhySim(HTML, RAN)
|
||||
elif action == 'DeployGenObject':
|
||||
|
||||
@@ -47,3 +47,4 @@
|
||||
- PingFromContainer
|
||||
- IperfFromContainer
|
||||
- StatsFromGenObject
|
||||
- LicenceAndFormattingCheck
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
<!--
|
||||
|
||||
Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
contributor license agreements. See the NOTICE file distributed with
|
||||
this work for additional information regarding copyright ownership.
|
||||
The OpenAirInterface Software Alliance licenses this file to You under
|
||||
the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.openairinterface.org/?page_id=698
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
contact@openairinterface.org
|
||||
|
||||
-->
|
||||
<testCaseList>
|
||||
<htmlTabRef>formatting-tab</htmlTabRef>
|
||||
<htmlTabName>License and Formatting Checks</htmlTabName>
|
||||
<htmlTabIcon>wrench</htmlTabIcon>
|
||||
<TestCaseRequestedList>
|
||||
000002
|
||||
</TestCaseRequestedList>
|
||||
<TestCaseExclusionList></TestCaseExclusionList>
|
||||
|
||||
<testCase id="000002">
|
||||
<class>LicenceAndFormattingCheck</class>
|
||||
<desc>License and Formatting Checks</desc>
|
||||
</testCase>
|
||||
|
||||
</testCaseList>
|
||||
Reference in New Issue
Block a user