Better test automation with cmsis build tools
Added a python script for automation of tests with cmsis build tools. Correction to some tests Float promotion issues in some f16 code.pull/75/head
Christophe Favergeon
3 years ago
parent
43aa2a9e7f
commit
f91c1f3c24
@ -0,0 +1,21 @@
|
|||||||
|
|
||||||
|
/*
|
||||||
|
* Auto generated Run-Time-Environment Configuration File
|
||||||
|
* *** Do not modify ! ***
|
||||||
|
*
|
||||||
|
* Project: 'test.Release+VHT_M55'
|
||||||
|
* Target: 'test.Release+VHT_M55'
|
||||||
|
*/
|
||||||
|
|
||||||
|
#ifndef RTE_COMPONENTS_H
|
||||||
|
#define RTE_COMPONENTS_H
|
||||||
|
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Define the Device Header File:
|
||||||
|
*/
|
||||||
|
#define CMSIS_device_header "ARMCM55.h"
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#endif /* RTE_COMPONENTS_H */
|
||||||
@ -1,24 +1,45 @@
|
|||||||
ECHO OFF
|
ECHO OFF
|
||||||
ECHO "Gen Test"
|
ECHO "Gen Test"
|
||||||
cd ..
|
cd ..
|
||||||
python processTests.py -f Output.pickle -e StatsTests
|
python processTests.py -f Output.pickle -e UnaryTestsF32
|
||||||
cd cmsis_build
|
cd cmsis_build
|
||||||
|
|
||||||
ECHO "Build"
|
REM ECHO "Build"
|
||||||
cbuild "test.Release+VHT-Corstone-300.cprj"
|
REM cbuild "test.Release+VHT-Corstone-300.cprj"
|
||||||
ECHO "Run"
|
REM ECHO "Run"
|
||||||
C:\Keil_v5\ARM\VHT\VHT_Corstone_SSE-300_Ethos-U55.exe -f configs/ARM_VHT_Corstone_300_config.txt -a cpu0="out\test\VHT-Corstone-300\Release\test.Release+VHT-Corstone-300.axf" > results_cs300_u55.txt
|
REM C:\Keil_v5\ARM\VHT_11.18.29\VHT_Corstone_SSE-300_Ethos-U55.exe -f configs/ARM_VHT_Corstone_300_config.txt -a cpu0="out\test\VHT-Corstone-300\Release\test.Release+VHT-Corstone-300.axf" > results_cs300_u55.txt
|
||||||
python ..\processResult.py -f ..\Output.pickle -e -r results_cs300_u55.txt
|
REM set MyError=%errorlevel%
|
||||||
|
REM python ..\processResult.py -f ..\Output.pickle -e -r results_cs300_u55.txt
|
||||||
|
REM if %MyError% neq 0 goto :error
|
||||||
|
REM goto :EOF
|
||||||
|
|
||||||
REM ECHO "Build"
|
REM ECHO "Build"
|
||||||
REM cbuild "test.Release+VHT_M7_UNROLLED.cprj"
|
REM cbuild "test.Release+VHT_M7_UNROLLED.cprj"
|
||||||
REM ECHO "Run"
|
REM ECHO "Run"
|
||||||
REM C:\Keil_v5\ARM\VHT\VHT_MPS2_Cortex-M7.exe -f configs/ARM_VHT_MPS2_M7DP_config.txt out\test\VHT_M7_UNROLLED\Release\test.Release+VHT_M7_UNROLLED.axf > results_m7_unrolled.txt
|
REM C:\Keil_v5\ARM\VHT_11.18.29\VHT_MPS2_Cortex-M7.exe -f configs/ARM_VHT_MPS2_M7DP_config.txt out\test\VHT_M7_UNROLLED\Release\test.Release+VHT_M7_UNROLLED.axf > results_m7_unrolled.txt
|
||||||
|
REM set MyError=%errorlevel%
|
||||||
REM python ..\processResult.py -f ..\Output.pickle -e -r results_m7_unrolled.txt
|
REM python ..\processResult.py -f ..\Output.pickle -e -r results_m7_unrolled.txt
|
||||||
|
REM if %MyError% neq 0 goto :error
|
||||||
|
REM goto :EOF
|
||||||
|
|
||||||
|
ECHO "Build"
|
||||||
|
cbuild "test.Release+VHT_M0P.cprj"
|
||||||
|
ECHO "Run"
|
||||||
|
C:\Keil_v5\ARM\VHT_11.18.29\VHT_MPS2_Cortex-M0plus.exe -f configs/ARM_VHT_MPS2_M0plus_config.txt out\test\VHT_M0P\Release\test.Release+VHT_M0P.axf > results_m0p.txt
|
||||||
|
set MyError=%errorlevel%
|
||||||
|
python ..\processResult.py -f ..\Output.pickle -e -r results_m0p.txt
|
||||||
|
if %MyError% neq 0 goto :error
|
||||||
|
goto :EOF
|
||||||
|
|
||||||
REM ECHO "Build"
|
REM ECHO "Build"
|
||||||
REM cbuild "test.Release+FVP_M55.cprj"
|
REM cbuild "test.Release+VHT_M55.cprj"
|
||||||
REM ECHO "Run"
|
REM ECHO "Run"
|
||||||
REM C:\Keil_v5\ARM\VHT\VHT_MPS2_Cortex-M55.exe -f configs/REM ARM_VHT_MPS2_M55_config.txt -a cpu0="REM out\test\FVP_M55\Release\test.Release+FVP_M55.elf" REM > results_m55.txt
|
REM C:\Keil_v5\ARM\VHT_11.18.29\VHT_MPS2_Cortex-M55.exe -f configs/ARM_VHT_MPS2_M55_config.txt -a cpu0="out\test\VHT_M55\Release\test.Release+VHT_M55.elf" > results_m55.txt
|
||||||
REM python ..\processResult.py -f ..\Output.pickle -e -r REM results_m55.txt
|
REM set MyError=%errorlevel%
|
||||||
REM
|
REM python ..\processResult.py -f ..\Output.pickle -e -r results_m55.txt
|
||||||
|
REM if %MyError% neq 0 goto :error
|
||||||
|
REM goto :EOF
|
||||||
|
|
||||||
|
:error
|
||||||
|
echo Failed running all tests with error %MyError%.
|
||||||
|
exit /b %errorlevel%
|
||||||
|
|||||||
@ -0,0 +1,266 @@
|
|||||||
|
import argparse
|
||||||
|
import pickle
|
||||||
|
import sys
|
||||||
|
import subprocess
|
||||||
|
import os
|
||||||
|
import colorama
|
||||||
|
from colorama import init,Fore, Back, Style
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description='Parse test description')
|
||||||
|
parser.add_argument('-avh', nargs='?',type = str, default="C:/Keil_v5/ARM/VHT_11.18.29", help="AVH folder")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
init()
|
||||||
|
|
||||||
|
sys.path.append("..")
|
||||||
|
|
||||||
|
from TestScripts.Parser import TreeElem
|
||||||
|
|
||||||
|
ERROR_OCCURED = False
|
||||||
|
|
||||||
|
def printTitle(s):
|
||||||
|
print("\n" + Fore.GREEN + Style.BRIGHT + s + Style.RESET_ALL)
|
||||||
|
|
||||||
|
def printSubTitle(s):
|
||||||
|
print("\n" + Fore.YELLOW + Style.BRIGHT + s + Style.RESET_ALL)
|
||||||
|
|
||||||
|
def printError(s):
|
||||||
|
print("\n" + Fore.RED + Style.BRIGHT + s + Style.RESET_ALL)
|
||||||
|
|
||||||
|
# Load Output.pickle files for the test description
|
||||||
|
def loadRoot(f):
|
||||||
|
root = None
|
||||||
|
with open(f,"rb") as inf:
|
||||||
|
root=pickle.load(inf)
|
||||||
|
return(root)
|
||||||
|
|
||||||
|
# Get test suites from the test descriptions
|
||||||
|
def getSuites(node,filterList,currentList=[]):
|
||||||
|
if node.kind==TreeElem.SUITE:
|
||||||
|
currentList.append(node.data["class"])
|
||||||
|
if node.kind==TreeElem.GROUP:
|
||||||
|
if not node.data["class"] in filterList:
|
||||||
|
for c in node.children:
|
||||||
|
getSuites(c,filterList,currentList)
|
||||||
|
|
||||||
|
class Result:
|
||||||
|
def __init__(self,msg,error=False):
|
||||||
|
self._error = error
|
||||||
|
self._msg = msg
|
||||||
|
|
||||||
|
@property
|
||||||
|
def error(self):
|
||||||
|
return self._error
|
||||||
|
|
||||||
|
@property
|
||||||
|
def msg(self):
|
||||||
|
return self._msg
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Run a command and get error or result
|
||||||
|
# For the test report we don't need the stderr
|
||||||
|
# in case of error since the test report is giving
|
||||||
|
# all the details. So, there is an option to
|
||||||
|
# disable the dump of stderr
|
||||||
|
def run(*args,mustPrint=False,dumpStdErr=True):
|
||||||
|
global ERROR_OCCURED
|
||||||
|
try:
|
||||||
|
result=subprocess.run(args,text=True,capture_output=True,timeout=600)
|
||||||
|
if result.returncode !=0 :
|
||||||
|
ERROR_OCCURED = True
|
||||||
|
if dumpStdErr:
|
||||||
|
return(Result(result.stderr + "\n\nSTDOUT:\n\n" + result.stdout,error=True))
|
||||||
|
else:
|
||||||
|
return(Result(result.stdout,error=True))
|
||||||
|
|
||||||
|
if mustPrint:
|
||||||
|
print(result.stdout)
|
||||||
|
return(Result(result.stdout))
|
||||||
|
except Exception as e:
|
||||||
|
ERROR_OCCURED = True
|
||||||
|
return(Result(str(e),error=True))
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Configuration file for AVH core
|
||||||
|
configFiles={
|
||||||
|
"CS310":"ARM_VHT_Corstone_310_config.txt",
|
||||||
|
"CS300":"ARM_VHT_Corstone_300_config.txt",
|
||||||
|
"M55":"ARM_VHT_MPS2_M55_config.txt",
|
||||||
|
"M33_DSP_FP":"ARM_VHT_MPS2_M33_DSP_FP_config.txt",
|
||||||
|
"M7DP":"ARM_VHT_MPS2_M7DP_config.txt",
|
||||||
|
"M4FP":"ARM_VHT_MPS2_M4FP_config.txt",
|
||||||
|
"M3":"ARM_VHT_MPS2_M3_config.txt",
|
||||||
|
"M23":"ARM_VHT_MPS2_M23_config.txt",
|
||||||
|
"M0plus":"ARM_VHT_MPS2_M0plus_config.txt",
|
||||||
|
}
|
||||||
|
|
||||||
|
# Windows executable
|
||||||
|
# (At some point this script will also support
|
||||||
|
# unix)
|
||||||
|
avhExe={
|
||||||
|
"CS310":"VHT_Corstone_SSE-310.exe",
|
||||||
|
"CS300":"VHT_Corstone_SSE-300_Ethos-U55.exe",
|
||||||
|
"M55":"VHT_MPS2_Cortex-M55.exe",
|
||||||
|
"M33_DSP_FP":"VHT_MPS2_Cortex-M33.exe",
|
||||||
|
"M7DP":"VHT_MPS2_Cortex-M7.exe",
|
||||||
|
"M4FP":"VHT_MPS2_Cortex-M4.exe",
|
||||||
|
"M3":"VHT_MPS2_Cortex-M3.exe",
|
||||||
|
"M23":"VHT_MPS2_Cortex-M23.exe",
|
||||||
|
"M0plus":"VHT_MPS2_Cortex-M0plus.exe",
|
||||||
|
}
|
||||||
|
|
||||||
|
AVHROOT = args.avh
|
||||||
|
|
||||||
|
# Run AVH
|
||||||
|
def runAVH(build,core):
|
||||||
|
axf="out/test/%s/Release/test.Release+%s.axf" % (build,build)
|
||||||
|
elf="out/test/%s/Release/test.Release+%s.elf" % (build,build)
|
||||||
|
app = axf
|
||||||
|
if os.path.exists(axf):
|
||||||
|
app = axf
|
||||||
|
if os.path.exists(elf):
|
||||||
|
app = elf
|
||||||
|
config = os.path.join("configs",configFiles[core])
|
||||||
|
avh = os.path.join(AVHROOT,avhExe[core])
|
||||||
|
res=run(avh,"-f",config,app)
|
||||||
|
return(res)
|
||||||
|
|
||||||
|
####################
|
||||||
|
|
||||||
|
# Test descriptions to use
|
||||||
|
tests=["../Output.pickle","../Output_f16.pickle"]
|
||||||
|
# Test group to avoid
|
||||||
|
filterList=["NNTests","ExampleTests"]
|
||||||
|
allSuites=[]
|
||||||
|
|
||||||
|
# Load all the test suite to run
|
||||||
|
for t in tests:
|
||||||
|
root=loadRoot(t)
|
||||||
|
suites=[]
|
||||||
|
getSuites(root,filterList,suites)
|
||||||
|
allSuites += [(x,t) for x in suites ]
|
||||||
|
|
||||||
|
# Test suite and output pickle needed to decode the result
|
||||||
|
#print(allSuites)
|
||||||
|
|
||||||
|
#allSuites=[("BasicTestsF32","../Output.pickle"),
|
||||||
|
#("BasicTestsQ31","../Output.pickle")]
|
||||||
|
|
||||||
|
#allSuites=[("StatsTestsQ7","../Output.pickle")]
|
||||||
|
|
||||||
|
# Solution and build file for all
|
||||||
|
# the tests
|
||||||
|
# It is a pair : csolution target type and AVH identification
|
||||||
|
# AVH identification is used to find the executable
|
||||||
|
# and the configuration file
|
||||||
|
solutions={
|
||||||
|
'testac6.csolution.yml':[
|
||||||
|
("VHT-Corstone-310","CS310"),
|
||||||
|
("VHT-Corstone-300","CS300"),
|
||||||
|
#("VHT_M33","M33_DSP_FP"),
|
||||||
|
("VHT_M7","M7DP"),
|
||||||
|
("VHT_M7_UNROLLED","M7DP"),
|
||||||
|
#("VHT_M4","M4FP"),
|
||||||
|
#("VHT_M3","M3"),
|
||||||
|
#("VHT_M23","M23"),
|
||||||
|
("VHT_M0P","M0plus")
|
||||||
|
],
|
||||||
|
'testgcc.csolution.yml':[
|
||||||
|
("VHT_M55","M55"),
|
||||||
|
#("VHT_M33","M33_DSP_FP"),
|
||||||
|
("VHT_M7","M7DP"),
|
||||||
|
("VHT_M7_UNROLLED","M7DP"),
|
||||||
|
#("VHT_M4","M4FP"),
|
||||||
|
#("VHT_M3","M3"),
|
||||||
|
#("VHT_M23","M23"),
|
||||||
|
("VHT_M0P","M0plus")
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
HTMLHEADER="""<html>
|
||||||
|
<header>
|
||||||
|
<title>CMSIS-DSP Test summary</title>
|
||||||
|
</header>
|
||||||
|
<body>
|
||||||
|
"""
|
||||||
|
|
||||||
|
HTMLFOOTER="""
|
||||||
|
</body></html>
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Run the tests and log the result
|
||||||
|
# in a summary.html file
|
||||||
|
with open("summary.html","w") as f:
|
||||||
|
print(HTMLHEADER,file=f)
|
||||||
|
for s in solutions:
|
||||||
|
printTitle("Process solution %s" % s)
|
||||||
|
run("csolution","convert","-s",s)
|
||||||
|
print("<h1>Solution %s</h1>" % s,file=f)
|
||||||
|
maxNbBuilds=len(solutions[s])
|
||||||
|
buildNb=0
|
||||||
|
for build,core in solutions[s]:
|
||||||
|
buildNb = buildNb + 1
|
||||||
|
print("<h2>Core %s</h2>" % build,file=f)
|
||||||
|
printTitle("Process core %s (%d/%d)" % (build,buildNb,maxNbBuilds))
|
||||||
|
buildFile="test.Release+%s.cprj" % build
|
||||||
|
nb = 0
|
||||||
|
maxNb = len(allSuites)
|
||||||
|
for s,pickle in allSuites:
|
||||||
|
nb = nb + 1
|
||||||
|
printSubTitle("Process suite %s (%d/%d)" % (s,nb,maxNb))
|
||||||
|
res=run(sys.executable,"../processTests.py","-gen","..","-p","../Patterns","-d","../Parameters","-f",pickle,"-e",s,mustPrint=True)
|
||||||
|
if res.error:
|
||||||
|
printError("Error processTests")
|
||||||
|
print("<p><font color=\"red\">Error generating %s</font></p><PRE>" % s,file=f)
|
||||||
|
print(res.msg,file=f)
|
||||||
|
print("</PRE>",file=f)
|
||||||
|
continue
|
||||||
|
if nb==1:
|
||||||
|
# -r is needed for first
|
||||||
|
# build when we switch
|
||||||
|
# between different solutions
|
||||||
|
# (Like one using AC6 and the other
|
||||||
|
# using gcc)
|
||||||
|
res=run("cbuild","-r",buildFile)
|
||||||
|
else:
|
||||||
|
res=run("cbuild",buildFile)
|
||||||
|
if res.error:
|
||||||
|
printError("Error cbuild")
|
||||||
|
print("<p><font color=\"red\">Error building %s</font></p><PRE>" % s,file=f)
|
||||||
|
print(res.msg,file=f)
|
||||||
|
print("</PRE>",file=f)
|
||||||
|
continue
|
||||||
|
printSubTitle("Run AVH")
|
||||||
|
res=runAVH(build,core)
|
||||||
|
if res.error:
|
||||||
|
printError("Error running AVH")
|
||||||
|
print("<p><font color=\"red\">Error running %s</font></p><PRE>" % s,file=f)
|
||||||
|
print(res.msg,file=f)
|
||||||
|
print("</PRE>",file=f)
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
with open("results.txt","w") as o:
|
||||||
|
print(res.msg,file=o)
|
||||||
|
res=run(sys.executable,"../processResult.py","-f",pickle,"-e","-ahtml","-r","results.txt",dumpStdErr=False)
|
||||||
|
if res.error:
|
||||||
|
printError("Error processResult")
|
||||||
|
print("<p><font color=\"red\">Error processing %s result</font></p><PRE>" % s,file=f)
|
||||||
|
print(res.msg,file=f)
|
||||||
|
print("</PRE>",file=f)
|
||||||
|
continue
|
||||||
|
else:
|
||||||
|
pass
|
||||||
|
# When no error the section is not
|
||||||
|
# included in final file
|
||||||
|
#print(res.msg,file=f)
|
||||||
|
print(HTMLFOOTER,file=f)
|
||||||
|
|
||||||
|
if ERROR_OCCURED:
|
||||||
|
sys.exit("Error occurred")
|
||||||
|
else:
|
||||||
|
sys.exit(0)
|
||||||
Loading…
Reference in New Issue