Better test automation with cmsis build tools
Added a python script for automation of tests with cmsis build tools. Correction to some tests Float promotion issues in some f16 code.pull/75/head
Christophe Favergeon
3 years ago
parent
43aa2a9e7f
commit
f91c1f3c24
@ -0,0 +1,21 @@
|
||||
|
||||
/*
|
||||
* Auto generated Run-Time-Environment Configuration File
|
||||
* *** Do not modify ! ***
|
||||
*
|
||||
* Project: 'test.Release+VHT_M55'
|
||||
* Target: 'test.Release+VHT_M55'
|
||||
*/
|
||||
|
||||
#ifndef RTE_COMPONENTS_H
|
||||
#define RTE_COMPONENTS_H
|
||||
|
||||
|
||||
/*
|
||||
* Define the Device Header File:
|
||||
*/
|
||||
#define CMSIS_device_header "ARMCM55.h"
|
||||
|
||||
|
||||
|
||||
#endif /* RTE_COMPONENTS_H */
|
||||
@ -1,24 +1,45 @@
|
||||
ECHO OFF
|
||||
ECHO "Gen Test"
|
||||
cd ..
|
||||
python processTests.py -f Output.pickle -e StatsTests
|
||||
python processTests.py -f Output.pickle -e UnaryTestsF32
|
||||
cd cmsis_build
|
||||
|
||||
ECHO "Build"
|
||||
cbuild "test.Release+VHT-Corstone-300.cprj"
|
||||
ECHO "Run"
|
||||
C:\Keil_v5\ARM\VHT\VHT_Corstone_SSE-300_Ethos-U55.exe -f configs/ARM_VHT_Corstone_300_config.txt -a cpu0="out\test\VHT-Corstone-300\Release\test.Release+VHT-Corstone-300.axf" > results_cs300_u55.txt
|
||||
python ..\processResult.py -f ..\Output.pickle -e -r results_cs300_u55.txt
|
||||
REM ECHO "Build"
|
||||
REM cbuild "test.Release+VHT-Corstone-300.cprj"
|
||||
REM ECHO "Run"
|
||||
REM C:\Keil_v5\ARM\VHT_11.18.29\VHT_Corstone_SSE-300_Ethos-U55.exe -f configs/ARM_VHT_Corstone_300_config.txt -a cpu0="out\test\VHT-Corstone-300\Release\test.Release+VHT-Corstone-300.axf" > results_cs300_u55.txt
|
||||
REM set MyError=%errorlevel%
|
||||
REM python ..\processResult.py -f ..\Output.pickle -e -r results_cs300_u55.txt
|
||||
REM if %MyError% neq 0 goto :error
|
||||
REM goto :EOF
|
||||
|
||||
REM ECHO "Build"
|
||||
REM cbuild "test.Release+VHT_M7_UNROLLED.cprj"
|
||||
REM ECHO "Run"
|
||||
REM C:\Keil_v5\ARM\VHT\VHT_MPS2_Cortex-M7.exe -f configs/ARM_VHT_MPS2_M7DP_config.txt out\test\VHT_M7_UNROLLED\Release\test.Release+VHT_M7_UNROLLED.axf > results_m7_unrolled.txt
|
||||
REM C:\Keil_v5\ARM\VHT_11.18.29\VHT_MPS2_Cortex-M7.exe -f configs/ARM_VHT_MPS2_M7DP_config.txt out\test\VHT_M7_UNROLLED\Release\test.Release+VHT_M7_UNROLLED.axf > results_m7_unrolled.txt
|
||||
REM set MyError=%errorlevel%
|
||||
REM python ..\processResult.py -f ..\Output.pickle -e -r results_m7_unrolled.txt
|
||||
REM if %MyError% neq 0 goto :error
|
||||
REM goto :EOF
|
||||
|
||||
ECHO "Build"
|
||||
cbuild "test.Release+VHT_M0P.cprj"
|
||||
ECHO "Run"
|
||||
C:\Keil_v5\ARM\VHT_11.18.29\VHT_MPS2_Cortex-M0plus.exe -f configs/ARM_VHT_MPS2_M0plus_config.txt out\test\VHT_M0P\Release\test.Release+VHT_M0P.axf > results_m0p.txt
|
||||
set MyError=%errorlevel%
|
||||
python ..\processResult.py -f ..\Output.pickle -e -r results_m0p.txt
|
||||
if %MyError% neq 0 goto :error
|
||||
goto :EOF
|
||||
|
||||
REM ECHO "Build"
|
||||
REM cbuild "test.Release+FVP_M55.cprj"
|
||||
REM cbuild "test.Release+VHT_M55.cprj"
|
||||
REM ECHO "Run"
|
||||
REM C:\Keil_v5\ARM\VHT\VHT_MPS2_Cortex-M55.exe -f configs/REM ARM_VHT_MPS2_M55_config.txt -a cpu0="REM out\test\FVP_M55\Release\test.Release+FVP_M55.elf" REM > results_m55.txt
|
||||
REM python ..\processResult.py -f ..\Output.pickle -e -r REM results_m55.txt
|
||||
REM
|
||||
REM C:\Keil_v5\ARM\VHT_11.18.29\VHT_MPS2_Cortex-M55.exe -f configs/ARM_VHT_MPS2_M55_config.txt -a cpu0="out\test\VHT_M55\Release\test.Release+VHT_M55.elf" > results_m55.txt
|
||||
REM set MyError=%errorlevel%
|
||||
REM python ..\processResult.py -f ..\Output.pickle -e -r results_m55.txt
|
||||
REM if %MyError% neq 0 goto :error
|
||||
REM goto :EOF
|
||||
|
||||
:error
|
||||
echo Failed running all tests with error %MyError%.
|
||||
exit /b %errorlevel%
|
||||
|
||||
@ -0,0 +1,266 @@
|
||||
import argparse
|
||||
import pickle
|
||||
import sys
|
||||
import subprocess
|
||||
import os
|
||||
import colorama
|
||||
from colorama import init,Fore, Back, Style
|
||||
|
||||
parser = argparse.ArgumentParser(description='Parse test description')
|
||||
parser.add_argument('-avh', nargs='?',type = str, default="C:/Keil_v5/ARM/VHT_11.18.29", help="AVH folder")
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
init()
|
||||
|
||||
sys.path.append("..")
|
||||
|
||||
from TestScripts.Parser import TreeElem
|
||||
|
||||
ERROR_OCCURED = False
|
||||
|
||||
def printTitle(s):
|
||||
print("\n" + Fore.GREEN + Style.BRIGHT + s + Style.RESET_ALL)
|
||||
|
||||
def printSubTitle(s):
|
||||
print("\n" + Fore.YELLOW + Style.BRIGHT + s + Style.RESET_ALL)
|
||||
|
||||
def printError(s):
|
||||
print("\n" + Fore.RED + Style.BRIGHT + s + Style.RESET_ALL)
|
||||
|
||||
# Load Output.pickle files for the test description
|
||||
def loadRoot(f):
|
||||
root = None
|
||||
with open(f,"rb") as inf:
|
||||
root=pickle.load(inf)
|
||||
return(root)
|
||||
|
||||
# Get test suites from the test descriptions
|
||||
def getSuites(node,filterList,currentList=[]):
|
||||
if node.kind==TreeElem.SUITE:
|
||||
currentList.append(node.data["class"])
|
||||
if node.kind==TreeElem.GROUP:
|
||||
if not node.data["class"] in filterList:
|
||||
for c in node.children:
|
||||
getSuites(c,filterList,currentList)
|
||||
|
||||
class Result:
|
||||
def __init__(self,msg,error=False):
|
||||
self._error = error
|
||||
self._msg = msg
|
||||
|
||||
@property
|
||||
def error(self):
|
||||
return self._error
|
||||
|
||||
@property
|
||||
def msg(self):
|
||||
return self._msg
|
||||
|
||||
|
||||
|
||||
# Run a command and get error or result
|
||||
# For the test report we don't need the stderr
|
||||
# in case of error since the test report is giving
|
||||
# all the details. So, there is an option to
|
||||
# disable the dump of stderr
|
||||
def run(*args,mustPrint=False,dumpStdErr=True):
|
||||
global ERROR_OCCURED
|
||||
try:
|
||||
result=subprocess.run(args,text=True,capture_output=True,timeout=600)
|
||||
if result.returncode !=0 :
|
||||
ERROR_OCCURED = True
|
||||
if dumpStdErr:
|
||||
return(Result(result.stderr + "\n\nSTDOUT:\n\n" + result.stdout,error=True))
|
||||
else:
|
||||
return(Result(result.stdout,error=True))
|
||||
|
||||
if mustPrint:
|
||||
print(result.stdout)
|
||||
return(Result(result.stdout))
|
||||
except Exception as e:
|
||||
ERROR_OCCURED = True
|
||||
return(Result(str(e),error=True))
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
# Configuration file for AVH core
|
||||
configFiles={
|
||||
"CS310":"ARM_VHT_Corstone_310_config.txt",
|
||||
"CS300":"ARM_VHT_Corstone_300_config.txt",
|
||||
"M55":"ARM_VHT_MPS2_M55_config.txt",
|
||||
"M33_DSP_FP":"ARM_VHT_MPS2_M33_DSP_FP_config.txt",
|
||||
"M7DP":"ARM_VHT_MPS2_M7DP_config.txt",
|
||||
"M4FP":"ARM_VHT_MPS2_M4FP_config.txt",
|
||||
"M3":"ARM_VHT_MPS2_M3_config.txt",
|
||||
"M23":"ARM_VHT_MPS2_M23_config.txt",
|
||||
"M0plus":"ARM_VHT_MPS2_M0plus_config.txt",
|
||||
}
|
||||
|
||||
# Windows executable
|
||||
# (At some point this script will also support
|
||||
# unix)
|
||||
avhExe={
|
||||
"CS310":"VHT_Corstone_SSE-310.exe",
|
||||
"CS300":"VHT_Corstone_SSE-300_Ethos-U55.exe",
|
||||
"M55":"VHT_MPS2_Cortex-M55.exe",
|
||||
"M33_DSP_FP":"VHT_MPS2_Cortex-M33.exe",
|
||||
"M7DP":"VHT_MPS2_Cortex-M7.exe",
|
||||
"M4FP":"VHT_MPS2_Cortex-M4.exe",
|
||||
"M3":"VHT_MPS2_Cortex-M3.exe",
|
||||
"M23":"VHT_MPS2_Cortex-M23.exe",
|
||||
"M0plus":"VHT_MPS2_Cortex-M0plus.exe",
|
||||
}
|
||||
|
||||
AVHROOT = args.avh
|
||||
|
||||
# Run AVH
|
||||
def runAVH(build,core):
|
||||
axf="out/test/%s/Release/test.Release+%s.axf" % (build,build)
|
||||
elf="out/test/%s/Release/test.Release+%s.elf" % (build,build)
|
||||
app = axf
|
||||
if os.path.exists(axf):
|
||||
app = axf
|
||||
if os.path.exists(elf):
|
||||
app = elf
|
||||
config = os.path.join("configs",configFiles[core])
|
||||
avh = os.path.join(AVHROOT,avhExe[core])
|
||||
res=run(avh,"-f",config,app)
|
||||
return(res)
|
||||
|
||||
####################
|
||||
|
||||
# Test descriptions to use
|
||||
tests=["../Output.pickle","../Output_f16.pickle"]
|
||||
# Test group to avoid
|
||||
filterList=["NNTests","ExampleTests"]
|
||||
allSuites=[]
|
||||
|
||||
# Load all the test suite to run
|
||||
for t in tests:
|
||||
root=loadRoot(t)
|
||||
suites=[]
|
||||
getSuites(root,filterList,suites)
|
||||
allSuites += [(x,t) for x in suites ]
|
||||
|
||||
# Test suite and output pickle needed to decode the result
|
||||
#print(allSuites)
|
||||
|
||||
#allSuites=[("BasicTestsF32","../Output.pickle"),
|
||||
#("BasicTestsQ31","../Output.pickle")]
|
||||
|
||||
#allSuites=[("StatsTestsQ7","../Output.pickle")]
|
||||
|
||||
# Solution and build file for all
|
||||
# the tests
|
||||
# It is a pair : csolution target type and AVH identification
|
||||
# AVH identification is used to find the executable
|
||||
# and the configuration file
|
||||
solutions={
|
||||
'testac6.csolution.yml':[
|
||||
("VHT-Corstone-310","CS310"),
|
||||
("VHT-Corstone-300","CS300"),
|
||||
#("VHT_M33","M33_DSP_FP"),
|
||||
("VHT_M7","M7DP"),
|
||||
("VHT_M7_UNROLLED","M7DP"),
|
||||
#("VHT_M4","M4FP"),
|
||||
#("VHT_M3","M3"),
|
||||
#("VHT_M23","M23"),
|
||||
("VHT_M0P","M0plus")
|
||||
],
|
||||
'testgcc.csolution.yml':[
|
||||
("VHT_M55","M55"),
|
||||
#("VHT_M33","M33_DSP_FP"),
|
||||
("VHT_M7","M7DP"),
|
||||
("VHT_M7_UNROLLED","M7DP"),
|
||||
#("VHT_M4","M4FP"),
|
||||
#("VHT_M3","M3"),
|
||||
#("VHT_M23","M23"),
|
||||
("VHT_M0P","M0plus")
|
||||
]
|
||||
}
|
||||
|
||||
HTMLHEADER="""<html>
|
||||
<header>
|
||||
<title>CMSIS-DSP Test summary</title>
|
||||
</header>
|
||||
<body>
|
||||
"""
|
||||
|
||||
HTMLFOOTER="""
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# Run the tests and log the result
|
||||
# in a summary.html file
|
||||
with open("summary.html","w") as f:
|
||||
print(HTMLHEADER,file=f)
|
||||
for s in solutions:
|
||||
printTitle("Process solution %s" % s)
|
||||
run("csolution","convert","-s",s)
|
||||
print("<h1>Solution %s</h1>" % s,file=f)
|
||||
maxNbBuilds=len(solutions[s])
|
||||
buildNb=0
|
||||
for build,core in solutions[s]:
|
||||
buildNb = buildNb + 1
|
||||
print("<h2>Core %s</h2>" % build,file=f)
|
||||
printTitle("Process core %s (%d/%d)" % (build,buildNb,maxNbBuilds))
|
||||
buildFile="test.Release+%s.cprj" % build
|
||||
nb = 0
|
||||
maxNb = len(allSuites)
|
||||
for s,pickle in allSuites:
|
||||
nb = nb + 1
|
||||
printSubTitle("Process suite %s (%d/%d)" % (s,nb,maxNb))
|
||||
res=run(sys.executable,"../processTests.py","-gen","..","-p","../Patterns","-d","../Parameters","-f",pickle,"-e",s,mustPrint=True)
|
||||
if res.error:
|
||||
printError("Error processTests")
|
||||
print("<p><font color=\"red\">Error generating %s</font></p><PRE>" % s,file=f)
|
||||
print(res.msg,file=f)
|
||||
print("</PRE>",file=f)
|
||||
continue
|
||||
if nb==1:
|
||||
# -r is needed for first
|
||||
# build when we switch
|
||||
# between different solutions
|
||||
# (Like one using AC6 and the other
|
||||
# using gcc)
|
||||
res=run("cbuild","-r",buildFile)
|
||||
else:
|
||||
res=run("cbuild",buildFile)
|
||||
if res.error:
|
||||
printError("Error cbuild")
|
||||
print("<p><font color=\"red\">Error building %s</font></p><PRE>" % s,file=f)
|
||||
print(res.msg,file=f)
|
||||
print("</PRE>",file=f)
|
||||
continue
|
||||
printSubTitle("Run AVH")
|
||||
res=runAVH(build,core)
|
||||
if res.error:
|
||||
printError("Error running AVH")
|
||||
print("<p><font color=\"red\">Error running %s</font></p><PRE>" % s,file=f)
|
||||
print(res.msg,file=f)
|
||||
print("</PRE>",file=f)
|
||||
continue
|
||||
else:
|
||||
with open("results.txt","w") as o:
|
||||
print(res.msg,file=o)
|
||||
res=run(sys.executable,"../processResult.py","-f",pickle,"-e","-ahtml","-r","results.txt",dumpStdErr=False)
|
||||
if res.error:
|
||||
printError("Error processResult")
|
||||
print("<p><font color=\"red\">Error processing %s result</font></p><PRE>" % s,file=f)
|
||||
print(res.msg,file=f)
|
||||
print("</PRE>",file=f)
|
||||
continue
|
||||
else:
|
||||
pass
|
||||
# When no error the section is not
|
||||
# included in final file
|
||||
#print(res.msg,file=f)
|
||||
print(HTMLFOOTER,file=f)
|
||||
|
||||
if ERROR_OCCURED:
|
||||
sys.exit("Error occurred")
|
||||
else:
|
||||
sys.exit(0)
|
||||
Loading…
Reference in New Issue