dengzhihao
/
CMSIS-DSP
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CMSIS-DSP: Improvement to the regression script for tests

Browse Source
pull/19/head
Christophe Favergeon 6 years ago
parent c33adc9485
commit 2653dba272
5 changed files with 99 additions and 112 deletions
Show Stats Download Patch File Download Diff File

3
Testing/CMakeLists.txt
Unescape Escape View File

@ -72,6 +72,7 @@ option(BENCHMARK "Benchmarking compiled" OFF)
option(EXTBENCH "Benchmarking with external traces" OFF) option(EXTBENCH "Benchmarking with external traces" OFF)
option(NN "NN Tests included" OFF) option(NN "NN Tests included" OFF)
option(REFLIB "Use already built reference lib" OFF) option(REFLIB "Use already built reference lib" OFF)
option(FLOAT16TESTS "Float16 tests" OFF)
project(Testing) project(Testing)
@ -221,7 +222,7 @@ set(TESTSRC
Source/Tests/ExampleCategoryQ7.cpp Source/Tests/ExampleCategoryQ7.cpp
) )
if ((NOT ARMAC5) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL))) if ((NOT ARMAC5) AND (FLOAT16TESTS) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL)))
set(TESTSRC16 set(TESTSRC16
Source/Tests/BasicTestsF16.cpp Source/Tests/BasicTestsF16.cpp
Source/Tests/TransformCF16.cpp Source/Tests/TransformCF16.cpp

26
Testing/TestScripts/Regression/Commands.py
Unescape Escape View File

@ -153,7 +153,7 @@ class BuildConfig:
# Launch cmake command. # Launch cmake command.
def createCMake(self,flags): def createCMake(self,flags,benchMode,platform):
with self.buildFolder() as b: with self.buildFolder() as b:
self.saveEnv() self.saveEnv()
msg("Create cmake for %s\n" % self.buildFolderName()) msg("Create cmake for %s\n" % self.buildFolderName())
@ -162,12 +162,18 @@ class BuildConfig:
cmd += ["-DCMAKE_PREFIX_PATH=%s" % self.compiler(), cmd += ["-DCMAKE_PREFIX_PATH=%s" % self.compiler(),
"-DCMAKE_TOOLCHAIN_FILE=%s" % toolchainCmake, "-DCMAKE_TOOLCHAIN_FILE=%s" % toolchainCmake,
"-DARM_CPU=%s" % self.core(), "-DARM_CPU=%s" % self.core(),
"-DPLATFORM=FVP" "-DPLATFORM=%s" % platform
] ]
cmd += flags cmd += flags
cmd += ["-DBENCHMARK=OFF", if benchMode:
"-DWRAPPER=OFF", msg("Benchmark mode\n")
"-DCONFIGTABLE=OFF", cmd += ["-DBENCHMARK=ON"]
cmd += ["-DWRAPPER=ON"]
else:
cmd += ["-DBENCHMARK=OFF"]
cmd += ["-DWRAPPER=OFF"]
cmd += ["-DCONFIGTABLE=OFF",
"-DROOT=%s" % self._rootFolder, "-DROOT=%s" % self._rootFolder,
"-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_BUILD_TYPE=Release",
"-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()] "-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()]
@ -310,7 +316,7 @@ class Test:
else: else:
return(TESTFAILED) return(TESTFAILED)
def runAndProcess(self,compiler,fvp): def runAndProcess(self,compiler,fvp,sim):
# If we can't parse test description we fail all tests # If we can't parse test description we fail all tests
self.processTest() self.processTest()
# Otherwise if only building or those tests are failing, we continue # Otherwise if only building or those tests are failing, we continue
@ -323,7 +329,7 @@ class Test:
# For other compilers only build is tests # For other compilers only build is tests
# Since full build is no more possible because of huge pattersn, # Since full build is no more possible because of huge pattersn,
# build is done per test suite. # build is done per test suite.
if compiler == "AC6": if sim:
if fvp is not None: if fvp is not None:
self.run(fvp) self.run(fvp)
return(self.processResult()) return(self.processResult())
@ -336,9 +342,9 @@ class Test:
# Preprocess the test description # Preprocess the test description
def preprocess(): def preprocess(desc):
msg("Process test description file\n") msg("Process test description file %s\n" % desc)
completed = subprocess.run(["python", "preprocess.py","-f","desc.txt"],timeout=3600) completed = subprocess.run(["python", "preprocess.py","-f",desc],timeout=3600)
check(completed) check(completed)
# Generate all missing C code by using all classes in the # Generate all missing C code by using all classes in the

2
Testing/processResult.py
Unescape Escape View File

@ -359,7 +359,7 @@ def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter):
benchFile = None benchFile = None
config="" config=""
if embedded: if embedded:
prefix = ".*S:[ ]" prefix = ".*[S]+:[ ]"
# Parse the result file. # Parse the result file.
# NORMAL mode is when we are parsing suite or group. # NORMAL mode is when we are parsing suite or group.

74
Testing/results.nb
Unescape Escape View File

@ -1,74 +0,0 @@
(* Content-type: application/vnd.wolfram.mathematica *)
(*** Wolfram Notebook File ***)
(* http://www.wolfram.com/nb *)
(* CreatedBy='Mathematica 12.0' *)
(*CacheID: 234*)
(* Internal cache information:
NotebookFileLineBreakTest
NotebookFileLineBreakTest
NotebookDataPosition[ 158, 7]
NotebookDataLength[ 2158, 66]
NotebookOptionsPosition[ 1770, 52]
NotebookOutlinePosition[ 2114, 67]
CellTagsIndexPosition[ 2071, 64]
WindowFrame->Normal*)
(* Beginning of Notebook Content *)
Notebook[{
Cell[BoxData[
RowBox[{
RowBox[{"m", "=",
RowBox[{"Dataset", "[",
RowBox[{
RowBox[{"Get", "[",
RowBox[{"FileNameJoin", "[",
RowBox[{"{",
RowBox[{
RowBox[{"NotebookDirectory", "[", "]"}], ",", "\"\<test.txt\>\""}],
"}"}], "]"}], "]"}], "[", "\"\<Root \>\"", "]"}], "]"}]}],
";"}]], "Input",
CellChangeTimes->{{3.7717360240194063`*^9, 3.771736045131504*^9}, {
3.771736686569442*^9, 3.771736690261133*^9}, {3.7717512765989704`*^9,
3.7717513417302694`*^9}, {3.771751417685506*^9, 3.7717514233643456`*^9}, {
3.7719129941602864`*^9, 3.7719130089901433`*^9}, 3.7719133429206567`*^9},
CellLabel->"In[1]:=",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"],
Cell[BoxData[
RowBox[{"Column", "[",
RowBox[{"Table", "[",
RowBox[{
RowBox[{"m", "[",
RowBox[{"[", "i", "]"}], "]"}], ",",
RowBox[{"{",
RowBox[{"i", ",", "1", ",",
RowBox[{
RowBox[{"Dimensions", "[", "m", "]"}], "[",
RowBox[{"[", "1", "]"}], "]"}]}], "}"}]}], "]"}], "]"}]], "Input",
CellChangeTimes->{{3.771913348787628*^9, 3.771913473780173*^9}},
CellLabel->"In[14]:=",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"]
},
WindowSize->{1063, 641},
WindowMargins->{{Automatic, 162}, {20, Automatic}},
FrontEndVersion->"12.0 for Microsoft Windows (64-bit) (April 8, 2019)",
StyleDefinitions->"Default.nb"
]
(* End of Notebook Content *)
(* Internal cache information *)
(*CellTagsOutline
CellTagsIndex->{}
*)
(*CellTagsIndex
CellTagsIndex->{}
*)
(*NotebookFileOutline
Notebook[{
Cell[558, 20, 726, 16, 28, "Input",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"],
Cell[1287, 38, 479, 12, 28, "Input",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"]
}
]
*)

106
Testing/runAllTests.py
Unescape Escape View File

@ -39,7 +39,47 @@ def updateTestStatus(testStatusForThisBuild,newTestStatus):
if testStatusForThisBuild == CALLFAILURE: if testStatusForThisBuild == CALLFAILURE:
return(testStatusForThisBuild) return(testStatusForThisBuild)
root = Path(os.getcwd()).parent.parent.parent # Analyze the configuration flags (like loopunroll etc ...)
def analyzeFlags(flags):
onoffFlags = []
for f in flags:
if type(f) is dict:
for var in f:
if type(f[var]) is bool:
if f[var]:
onoffFlags.append(["-D%s=ON" % (var)])
else:
onoffFlags.append(["-D%s=OFF" % (var)])
else:
onoffFlags.append(["-D%s=%s" % (var,f[var])])
else:
onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"])
allConfigs=cartesian(*onoffFlags)
return(allConfigs)
# Extract the cmake for a specific compiler
# and the flag configuration to use for this compiler.
# This flags configuration will override the global one
def analyzeToolchain(toolchain, globalConfig):
config=globalConfig
cmake=""
sim=True
if type(toolchain) is str:
cmake=toolchain
else:
for t in toolchain:
if type(t) is dict:
if "FLAGS" in t:
hasConfig=True
config = analyzeFlags(t["FLAGS"])
if "SIM" in t:
sim = t["SIM"]
if type(t) is str:
cmake=t
return(cmake,config,sim)
def cartesian(*somelists): def cartesian(*somelists):
@ -48,6 +88,9 @@ def cartesian(*somelists):
r.append(list(element)) r.append(list(element))
return(r) return(r)
root = Path(os.getcwd()).parent.parent.parent
testFailed = 0 testFailed = 0
init() init()
@ -55,11 +98,13 @@ init()
parser = argparse.ArgumentParser(description='Parse test description') parser = argparse.ArgumentParser(description='Parse test description')
parser.add_argument('-i', nargs='?',type = str, default="testrunConfig.yaml",help="Config file") parser.add_argument('-i', nargs='?',type = str, default="testrunConfig.yaml",help="Config file")
parser.add_argument('-r', nargs='?',type = str, default=root, help="Root folder") parser.add_argument('-r', nargs='?',type = str, default=root, help="Root folder")
parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launchign in parallel") parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launching in parallel")
parser.add_argument('-b', action='store_true', help="Benchmark mode")
parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file")
parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running")
args = parser.parse_args() args = parser.parse_args()
with open(args.i,"r") as f: with open(args.i,"r") as f:
config=yaml.safe_load(f) config=yaml.safe_load(f)
@ -67,16 +112,14 @@ with open(args.i,"r") as f:
#print(config["IMPLIEDFLAGS"]) #print(config["IMPLIEDFLAGS"])
flags = config["FLAGS"]
onoffFlags = []
for f in flags:
onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"])
allConfigs=cartesian(*onoffFlags)
flags = config["FLAGS"]
allConfigs = analyzeFlags(flags)
if DEBUGMODE: if DEBUGMODE:
allConfigs=[allConfigs[0]] allConfigs=[allConfigs[0]]
failedBuild = {} failedBuild = {}
# Test all builds # Test all builds
@ -97,31 +140,34 @@ def logFailedBuild(root,f):
status.write("%s : Subprocess failure\n" % build) status.write("%s : Subprocess failure\n" % build)
def buildAndTest(compiler): def buildAndTest(compiler,theConfig,cmake,sim):
# Run all tests for AC6 # Run all tests for AC6
try: try:
for core in config['CORES']: for core in config['CORES']:
configNb = 0 configNb = 0
if compiler in config['CORES'][core]: if compiler in config['CORES'][core]:
for flagConfig in allConfigs: msg("Testing Core %s\n" % core)
for flagConfig in theConfig:
folderCreated = False folderCreated = False
configNb = configNb + 1 configNb = configNb + 1
buildStr = "build_%s_%s_%d" % (compiler,core,configNb) buildStr = "build_%s_%s_%d" % (compiler,core,configNb)
toUnset = None toUnset = None
toSet = None toSet = None
if compiler in config['UNSET']: if 'UNSET' in config:
if core in config['UNSET'][compiler]: if compiler in config['UNSET']:
toUnset = config['UNSET'][compiler][core] if core in config['UNSET'][compiler]:
toUnset = config['UNSET'][compiler][core]
if compiler in config['SET']: if 'SET' in config:
if core in config['SET'][compiler]: if compiler in config['SET']:
toSet = config['SET'][compiler][core] if core in config['SET'][compiler]:
toSet = config['SET'][compiler][core]
build = BuildConfig(toUnset,toSet,args.r, build = BuildConfig(toUnset,toSet,args.r,
buildStr, buildStr,
config['COMPILERS'][core][compiler], config['COMPILERS'][core][compiler],
config['TOOLCHAINS'][compiler], cmake,
config['CORES'][core][compiler], config['CORES'][core][compiler],
config["CMAKE"] config["CMAKE"]
) )
@ -140,16 +186,21 @@ def buildAndTest(compiler):
try: try:
# This is saving the flag configuration # This is saving the flag configuration
build.createArchive(flags) build.createArchive(flags)
msg("Config " + str(flagConfig) + "\n")
build.createCMake(flags)
build.createCMake(flags,args.b,args.p)
for test in config["TESTS"]: for test in config["TESTS"]:
msg(test["testName"]+"\n") msg(test["testName"]+"\n")
testClass=test["testClass"] testClass=test["testClass"]
test = build.getTest(testClass) test = build.getTest(testClass)
fvp = None fvp = None
if core in config['FVP']: if 'FVP' in config:
fvp = config['FVP'][core] if core in config['FVP']:
newTestStatus = test.runAndProcess(compiler,fvp) fvp = config['FVP'][core]
if 'SIM' in config:
if core in config['SIM']:
fvp = config['SIM'][core]
newTestStatus = test.runAndProcess(compiler,fvp,sim)
testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus) testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus)
if testStatusForThisBuild != NOTESTFAILED: if testStatusForThisBuild != NOTESTFAILED:
failedBuild[buildStr] = testStatusForThisBuild failedBuild[buildStr] = testStatusForThisBuild
@ -175,12 +226,15 @@ def buildAndTest(compiler):
############## Builds for all toolchains ############## Builds for all toolchains
if not DEBUGMODE: if not DEBUGMODE:
preprocess() preprocess(args.f)
generateAllCCode() generateAllCCode()
for t in config["TOOLCHAINS"]: for t in config["TOOLCHAINS"]:
msg("Testing toolchain %s\n" % t) cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs)
buildAndTest(t) msg("Testing toolchain %s\n" % cmake)
buildAndTest(t,localConfig,cmake,sim)
exit(1)
logFailedBuild(args.r,failedBuild) logFailedBuild(args.r,failedBuild)
sys.exit(testFailed) sys.exit(testFailed)

Write Preview
Loading…
Cancel
Save