dengzhihao
/
CMSIS-DSP
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CMSIS-DSP: Improvement to the regression script for tests

Browse Source
pull/19/head
Christophe Favergeon 6 years ago
parent c33adc9485
commit 2653dba272
5 changed files with 99 additions and 112 deletions
Show Stats Download Patch File Download Diff File

3
Testing/CMakeLists.txt
Unescape Escape View File

@ -72,6 +72,7 @@ option(BENCHMARK "Benchmarking compiled" OFF)
option(EXTBENCH "Benchmarking with external traces" OFF)
option(NN "NN Tests included" OFF)
option(REFLIB "Use already built reference lib" OFF)
option(FLOAT16TESTS "Float16 tests" OFF)
project(Testing)
@ -221,7 +222,7 @@ set(TESTSRC
Source/Tests/ExampleCategoryQ7.cpp
)
if ((NOT ARMAC5) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL)))
if ((NOT ARMAC5) AND (FLOAT16TESTS) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL)))
set(TESTSRC16
Source/Tests/BasicTestsF16.cpp
Source/Tests/TransformCF16.cpp

26
Testing/TestScripts/Regression/Commands.py
Unescape Escape View File

@ -153,7 +153,7 @@ class BuildConfig:
# Launch cmake command.
def createCMake(self,flags):
def createCMake(self,flags,benchMode,platform):
with self.buildFolder() as b:
self.saveEnv()
msg("Create cmake for %s\n" % self.buildFolderName())
@ -162,12 +162,18 @@ class BuildConfig:
cmd += ["-DCMAKE_PREFIX_PATH=%s" % self.compiler(),
"-DCMAKE_TOOLCHAIN_FILE=%s" % toolchainCmake,
"-DARM_CPU=%s" % self.core(),
"-DPLATFORM=FVP"
"-DPLATFORM=%s" % platform
]
cmd += flags
cmd += ["-DBENCHMARK=OFF",
"-DWRAPPER=OFF",
"-DCONFIGTABLE=OFF",
if benchMode:
msg("Benchmark mode\n")
cmd += ["-DBENCHMARK=ON"]
cmd += ["-DWRAPPER=ON"]
else:
cmd += ["-DBENCHMARK=OFF"]
cmd += ["-DWRAPPER=OFF"]
cmd += ["-DCONFIGTABLE=OFF",
"-DROOT=%s" % self._rootFolder,
"-DCMAKE_BUILD_TYPE=Release",
"-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()]
@ -310,7 +316,7 @@ class Test:
else:
return(TESTFAILED)
def runAndProcess(self,compiler,fvp):
def runAndProcess(self,compiler,fvp,sim):
# If we can't parse test description we fail all tests
self.processTest()
# Otherwise if only building or those tests are failing, we continue
@ -323,7 +329,7 @@ class Test:
# For other compilers only build is tests
# Since full build is no more possible because of huge pattersn,
# build is done per test suite.
if compiler == "AC6":
if sim:
if fvp is not None:
self.run(fvp)
return(self.processResult())
@ -336,9 +342,9 @@ class Test:
# Preprocess the test description
def preprocess():
msg("Process test description file\n")
completed = subprocess.run(["python", "preprocess.py","-f","desc.txt"],timeout=3600)
def preprocess(desc):
msg("Process test description file %s\n" % desc)
completed = subprocess.run(["python", "preprocess.py","-f",desc],timeout=3600)
check(completed)
# Generate all missing C code by using all classes in the

2
Testing/processResult.py
Unescape Escape View File

@ -359,7 +359,7 @@ def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter):
benchFile = None
config=""
if embedded:
prefix = ".*S:[ ]"
prefix = ".*[S]+:[ ]"
# Parse the result file.
# NORMAL mode is when we are parsing suite or group.

74
Testing/results.nb
Unescape Escape View File

@ -1,74 +0,0 @@
(* Content-type: application/vnd.wolfram.mathematica *)
(*** Wolfram Notebook File ***)
(* http://www.wolfram.com/nb *)
(* CreatedBy='Mathematica 12.0' *)
(*CacheID: 234*)
(* Internal cache information:
NotebookFileLineBreakTest
NotebookFileLineBreakTest
NotebookDataPosition[ 158, 7]
NotebookDataLength[ 2158, 66]
NotebookOptionsPosition[ 1770, 52]
NotebookOutlinePosition[ 2114, 67]
CellTagsIndexPosition[ 2071, 64]
WindowFrame->Normal*)
(* Beginning of Notebook Content *)
Notebook[{
Cell[BoxData[
RowBox[{
RowBox[{"m", "=",
RowBox[{"Dataset", "[",
RowBox[{
RowBox[{"Get", "[",
RowBox[{"FileNameJoin", "[",
RowBox[{"{",
RowBox[{
RowBox[{"NotebookDirectory", "[", "]"}], ",", "\"\<test.txt\>\""}],
"}"}], "]"}], "]"}], "[", "\"\<Root \>\"", "]"}], "]"}]}],
";"}]], "Input",
CellChangeTimes->{{3.7717360240194063`*^9, 3.771736045131504*^9}, {
3.771736686569442*^9, 3.771736690261133*^9}, {3.7717512765989704`*^9,
3.7717513417302694`*^9}, {3.771751417685506*^9, 3.7717514233643456`*^9}, {
3.7719129941602864`*^9, 3.7719130089901433`*^9}, 3.7719133429206567`*^9},
CellLabel->"In[1]:=",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"],
Cell[BoxData[
RowBox[{"Column", "[",
RowBox[{"Table", "[",
RowBox[{
RowBox[{"m", "[",
RowBox[{"[", "i", "]"}], "]"}], ",",
RowBox[{"{",
RowBox[{"i", ",", "1", ",",
RowBox[{
RowBox[{"Dimensions", "[", "m", "]"}], "[",
RowBox[{"[", "1", "]"}], "]"}]}], "}"}]}], "]"}], "]"}]], "Input",
CellChangeTimes->{{3.771913348787628*^9, 3.771913473780173*^9}},
CellLabel->"In[14]:=",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"]
},
WindowSize->{1063, 641},
WindowMargins->{{Automatic, 162}, {20, Automatic}},
FrontEndVersion->"12.0 for Microsoft Windows (64-bit) (April 8, 2019)",
StyleDefinitions->"Default.nb"
]
(* End of Notebook Content *)
(* Internal cache information *)
(*CellTagsOutline
CellTagsIndex->{}
*)
(*CellTagsIndex
CellTagsIndex->{}
*)
(*NotebookFileOutline
Notebook[{
Cell[558, 20, 726, 16, 28, "Input",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"],
Cell[1287, 38, 479, 12, 28, "Input",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"]
}
]
*)

106
Testing/runAllTests.py
Unescape Escape View File

@ -39,7 +39,47 @@ def updateTestStatus(testStatusForThisBuild,newTestStatus):
if testStatusForThisBuild == CALLFAILURE:
return(testStatusForThisBuild)
root = Path(os.getcwd()).parent.parent.parent
# Analyze the configuration flags (like loopunroll etc ...)
def analyzeFlags(flags):
onoffFlags = []
for f in flags:
if type(f) is dict:
for var in f:
if type(f[var]) is bool:
if f[var]:
onoffFlags.append(["-D%s=ON" % (var)])
else:
onoffFlags.append(["-D%s=OFF" % (var)])
else:
onoffFlags.append(["-D%s=%s" % (var,f[var])])
else:
onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"])
allConfigs=cartesian(*onoffFlags)
return(allConfigs)
# Extract the cmake for a specific compiler
# and the flag configuration to use for this compiler.
# This flags configuration will override the global one
def analyzeToolchain(toolchain, globalConfig):
config=globalConfig
cmake=""
sim=True
if type(toolchain) is str:
cmake=toolchain
else:
for t in toolchain:
if type(t) is dict:
if "FLAGS" in t:
hasConfig=True
config = analyzeFlags(t["FLAGS"])
if "SIM" in t:
sim = t["SIM"]
if type(t) is str:
cmake=t
return(cmake,config,sim)
def cartesian(*somelists):
@ -48,6 +88,9 @@ def cartesian(*somelists):
r.append(list(element))
return(r)
root = Path(os.getcwd()).parent.parent.parent
testFailed = 0
init()
@ -55,11 +98,13 @@ init()
parser = argparse.ArgumentParser(description='Parse test description')
parser.add_argument('-i', nargs='?',type = str, default="testrunConfig.yaml",help="Config file")
parser.add_argument('-r', nargs='?',type = str, default=root, help="Root folder")
parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launchign in parallel")
parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launching in parallel")
parser.add_argument('-b', action='store_true', help="Benchmark mode")
parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file")
parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running")
args = parser.parse_args()
with open(args.i,"r") as f:
config=yaml.safe_load(f)
@ -67,16 +112,14 @@ with open(args.i,"r") as f:
#print(config["IMPLIEDFLAGS"])
flags = config["FLAGS"]
onoffFlags = []
for f in flags:
onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"])
allConfigs=cartesian(*onoffFlags)
flags = config["FLAGS"]
allConfigs = analyzeFlags(flags)
if DEBUGMODE:
allConfigs=[allConfigs[0]]
failedBuild = {}
# Test all builds
@ -97,31 +140,34 @@ def logFailedBuild(root,f):
status.write("%s : Subprocess failure\n" % build)
def buildAndTest(compiler):
def buildAndTest(compiler,theConfig,cmake,sim):
# Run all tests for AC6
try:
for core in config['CORES']:
configNb = 0
if compiler in config['CORES'][core]:
for flagConfig in allConfigs:
msg("Testing Core %s\n" % core)
for flagConfig in theConfig:
folderCreated = False
configNb = configNb + 1
buildStr = "build_%s_%s_%d" % (compiler,core,configNb)
toUnset = None
toSet = None
if compiler in config['UNSET']:
if core in config['UNSET'][compiler]:
toUnset = config['UNSET'][compiler][core]
if 'UNSET' in config:
if compiler in config['UNSET']:
if core in config['UNSET'][compiler]:
toUnset = config['UNSET'][compiler][core]
if compiler in config['SET']:
if core in config['SET'][compiler]:
toSet = config['SET'][compiler][core]
if 'SET' in config:
if compiler in config['SET']:
if core in config['SET'][compiler]:
toSet = config['SET'][compiler][core]
build = BuildConfig(toUnset,toSet,args.r,
buildStr,
config['COMPILERS'][core][compiler],
config['TOOLCHAINS'][compiler],
cmake,
config['CORES'][core][compiler],
config["CMAKE"]
)
@ -140,16 +186,21 @@ def buildAndTest(compiler):
try:
# This is saving the flag configuration
build.createArchive(flags)
build.createCMake(flags)
msg("Config " + str(flagConfig) + "\n")
build.createCMake(flags,args.b,args.p)
for test in config["TESTS"]:
msg(test["testName"]+"\n")
testClass=test["testClass"]
test = build.getTest(testClass)
fvp = None
if core in config['FVP']:
fvp = config['FVP'][core]
newTestStatus = test.runAndProcess(compiler,fvp)
if 'FVP' in config:
if core in config['FVP']:
fvp = config['FVP'][core]
if 'SIM' in config:
if core in config['SIM']:
fvp = config['SIM'][core]
newTestStatus = test.runAndProcess(compiler,fvp,sim)
testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus)
if testStatusForThisBuild != NOTESTFAILED:
failedBuild[buildStr] = testStatusForThisBuild
@ -175,12 +226,15 @@ def buildAndTest(compiler):
############## Builds for all toolchains
if not DEBUGMODE:
preprocess()
preprocess(args.f)
generateAllCCode()
for t in config["TOOLCHAINS"]:
msg("Testing toolchain %s\n" % t)
buildAndTest(t)
cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs)
msg("Testing toolchain %s\n" % cmake)
buildAndTest(t,localConfig,cmake,sim)
exit(1)
logFailedBuild(args.r,failedBuild)
sys.exit(testFailed)

Write Preview
Loading…
Cancel
Save