From 2653dba2724a015dd44b574a4bfb85f323ce63bb Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Mon, 27 Apr 2020 14:51:08 +0200 Subject: [PATCH] CMSIS-DSP: Improvement to the regression script for tests --- Testing/CMakeLists.txt | 3 +- Testing/TestScripts/Regression/Commands.py | 26 +++-- Testing/processResult.py | 2 +- Testing/results.nb | 74 -------------- Testing/runAllTests.py | 106 ++++++++++++++++----- 5 files changed, 99 insertions(+), 112 deletions(-) delete mode 100644 Testing/results.nb diff --git a/Testing/CMakeLists.txt b/Testing/CMakeLists.txt index 8a3c628e..b7c619c3 100644 --- a/Testing/CMakeLists.txt +++ b/Testing/CMakeLists.txt @@ -72,6 +72,7 @@ option(BENCHMARK "Benchmarking compiled" OFF) option(EXTBENCH "Benchmarking with external traces" OFF) option(NN "NN Tests included" OFF) option(REFLIB "Use already built reference lib" OFF) +option(FLOAT16TESTS "Float16 tests" OFF) project(Testing) @@ -221,7 +222,7 @@ set(TESTSRC Source/Tests/ExampleCategoryQ7.cpp ) -if ((NOT ARMAC5) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL))) +if ((NOT ARMAC5) AND (FLOAT16TESTS) AND ((FLOAT16) OR (MVEF) OR (HELIUM) OR (NEON) OR (NEONEXPERIMENTAL))) set(TESTSRC16 Source/Tests/BasicTestsF16.cpp Source/Tests/TransformCF16.cpp diff --git a/Testing/TestScripts/Regression/Commands.py b/Testing/TestScripts/Regression/Commands.py index f3deaf30..6ccccb56 100755 --- a/Testing/TestScripts/Regression/Commands.py +++ b/Testing/TestScripts/Regression/Commands.py @@ -153,7 +153,7 @@ class BuildConfig: # Launch cmake command. - def createCMake(self,flags): + def createCMake(self,flags,benchMode,platform): with self.buildFolder() as b: self.saveEnv() msg("Create cmake for %s\n" % self.buildFolderName()) @@ -162,12 +162,18 @@ class BuildConfig: cmd += ["-DCMAKE_PREFIX_PATH=%s" % self.compiler(), "-DCMAKE_TOOLCHAIN_FILE=%s" % toolchainCmake, "-DARM_CPU=%s" % self.core(), - "-DPLATFORM=FVP" + "-DPLATFORM=%s" % platform ] cmd += flags - cmd += ["-DBENCHMARK=OFF", - "-DWRAPPER=OFF", - "-DCONFIGTABLE=OFF", + if benchMode: + msg("Benchmark mode\n") + cmd += ["-DBENCHMARK=ON"] + cmd += ["-DWRAPPER=ON"] + else: + cmd += ["-DBENCHMARK=OFF"] + cmd += ["-DWRAPPER=OFF"] + + cmd += ["-DCONFIGTABLE=OFF", "-DROOT=%s" % self._rootFolder, "-DCMAKE_BUILD_TYPE=Release", "-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()] @@ -310,7 +316,7 @@ class Test: else: return(TESTFAILED) - def runAndProcess(self,compiler,fvp): + def runAndProcess(self,compiler,fvp,sim): # If we can't parse test description we fail all tests self.processTest() # Otherwise if only building or those tests are failing, we continue @@ -323,7 +329,7 @@ class Test: # For other compilers only build is tests # Since full build is no more possible because of huge pattersn, # build is done per test suite. - if compiler == "AC6": + if sim: if fvp is not None: self.run(fvp) return(self.processResult()) @@ -336,9 +342,9 @@ class Test: # Preprocess the test description -def preprocess(): - msg("Process test description file\n") - completed = subprocess.run(["python", "preprocess.py","-f","desc.txt"],timeout=3600) +def preprocess(desc): + msg("Process test description file %s\n" % desc) + completed = subprocess.run(["python", "preprocess.py","-f",desc],timeout=3600) check(completed) # Generate all missing C code by using all classes in the diff --git a/Testing/processResult.py b/Testing/processResult.py index c1469e0f..a914cfb4 100644 --- a/Testing/processResult.py +++ b/Testing/processResult.py @@ -359,7 +359,7 @@ def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter): benchFile = None config="" if embedded: - prefix = ".*S:[ ]" + prefix = ".*[S]+:[ ]" # Parse the result file. # NORMAL mode is when we are parsing suite or group. diff --git a/Testing/results.nb b/Testing/results.nb deleted file mode 100644 index 83218511..00000000 --- a/Testing/results.nb +++ /dev/null @@ -1,74 +0,0 @@ -(* Content-type: application/vnd.wolfram.mathematica *) - -(*** Wolfram Notebook File ***) -(* http://www.wolfram.com/nb *) - -(* CreatedBy='Mathematica 12.0' *) - -(*CacheID: 234*) -(* Internal cache information: -NotebookFileLineBreakTest -NotebookFileLineBreakTest -NotebookDataPosition[ 158, 7] -NotebookDataLength[ 2158, 66] -NotebookOptionsPosition[ 1770, 52] -NotebookOutlinePosition[ 2114, 67] -CellTagsIndexPosition[ 2071, 64] -WindowFrame->Normal*) - -(* Beginning of Notebook Content *) -Notebook[{ -Cell[BoxData[ - RowBox[{ - RowBox[{"m", "=", - RowBox[{"Dataset", "[", - RowBox[{ - RowBox[{"Get", "[", - RowBox[{"FileNameJoin", "[", - RowBox[{"{", - RowBox[{ - RowBox[{"NotebookDirectory", "[", "]"}], ",", "\"\\""}], - "}"}], "]"}], "]"}], "[", "\"\\"", "]"}], "]"}]}], - ";"}]], "Input", - CellChangeTimes->{{3.7717360240194063`*^9, 3.771736045131504*^9}, { - 3.771736686569442*^9, 3.771736690261133*^9}, {3.7717512765989704`*^9, - 3.7717513417302694`*^9}, {3.771751417685506*^9, 3.7717514233643456`*^9}, { - 3.7719129941602864`*^9, 3.7719130089901433`*^9}, 3.7719133429206567`*^9}, - CellLabel->"In[1]:=",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"], - -Cell[BoxData[ - RowBox[{"Column", "[", - RowBox[{"Table", "[", - RowBox[{ - RowBox[{"m", "[", - RowBox[{"[", "i", "]"}], "]"}], ",", - RowBox[{"{", - RowBox[{"i", ",", "1", ",", - RowBox[{ - RowBox[{"Dimensions", "[", "m", "]"}], "[", - RowBox[{"[", "1", "]"}], "]"}]}], "}"}]}], "]"}], "]"}]], "Input", - CellChangeTimes->{{3.771913348787628*^9, 3.771913473780173*^9}}, - CellLabel->"In[14]:=",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"] -}, -WindowSize->{1063, 641}, -WindowMargins->{{Automatic, 162}, {20, Automatic}}, -FrontEndVersion->"12.0 for Microsoft Windows (64-bit) (April 8, 2019)", -StyleDefinitions->"Default.nb" -] -(* End of Notebook Content *) - -(* Internal cache information *) -(*CellTagsOutline -CellTagsIndex->{} -*) -(*CellTagsIndex -CellTagsIndex->{} -*) -(*NotebookFileOutline -Notebook[{ -Cell[558, 20, 726, 16, 28, "Input",ExpressionUUID->"a3ff6af3-d397-4a63-9a0c-239444d6be9a"], -Cell[1287, 38, 479, 12, 28, "Input",ExpressionUUID->"6cdbb696-26f9-45b5-88dc-77262be1f94d"] -} -] -*) - diff --git a/Testing/runAllTests.py b/Testing/runAllTests.py index e67531a1..816f70b9 100755 --- a/Testing/runAllTests.py +++ b/Testing/runAllTests.py @@ -39,7 +39,47 @@ def updateTestStatus(testStatusForThisBuild,newTestStatus): if testStatusForThisBuild == CALLFAILURE: return(testStatusForThisBuild) -root = Path(os.getcwd()).parent.parent.parent +# Analyze the configuration flags (like loopunroll etc ...) +def analyzeFlags(flags): + + onoffFlags = [] + for f in flags: + if type(f) is dict: + for var in f: + if type(f[var]) is bool: + if f[var]: + onoffFlags.append(["-D%s=ON" % (var)]) + else: + onoffFlags.append(["-D%s=OFF" % (var)]) + else: + onoffFlags.append(["-D%s=%s" % (var,f[var])]) + else: + onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"]) + + allConfigs=cartesian(*onoffFlags) + return(allConfigs) + +# Extract the cmake for a specific compiler +# and the flag configuration to use for this compiler. +# This flags configuration will override the global one +def analyzeToolchain(toolchain, globalConfig): + config=globalConfig + cmake="" + sim=True + if type(toolchain) is str: + cmake=toolchain + else: + for t in toolchain: + if type(t) is dict: + if "FLAGS" in t: + hasConfig=True + config = analyzeFlags(t["FLAGS"]) + if "SIM" in t: + sim = t["SIM"] + if type(t) is str: + cmake=t + return(cmake,config,sim) + def cartesian(*somelists): @@ -48,6 +88,9 @@ def cartesian(*somelists): r.append(list(element)) return(r) +root = Path(os.getcwd()).parent.parent.parent + + testFailed = 0 init() @@ -55,11 +98,13 @@ init() parser = argparse.ArgumentParser(description='Parse test description') parser.add_argument('-i', nargs='?',type = str, default="testrunConfig.yaml",help="Config file") parser.add_argument('-r', nargs='?',type = str, default=root, help="Root folder") -parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launchign in parallel") +parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launching in parallel") +parser.add_argument('-b', action='store_true', help="Benchmark mode") +parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file") +parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running") args = parser.parse_args() - with open(args.i,"r") as f: config=yaml.safe_load(f) @@ -67,16 +112,14 @@ with open(args.i,"r") as f: #print(config["IMPLIEDFLAGS"]) -flags = config["FLAGS"] -onoffFlags = [] -for f in flags: - onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"]) -allConfigs=cartesian(*onoffFlags) + + +flags = config["FLAGS"] +allConfigs = analyzeFlags(flags) if DEBUGMODE: allConfigs=[allConfigs[0]] - failedBuild = {} # Test all builds @@ -97,31 +140,34 @@ def logFailedBuild(root,f): status.write("%s : Subprocess failure\n" % build) -def buildAndTest(compiler): +def buildAndTest(compiler,theConfig,cmake,sim): # Run all tests for AC6 try: for core in config['CORES']: configNb = 0 if compiler in config['CORES'][core]: - for flagConfig in allConfigs: + msg("Testing Core %s\n" % core) + for flagConfig in theConfig: folderCreated = False configNb = configNb + 1 buildStr = "build_%s_%s_%d" % (compiler,core,configNb) toUnset = None toSet = None - if compiler in config['UNSET']: - if core in config['UNSET'][compiler]: - toUnset = config['UNSET'][compiler][core] + if 'UNSET' in config: + if compiler in config['UNSET']: + if core in config['UNSET'][compiler]: + toUnset = config['UNSET'][compiler][core] - if compiler in config['SET']: - if core in config['SET'][compiler]: - toSet = config['SET'][compiler][core] + if 'SET' in config: + if compiler in config['SET']: + if core in config['SET'][compiler]: + toSet = config['SET'][compiler][core] build = BuildConfig(toUnset,toSet,args.r, buildStr, config['COMPILERS'][core][compiler], - config['TOOLCHAINS'][compiler], + cmake, config['CORES'][core][compiler], config["CMAKE"] ) @@ -140,16 +186,21 @@ def buildAndTest(compiler): try: # This is saving the flag configuration build.createArchive(flags) - - build.createCMake(flags) + msg("Config " + str(flagConfig) + "\n") + + build.createCMake(flags,args.b,args.p) for test in config["TESTS"]: msg(test["testName"]+"\n") testClass=test["testClass"] test = build.getTest(testClass) fvp = None - if core in config['FVP']: - fvp = config['FVP'][core] - newTestStatus = test.runAndProcess(compiler,fvp) + if 'FVP' in config: + if core in config['FVP']: + fvp = config['FVP'][core] + if 'SIM' in config: + if core in config['SIM']: + fvp = config['SIM'][core] + newTestStatus = test.runAndProcess(compiler,fvp,sim) testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus) if testStatusForThisBuild != NOTESTFAILED: failedBuild[buildStr] = testStatusForThisBuild @@ -175,12 +226,15 @@ def buildAndTest(compiler): ############## Builds for all toolchains if not DEBUGMODE: - preprocess() + preprocess(args.f) generateAllCCode() for t in config["TOOLCHAINS"]: - msg("Testing toolchain %s\n" % t) - buildAndTest(t) + cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs) + msg("Testing toolchain %s\n" % cmake) + buildAndTest(t,localConfig,cmake,sim) + +exit(1) logFailedBuild(args.r,failedBuild) sys.exit(testFailed)