From 23eaafccf9c4aa6c5359210e5223d11dc66830cf Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Wed, 6 May 2020 12:45:50 +0200 Subject: [PATCH] CMSIS-DSP: Updated regression script for tests. --- Testing/TestScripts/Regression/Commands.py | 80 ++++++++++++++++++++-- Testing/createDb.sql | 24 ++++--- Testing/examples.sql | 23 +++++-- Testing/runAllTests.py | 28 +++++++- 4 files changed, 131 insertions(+), 24 deletions(-) diff --git a/Testing/TestScripts/Regression/Commands.py b/Testing/TestScripts/Regression/Commands.py index 128fdb6d..6c970e19 100755 --- a/Testing/TestScripts/Regression/Commands.py +++ b/Testing/TestScripts/Regression/Commands.py @@ -13,12 +13,33 @@ import sys DEBUGMODE = False KEEPBUILDFOLDER = False +DEBUGLIST=[ +"-DBASICMATH=ON", +"-DCOMPLEXMATH=OFF", +"-DCONTROLLER=OFF", +"-DFASTMATH=OFF", +"-DFILTERING=OFF", +"-DMATRIX=OFF", +"-DSTATISTICS=OFF", +"-DSUPPORT=OFF", +"-DTRANSFORM=OFF", +"-DSVM=OFF", +"-DBAYES=OFF", +"-DDISTANCE=OFF"] + NOTESTFAILED = 0 MAKEFAILED = 1 TESTFAILED = 2 FLOWFAILURE = 3 CALLFAILURE = 4 +def setDebugMode(): + global DEBUGMODE + DEBUGMODE=True + +def isDebugMode(): + return(DEBUGMODE) + def joinit(iterable, delimiter): it = iter(iterable) @@ -27,6 +48,17 @@ def joinit(iterable, delimiter): yield delimiter yield x +def addToDb(db,desc): + msg("Add %s to summary database\n" % desc) + completed = subprocess.run([sys.executable, "addToDB.py","-o",db,desc], timeout=3600) + check(completed) + +def addToRegDb(db,desc): + msg("Add %s to regression database\n" % desc) + completed = subprocess.run([sys.executable, "addToRegDB.py","-o",db,desc], timeout=3600) + check(completed) + + class TestFlowFailure(Exception): def __init__(self,completed): self._errorcode = completed.returncode @@ -158,6 +190,8 @@ class BuildConfig: def createCMake(self,flags,benchMode,platform): with self.buildFolder() as b: self.saveEnv() + if benchMode: + msg("Benchmark mode\n") msg("Create cmake for %s\n" % self.buildFolderName()) toolchainCmake = os.path.join(self.toolChainPath(),self.toolChainFile()) cmd = [self._cmake] @@ -167,8 +201,11 @@ class BuildConfig: "-DPLATFORM=%s" % platform ] cmd += flags + + if DEBUGMODE: + cmd += DEBUGLIST + if benchMode: - msg("Benchmark mode\n") cmd += ["-DBENCHMARK=ON"] cmd += ["-DWRAPPER=ON"] else: @@ -180,6 +217,9 @@ class BuildConfig: "-DCMAKE_BUILD_TYPE=Release", "-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()] + if DEBUGMODE: + print(cmd) + with open(os.path.join(self.archiveLogPath(),"cmakecmd.txt"),"w") as cmakecmd: cmakecmd.write("".join(joinit(cmd," "))) @@ -296,12 +336,15 @@ class Test: # the build folder for this test # # We need a timeout and detect failed run - def run(self,fvp): + def run(self,fvp,benchmode): + timeoutVal=3600 + if benchmode: + timeoutVal = 3600 * 4 completed = None with self.buildConfig().buildFolder() as b: msg(" Run %s\n" % self.testName() ) with open(self.resultName(),"w") as results: - completed=subprocess.run(fvp.split(),stdout=results,timeout=3600) + completed=subprocess.run(fvp.split(),stdout=results,timeout=timeoutVal) check(completed) # Process results of the given tests @@ -318,7 +361,19 @@ class Test: else: return(TESTFAILED) - def runAndProcess(self,compiler,fvp,sim): + # Compute the regression data + def computeSummaryStat(self): + msg(" Compute regressions for %s\n" % self.testName()) + with open(os.path.join(self.buildConfig().archiveResultPath(),"processedResult_%s.txt" % self.testName()),"w") as presult: + completed=subprocess.run([sys.executable,"summaryBench.py","-r",self.getResultPath()],stdout=presult,timeout=3600) + # When a test fail, the regression is continuing but we + # track that a test has failed + if completed.returncode==0: + return(NOTESTFAILED) + else: + return(TESTFAILED) + + def runAndProcess(self,compiler,fvp,sim,benchmode,db,regdb): # If we can't parse test description we fail all tests self.processTest() # Otherwise if only building or those tests are failing, we continue @@ -333,8 +388,15 @@ class Test: # build is done per test suite. if sim: if fvp is not None: - self.run(fvp) - return(self.processResult()) + self.run(fvp,benchmode) + error=self.processResult() + if benchmode and (error == NOTESTFAILED): + error = self.computeSummaryStat() + if db is not None: + addToDb(db,self.testName()) + if regdb is not None: + addToRegDb(regdb,self.testName()) + return(error) else: msg("No FVP available") return(NOTESTFAILED) @@ -356,6 +418,12 @@ def generateAllCCode(): completed = subprocess.run([sys.executable,"processTests.py", "-e"],timeout=3600) check(completed) +# Create db +def createDb(sqlite,desc): + msg("Create database %s\n" % desc) + with open("createDb.sql") as db: + completed = subprocess.run([sqlite, desc],stdin=db, timeout=3600) + check(completed) diff --git a/Testing/createDb.sql b/Testing/createDb.sql index 3e84e3cf..eca7a75e 100755 --- a/Testing/createDb.sql +++ b/Testing/createDb.sql @@ -74,13 +74,17 @@ INSERT INTO COMPILERKIND VALUES(2,"GCC"); INSERT INTO CORE VALUES(1,"m0","ARMCM0"); INSERT INTO CORE VALUES(2,"m0p","ARMCM0P"); INSERT INTO CORE VALUES(3,"m3","ARMCM3"); -INSERT INTO CORE VALUES(4,"m4f","ARMCM4_FP"); -INSERT INTO CORE VALUES(5,"m7","ARMCM7_DP"); -INSERT INTO CORE VALUES(6,"m23","ARMCM23"); -INSERT INTO CORE VALUES(7,"m33","ARMCM33_DSP_FP"); -INSERT INTO CORE VALUES(8,"m35","ARMCM35P_DSP_FP"); -INSERT INTO CORE VALUES(9,"a5","ARMCA5"); -INSERT INTO CORE VALUES(10,"a7","ARMCA7"); -INSERT INTO CORE VALUES(11,"a9","ARMCA9"); -INSERT INTO CORE VALUES(12,"a15","ARMCA15"); -INSERT INTO CORE VALUES(13,"helium","ARMv81MML_DSP_DP_MVE_FP"); +INSERT INTO CORE VALUES(4,"m4","ARMCM4"); +INSERT INTO CORE VALUES(5,"m4f","ARMCM4_FP"); +INSERT INTO CORE VALUES(6,"m7","ARMCM7_DP"); +INSERT INTO CORE VALUES(7,"m23","ARMCM23"); +INSERT INTO CORE VALUES(8,"m33","ARMCM33_DSP_FP"); +INSERT INTO CORE VALUES(9,"m35","ARMCM35P_DSP_FP"); +INSERT INTO CORE VALUES(10,"a5","ARMCA5"); +INSERT INTO CORE VALUES(11,"a7","ARMCA7"); +INSERT INTO CORE VALUES(12,"a9","ARMCA9"); +INSERT INTO CORE VALUES(13,"a15","ARMCA15"); +INSERT INTO CORE VALUES(14,"m55","ARMv81MML_DSP_DP_MVE_FP"); + +.quit + diff --git a/Testing/examples.sql b/Testing/examples.sql index 7237c7e7..e4c201c6 100755 --- a/Testing/examples.sql +++ b/Testing/examples.sql @@ -3,18 +3,29 @@ Build the table with the platform, compiler and core names. */ +.headers ON +.mode csv + /* -select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,DATE - from BasicBenchmarks +select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE + from BasicMathsBenchmarksF32 INNER JOIN CATEGORY USING(categoryid) INNER JOIN PLATFORM USING(platformid) INNER JOIN CORE USING(coreid) INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) ; - */ +select Regression,MAX,MAXREGCOEF,CATEGORY.category,NAME,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE + from BasicMathsBenchmarksF32 + INNER JOIN CATEGORY USING(categoryid) + INNER JOIN PLATFORM USING(platformid) + INNER JOIN CORE USING(coreid) + INNER JOIN COMPILER USING(compilerid) + INNER JOIN COMPILERKIND USING(compilerkindid) + ; + /* Compute the max cycles for a test configuration (category + name) @@ -45,11 +56,13 @@ See diff.sql for example */ -select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,DATE - from BasicBenchmarks +/* +select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE + from BasicMathsBenchmarksF32 INNER JOIN CATEGORY USING(categoryid) INNER JOIN PLATFORM USING(platformid) INNER JOIN CORE USING(coreid) INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILERKIND USING(compilerkindid) WHERE DATE BETWEEN datetime('now','localtime','-10 minutes') AND datetime('now', 'localtime'); +*/ \ No newline at end of file diff --git a/Testing/runAllTests.py b/Testing/runAllTests.py index 0ff06cb6..d1455ed6 100755 --- a/Testing/runAllTests.py +++ b/Testing/runAllTests.py @@ -103,8 +103,28 @@ parser.add_argument('-b', action='store_true', help="Benchmark mode") parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file") parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running") +parser.add_argument('-db', nargs='?',type = str,help="Benchmark database") +parser.add_argument('-regdb', nargs='?',type = str,help="Regression database") +parser.add_argument('-sqlite', nargs='?',default="/usr/bin/sqlite3",type = str,help="Regression database") + +parser.add_argument('-debug', action='store_true', help="Debug mode") + args = parser.parse_args() +if args.debug: + setDebugMode() + +# Create missing database files +# if the db arguments are specified +if args.db is not None: + if not os.path.exists(args.db): + createDb(args.sqlite,args.db) + +if args.regdb is not None: + if not os.path.exists(args.regdb): + createDb(args.sqlite,args.regdb) + + with open(args.i,"r") as f: config=yaml.safe_load(f) @@ -118,7 +138,7 @@ with open(args.i,"r") as f: flags = config["FLAGS"] allConfigs = analyzeFlags(flags) -if DEBUGMODE: +if isDebugMode(): allConfigs=[allConfigs[0]] failedBuild = {} @@ -200,7 +220,7 @@ def buildAndTest(compiler,theConfig,cmake,sim): if 'SIM' in config: if core in config['SIM']: fvp = config['SIM'][core] - newTestStatus = test.runAndProcess(compiler,fvp,sim) + newTestStatus = test.runAndProcess(compiler,fvp,sim,args.b,args.db,args.regdb) testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus) if testStatusForThisBuild != NOTESTFAILED: failedBuild[buildStr] = testStatusForThisBuild @@ -225,9 +245,11 @@ def buildAndTest(compiler,theConfig,cmake,sim): ############## Builds for all toolchains -if not DEBUGMODE: +if not isDebugMode(): preprocess(args.f) generateAllCCode() +else: + msg("Debug Mode\n") for t in config["TOOLCHAINS"]: cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs)