dengzhihao
/
CMSIS-DSP
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CMSIS-DSP: Updated regression script for tests.

Browse Source
pull/19/head
Christophe Favergeon 6 years ago
parent 21383c9dd2
commit 23eaafccf9
4 changed files with 131 additions and 24 deletions
Show Stats Download Patch File Download Diff File

80
Testing/TestScripts/Regression/Commands.py
Unescape Escape View File

@ -13,12 +13,33 @@ import sys
DEBUGMODE = False DEBUGMODE = False
KEEPBUILDFOLDER = False KEEPBUILDFOLDER = False
DEBUGLIST=[
"-DBASICMATH=ON",
"-DCOMPLEXMATH=OFF",
"-DCONTROLLER=OFF",
"-DFASTMATH=OFF",
"-DFILTERING=OFF",
"-DMATRIX=OFF",
"-DSTATISTICS=OFF",
"-DSUPPORT=OFF",
"-DTRANSFORM=OFF",
"-DSVM=OFF",
"-DBAYES=OFF",
"-DDISTANCE=OFF"]
NOTESTFAILED = 0 NOTESTFAILED = 0
MAKEFAILED = 1 MAKEFAILED = 1
TESTFAILED = 2 TESTFAILED = 2
FLOWFAILURE = 3 FLOWFAILURE = 3
CALLFAILURE = 4 CALLFAILURE = 4
def setDebugMode():
global DEBUGMODE
DEBUGMODE=True
def isDebugMode():
return(DEBUGMODE)
def joinit(iterable, delimiter): def joinit(iterable, delimiter):
it = iter(iterable) it = iter(iterable)
@ -27,6 +48,17 @@ def joinit(iterable, delimiter):
yield delimiter yield delimiter
yield x yield x
def addToDb(db,desc):
msg("Add %s to summary database\n" % desc)
completed = subprocess.run([sys.executable, "addToDB.py","-o",db,desc], timeout=3600)
check(completed)
def addToRegDb(db,desc):
msg("Add %s to regression database\n" % desc)
completed = subprocess.run([sys.executable, "addToRegDB.py","-o",db,desc], timeout=3600)
check(completed)
class TestFlowFailure(Exception): class TestFlowFailure(Exception):
def __init__(self,completed): def __init__(self,completed):
self._errorcode = completed.returncode self._errorcode = completed.returncode
@ -158,6 +190,8 @@ class BuildConfig:
def createCMake(self,flags,benchMode,platform): def createCMake(self,flags,benchMode,platform):
with self.buildFolder() as b: with self.buildFolder() as b:
self.saveEnv() self.saveEnv()
if benchMode:
msg("Benchmark mode\n")
msg("Create cmake for %s\n" % self.buildFolderName()) msg("Create cmake for %s\n" % self.buildFolderName())
toolchainCmake = os.path.join(self.toolChainPath(),self.toolChainFile()) toolchainCmake = os.path.join(self.toolChainPath(),self.toolChainFile())
cmd = [self._cmake] cmd = [self._cmake]
@ -167,8 +201,11 @@ class BuildConfig:
"-DPLATFORM=%s" % platform "-DPLATFORM=%s" % platform
] ]
cmd += flags cmd += flags
if DEBUGMODE:
cmd += DEBUGLIST
if benchMode: if benchMode:
msg("Benchmark mode\n")
cmd += ["-DBENCHMARK=ON"] cmd += ["-DBENCHMARK=ON"]
cmd += ["-DWRAPPER=ON"] cmd += ["-DWRAPPER=ON"]
else: else:
@ -180,6 +217,9 @@ class BuildConfig:
"-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_BUILD_TYPE=Release",
"-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()] "-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()]
if DEBUGMODE:
print(cmd)
with open(os.path.join(self.archiveLogPath(),"cmakecmd.txt"),"w") as cmakecmd: with open(os.path.join(self.archiveLogPath(),"cmakecmd.txt"),"w") as cmakecmd:
cmakecmd.write("".join(joinit(cmd," "))) cmakecmd.write("".join(joinit(cmd," ")))
@ -296,12 +336,15 @@ class Test:
# the build folder for this test # the build folder for this test
# #
# We need a timeout and detect failed run # We need a timeout and detect failed run
def run(self,fvp): def run(self,fvp,benchmode):
timeoutVal=3600
if benchmode:
timeoutVal = 3600 * 4
completed = None completed = None
with self.buildConfig().buildFolder() as b: with self.buildConfig().buildFolder() as b:
msg(" Run %s\n" % self.testName() ) msg(" Run %s\n" % self.testName() )
with open(self.resultName(),"w") as results: with open(self.resultName(),"w") as results:
completed=subprocess.run(fvp.split(),stdout=results,timeout=3600) completed=subprocess.run(fvp.split(),stdout=results,timeout=timeoutVal)
check(completed) check(completed)
# Process results of the given tests # Process results of the given tests
@ -318,7 +361,19 @@ class Test:
else: else:
return(TESTFAILED) return(TESTFAILED)
def runAndProcess(self,compiler,fvp,sim): # Compute the regression data
def computeSummaryStat(self):
msg(" Compute regressions for %s\n" % self.testName())
with open(os.path.join(self.buildConfig().archiveResultPath(),"processedResult_%s.txt" % self.testName()),"w") as presult:
completed=subprocess.run([sys.executable,"summaryBench.py","-r",self.getResultPath()],stdout=presult,timeout=3600)
# When a test fail, the regression is continuing but we
# track that a test has failed
if completed.returncode==0:
return(NOTESTFAILED)
else:
return(TESTFAILED)
def runAndProcess(self,compiler,fvp,sim,benchmode,db,regdb):
# If we can't parse test description we fail all tests # If we can't parse test description we fail all tests
self.processTest() self.processTest()
# Otherwise if only building or those tests are failing, we continue # Otherwise if only building or those tests are failing, we continue
@ -333,8 +388,15 @@ class Test:
# build is done per test suite. # build is done per test suite.
if sim: if sim:
if fvp is not None: if fvp is not None:
self.run(fvp) self.run(fvp,benchmode)
return(self.processResult()) error=self.processResult()
if benchmode and (error == NOTESTFAILED):
error = self.computeSummaryStat()
if db is not None:
addToDb(db,self.testName())
if regdb is not None:
addToRegDb(regdb,self.testName())
return(error)
else: else:
msg("No FVP available") msg("No FVP available")
return(NOTESTFAILED) return(NOTESTFAILED)
@ -356,6 +418,12 @@ def generateAllCCode():
completed = subprocess.run([sys.executable,"processTests.py", "-e"],timeout=3600) completed = subprocess.run([sys.executable,"processTests.py", "-e"],timeout=3600)
check(completed) check(completed)
# Create db
def createDb(sqlite,desc):
msg("Create database %s\n" % desc)
with open("createDb.sql") as db:
completed = subprocess.run([sqlite, desc],stdin=db, timeout=3600)
check(completed)

24
Testing/createDb.sql
Unescape Escape View File

@ -74,13 +74,17 @@ INSERT INTO COMPILERKIND VALUES(2,"GCC");
INSERT INTO CORE VALUES(1,"m0","ARMCM0"); INSERT INTO CORE VALUES(1,"m0","ARMCM0");
INSERT INTO CORE VALUES(2,"m0p","ARMCM0P"); INSERT INTO CORE VALUES(2,"m0p","ARMCM0P");
INSERT INTO CORE VALUES(3,"m3","ARMCM3"); INSERT INTO CORE VALUES(3,"m3","ARMCM3");
INSERT INTO CORE VALUES(4,"m4f","ARMCM4_FP"); INSERT INTO CORE VALUES(4,"m4","ARMCM4");
INSERT INTO CORE VALUES(5,"m7","ARMCM7_DP"); INSERT INTO CORE VALUES(5,"m4f","ARMCM4_FP");
INSERT INTO CORE VALUES(6,"m23","ARMCM23"); INSERT INTO CORE VALUES(6,"m7","ARMCM7_DP");
INSERT INTO CORE VALUES(7,"m33","ARMCM33_DSP_FP"); INSERT INTO CORE VALUES(7,"m23","ARMCM23");
INSERT INTO CORE VALUES(8,"m35","ARMCM35P_DSP_FP"); INSERT INTO CORE VALUES(8,"m33","ARMCM33_DSP_FP");
INSERT INTO CORE VALUES(9,"a5","ARMCA5"); INSERT INTO CORE VALUES(9,"m35","ARMCM35P_DSP_FP");
INSERT INTO CORE VALUES(10,"a7","ARMCA7"); INSERT INTO CORE VALUES(10,"a5","ARMCA5");
INSERT INTO CORE VALUES(11,"a9","ARMCA9"); INSERT INTO CORE VALUES(11,"a7","ARMCA7");
INSERT INTO CORE VALUES(12,"a15","ARMCA15"); INSERT INTO CORE VALUES(12,"a9","ARMCA9");
INSERT INTO CORE VALUES(13,"helium","ARMv81MML_DSP_DP_MVE_FP"); INSERT INTO CORE VALUES(13,"a15","ARMCA15");
INSERT INTO CORE VALUES(14,"m55","ARMv81MML_DSP_DP_MVE_FP");
.quit

23
Testing/examples.sql
Unescape Escape View File

@ -3,18 +3,29 @@
Build the table with the platform, compiler and core names. Build the table with the platform, compiler and core names.
*/ */
.headers ON
.mode csv
/* /*
select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,DATE select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE
from BasicBenchmarks from BasicMathsBenchmarksF32
INNER JOIN CATEGORY USING(categoryid) INNER JOIN CATEGORY USING(categoryid)
INNER JOIN PLATFORM USING(platformid) INNER JOIN PLATFORM USING(platformid)
INNER JOIN CORE USING(coreid) INNER JOIN CORE USING(coreid)
INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN COMPILERKIND USING(compilerkindid)
; ;
*/ */
select Regression,MAX,MAXREGCOEF,CATEGORY.category,NAME,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE
from BasicMathsBenchmarksF32
INNER JOIN CATEGORY USING(categoryid)
INNER JOIN PLATFORM USING(platformid)
INNER JOIN CORE USING(coreid)
INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid)
;
/* /*
Compute the max cycles for a test configuration (category + name) Compute the max cycles for a test configuration (category + name)
@ -45,11 +56,13 @@ See diff.sql for example
*/ */
select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,DATE /*
from BasicBenchmarks select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE
from BasicMathsBenchmarksF32
INNER JOIN CATEGORY USING(categoryid) INNER JOIN CATEGORY USING(categoryid)
INNER JOIN PLATFORM USING(platformid) INNER JOIN PLATFORM USING(platformid)
INNER JOIN CORE USING(coreid) INNER JOIN CORE USING(coreid)
INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN COMPILERKIND USING(compilerkindid)
WHERE DATE BETWEEN datetime('now','localtime','-10 minutes') AND datetime('now', 'localtime'); WHERE DATE BETWEEN datetime('now','localtime','-10 minutes') AND datetime('now', 'localtime');
*/

28
Testing/runAllTests.py
Unescape Escape View File

@ -103,8 +103,28 @@ parser.add_argument('-b', action='store_true', help="Benchmark mode")
parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file") parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file")
parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running") parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running")
parser.add_argument('-db', nargs='?',type = str,help="Benchmark database")
parser.add_argument('-regdb', nargs='?',type = str,help="Regression database")
parser.add_argument('-sqlite', nargs='?',default="/usr/bin/sqlite3",type = str,help="Regression database")
parser.add_argument('-debug', action='store_true', help="Debug mode")
args = parser.parse_args() args = parser.parse_args()
if args.debug:
setDebugMode()
# Create missing database files
# if the db arguments are specified
if args.db is not None:
if not os.path.exists(args.db):
createDb(args.sqlite,args.db)
if args.regdb is not None:
if not os.path.exists(args.regdb):
createDb(args.sqlite,args.regdb)
with open(args.i,"r") as f: with open(args.i,"r") as f:
config=yaml.safe_load(f) config=yaml.safe_load(f)
@ -118,7 +138,7 @@ with open(args.i,"r") as f:
flags = config["FLAGS"] flags = config["FLAGS"]
allConfigs = analyzeFlags(flags) allConfigs = analyzeFlags(flags)
if DEBUGMODE: if isDebugMode():
allConfigs=[allConfigs[0]] allConfigs=[allConfigs[0]]
failedBuild = {} failedBuild = {}
@ -200,7 +220,7 @@ def buildAndTest(compiler,theConfig,cmake,sim):
if 'SIM' in config: if 'SIM' in config:
if core in config['SIM']: if core in config['SIM']:
fvp = config['SIM'][core] fvp = config['SIM'][core]
newTestStatus = test.runAndProcess(compiler,fvp,sim) newTestStatus = test.runAndProcess(compiler,fvp,sim,args.b,args.db,args.regdb)
testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus) testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus)
if testStatusForThisBuild != NOTESTFAILED: if testStatusForThisBuild != NOTESTFAILED:
failedBuild[buildStr] = testStatusForThisBuild failedBuild[buildStr] = testStatusForThisBuild
@ -225,9 +245,11 @@ def buildAndTest(compiler,theConfig,cmake,sim):
############## Builds for all toolchains ############## Builds for all toolchains
if not DEBUGMODE: if not isDebugMode():
preprocess(args.f) preprocess(args.f)
generateAllCCode() generateAllCCode()
else:
msg("Debug Mode\n")
for t in config["TOOLCHAINS"]: for t in config["TOOLCHAINS"]:
cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs) cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs)

Write Preview
Loading…
Cancel
Save