dengzhihao
/
CMSIS-DSP
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CMSIS-DSP: Updated regression script for tests.

Browse Source
pull/19/head
Christophe Favergeon 6 years ago
parent 21383c9dd2
commit 23eaafccf9
4 changed files with 131 additions and 24 deletions
Show Stats Download Patch File Download Diff File

80
Testing/TestScripts/Regression/Commands.py
Unescape Escape View File

@ -13,12 +13,33 @@ import sys
DEBUGMODE = False
KEEPBUILDFOLDER = False
DEBUGLIST=[
"-DBASICMATH=ON",
"-DCOMPLEXMATH=OFF",
"-DCONTROLLER=OFF",
"-DFASTMATH=OFF",
"-DFILTERING=OFF",
"-DMATRIX=OFF",
"-DSTATISTICS=OFF",
"-DSUPPORT=OFF",
"-DTRANSFORM=OFF",
"-DSVM=OFF",
"-DBAYES=OFF",
"-DDISTANCE=OFF"]
NOTESTFAILED = 0
MAKEFAILED = 1
TESTFAILED = 2
FLOWFAILURE = 3
CALLFAILURE = 4
def setDebugMode():
global DEBUGMODE
DEBUGMODE=True
def isDebugMode():
return(DEBUGMODE)
def joinit(iterable, delimiter):
it = iter(iterable)
@ -27,6 +48,17 @@ def joinit(iterable, delimiter):
yield delimiter
yield x
def addToDb(db,desc):
msg("Add %s to summary database\n" % desc)
completed = subprocess.run([sys.executable, "addToDB.py","-o",db,desc], timeout=3600)
check(completed)
def addToRegDb(db,desc):
msg("Add %s to regression database\n" % desc)
completed = subprocess.run([sys.executable, "addToRegDB.py","-o",db,desc], timeout=3600)
check(completed)
class TestFlowFailure(Exception):
def __init__(self,completed):
self._errorcode = completed.returncode
@ -158,6 +190,8 @@ class BuildConfig:
def createCMake(self,flags,benchMode,platform):
with self.buildFolder() as b:
self.saveEnv()
if benchMode:
msg("Benchmark mode\n")
msg("Create cmake for %s\n" % self.buildFolderName())
toolchainCmake = os.path.join(self.toolChainPath(),self.toolChainFile())
cmd = [self._cmake]
@ -167,8 +201,11 @@ class BuildConfig:
"-DPLATFORM=%s" % platform
]
cmd += flags
if DEBUGMODE:
cmd += DEBUGLIST
if benchMode:
msg("Benchmark mode\n")
cmd += ["-DBENCHMARK=ON"]
cmd += ["-DWRAPPER=ON"]
else:
@ -180,6 +217,9 @@ class BuildConfig:
"-DCMAKE_BUILD_TYPE=Release",
"-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()]
if DEBUGMODE:
print(cmd)
with open(os.path.join(self.archiveLogPath(),"cmakecmd.txt"),"w") as cmakecmd:
cmakecmd.write("".join(joinit(cmd," ")))
@ -296,12 +336,15 @@ class Test:
# the build folder for this test
#
# We need a timeout and detect failed run
def run(self,fvp):
def run(self,fvp,benchmode):
timeoutVal=3600
if benchmode:
timeoutVal = 3600 * 4
completed = None
with self.buildConfig().buildFolder() as b:
msg(" Run %s\n" % self.testName() )
with open(self.resultName(),"w") as results:
completed=subprocess.run(fvp.split(),stdout=results,timeout=3600)
completed=subprocess.run(fvp.split(),stdout=results,timeout=timeoutVal)
check(completed)
# Process results of the given tests
@ -318,7 +361,19 @@ class Test:
else:
return(TESTFAILED)
def runAndProcess(self,compiler,fvp,sim):
# Compute the regression data
def computeSummaryStat(self):
msg(" Compute regressions for %s\n" % self.testName())
with open(os.path.join(self.buildConfig().archiveResultPath(),"processedResult_%s.txt" % self.testName()),"w") as presult:
completed=subprocess.run([sys.executable,"summaryBench.py","-r",self.getResultPath()],stdout=presult,timeout=3600)
# When a test fail, the regression is continuing but we
# track that a test has failed
if completed.returncode==0:
return(NOTESTFAILED)
else:
return(TESTFAILED)
def runAndProcess(self,compiler,fvp,sim,benchmode,db,regdb):
# If we can't parse test description we fail all tests
self.processTest()
# Otherwise if only building or those tests are failing, we continue
@ -333,8 +388,15 @@ class Test:
# build is done per test suite.
if sim:
if fvp is not None:
self.run(fvp)
return(self.processResult())
self.run(fvp,benchmode)
error=self.processResult()
if benchmode and (error == NOTESTFAILED):
error = self.computeSummaryStat()
if db is not None:
addToDb(db,self.testName())
if regdb is not None:
addToRegDb(regdb,self.testName())
return(error)
else:
msg("No FVP available")
return(NOTESTFAILED)
@ -356,6 +418,12 @@ def generateAllCCode():
completed = subprocess.run([sys.executable,"processTests.py", "-e"],timeout=3600)
check(completed)
# Create db
def createDb(sqlite,desc):
msg("Create database %s\n" % desc)
with open("createDb.sql") as db:
completed = subprocess.run([sqlite, desc],stdin=db, timeout=3600)
check(completed)

24
Testing/createDb.sql
Unescape Escape View File

@ -74,13 +74,17 @@ INSERT INTO COMPILERKIND VALUES(2,"GCC");
INSERT INTO CORE VALUES(1,"m0","ARMCM0");
INSERT INTO CORE VALUES(2,"m0p","ARMCM0P");
INSERT INTO CORE VALUES(3,"m3","ARMCM3");
INSERT INTO CORE VALUES(4,"m4f","ARMCM4_FP");
INSERT INTO CORE VALUES(5,"m7","ARMCM7_DP");
INSERT INTO CORE VALUES(6,"m23","ARMCM23");
INSERT INTO CORE VALUES(7,"m33","ARMCM33_DSP_FP");
INSERT INTO CORE VALUES(8,"m35","ARMCM35P_DSP_FP");
INSERT INTO CORE VALUES(9,"a5","ARMCA5");
INSERT INTO CORE VALUES(10,"a7","ARMCA7");
INSERT INTO CORE VALUES(11,"a9","ARMCA9");
INSERT INTO CORE VALUES(12,"a15","ARMCA15");
INSERT INTO CORE VALUES(13,"helium","ARMv81MML_DSP_DP_MVE_FP");
INSERT INTO CORE VALUES(4,"m4","ARMCM4");
INSERT INTO CORE VALUES(5,"m4f","ARMCM4_FP");
INSERT INTO CORE VALUES(6,"m7","ARMCM7_DP");
INSERT INTO CORE VALUES(7,"m23","ARMCM23");
INSERT INTO CORE VALUES(8,"m33","ARMCM33_DSP_FP");
INSERT INTO CORE VALUES(9,"m35","ARMCM35P_DSP_FP");
INSERT INTO CORE VALUES(10,"a5","ARMCA5");
INSERT INTO CORE VALUES(11,"a7","ARMCA7");
INSERT INTO CORE VALUES(12,"a9","ARMCA9");
INSERT INTO CORE VALUES(13,"a15","ARMCA15");
INSERT INTO CORE VALUES(14,"m55","ARMv81MML_DSP_DP_MVE_FP");
.quit

23
Testing/examples.sql
Unescape Escape View File

@ -3,18 +3,29 @@
Build the table with the platform, compiler and core names.
*/
.headers ON
.mode csv
/*
select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,DATE
from BasicBenchmarks
select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE
from BasicMathsBenchmarksF32
INNER JOIN CATEGORY USING(categoryid)
INNER JOIN PLATFORM USING(platformid)
INNER JOIN CORE USING(coreid)
INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid)
;
*/
select Regression,MAX,MAXREGCOEF,CATEGORY.category,NAME,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE
from BasicMathsBenchmarksF32
INNER JOIN CATEGORY USING(categoryid)
INNER JOIN PLATFORM USING(platformid)
INNER JOIN CORE USING(coreid)
INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid)
;
/*
Compute the max cycles for a test configuration (category + name)
@ -45,11 +56,13 @@ See diff.sql for example
*/
select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,DATE
from BasicBenchmarks
/*
select NB,CATEGORY.category,NAME,CYCLES,PLATFORM.platform,CORE.core,COMPILERKIND.compiler,COMPILER.version,BasicMathsBenchmarksF32.DATE
from BasicMathsBenchmarksF32
INNER JOIN CATEGORY USING(categoryid)
INNER JOIN PLATFORM USING(platformid)
INNER JOIN CORE USING(coreid)
INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid)
WHERE DATE BETWEEN datetime('now','localtime','-10 minutes') AND datetime('now', 'localtime');
*/

28
Testing/runAllTests.py
Unescape Escape View File

@ -103,8 +103,28 @@ parser.add_argument('-b', action='store_true', help="Benchmark mode")
parser.add_argument('-f', nargs='?',type = str, default="desc.txt",help="Test description file")
parser.add_argument('-p', nargs='?',type = str, default="FVP",help="Platform for running")
parser.add_argument('-db', nargs='?',type = str,help="Benchmark database")
parser.add_argument('-regdb', nargs='?',type = str,help="Regression database")
parser.add_argument('-sqlite', nargs='?',default="/usr/bin/sqlite3",type = str,help="Regression database")
parser.add_argument('-debug', action='store_true', help="Debug mode")
args = parser.parse_args()
if args.debug:
setDebugMode()
# Create missing database files
# if the db arguments are specified
if args.db is not None:
if not os.path.exists(args.db):
createDb(args.sqlite,args.db)
if args.regdb is not None:
if not os.path.exists(args.regdb):
createDb(args.sqlite,args.regdb)
with open(args.i,"r") as f:
config=yaml.safe_load(f)
@ -118,7 +138,7 @@ with open(args.i,"r") as f:
flags = config["FLAGS"]
allConfigs = analyzeFlags(flags)
if DEBUGMODE:
if isDebugMode():
allConfigs=[allConfigs[0]]
failedBuild = {}
@ -200,7 +220,7 @@ def buildAndTest(compiler,theConfig,cmake,sim):
if 'SIM' in config:
if core in config['SIM']:
fvp = config['SIM'][core]
newTestStatus = test.runAndProcess(compiler,fvp,sim)
newTestStatus = test.runAndProcess(compiler,fvp,sim,args.b,args.db,args.regdb)
testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus)
if testStatusForThisBuild != NOTESTFAILED:
failedBuild[buildStr] = testStatusForThisBuild
@ -225,9 +245,11 @@ def buildAndTest(compiler,theConfig,cmake,sim):
############## Builds for all toolchains
if not DEBUGMODE:
if not isDebugMode():
preprocess(args.f)
generateAllCCode()
else:
msg("Debug Mode\n")
for t in config["TOOLCHAINS"]:
cmake,localConfig,sim = analyzeToolchain(config["TOOLCHAINS"][t],allConfigs)

Write Preview
Loading…
Cancel
Save