dengzhihao
/
CMSIS-DSP
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

CMSIS-DSP: Improved test reporting script

Browse Source
pull/19/head
Christophe Favergeon 5 years ago
parent 68e5b742ba
commit 443e72fdb2
2 changed files with 193 additions and 65 deletions
Show Stats Download Patch File Download Diff File

21
Testing/TestScripts/doc/Format.py
Unescape Escape View File

@ -1,7 +1,9 @@
import math import math
from datetime import date from datetime import date
NORMALFORMAT=0
BYCFORMAT=1
BYDFORMAT=2
def joinit(iterable, delimiter): def joinit(iterable, delimiter):
it = iter(iterable) it = iter(iterable)
@ -467,10 +469,13 @@ class HTMLToc:
def leaveDocument(self,document): def leaveDocument(self,document):
self._output.write("</ul></div>%s\n" % script) self._output.write("</ul></div>%s\n" % script)
def permutation(ordered,unordered): def permutation(ordered,unordered,mode):
result=[] result=[]
restricted=[] restricted=[]
for c in ORDEREDCORES: order = ORDEREDCORES
if mode == BYDFORMAT:
order = ORDEREDTYPES
for c in order:
if c in unordered: if c in unordered:
restricted.append(c) restricted.append(c)
@ -548,8 +553,10 @@ myhist(thehdata%d,"#hi%d");
self._output.write(str(col)) self._output.write(str(col))
self._output.write("</th>\n") self._output.write("</th>\n")
if self._reorder: if self._reorder == NORMALFORMAT:
perm,restricted=permutation(ORDEREDCORES,table.cores) perm,restricted=permutation(ORDEREDCORES,table.cores,self._reorder)
elif self._reorder == BYDFORMAT:
perm,restricted=permutation(ORDEREDTYPES,table.cores,self._reorder)
else: else:
restricted = table.cores restricted = table.cores
@ -576,7 +583,9 @@ myhist(thehdata%d,"#hi%d");
params=row[0:nbParams] params=row[0:nbParams]
values=row[nbParams:] values=row[nbParams:]
if self._reorder: if self._reorder == NORMALFORMAT:
row = params + reorder(perm,values)
elif self._reorder == BYDFORMAT:
row = params + reorder(perm,values) row = params + reorder(perm,values)
else: else:
row = params + values row = params + values

237
Testing/extractDb.py
Unescape Escape View File

@ -118,6 +118,12 @@ def getExistingTypes(benchTable):
result=[x[0] for x in r] result=[x[0] for x in r]
return(result) return(result)
# Get existing cores in a table
def getAllExistingCores(benchTable):
r=c.execute("select distinct coreid from %s WHERE %s order by coreid desc " % (benchTable,runidCMD),runidval).fetchall()
result=[x[0] for x in r]
return(result)
def getrunIDDetails(): def getrunIDDetails():
tables=getBenchTables() tables=getBenchTables()
r=[] r=[]
@ -137,6 +143,9 @@ if args.details:
# Get compilers from specific type and table # Get compilers from specific type and table
allCompilers="""select distinct compilerid from %s WHERE typeid=?""" allCompilers="""select distinct compilerid from %s WHERE typeid=?"""
# Get compilers from specific type and table
allCompilerForCore="""select distinct compilerid from %s WHERE coreid=?"""
# Get compilers from specific type and table # Get compilers from specific type and table
allCores="""select distinct coreid from %s WHERE typeid=? AND (%s)""" allCores="""select distinct coreid from %s WHERE typeid=? AND (%s)"""
@ -152,6 +161,13 @@ def getExistingCompiler(benchTable,typeid):
r=c.execute(allCompilers % benchTable,(typeid,)).fetchall() r=c.execute(allCompilers % benchTable,(typeid,)).fetchall()
return([x[0] for x in r]) return([x[0] for x in r])
# Get existing compiler in a table for a specific core
# (In case report is structured by core)
def getExistingCompilerForCore(benchTable,coreid):
r=c.execute(allCompilerForCore % benchTable,(coreid,)).fetchall()
return([x[0] for x in r])
def getExistingCores(benchTable,typeid): def getExistingCores(benchTable,typeid):
vals = (typeid,) + runidval vals = (typeid,) + runidval
r=c.execute(allCores % (benchTable,runidCMD),vals).fetchall() r=c.execute(allCores % (benchTable,runidCMD),vals).fetchall()
@ -178,6 +194,19 @@ def diff(first, second):
return [item for item in first if item not in second] return [item for item in first if item not in second]
# Command to get data for specific compiler
# and type
benchCmdForCoreCompiler="""select %s from %s
INNER JOIN CATEGORY USING(categoryid)
INNER JOIN PLATFORM USING(platformid)
INNER JOIN CORE USING(coreid)
INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid)
INNER JOIN TYPE USING(typeid)
INNER JOIN TESTNAME USING(testnameid)
WHERE coreid=? AND compilerid = ? AND (%s)
"""
# Command to get data for specific core # Command to get data for specific core
# and type # and type
historyCmd="""select %s from %s historyCmd="""select %s from %s
@ -230,16 +259,27 @@ benchCmdForCompiler="""select %s from %s
# Command to get test names for specific compiler # Command to get test names for specific compiler
# and type # and type
benchNamesForCore="""select distinct ID,name from %s benchNamesForCore="""select distinct name from %s
INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN COMPILERKIND USING(compilerkindid)
INNER JOIN TYPE USING(typeid) INNER JOIN TYPE USING(typeid)
INNER JOIN TESTNAME USING(testnameid) INNER JOIN TESTNAME USING(testnameid)
WHERE coreid=? AND typeid = ? AND (%s) WHERE coreid=? AND typeid = ? AND (%s)
""" """
# Command to get test names for specific core
# and compiler
benchNamesForCoreCompiler="""select distinct name from %s
INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid)
INNER JOIN TYPE USING(typeid)
INNER JOIN TESTNAME USING(testnameid)
WHERE coreid=? AND compilerid = ? AND (%s)
"""
# Command to get test names for specific compiler # Command to get test names for specific compiler
# and type # and type
benchNamesForCompiler="""select distinct ID,name from %s benchNamesForCompiler="""select distinct name from %s
INNER JOIN COMPILER USING(compilerid) INNER JOIN COMPILER USING(compilerid)
INNER JOIN COMPILERKIND USING(compilerkindid) INNER JOIN COMPILERKIND USING(compilerkindid)
INNER JOIN TYPE USING(typeid) INNER JOIN TYPE USING(typeid)
@ -273,12 +313,20 @@ def isNotIDColumn(col):
else: else:
return(True) return(True)
# Get test names
# for specific core and compiler (for the data)
def getTestNamesForCoreCompiler(benchTable,compilerid,core):
vals=(core,compilerid) + runidval
result=c.execute(benchNamesForCoreCompiler % (benchTable,runidCMD),vals).fetchall()
names=[x[0] for x in list(result)]
return(names)
# Get test names # Get test names
# for specific typeid and core (for the data) # for specific typeid and core (for the data)
def getTestNamesForCore(benchTable,core,typeid): def getTestNamesForCore(benchTable,core,typeid):
vals=(core,typeid) + runidval vals=(core,typeid) + runidval
result=c.execute(benchNamesForCore % (benchTable,runidCMD),vals).fetchall() result=c.execute(benchNamesForCore % (benchTable,runidCMD),vals).fetchall()
names=[(x[0],x[1]) for x in list(result)] names=[x[0] for x in list(result)]
return(names) return(names)
# Get test names # Get test names
@ -286,7 +334,7 @@ def getTestNamesForCore(benchTable,core,typeid):
def getTestNamesForCompiler(benchTable,comp,typeid): def getTestNamesForCompiler(benchTable,comp,typeid):
vals=(comp,typeid) + runidval vals=(comp,typeid) + runidval
result=c.execute(benchNamesForCompiler % (benchTable,runidCMD),vals).fetchall() result=c.execute(benchNamesForCompiler % (benchTable,runidCMD),vals).fetchall()
names=[(x[0],x[1]) for x in list(result)] names=[x[0] for x in list(result)]
return(names) return(names)
# Command to get data for specific core # Command to get data for specific core
@ -301,9 +349,18 @@ nbElemsInBenchAndTypeAndCompilerCmd="""select count(*) from %s
WHERE compilerid=? AND typeid = ? AND (%s) WHERE compilerid=? AND typeid = ? AND (%s)
""" """
# Command to get data for specific compiler
# and type
nbElemsInBenchAndCoreAndCompilerCmd="""select count(*) from %s
WHERE compilerid=? AND coreid = ? AND (%s)
"""
nbElemsInBenchAndTypeCmd="""select count(*) from %s nbElemsInBenchAndTypeCmd="""select count(*) from %s
WHERE typeid = ? AND (%s) WHERE typeid = ? AND (%s)
""" """
nbElemsInBenchAndCoreCmd="""select count(*) from %s
WHERE coreid = ? AND (%s)
"""
nbElemsInBenchCmd="""select count(*) from %s nbElemsInBenchCmd="""select count(*) from %s
WHERE %s WHERE %s
@ -330,11 +387,22 @@ def getNbElemsInBenchAndTypeAndCompilerCmd(benchTable,comp,typeid):
result=c.execute(nbElemsInBenchAndTypeAndCompilerCmd % (benchTable,runidCMD),vals).fetchone() result=c.execute(nbElemsInBenchAndTypeAndCompilerCmd % (benchTable,runidCMD),vals).fetchone()
return(result[0]) return(result[0])
# Get nb elems in a table
def getNbElemsInBenchAndCoreAndCompilerCmd(benchTable,comp,coreid):
vals=(comp,coreid) + runidval
result=c.execute(nbElemsInBenchAndCoreAndCompilerCmd % (benchTable,runidCMD),vals).fetchone()
return(result[0])
def getNbElemsInBenchAndTypeCmd(benchTable,typeid): def getNbElemsInBenchAndTypeCmd(benchTable,typeid):
vals=(typeid,) + runidval vals=(typeid,) + runidval
result=c.execute(nbElemsInBenchAndTypeCmd % (benchTable,runidCMD),vals).fetchone() result=c.execute(nbElemsInBenchAndTypeCmd % (benchTable,runidCMD),vals).fetchone()
return(result[0]) return(result[0])
def getNbElemsInBenchAndCoreCmd(benchTable,coreid):
vals=(coreid,) + runidval
result=c.execute(nbElemsInBenchAndCoreCmd % (benchTable,runidCMD),vals).fetchone()
return(result[0])
def getNbElemsInBenchCmd(benchTable): def getNbElemsInBenchCmd(benchTable):
result=c.execute(nbElemsInBenchCmd % (benchTable,runidCMD),runidval).fetchone() result=c.execute(nbElemsInBenchCmd % (benchTable,runidCMD),runidval).fetchone()
return(result[0]) return(result[0])
@ -345,7 +413,7 @@ def getColNamesAndHistory(benchTable,compiler,core,typeid,testid):
cursor=c.cursor() cursor=c.cursor()
result=cursor.execute(benchCmdColumns % (benchTable)) result=cursor.execute(benchCmdColumns % (benchTable))
cols= [member[0] for member in cursor.description] cols= [member[0] for member in cursor.description]
keepCols = ['name','runid'] + [c for c in diff(cols , REMOVECOLUMNSFORHISTORY) if isNotIDColumn(c)] keepCols = ['name'] + [c for c in diff(cols , REMOVECOLUMNSFORHISTORY) if isNotIDColumn(c)]
keepColsStr = "".join(joinit(keepCols,",")) keepColsStr = "".join(joinit(keepCols,","))
vals=(compiler,core,typeid,testid,runid) vals=(compiler,core,typeid,testid,runid)
result=cursor.execute(historyCmd % (keepColsStr,benchTable),vals) result=cursor.execute(historyCmd % (keepColsStr,benchTable),vals)
@ -365,6 +433,18 @@ def getColNamesAndDataForCore(benchTable,core,typeid):
vals =np.array([list(x) for x in list(result)]) vals =np.array([list(x) for x in list(result)])
return(keepCols,vals) return(keepCols,vals)
# Get names of columns and data for a table
# for specific coreid and compilerid (for the data)
def getColNamesAndDataForCoreCompiler(benchTable,compilerid,core):
cursor=c.cursor()
result=cursor.execute(benchCmdColumns % (benchTable))
cols= [member[0] for member in cursor.description]
keepCols = ['name','type'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)]
keepColsStr = "".join(joinit(keepCols,","))
vals=(core,compilerid) + runidval
result=cursor.execute(benchCmdForCoreCompiler % (keepColsStr,benchTable,runidCMD),vals)
vals =np.array([list(x) for x in list(result)])
return(keepCols,vals)
# Get names of columns and data for a table # Get names of columns and data for a table
# for specific typeid and compiler (for the data) # for specific typeid and compiler (for the data)
@ -469,6 +549,16 @@ def getHistory(desc,testid,indexCols):
hist=History(series,runid) hist=History(series,runid)
return(hist) return(hist)
def convertRowToInt(r):
result=[]
for e in r:
if type(e) is float:
result.append(int(e))
else:
result.append(e)
return(result)
def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals): def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals):
if vals.size != 0: if vals.size != 0:
ref=pd.DataFrame(vals,columns=cols) ref=pd.DataFrame(vals,columns=cols)
@ -486,14 +576,13 @@ def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals):
indexCols=diff(cols,byname + ['Regression','MAXREGCOEF','MAX'] + section) indexCols=diff(cols,byname + ['Regression','MAXREGCOEF','MAX'] + section)
valList = ['Regression'] valList = ['Regression']
else: else:
ref['CYCLES']=pd.to_numeric(ref['CYCLES']) ref['CYCLES']=pd.to_numeric(ref['CYCLES']).round(decimals=0)
indexCols=diff(cols,byname + ['CYCLES'] + section) indexCols=diff(cols,byname + ['CYCLES'] + section)
valList = ['CYCLES'] valList = ['CYCLES']
for name in testNames:
for testid,name in testNames:
if args.r: if args.r:
testSection = Section(name) testSection = Section(name)
typeSection.addSection(testSection) typeSection.addSection(testSection)
@ -536,30 +625,33 @@ def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals):
else: else:
data=ref.pivot_table(index=indexCols, columns=byname, data=ref.pivot_table(index=indexCols, columns=byname,
values=valList, aggfunc='first') values=valList, aggfunc='first')
data=data.sort_values(toSort) data=data.sort_values(toSort)
#print(list(data.columns)) #print(list(data.columns))
columnsID = [formatColumnName(c[1:]) for c in list(data.columns)]
columns = diff(indexCols,['name'])
testSection = Section(name) testSection = Section(name)
typeSection.addSection(testSection) typeSection.addSection(testSection)
dataForFunc=data.loc[name]
dataForFunc = dataForFunc.dropna(axis=1)
columnsID = [formatColumnName(c[1:]) for c in list(dataForFunc.columns)]
columns = diff(indexCols,['name'])
dataTable=Table(columns,columnsID) dataTable=Table(columns,columnsID)
testSection.addContent(dataTable) testSection.addContent(dataTable)
dataForFunc=data.loc[name]
if type(dataForFunc) is pd.DataFrame: if type(dataForFunc) is pd.DataFrame:
for row in dataForFunc.itertuples(): for row in dataForFunc.itertuples():
row=list(row)
if type(row[0]) is int: if type(row[0]) is int:
row=[row[0]] + row[1:] row=list([row[0]] + list(row[1:]))
else: else:
row=list(row[0]) + row[1:] row=list(row[0]) + list(row[1:])
dataTable.addRow(row) dataTable.addRow(convertRowToInt(row))
else: else:
dataTable.addRow(dataForFunc) dataTable.addRow(convertRowToInt(dataForFunc))
# Add a report for each table # Add a report for each table
def addReportFor(document,benchName): def addReportFor(document,benchName):
@ -569,44 +661,69 @@ def addReportFor(document,benchName):
benchSection = Section(categoryName) benchSection = Section(categoryName)
document.addSection(benchSection) document.addSection(benchSection)
print("Process %s\n" % benchName) print("Process %s\n" % benchName)
allTypes = getExistingTypes(benchName) if args.byd:
# Add report for each type allCores=getAllExistingCores(benchName)
for aTypeID in allTypes: for aCoreID in allCores:
nbElems = getNbElemsInBenchAndTypeCmd(benchName,aTypeID) nbElems = getNbElemsInBenchAndCoreCmd(benchName,aCoreID)
if nbElems > 0: if nbElems > 0:
typeName = getTypeName(aTypeID) coreName=getCoreDesc(aCoreID)
typeSection = Section(typeName) coreSection = Section("%s" % coreName)
benchSection.addSection(typeSection) benchSection.addSection(coreSection)
if args.byc: allCompilers = getExistingCompilerForCore(benchName,aCoreID)
## Add report for each core for compiler in allCompilers:
allCores = getExistingCores(benchName,aTypeID) #print(compiler)
for core in allCores: nbElems = getNbElemsInBenchAndCoreAndCompilerCmd(benchName,compiler,aCoreID)
#print(core)
nbElems = getNbElemsInBenchAndTypeAndCoreCmd(benchName,core,aTypeID) # Print test results for table, type, compiler
# Print test results for table, type, compiler if nbElems > 0:
if nbElems > 0: compilerName,version=getCompilerDesc(compiler)
coreName=getCoreDesc(core) compilerSection = Section("%s (%s)" % (compilerName,version))
coreSection = Section("%s" % coreName) coreSection.addSection(compilerSection)
typeSection.addSection(coreSection) cols,vals=getColNamesAndDataForCoreCompiler(benchName,compiler,aCoreID)
cols,vals=getColNamesAndDataForCore(benchName,core,aTypeID) desc=(benchName,compiler,aCoreID)
desc=(benchName,core,aTypeID) names=getTestNamesForCoreCompiler(benchName,compiler,aCoreID)
names=getTestNamesForCore(benchName,core,aTypeID)
formatTableBy(desc,['compiler','version'],['core'],coreSection,names,cols,vals) formatTableBy(desc,['type'],['core','version','compiler'],compilerSection,names,cols,vals)
else:
## Add report for each compiler else:
allCompilers = getExistingCompiler(benchName,aTypeID) allTypes = getExistingTypes(benchName)
for compiler in allCompilers: # Add report for each type
#print(compiler) for aTypeID in allTypes:
nbElems = getNbElemsInBenchAndTypeAndCompilerCmd(benchName,compiler,aTypeID) nbElems = getNbElemsInBenchAndTypeCmd(benchName,aTypeID)
# Print test results for table, type, compiler if nbElems > 0:
if nbElems > 0: typeName = getTypeName(aTypeID)
compilerName,version=getCompilerDesc(compiler) typeSection = Section(typeName)
compilerSection = Section("%s (%s)" % (compilerName,version)) benchSection.addSection(typeSection)
typeSection.addSection(compilerSection) if args.byc:
cols,vals=getColNamesAndDataForCompiler(benchName,compiler,aTypeID) ## Add report for each core
desc=(benchName,compiler,aTypeID) allCores = getExistingCores(benchName,aTypeID)
names=getTestNamesForCompiler(benchName,compiler,aTypeID) for core in allCores:
formatTableBy(desc,['core'],['version','compiler'],compilerSection,names,cols,vals) #print(core)
nbElems = getNbElemsInBenchAndTypeAndCoreCmd(benchName,core,aTypeID)
# Print test results for table, type, compiler
if nbElems > 0:
coreName=getCoreDesc(core)
coreSection = Section("%s" % coreName)
typeSection.addSection(coreSection)
cols,vals=getColNamesAndDataForCore(benchName,core,aTypeID)
desc=(benchName,core,aTypeID)
names=getTestNamesForCore(benchName,core,aTypeID)
formatTableBy(desc,['compiler','version'],['core'],coreSection,names,cols,vals)
else:
## Add report for each compiler
allCompilers = getExistingCompiler(benchName,aTypeID)
for compiler in allCompilers:
#print(compiler)
nbElems = getNbElemsInBenchAndTypeAndCompilerCmd(benchName,compiler,aTypeID)
# Print test results for table, type, compiler
if nbElems > 0:
compilerName,version=getCompilerDesc(compiler)
compilerSection = Section("%s (%s)" % (compilerName,version))
typeSection.addSection(compilerSection)
cols,vals=getColNamesAndDataForCompiler(benchName,compiler,aTypeID)
desc=(benchName,compiler,aTypeID)
names=getTestNamesForCompiler(benchName,compiler,aTypeID)
formatTableBy(desc,['core'],['version','compiler'],compilerSection,names,cols,vals)
@ -686,9 +803,11 @@ try:
if args.t=="md": if args.t=="md":
document.accept(Markdown(output)) document.accept(Markdown(output))
if args.t=="html": if args.t=="html":
reorder=True reorder=NORMALFORMAT
if args.byc: if args.byc:
reorder=False reorder=BYCFORMAT
if args.byd:
reorder=BYDFORMAT
document.accept(HTML(output,args.r,reorder)) document.accept(HTML(output,args.r,reorder))
finally: finally:

Write Preview
Loading…
Cancel
Save