CMSIS-DSP: Improvement to testing scripts
Christophe Favergeon
6 years ago
parent
70f186b81d
commit
c2ca0dd2f8
File diff suppressed because it is too large
Load Diff
@ -0,0 +1,283 @@
|
|||||||
|
import argparse
|
||||||
|
import sqlite3
|
||||||
|
import re
|
||||||
|
import pandas as pd
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
# Command to get last runid
|
||||||
|
lastID="""SELECT runid FROM RUN ORDER BY runid DESC LIMIT 1
|
||||||
|
"""
|
||||||
|
|
||||||
|
def getLastRunID():
|
||||||
|
r=c.execute(lastID)
|
||||||
|
return(int(r.fetchone()[0]))
|
||||||
|
|
||||||
|
|
||||||
|
runid = 1
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description='Generate summary benchmarks')
|
||||||
|
|
||||||
|
parser.add_argument('-b', nargs='?',type = str, default="bench.db", help="Benchmark database")
|
||||||
|
parser.add_argument('-o', nargs='?',type = str, default="full.md", help="Full summary")
|
||||||
|
parser.add_argument('-r', action='store_true', help="Regression database")
|
||||||
|
|
||||||
|
# For runid or runid range
|
||||||
|
parser.add_argument('others', nargs=argparse.REMAINDER)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
c = sqlite3.connect(args.b)
|
||||||
|
|
||||||
|
if args.others:
|
||||||
|
runid=int(args.others[0])
|
||||||
|
else:
|
||||||
|
runid=getLastRunID()
|
||||||
|
|
||||||
|
# We extract data only from data tables
|
||||||
|
# Those tables below are used for descriptions
|
||||||
|
REMOVETABLES=['RUN','CORE', 'PLATFORM', 'COMPILERKIND', 'COMPILER', 'TYPE', 'CATEGORY', 'CONFIG']
|
||||||
|
|
||||||
|
# This is assuming the database is generated by the regression script
|
||||||
|
# So platform is the same for all benchmarks.
|
||||||
|
# Category and type is coming from the test name in the yaml
|
||||||
|
# So no need to add this information here
|
||||||
|
# Name is removed here because it is added at the beginning
|
||||||
|
REMOVECOLUMNS=['runid','NAME','type','platform','category','coredef','OPTIMIZED','HARDFP','FASTMATH','NEON','HELIUM','UNROLL','ROUNDING','DATE','compilerkindid','date','categoryid', 'ID', 'platformid', 'coreid', 'compilerid', 'typeid']
|
||||||
|
|
||||||
|
# Get existing benchmark tables
|
||||||
|
def getBenchTables():
|
||||||
|
r=c.execute("SELECT name FROM sqlite_master WHERE type='table'")
|
||||||
|
benchtables=[]
|
||||||
|
for table in r:
|
||||||
|
if not table[0] in REMOVETABLES:
|
||||||
|
benchtables.append(table[0])
|
||||||
|
return(benchtables)
|
||||||
|
|
||||||
|
# get existing types in a table
|
||||||
|
def getExistingTypes(benchTable):
|
||||||
|
r=c.execute("select distinct typeid from %s" % benchTable).fetchall()
|
||||||
|
result=[x[0] for x in r]
|
||||||
|
return(result)
|
||||||
|
|
||||||
|
# Get compilers from specific type and table
|
||||||
|
versioncompiler="""select distinct compiler,version from %s
|
||||||
|
INNER JOIN COMPILER USING(compilerid)
|
||||||
|
INNER JOIN COMPILERKIND USING(compilerkindid) WHERE typeid=?"""
|
||||||
|
|
||||||
|
# Get existing compiler in a table for a specific type
|
||||||
|
# (In case report is structured by types)
|
||||||
|
def getExistingCompiler(benchTable,typeid):
|
||||||
|
r=c.execute(versioncompiler % benchTable,(typeid,)).fetchall()
|
||||||
|
return(r)
|
||||||
|
|
||||||
|
# Get type name from type id
|
||||||
|
def getTypeName(typeid):
|
||||||
|
r=c.execute("select type from TYPE where typeid=?",(typeid,)).fetchone()
|
||||||
|
return(r[0])
|
||||||
|
|
||||||
|
# Diff of 2 lists
|
||||||
|
def diff(first, second):
|
||||||
|
second = set(second)
|
||||||
|
return [item for item in first if item not in second]
|
||||||
|
|
||||||
|
|
||||||
|
# Command to get data for specific compiler
|
||||||
|
# and type
|
||||||
|
benchCmd="""select %s from %s
|
||||||
|
INNER JOIN CATEGORY USING(categoryid)
|
||||||
|
INNER JOIN PLATFORM USING(platformid)
|
||||||
|
INNER JOIN CORE USING(coreid)
|
||||||
|
INNER JOIN COMPILER USING(compilerid)
|
||||||
|
INNER JOIN COMPILERKIND USING(compilerkindid)
|
||||||
|
INNER JOIN TYPE USING(typeid)
|
||||||
|
WHERE compiler=? AND VERSION=? AND typeid = ? AND runid = ?
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Command to get test names for specific compiler
|
||||||
|
# and type
|
||||||
|
benchNames="""select distinct NAME from %s
|
||||||
|
INNER JOIN COMPILER USING(compilerid)
|
||||||
|
INNER JOIN COMPILERKIND USING(compilerkindid)
|
||||||
|
INNER JOIN TYPE USING(typeid)
|
||||||
|
WHERE compiler=? AND VERSION=? AND typeid = ? AND runid = ?
|
||||||
|
"""
|
||||||
|
|
||||||
|
# Command to get columns for specific table
|
||||||
|
benchCmdColumns="""select * from %s
|
||||||
|
INNER JOIN CATEGORY USING(categoryid)
|
||||||
|
INNER JOIN PLATFORM USING(platformid)
|
||||||
|
INNER JOIN CORE USING(coreid)
|
||||||
|
INNER JOIN COMPILER USING(compilerid)
|
||||||
|
INNER JOIN COMPILERKIND USING(compilerkindid)
|
||||||
|
INNER JOIN TYPE USING(typeid)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def joinit(iterable, delimiter):
|
||||||
|
it = iter(iterable)
|
||||||
|
yield next(it)
|
||||||
|
for x in it:
|
||||||
|
yield delimiter
|
||||||
|
yield x
|
||||||
|
|
||||||
|
# Is not a column name finishing by id
|
||||||
|
# (often primary key for thetable)
|
||||||
|
def isNotIDColumn(col):
|
||||||
|
if re.match(r'^.*id$',col):
|
||||||
|
return(False)
|
||||||
|
else:
|
||||||
|
return(True)
|
||||||
|
|
||||||
|
# Get test names
|
||||||
|
# for specific typeid and compiler (for the data)
|
||||||
|
def getTestNames(benchTable,comp,typeid):
|
||||||
|
vals=(comp[0],comp[1],typeid,runid)
|
||||||
|
result=c.execute(benchNames % benchTable,vals).fetchall()
|
||||||
|
return([x[0] for x in list(result)])
|
||||||
|
|
||||||
|
# Get names of columns and data for a table
|
||||||
|
# for specific typeid and compiler (for the data)
|
||||||
|
def getColNamesAndData(benchTable,comp,typeid):
|
||||||
|
cursor=c.cursor()
|
||||||
|
result=cursor.execute(benchCmdColumns % (benchTable))
|
||||||
|
cols= [member[0] for member in cursor.description]
|
||||||
|
keepCols = ['NAME'] + [c for c in diff(cols , REMOVECOLUMNS) if isNotIDColumn(c)]
|
||||||
|
keepColsStr = "".join(joinit(keepCols,","))
|
||||||
|
vals=(comp[0],comp[1],typeid,runid)
|
||||||
|
result=cursor.execute(benchCmd % (keepColsStr,benchTable),vals)
|
||||||
|
vals =np.array([list(x) for x in list(result)])
|
||||||
|
return(keepCols,vals)
|
||||||
|
|
||||||
|
# Write columns in markdown format
|
||||||
|
def writeColumns(f,cols):
|
||||||
|
colStr = "".join(joinit(cols,"|"))
|
||||||
|
f.write("|")
|
||||||
|
f.write(colStr)
|
||||||
|
f.write("|\n")
|
||||||
|
sepStr="".join(joinit([":-:" for x in cols],"|"))
|
||||||
|
f.write("|")
|
||||||
|
f.write(sepStr)
|
||||||
|
f.write("|\n")
|
||||||
|
|
||||||
|
# Write row in markdown format
|
||||||
|
def writeRow(f,row):
|
||||||
|
row=[str(x) for x in row]
|
||||||
|
rowStr = "".join(joinit(row,"|"))
|
||||||
|
f.write("|")
|
||||||
|
f.write(rowStr)
|
||||||
|
f.write("|\n")
|
||||||
|
|
||||||
|
PARAMS=["NB","NumTaps", "NBA", "NBB", "Factor", "NumStages","VECDIM","NBR","NBC","NBI","IFFT", "BITREV"]
|
||||||
|
|
||||||
|
def regressionTableFor(name,output,ref,toSort,indexCols,field):
|
||||||
|
data=ref.pivot_table(index=indexCols, columns='core',
|
||||||
|
values=[field], aggfunc='first')
|
||||||
|
|
||||||
|
data=data.sort_values(toSort)
|
||||||
|
|
||||||
|
cores = [c[1] for c in list(data.columns)]
|
||||||
|
columns = diff(indexCols,['NAME']) + cores
|
||||||
|
|
||||||
|
writeColumns(output,columns)
|
||||||
|
dataForFunc=data.loc[name]
|
||||||
|
if type(dataForFunc) is pd.DataFrame:
|
||||||
|
for row in dataForFunc.itertuples():
|
||||||
|
row=list(row)
|
||||||
|
if type(row[0]) is int:
|
||||||
|
row=[row[0]] + row[1:]
|
||||||
|
else:
|
||||||
|
row=list(row[0]) + row[1:]
|
||||||
|
writeRow(output,row)
|
||||||
|
else:
|
||||||
|
writeRow(output,dataForFunc)
|
||||||
|
|
||||||
|
def formatTableByCore(output,testNames,cols,vals):
|
||||||
|
if vals.size != 0:
|
||||||
|
ref=pd.DataFrame(vals,columns=cols)
|
||||||
|
toSort=["NAME"]
|
||||||
|
|
||||||
|
for param in PARAMS:
|
||||||
|
if param in ref.columns:
|
||||||
|
ref[param]=pd.to_numeric(ref[param])
|
||||||
|
toSort.append(param)
|
||||||
|
if args.r:
|
||||||
|
# Regression table
|
||||||
|
ref['MAX']=pd.to_numeric(ref['MAX'])
|
||||||
|
ref['MAXREGCOEF']=pd.to_numeric(ref['MAXREGCOEF'])
|
||||||
|
|
||||||
|
indexCols=diff(cols,['core','Regression','MAXREGCOEF','MAX','version','compiler'])
|
||||||
|
valList = ['Regression']
|
||||||
|
else:
|
||||||
|
ref['CYCLES']=pd.to_numeric(ref['CYCLES'])
|
||||||
|
|
||||||
|
indexCols=diff(cols,['core','CYCLES','version','compiler'])
|
||||||
|
valList = ['CYCLES']
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
for name in testNames:
|
||||||
|
if args.r:
|
||||||
|
output.write("#### %s\n" % name)
|
||||||
|
|
||||||
|
output.write("##### Regression\n" )
|
||||||
|
regressionTableFor(name,output,ref,toSort,indexCols,'Regression')
|
||||||
|
|
||||||
|
output.write("##### Max cycles\n" )
|
||||||
|
regressionTableFor(name,output,ref,toSort,indexCols,'MAX')
|
||||||
|
|
||||||
|
output.write("##### Max Reg Coef\n" )
|
||||||
|
regressionTableFor(name,output,ref,toSort,indexCols,'MAXREGCOEF')
|
||||||
|
|
||||||
|
else:
|
||||||
|
data=ref.pivot_table(index=indexCols, columns='core',
|
||||||
|
values=valList, aggfunc='first')
|
||||||
|
|
||||||
|
data=data.sort_values(toSort)
|
||||||
|
|
||||||
|
cores = [c[1] for c in list(data.columns)]
|
||||||
|
columns = diff(indexCols,['NAME']) + cores
|
||||||
|
|
||||||
|
output.write("#### %s\n" % name)
|
||||||
|
writeColumns(output,columns)
|
||||||
|
dataForFunc=data.loc[name]
|
||||||
|
if type(dataForFunc) is pd.DataFrame:
|
||||||
|
for row in dataForFunc.itertuples():
|
||||||
|
row=list(row)
|
||||||
|
if type(row[0]) is int:
|
||||||
|
row=[row[0]] + row[1:]
|
||||||
|
else:
|
||||||
|
row=list(row[0]) + row[1:]
|
||||||
|
writeRow(output,row)
|
||||||
|
else:
|
||||||
|
writeRow(output,dataForFunc)
|
||||||
|
|
||||||
|
# Add a report for each table
|
||||||
|
def addReportFor(output,benchName):
|
||||||
|
print("Process %s\n" % benchName)
|
||||||
|
output.write("# %s\n" % benchName)
|
||||||
|
allTypes = getExistingTypes(benchName)
|
||||||
|
# Add report for each type
|
||||||
|
for aTypeID in allTypes:
|
||||||
|
typeName = getTypeName(aTypeID)
|
||||||
|
output.write("## %s\n" % typeName)
|
||||||
|
## Add report for each compiler
|
||||||
|
allCompilers = getExistingCompiler(benchName,aTypeID)
|
||||||
|
for compiler in allCompilers:
|
||||||
|
#print(compiler)
|
||||||
|
output.write("### %s (%s)\n" % compiler)
|
||||||
|
cols,vals=getColNamesAndData(benchName,compiler,aTypeID)
|
||||||
|
names=getTestNames(benchName,compiler,aTypeID)
|
||||||
|
formatTableByCore(output,names,cols,vals)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
with open(args.o,"w") as output:
|
||||||
|
benchtables=getBenchTables()
|
||||||
|
for bench in benchtables:
|
||||||
|
addReportFor(output,bench)
|
||||||
|
finally:
|
||||||
|
c.close()
|
||||||
|
|
||||||
|
|
||||||
@ -1,70 +0,0 @@
|
|||||||
@ECHO OFF
|
|
||||||
|
|
||||||
echo "Basic Maths"
|
|
||||||
python processTests.py -e BasicBenchmarks
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Complex Maths"
|
|
||||||
python processTests.py -e ComplexBenchmarks
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "FIR"
|
|
||||||
python processTests.py -e FIR
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Convolution / Correlation"
|
|
||||||
python processTests.py -e MISC
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Decimation / Interpolation"
|
|
||||||
python processTests.py -e DECIM
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "BiQuad"
|
|
||||||
python processTests.py -e BIQUAD
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Controller"
|
|
||||||
python processTests.py -e Controller
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Fast Math"
|
|
||||||
python processTests.py -e FastMath
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Barycenter"
|
|
||||||
python processTests.py -e SupportBarF32
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Support"
|
|
||||||
python processTests.py -e Support
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Unary Matrix"
|
|
||||||
python processTests.py -e Unary
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Binary Matrix"
|
|
||||||
python processTests.py -e Binary
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
echo "Transform"
|
|
||||||
python processTests.py -e Transform
|
|
||||||
call:runBench
|
|
||||||
|
|
||||||
EXIT /B
|
|
||||||
|
|
||||||
:runBench
|
|
||||||
REM pushd build_m7
|
|
||||||
REM pushd build_m0
|
|
||||||
pushd build_a5
|
|
||||||
make
|
|
||||||
REM "C:\Program Files\ARM\Development Studio 2019.0\sw\models\bin\FVP_MPS2_Cortex-M7.exe" -a Testing > result.txt
|
|
||||||
REM "C:\Program Files\ARM\Development Studio 2019.0\sw\models\bin\FVP_MPS2_Cortex-M0.exe" -a Testing > result.txt
|
|
||||||
"C:\Program Files\ARM\Development Studio 2019.0\sw\models\bin\FVP_VE_Cortex-A5x1.exe" -a Testing > result.txt
|
|
||||||
popd
|
|
||||||
echo "Parse result"
|
|
||||||
REM python processResult.py -e -r build_m7\result.txt
|
|
||||||
REM python processResult.py -e -r build_m0\result.txt
|
|
||||||
python processResult.py -e -r build_a5\result.txt
|
|
||||||
goto:eof
|
|
||||||
@ -1,105 +0,0 @@
|
|||||||
import os
|
|
||||||
import os.path
|
|
||||||
import subprocess
|
|
||||||
import colorama
|
|
||||||
from colorama import init,Fore, Back, Style
|
|
||||||
import argparse
|
|
||||||
|
|
||||||
GROUPS = [
|
|
||||||
"BasicBenchmarks",
|
|
||||||
"ComplexBenchmarks",
|
|
||||||
"FIR",
|
|
||||||
"MISC",
|
|
||||||
"DECIM",
|
|
||||||
"BIQUAD",
|
|
||||||
"Controller",
|
|
||||||
"FastMath",
|
|
||||||
"SupportBarF32",
|
|
||||||
"Support",
|
|
||||||
"Unary",
|
|
||||||
"Binary",
|
|
||||||
"Transform"
|
|
||||||
]
|
|
||||||
|
|
||||||
init()
|
|
||||||
|
|
||||||
def msg(t):
|
|
||||||
print(Fore.CYAN + t + Style.RESET_ALL)
|
|
||||||
|
|
||||||
def processTest(test):
|
|
||||||
subprocess.call(["python","processTests.py","-e",test])
|
|
||||||
|
|
||||||
def addToDB(cmd):
|
|
||||||
for g in GROUPS:
|
|
||||||
msg("Add group %s" % g)
|
|
||||||
subprocess.call(["python",cmd,g])
|
|
||||||
|
|
||||||
def run(build,fvp,custom=None):
|
|
||||||
result = "results.txt"
|
|
||||||
resultPath = os.path.join(build,result)
|
|
||||||
|
|
||||||
current=os.getcwd()
|
|
||||||
try:
|
|
||||||
msg("Build" )
|
|
||||||
os.chdir(build)
|
|
||||||
subprocess.call(["make"])
|
|
||||||
msg("Run")
|
|
||||||
with open(result,"w") as results:
|
|
||||||
if custom:
|
|
||||||
subprocess.call([fvp] + custom,stdout=results)
|
|
||||||
else:
|
|
||||||
subprocess.call([fvp,"-a","Testing"],stdout=results)
|
|
||||||
finally:
|
|
||||||
os.chdir(current)
|
|
||||||
|
|
||||||
msg("Parse result")
|
|
||||||
subprocess.call(["python","processResult.py","-e","-r",resultPath])
|
|
||||||
|
|
||||||
msg("Regression computations")
|
|
||||||
subprocess.call(["python","summaryBench.py","-r",resultPath])
|
|
||||||
|
|
||||||
msg("Add results to benchmark database")
|
|
||||||
addToDB("addToDB.py")
|
|
||||||
|
|
||||||
msg("Add results to regression database")
|
|
||||||
addToDB("addToRegDB.py")
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def processAndRun(buildfolder,fvp,custom=None):
|
|
||||||
processTest("DSPBenchmarks")
|
|
||||||
run(buildfolder,fvp,custom=custom)
|
|
||||||
|
|
||||||
parser = argparse.ArgumentParser(description='Parse test description')
|
|
||||||
parser.add_argument('-f', nargs='?',type = str, default="build_benchmark_m7", help="Build folder")
|
|
||||||
parser.add_argument('-v', nargs='?',type = str, default="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe", help="Fast Model")
|
|
||||||
parser.add_argument('-c', nargs='?',type = str, help="Custom args")
|
|
||||||
|
|
||||||
args = parser.parse_args()
|
|
||||||
|
|
||||||
if args.f is not None:
|
|
||||||
BUILDFOLDER=args.f
|
|
||||||
else:
|
|
||||||
BUILDFOLDER="build_benchmark_m7"
|
|
||||||
|
|
||||||
if args.v is not None:
|
|
||||||
FVP=args.v
|
|
||||||
else:
|
|
||||||
FVP="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe"
|
|
||||||
|
|
||||||
|
|
||||||
if args.c:
|
|
||||||
custom = args.c.split()
|
|
||||||
else:
|
|
||||||
custom = None
|
|
||||||
|
|
||||||
print(Fore.RED + "bench.db and reg.db databases must exist before running this script" + Style.RESET_ALL)
|
|
||||||
|
|
||||||
msg("Process benchmark description file")
|
|
||||||
subprocess.call(["python", "preprocess.py","-f","bench.txt"])
|
|
||||||
|
|
||||||
msg("Generate all missing C files")
|
|
||||||
subprocess.call(["python","processTests.py", "-e"])
|
|
||||||
|
|
||||||
|
|
||||||
processAndRun(BUILDFOLDER,FVP,custom=custom)
|
|
||||||
Loading…
Reference in New Issue