From 23f969c6c70502bd2cbcdb33aca1c915540d1dc5 Mon Sep 17 00:00:00 2001 From: Christophe Favergeon Date: Fri, 7 Feb 2020 11:25:11 +0100 Subject: [PATCH] CMSIS-DSP: Added M55 to test framework. Added a regression script to test several configurations of the CMSIS-DSP. --- Platforms/FVP/ARMCA5/LinkScripts/AC5/lnk.sct | 2 +- .../FVP/ARMCA5/Startup/AC5/startup_ARMCA5.c | 5 + Testing/.gitignore | 4 +- Testing/README.md | 5 + Testing/Source/Benchmarks/TransformF32.cpp | 1 + Testing/TestScripts/Regression/Commands.py | 349 ++++++++++++++++++ Testing/addAllBenchToDatabase.bat | 26 +- Testing/addAllBenchToRegressionDatabase.bat | 26 +- Testing/addToDB.py | 4 +- Testing/addToRegDB.py | 2 +- Testing/processResult.py | 10 + Testing/runAllBenchmarks.bat | 32 +- Testing/runAllBenchmarks.py | 105 ++++++ Testing/runAllTests.py | 261 +++++++------ Testing/summaryBench.py | 2 +- Testing/testmain.cpp | 3 +- Toolchain/AC6.cmake | 5 + configBoot.cmake | 6 +- configCore.cmake | 10 + configLib.cmake | 2 +- configPlatform.cmake | 9 + gcc.cmake | 4 +- 22 files changed, 714 insertions(+), 159 deletions(-) create mode 100755 Testing/TestScripts/Regression/Commands.py create mode 100755 Testing/runAllBenchmarks.py diff --git a/Platforms/FVP/ARMCA5/LinkScripts/AC5/lnk.sct b/Platforms/FVP/ARMCA5/LinkScripts/AC5/lnk.sct index 41e562cb..7eba725f 100755 --- a/Platforms/FVP/ARMCA5/LinkScripts/AC5/lnk.sct +++ b/Platforms/FVP/ARMCA5/LinkScripts/AC5/lnk.sct @@ -1,4 +1,4 @@ -#! armclang -E --target=arm-arm-none-eabi -mcpu=cortex-a5 -xc +#! armcc -E ;************************************************** ; Copyright (c) 2017 ARM Ltd. All rights reserved. ;************************************************** diff --git a/Platforms/FVP/ARMCA5/Startup/AC5/startup_ARMCA5.c b/Platforms/FVP/ARMCA5/Startup/AC5/startup_ARMCA5.c index 84a122b4..17da1c86 100755 --- a/Platforms/FVP/ARMCA5/Startup/AC5/startup_ARMCA5.c +++ b/Platforms/FVP/ARMCA5/Startup/AC5/startup_ARMCA5.c @@ -58,6 +58,8 @@ void FIQ_Handler (void) __attribute__ ((weak, alias("Default_Handler"))); Exception / Interrupt Vector Table *----------------------------------------------------------------------------*/ void Vectors(void) { + volatile int i; +#if 0 __ASM volatile( "LDR __current_pc, =Reset_Handler \n" "LDR __current_pc, =Undef_Handler \n" @@ -68,12 +70,14 @@ void Vectors(void) { "LDR __current_pc, =IRQ_Handler \n" "LDR __current_pc, =FIQ_Handler \n" ); +#endif } /*---------------------------------------------------------------------------- Reset Handler called on controller reset *----------------------------------------------------------------------------*/ void Reset_Handler(void) { +#if 0 __ASM volatile( // Mask interrupts @@ -128,6 +132,7 @@ void Reset_Handler(void) { // Call __main "BL __main \n" ); +#endif } /*---------------------------------------------------------------------------- diff --git a/Testing/.gitignore b/Testing/.gitignore index 02b57a45..6c37ff63 100644 --- a/Testing/.gitignore +++ b/Testing/.gitignore @@ -14,4 +14,6 @@ TestDesc.txt currentConfig.csv test.txt __pycache__ -bugcheck.py \ No newline at end of file +bugcheck.py +fulltests/ +testUNIXrunConfig.yaml diff --git a/Testing/README.md b/Testing/README.md index 163f19cc..48598f52 100644 --- a/Testing/README.md +++ b/Testing/README.md @@ -300,6 +300,11 @@ If you want to compute summary statistics with regression: pip install numpy pip install panda +If you want to run the script which is launching all the tests on all possible configurations then +you'll need yaml: + + pip install pyyaml + ### Generate the test patterns in Patterns folder We have archived lot of test patterns on github. So this step is needed only if you write new test patterns. diff --git a/Testing/Source/Benchmarks/TransformF32.cpp b/Testing/Source/Benchmarks/TransformF32.cpp index 6f817b0b..d742ae8d 100755 --- a/Testing/Source/Benchmarks/TransformF32.cpp +++ b/Testing/Source/Benchmarks/TransformF32.cpp @@ -1,5 +1,6 @@ #include "TransformF32.h" #include "Error.h" +#include "arm_math.h" #include "arm_const_structs.h" const arm_cfft_instance_f32 *arm_cfft_get_instance_f32(uint16_t fftLen) diff --git a/Testing/TestScripts/Regression/Commands.py b/Testing/TestScripts/Regression/Commands.py new file mode 100755 index 00000000..9fdd760e --- /dev/null +++ b/Testing/TestScripts/Regression/Commands.py @@ -0,0 +1,349 @@ +import subprocess +import colorama +from colorama import init,Fore, Back, Style +import argparse +import os +import os.path +from contextlib import contextmanager +import shutil +import glob +from pathlib import Path + +DEBUGMODE = False + +NOTESTFAILED = 0 +MAKEFAILED = 1 +TESTFAILED = 2 +FLOWFAILURE = 3 +CALLFAILURE = 4 + + +def joinit(iterable, delimiter): + it = iter(iterable) + yield next(it) + for x in it: + yield delimiter + yield x + +class TestFlowFailure(Exception): + def __init__(self,completed): + self._errorcode = completed.returncode + + def errorCode(self): + return(self._errorcode) + +class CallFailure(Exception): + pass + +def check(n): + #print(n) + if n is not None: + if n.returncode != 0: + raise TestFlowFailure(n) + else: + raise CallFailure() + +def msg(t): + print(Fore.CYAN + t + Style.RESET_ALL) + +def errorMsg(t): + print(Fore.RED + t + Style.RESET_ALL) + +def fullTestFolder(rootFolder): + return(os.path.join(rootFolder,"CMSIS","DSP","Testing","fulltests")) + +class BuildConfig: + def __init__(self,toUnset,rootFolder,buildFolder,compiler,toolchain,core,cmake): + self._toUnset = toUnset + self._buildFolder = buildFolder + self._rootFolder = os.path.abspath(rootFolder) + self._dspFolder = os.path.join(self._rootFolder,"CMSIS","DSP") + self._testingFolder = os.path.join(self._dspFolder,"Testing") + self._fullTests = os.path.join(self._testingFolder,"fulltests") + self._compiler = compiler + self._toolchain = toolchain + self._core = core + self._cmake = cmake + self._savedEnv = {} + + def compiler(self): + return(self._compiler) + + def toolChainFile(self): + return(self._toolchain) + + def core(self): + return(self._core) + + def path(self): + return(os.path.join(self._fullTests,self._buildFolder)) + + def archivePath(self): + return(os.path.join(self._fullTests,"archive",self._buildFolder)) + + def archiveResultPath(self): + return(os.path.join(self._fullTests,"archive",self._buildFolder,"results")) + + def archiveLogPath(self): + return(os.path.join(self._fullTests,"archive",self._buildFolder,"logs")) + + def archiveErrorPath(self): + return(os.path.join(self._fullTests,"archive",self._buildFolder,"errors")) + + def toolChainPath(self): + return(self._dspFolder) + + def cmakeFilePath(self): + return(self._testingFolder) + + def buildFolderName(self): + return(self._buildFolder) + + def saveEnv(self): + if self._toUnset is not None: + for v in self._toUnset: + self._savedEnv[v] = os.environ[v] + del os.environ[v] + + + def restoreEnv(self): + if self._toUnset is not None: + for v in self._toUnset: + os.environ[v] = self._savedEnv[v] + self._savedEnv = {} + + # Build for a folder + # We need to be able to detect failed build + def build(self,test): + completed=None + # Save and unset some environment variables + self.saveEnv() + with self.buildFolder() as b: + msg(" Build %s\n" % self.buildFolderName()) + with open(os.path.join(self.archiveLogPath(),"makelog_%s.txt" % test),"w") as makelog: + with open(os.path.join(self.archiveErrorPath(),"makeerror_%s.txt" % test),"w") as makeerr: + if DEBUGMODE: + completed=subprocess.run(["make","-j8","VERBOSE=1"],timeout=3600) + else: + completed=subprocess.run(["make","-j8","VERBOSE=1"],stdout=makelog,stderr=makeerr,timeout=3600) + # Restore environment variables + self.restoreEnv() + check(completed) + + def getTest(self,test): + return(Test(self,test)) + + + + # Launch cmake command. + def createCMake(self,flags): + with self.buildFolder() as b: + self.saveEnv() + msg("Create cmake for %s\n" % self.buildFolderName()) + toolchainCmake = os.path.join(self.toolChainPath(),self.toolChainFile()) + cmd = [self._cmake] + cmd += ["-DCMAKE_PREFIX_PATH=%s" % self.compiler(), + "-DCMAKE_TOOLCHAIN_FILE=%s" % toolchainCmake, + "-DARM_CPU=%s" % self.core(), + "-DPLATFORM=FVP" + ] + cmd += flags + cmd += ["-DBENCHMARK=OFF", + "-DFULLYCONNECTED=OFF", + "-DCONVOLUTION=OFF", + "-DACTIVATION=OFF", + "-DPOOLING=OFF", + "-DSOFTMAX=OFF", + "-DNNSUPPORT=OFF", + "-DBASICMATHSNN=OFF", + "-DRESHAPE=OFF", + "-DCONCATENATION=OFF", + "-DWRAPPER=OFF", + "-DCONFIGTABLE=OFF", + "-DROOT=%s" % self._rootFolder, + "-DCMAKE_BUILD_TYPE=Release", + "-G", "Unix Makefiles" ,"%s" % self.cmakeFilePath()] + + with open(os.path.join(self.archiveLogPath(),"cmakecmd.txt"),"w") as cmakecmd: + cmakecmd.write("".join(joinit(cmd," "))) + + with open(os.path.join(self.archiveLogPath(),"cmakelog.txt"),"w") as cmakelog: + with open(os.path.join(self.archiveErrorPath(),"cmakeerror.txt"),"w") as cmakeerr: + completed=subprocess.run(cmd, stdout=cmakelog,stderr=cmakeerr, timeout=3600) + self.restoreEnv() + check(completed) + + + # Create the build folder if missing + def createFolder(self): + os.makedirs(self.path(),exist_ok=True) + + def createArchive(self, flags): + os.makedirs(self.archivePath(),exist_ok=True) + os.makedirs(self.archiveResultPath(),exist_ok=True) + os.makedirs(self.archiveErrorPath(),exist_ok=True) + os.makedirs(self.archiveLogPath(),exist_ok=True) + with open(os.path.join(self.archivePath(),"flags.txt"),"w") as f: + for flag in flags: + f.write(flag) + f.write("\n") + + + # Delete the build folder + def cleanFolder(self): + print("Delete %s\n" % self.path()) + #DEBUG + if not DEBUGMODE: + shutil.rmtree(self.path()) + + # Archive results and currentConfig.csv to another folder + def archiveResults(self): + results=glob.glob(os.path.join(self.path(),"results_*")) + for result in results: + dst=os.path.join(self.archiveResultPath(),os.path.basename(result)) + shutil.copy(result,dst) + + src = os.path.join(self.path(),"currentConfig.csv") + dst = os.path.join(self.archiveResultPath(),os.path.basename(src)) + shutil.copy(src,dst) + + + @contextmanager + def buildFolder(self): + current=os.getcwd() + try: + os.chdir(self.path() ) + yield self.path() + finally: + os.chdir(current) + + @contextmanager + def archiveFolder(self): + current=os.getcwd() + try: + os.chdir(self.archivePath() ) + yield self.archivePath() + finally: + os.chdir(current) + + @contextmanager + def resultFolder(self): + current=os.getcwd() + try: + os.chdir(self.archiveResultPath()) + yield self.archiveResultPath() + finally: + os.chdir(current) + + @contextmanager + def logFolder(self): + current=os.getcwd() + try: + os.chdir(self.archiveLogPath()) + yield self.archiveLogPath() + finally: + os.chdir(current) + + @contextmanager + def errorFolder(self): + current=os.getcwd() + try: + os.chdir(self.archiveErrorPath()) + yield self.archiveErrorPath() + finally: + os.chdir(current) + +class Test: + def __init__(self,build,test): + self._test = test + self._buildConfig = build + + def buildConfig(self): + return(self._buildConfig) + + def testName(self): + return(self._test) + + # Process a test from the test description file + def processTest(self): + completed=subprocess.run(["python","processTests.py","-e",self.testName()],timeout=3600) + check(completed) + + def getResultPath(self): + return(os.path.join(self.buildConfig().path() ,self.resultName())) + + def resultName(self): + return("results_%s.txt" % self.testName()) + + # Run a specific test in the current folder + # A specific results.txt file is created in + # the build folder for this test + # + # We need a timeout and detect failed run + def run(self,fvp): + completed = None + with self.buildConfig().buildFolder() as b: + msg(" Run %s\n" % self.testName() ) + with open(self.resultName(),"w") as results: + completed=subprocess.run(fvp.split(),stdout=results,timeout=3600) + check(completed) + + # Process results of the given tests + # in given build folder + # We need to detect failed tests + def processResult(self): + msg(" Parse result for %s\n" % self.testName()) + with open(os.path.join(self.buildConfig().archiveResultPath(),"processedResult_%s.txt" % self.testName()),"w") as presult: + completed=subprocess.run(["python","processResult.py","-e","-r",self.getResultPath()],stdout=presult,timeout=3600) + # When a test fail, the regression is continuing but we + # track that a test has failed + if completed.returncode==0: + return(NOTESTFAILED) + else: + return(TESTFAILED) + + def runAndProcess(self,compiler,fvp): + # If we can't parse test description we fail all tests + self.processTest() + # Otherwise if only building or those tests are failing, we continue + # with other tests + try: + self.buildConfig().build(self.testName()) + except: + return(MAKEFAILED) + # We run tests only for AC6 + # For other compilers only build is tests + # Since full build is no more possible because of huge pattersn, + # build is done per test suite. + if compiler == "AC6": + if fvp is not None: + self.run(fvp) + return(self.processResult()) + else: + msg("No FVP available") + return(NOTESTFAILED) + else: + return(NOTESTFAILED) + + + +# Preprocess the test description +def preprocess(): + msg("Process test description file\n") + completed = subprocess.run(["python", "preprocess.py","-f","desc.txt"],timeout=3600) + check(completed) + +# Generate all missing C code by using all classes in the +# test description file +def generateAllCCode(): + msg("Generate all missing C files\n") + completed = subprocess.run(["python","processTests.py", "-e"],timeout=3600) + check(completed) + + + + + + + + diff --git a/Testing/addAllBenchToDatabase.bat b/Testing/addAllBenchToDatabase.bat index 4534dc9f..fbd8440e 100755 --- a/Testing/addAllBenchToDatabase.bat +++ b/Testing/addAllBenchToDatabase.bat @@ -1,26 +1,26 @@ echo "Basic Maths" -python addToDB.py -f bench.txt BasicBenchmarks +python addToDB.py BasicBenchmarks echo "Complex Maths" -python addToDB.py -f bench.txt ComplexBenchmarks +python addToDB.py ComplexBenchmarks echo "FIR" -python addToDB.py -f bench.txt FIR +python addToDB.py FIR echo "Convolution / Correlation" -python addToDB.py -f bench.txt MISC +python addToDB.py MISC echo "Decimation / Interpolation" -python addToDB.py -f bench.txt DECIM +python addToDB.py DECIM echo "BiQuad" -python addToDB.py -f bench.txt BIQUAD +python addToDB.py BIQUAD echo "Controller" -python addToDB.py -f bench.txt Controller +python addToDB.py Controller echo "Fast Math" -python addToDB.py -f bench.txt FastMath +python addToDB.py FastMath echo "Barycenter" -python addToDB.py -f bench.txt SupportBarF32 +python addToDB.py SupportBarF32 echo "Support" -python addToDB.py -f bench.txt Support +python addToDB.py Support echo "Unary Matrix" -python addToDB.py -f bench.txt Unary +python addToDB.py Unary echo "Binary Matrix" -python addToDB.py -f bench.txt Binary +python addToDB.py Binary echo "Transform" -python addToDB.py -f bench.txt Transform \ No newline at end of file +python addToDB.py Transform \ No newline at end of file diff --git a/Testing/addAllBenchToRegressionDatabase.bat b/Testing/addAllBenchToRegressionDatabase.bat index eee59703..677bc121 100755 --- a/Testing/addAllBenchToRegressionDatabase.bat +++ b/Testing/addAllBenchToRegressionDatabase.bat @@ -1,26 +1,26 @@ echo "Basic Maths" -python addToRegDB.py -f bench.txt BasicBenchmarks +python addToRegDB.py BasicBenchmarks echo "Complex Maths" -python addToRegDB.py -f bench.txt ComplexBenchmarks +python addToRegDB.py ComplexBenchmarks echo "FIR" -python addToRegDB.py -f bench.txt FIR +python addToRegDB.py FIR echo "Convolution / Correlation" -python addToRegDB.py -f bench.txt MISC +python addToRegDB.py MISC echo "Decimation / Interpolation" -python addToRegDB.py -f bench.txt DECIM +python addToRegDB.py DECIM echo "BiQuad" -python addToRegDB.py -f bench.txt BIQUAD +python addToRegDB.py BIQUAD echo "Controller" -python addToRegDB.py -f bench.txt Controller +python addToRegDB.py Controller echo "Fast Math" -python addToRegDB.py -f bench.txt FastMath +python addToRegDB.py FastMath echo "Barycenter" -python addToRegDB.py -f bench.txt SupportBarF32 +python addToRegDB.py SupportBarF32 echo "Support" -python addToRegDB.py -f bench.txt Support +python addToRegDB.py Support echo "Unary Matrix" -python addToRegDB.py -f bench.txt Unary +python addToRegDB.py Unary echo "Binary Matrix" -python addToRegDB.py -f bench.txt Binary +python addToRegDB.py Binary echo "Transform" -python addToRegDB.py -f bench.txt Transform \ No newline at end of file +python addToRegDB.py Transform \ No newline at end of file diff --git a/Testing/addToDB.py b/Testing/addToDB.py index 5b2d117e..fb72f81a 100755 --- a/Testing/addToDB.py +++ b/Testing/addToDB.py @@ -267,7 +267,7 @@ def addOneBenchmark(elem,fullPath,db,group): else: tableName = elem.data["class"] conn = sqlite3.connect(db) - #createTableIfMissing(conn,elem,tableName,full) + createTableIfMissing(conn,elem,tableName,full) config = addRows(conn,elem,tableName,full) addConfig(conn,config,fullDate) conn.close() @@ -287,7 +287,7 @@ def addToDB(benchmark,dbpath,elem,group): parser = argparse.ArgumentParser(description='Generate summary benchmarks') -parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Test description file path") +parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="File path") parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path") #parser.add_argument('-e', action='store_true', help="Embedded test") parser.add_argument('-o', nargs='?',type = str, default="bench.db", help="Benchmark database") diff --git a/Testing/addToRegDB.py b/Testing/addToRegDB.py index cf21085e..9fdbe7df 100755 --- a/Testing/addToRegDB.py +++ b/Testing/addToRegDB.py @@ -298,7 +298,7 @@ def addToDB(benchmark,dbpath,elem,group): parser = argparse.ArgumentParser(description='Generate summary benchmarks') -parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Test description file path") +parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="File path") parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path") #parser.add_argument('-e', action='store_true', help="Embedded test") parser.add_argument('-o', nargs='?',type = str, default="reg.db", help="Regression benchmark database") diff --git a/Testing/processResult.py b/Testing/processResult.py index 4d979c9b..c1469e0f 100644 --- a/Testing/processResult.py +++ b/Testing/processResult.py @@ -11,6 +11,9 @@ import csv import TestScripts.ParseTrace import colorama from colorama import init,Fore, Back, Style +import sys + +resultStatus=0 init() @@ -336,6 +339,7 @@ def getCyclesFromTrace(trace): return(TestScripts.ParseTrace.getCycles(trace)) def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter): + global resultStatus calibration = 0 if trace: # First cycle in the trace is the calibration data @@ -510,6 +514,8 @@ def analyseResult(resultPath,root,results,embedded,benchmark,trace,formatter): params="" writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config) # Format the node + if not passed: + resultStatus=1 formatter.printTest(elem,theId,theError,errorDetail,theLine,passed,cycles,params) @@ -552,6 +558,8 @@ parser.add_argument('-t', nargs='?',type = str, default=None, help="External tra args = parser.parse_args() + + if args.f is not None: #p = parse.Parser() # Parse the test description file @@ -568,6 +576,8 @@ if args.f is not None: # In FPGA mode, extract output files from stdout (result file) with open(args.r,"r") as results: extractDataFiles(results,args.o) + + sys.exit(resultStatus) else: parser.print_help() \ No newline at end of file diff --git a/Testing/runAllBenchmarks.bat b/Testing/runAllBenchmarks.bat index 7438b765..3074ddcf 100755 --- a/Testing/runAllBenchmarks.bat +++ b/Testing/runAllBenchmarks.bat @@ -1,55 +1,55 @@ @ECHO OFF echo "Basic Maths" -python processTests.py -f bench.txt -e BasicBenchmarks +python processTests.py -e BasicBenchmarks call:runBench echo "Complex Maths" -python processTests.py -f bench.txt -e ComplexBenchmarks +python processTests.py -e ComplexBenchmarks call:runBench echo "FIR" -python processTests.py -f bench.txt -e FIR +python processTests.py -e FIR call:runBench echo "Convolution / Correlation" -python processTests.py -f bench.txt -e MISC +python processTests.py -e MISC call:runBench echo "Decimation / Interpolation" -python processTests.py -f bench.txt -e DECIM +python processTests.py -e DECIM call:runBench echo "BiQuad" -python processTests.py -f bench.txt -e BIQUAD +python processTests.py -e BIQUAD call:runBench echo "Controller" -python processTests.py -f bench.txt -e Controller +python processTests.py -e Controller call:runBench echo "Fast Math" -python processTests.py -f bench.txt -e FastMath +python processTests.py -e FastMath call:runBench echo "Barycenter" -python processTests.py -f bench.txt -e SupportBarF32 +python processTests.py -e SupportBarF32 call:runBench echo "Support" -python processTests.py -f bench.txt -e Support +python processTests.py -e Support call:runBench echo "Unary Matrix" -python processTests.py -f bench.txt -e Unary +python processTests.py -e Unary call:runBench echo "Binary Matrix" -python processTests.py -f bench.txt -e Binary +python processTests.py -e Binary call:runBench echo "Transform" -python processTests.py -f bench.txt -e Transform +python processTests.py -e Transform call:runBench EXIT /B @@ -64,7 +64,7 @@ REM "C:\Program Files\ARM\Development Studio 2019.0\sw\models\bin\FVP_MPS2_Corte "C:\Program Files\ARM\Development Studio 2019.0\sw\models\bin\FVP_VE_Cortex-A5x1.exe" -a Testing > result.txt popd echo "Parse result" -REM python processResult.py -f bench.txt -e -r build_m7\result.txt -REM python processResult.py -f bench.txt -e -r build_m0\result.txt -python processResult.py -f bench.txt -e -r build_a5\result.txt +REM python processResult.py -e -r build_m7\result.txt +REM python processResult.py -e -r build_m0\result.txt +python processResult.py -e -r build_a5\result.txt goto:eof \ No newline at end of file diff --git a/Testing/runAllBenchmarks.py b/Testing/runAllBenchmarks.py new file mode 100755 index 00000000..b62a15ca --- /dev/null +++ b/Testing/runAllBenchmarks.py @@ -0,0 +1,105 @@ +import os +import os.path +import subprocess +import colorama +from colorama import init,Fore, Back, Style +import argparse + +GROUPS = [ +"BasicBenchmarks", +"ComplexBenchmarks", +"FIR", +"MISC", +"DECIM", +"BIQUAD", +"Controller", +"FastMath", +"SupportBarF32", +"Support", +"Unary", +"Binary", +"Transform" +] + +init() + +def msg(t): + print(Fore.CYAN + t + Style.RESET_ALL) + +def processTest(test): + subprocess.call(["python","processTests.py","-e",test]) + +def addToDB(cmd): + for g in GROUPS: + msg("Add group %s" % g) + subprocess.call(["python",cmd,g]) + +def run(build,fvp,custom=None): + result = "results.txt" + resultPath = os.path.join(build,result) + + current=os.getcwd() + try: + msg("Build" ) + os.chdir(build) + subprocess.call(["make"]) + msg("Run") + with open(result,"w") as results: + if custom: + subprocess.call([fvp] + custom,stdout=results) + else: + subprocess.call([fvp,"-a","Testing"],stdout=results) + finally: + os.chdir(current) + + msg("Parse result") + subprocess.call(["python","processResult.py","-e","-r",resultPath]) + + msg("Regression computations") + subprocess.call(["python","summaryBench.py","-r",resultPath]) + + msg("Add results to benchmark database") + addToDB("addToDB.py") + + msg("Add results to regression database") + addToDB("addToRegDB.py") + + + +def processAndRun(buildfolder,fvp,custom=None): + processTest("DSPBenchmarks") + run(buildfolder,fvp,custom=custom) + +parser = argparse.ArgumentParser(description='Parse test description') +parser.add_argument('-f', nargs='?',type = str, default="build_benchmark_m7", help="Build folder") +parser.add_argument('-v', nargs='?',type = str, default="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe", help="Fast Model") +parser.add_argument('-c', nargs='?',type = str, help="Custom args") + +args = parser.parse_args() + +if args.f is not None: + BUILDFOLDER=args.f +else: + BUILDFOLDER="build_benchmark_m7" + +if args.v is not None: + FVP=args.v +else: + FVP="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe" + + +if args.c: + custom = args.c.split() +else: + custom = None + +print(Fore.RED + "bench.db and reg.db databases must exist before running this script" + Style.RESET_ALL) + +msg("Process benchmark description file") +subprocess.call(["python", "preprocess.py","-f","bench.txt"]) + +msg("Generate all missing C files") +subprocess.call(["python","processTests.py", "-e"]) + + +processAndRun(BUILDFOLDER,FVP,custom=custom) diff --git a/Testing/runAllTests.py b/Testing/runAllTests.py index 1e07c8fa..08f9f986 100755 --- a/Testing/runAllTests.py +++ b/Testing/runAllTests.py @@ -4,126 +4,177 @@ import subprocess import colorama from colorama import init,Fore, Back, Style import argparse +from TestScripts.Regression.Commands import * +import yaml +import sys +import itertools +from pathlib import Path + + +# Small state machine +def updateTestStatus(testStatusForThisBuild,newTestStatus): + if testStatusForThisBuild == NOTESTFAILED: + if newTestStatus == NOTESTFAILED: + return(NOTESTFAILED) + if newTestStatus == MAKEFAILED: + return(MAKEFAILED) + if newTestStatus == TESTFAILED: + return(TESTFAILED) + if testStatusForThisBuild == MAKEFAILED: + if newTestStatus == NOTESTFAILED: + return(MAKEFAILED) + if newTestStatus == MAKEFAILED: + return(MAKEFAILED) + if newTestStatus == TESTFAILED: + return(TESTFAILED) + if testStatusForThisBuild == TESTFAILED: + if newTestStatus == NOTESTFAILED: + return(TESTFAILED) + if newTestStatus == MAKEFAILED: + return(TESTFAILED) + if newTestStatus == TESTFAILED: + return(TESTFAILED) + if testStatusForThisBuild == FLOWFAILURE: + return(testStatusForThisBuild) + if testStatusForThisBuild == CALLFAILURE: + return(testStatusForThisBuild) + +root = Path(os.getcwd()).parent.parent.parent + + +def cartesian(*somelists): + r=[] + for element in itertools.product(*somelists): + r.append(list(element)) + return(r) + +testFailed = 0 init() -def msg(t): - print(Fore.CYAN + t + Style.RESET_ALL) - -def processTest(test): - subprocess.call(["python","processTests.py","-e",test]) - -def build(build,fvp,test,custom=None): - result = "results_%s.txt" % test - resultPath = os.path.join(build,result) - - current=os.getcwd() - try: - msg("Build %s" % test) - os.chdir(build) - subprocess.call(["make"]) - msg("Run %s" % test) - with open(result,"w") as results: - if custom: - subprocess.call([fvp] + custom,stdout=results) - else: - subprocess.call([fvp,"-a","Testing"],stdout=results) - finally: - os.chdir(current) - - msg("Parse result for %s" % test) - subprocess.call(["python","processResult.py","-e","-r",resultPath]) - -def processAndRun(buildfolder,fvp,test,custom=None): - processTest(test) - build(buildfolder,fvp,test,custom=custom) - parser = argparse.ArgumentParser(description='Parse test description') -parser.add_argument('-f', nargs='?',type = str, default="build_m7", help="Build folder") -parser.add_argument('-v', nargs='?',type = str, default="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe", help="Fast Model") -parser.add_argument('-c', nargs='?',type = str, help="Custom args") +parser.add_argument('-i', nargs='?',type = str, default="testrunConfig.yaml",help="Config file") +parser.add_argument('-r', nargs='?',type = str, default=root, help="Root folder") +parser.add_argument('-n', nargs='?',type = int, default=0, help="ID value when launchign in parallel") args = parser.parse_args() -if args.f is not None: - BUILDFOLDER=args.f -else: - BUILDFOLDER="build_m7" - -if args.v is not None: - FVP=args.v -else: - FVP="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe" - - -if args.c: - custom = args.c.split() -else: - custom = None - -msg("Process test description file") -subprocess.call(["python", "preprocess.py","-f","desc.txt"]) - -msg("Generate all missing C files") -subprocess.call(["python","processTests.py", "-e"]) -msg("Statistics Tests") -processAndRun(BUILDFOLDER,FVP,"StatsTests",custom=custom) +with open(args.i,"r") as f: + config=yaml.safe_load(f) -msg("Support Tests") -processAndRun(BUILDFOLDER,FVP,"SupportTests",custom=custom) +#print(config) -msg("Support Bar Tests F32") -processAndRun(BUILDFOLDER,FVP,"SupportBarTestsF32",custom=custom) +#print(config["IMPLIEDFLAGS"]) -msg("Basic Tests") -processAndRun(BUILDFOLDER,FVP,"BasicTests",custom=custom) +flags = config["FLAGS"] +onoffFlags = [] +for f in flags: + onoffFlags.append(["-D" + f +"=ON","-D" + f +"=OFF"]) -msg("Interpolation Tests") -processAndRun(BUILDFOLDER,FVP,"InterpolationTests",custom=custom) +allConfigs=cartesian(*onoffFlags) -msg("Complex Tests") -processAndRun(BUILDFOLDER,FVP,"ComplexTests",custom=custom) +if DEBUGMODE: + allConfigs=[allConfigs[0]] + -msg("Fast Maths Tests") -processAndRun(BUILDFOLDER,FVP,"FastMath",custom=custom) +failedBuild = {} +# Test all builds -msg("SVM Tests") -processAndRun(BUILDFOLDER,FVP,"SVMTests",custom=custom) +folderCreated=False -msg("Bayes Tests") -processAndRun(BUILDFOLDER,FVP,"BayesTests",custom=custom) +def logFailedBuild(root,f): + with open(os.path.join(fullTestFolder(root),"buildStatus_%d.txt" % args.n),"w") as status: + for build in f: + s = f[build] + if s == MAKEFAILED: + status.write("%s : Make failure\n" % build) + if s == TESTFAILED: + status.write("%s : Test failure\n" % build) + if s == FLOWFAILURE: + status.write("%s : Flow failure\n" % build) + if s == CALLFAILURE: + status.write("%s : Subprocess failure\n" % build) -msg("Distance Tests") -processAndRun(BUILDFOLDER,FVP,"DistanceTests",custom=custom) -msg("Filtering Tests") -processAndRun(BUILDFOLDER,FVP,"FilteringTests",custom=custom) - -msg("Matrix Tests") -processAndRun(BUILDFOLDER,FVP,"MatrixTests",custom=custom) - -# Too many patterns to run the full transform directly -msg("Transform Tests CF64") -processAndRun(BUILDFOLDER,FVP,"TransformCF64",custom=custom) - -msg("Transform Tests RF64") -processAndRun(BUILDFOLDER,FVP,"TransformRF64",custom=custom) - -msg("Transform Tests CF32") -processAndRun(BUILDFOLDER,FVP,"TransformCF32",custom=custom) - -msg("Transform Tests RF32") -processAndRun(BUILDFOLDER,FVP,"TransformRF32",custom=custom) - -msg("Transform Tests CQ31") -processAndRun(BUILDFOLDER,FVP,"TransformCQ31",custom=custom) - -msg("Transform Tests RQ31") -processAndRun(BUILDFOLDER,FVP,"TransformRQ31",custom=custom) - -msg("Transform Tests CQ15") -processAndRun(BUILDFOLDER,FVP,"TransformCQ15",custom=custom) - -msg("Transform Tests RQ15") -processAndRun(BUILDFOLDER,FVP,"TransformRQ15",custom=custom) \ No newline at end of file +def buildAndTest(compiler): + # Run all tests for AC6 + try: + for core in config['CORES']: + configNb = 0 + if compiler in config['CORES'][core]: + for flagConfig in allConfigs: + folderCreated = False + configNb = configNb + 1 + buildStr = "build_%s_%s_%d" % (compiler,core,configNb) + toUnset = None + if compiler in config['UNSET']: + if core in config['UNSET'][compiler]: + toUnset = config['UNSET'][compiler][core] + build = BuildConfig(toUnset,args.r, + buildStr, + config['COMPILERS'][core][compiler], + config['TOOLCHAINS'][compiler], + config['CORES'][core][compiler], + config["CMAKE"] + ) + + flags = [] + if core in config["IMPLIEDFLAGS"]: + flags += config["IMPLIEDFLAGS"][core] + flags += flagConfig + + if compiler in config["IMPLIEDFLAGS"]: + flags += config["IMPLIEDFLAGS"][compiler] + + build.createFolder() + # Run all tests for the build + testStatusForThisBuild = NOTESTFAILED + try: + # This is saving the flag configuration + build.createArchive(flags) + + build.createCMake(flags) + for test in config["TESTS"]: + msg(test["testName"]+"\n") + testClass=test["testClass"] + test = build.getTest(testClass) + fvp = None + if core in config['FVP']: + fvp = config['FVP'][core] + newTestStatus = test.runAndProcess(compiler,fvp) + testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus) + if testStatusForThisBuild != NOTESTFAILED: + failedBuild[buildStr] = testStatusForThisBuild + # Final script status + testFailed = 1 + build.archiveResults() + finally: + build.cleanFolder() + else: + msg("No toolchain %s for core %s" % (compiler,core)) + + except TestFlowFailure as flow: + errorMsg("Error flow id %d\n" % flow.errorCode()) + failedBuild[buildStr] = FLOWFAILURE + logFailedBuild(args.r,failedBuild) + sys.exit(1) + except CallFailure: + errorMsg("Call failure\n") + failedBuild[buildStr] = CALLFAILURE + logFailedBuild(args.r,failedBuild) + sys.exit(1) + +############## Builds for all toolchains + +if not DEBUGMODE: + preprocess() + generateAllCCode() + +for t in config["TOOLCHAINS"]: + msg("Testing toolchain %s\n" % t) + buildAndTest(t) + +logFailedBuild(args.r,failedBuild) +sys.exit(testFailed) + \ No newline at end of file diff --git a/Testing/summaryBench.py b/Testing/summaryBench.py index 10aef20c..4cb9ac1b 100644 --- a/Testing/summaryBench.py +++ b/Testing/summaryBench.py @@ -107,7 +107,7 @@ def extractBenchmarks(resultPath,benchmark,elem): parser = argparse.ArgumentParser(description='Generate summary benchmarks') -parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Test description file path") +parser.add_argument('-f', nargs='?',type = str, default="Output.pickle", help="Test description cache") parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path") # Needed to find the currentConfig.csv and know the headers parser.add_argument('-r', nargs='?',type = str, default=None, help="Result file path") diff --git a/Testing/testmain.cpp b/Testing/testmain.cpp index 3186030e..a7ff79c4 100644 --- a/Testing/testmain.cpp +++ b/Testing/testmain.cpp @@ -52,7 +52,8 @@ int testmain() // An IO runner is driven by some IO // In future one may have a client/server runner driven // by a server running on a host. - Client::IORunner runner(&io,&mgr,Testing::kTestAndDump); + //Client::IORunner runner(&io,&mgr,Testing::kTestAndDump); + Client::IORunner runner(&io,&mgr,Testing::kTestOnly); // Root object containing all the tests diff --git a/Toolchain/AC6.cmake b/Toolchain/AC6.cmake index 6cde1df3..ec988529 100644 --- a/Toolchain/AC6.cmake +++ b/Toolchain/AC6.cmake @@ -34,6 +34,11 @@ function(compilerSpecificCompileOptions PROJECTNAME ROOT) # Core specific config + if (ARM_CPU STREQUAL "cortex-m55" ) + target_compile_options(${PROJECTNAME} PUBLIC "-fshort-enums") + target_compile_options(${PROJECTNAME} PUBLIC "-fshort-wchar") + endif() + if (ARM_CPU STREQUAL "cortex-m33" ) target_compile_options(${PROJECTNAME} PUBLIC "-mfpu=fpv5-sp-d16") endif() diff --git a/configBoot.cmake b/configBoot.cmake index ef818300..b2fb3bf5 100755 --- a/configBoot.cmake +++ b/configBoot.cmake @@ -54,9 +54,11 @@ function(configboot PROJECT_NAME ROOT PLATFORMFOLDER) # # Cortex M # - if (ARM_CPU MATCHES "^[cC]ortex-[Mm].*$") + # C startup for M55 boot code + if (ARM_CPU MATCHES "^[cC]ortex-[mM]55([^0-9].*)?$") + cortexm(${CORE} ${PROJECT_NAME} ${ROOT} ${PLATFORMFOLDER} ON) + elseif (ARM_CPU MATCHES "^[cC]ortex-[Mm].*$") cortexm(${CORE} ${PROJECT_NAME} ${ROOT} ${PLATFORMFOLDER} OFF) - endif() diff --git a/configCore.cmake b/configCore.cmake index d3d387d6..04a984de 100644 --- a/configCore.cmake +++ b/configCore.cmake @@ -85,6 +85,16 @@ function(configcore PROJECTNAME ROOT) # # CORTEX-M # + + if (ARM_CPU MATCHES "^[cC]ortex-[mM]55([^0-9].*)?$") + target_include_directories(${PROJECTNAME} PUBLIC "${ROOT}/CMSIS/Core/Include") + target_compile_definitions(${PROJECTNAME} PUBLIC CORTEXM) + target_compile_definitions(${PROJECTNAME} PRIVATE ARMv81MML_DSP_DP_MVE_FP) + + SET(HARDFP ON) + SET(LITTLEENDIAN ON) + SET(COREID ARMv81MML_DSP_DP_MVE_FP PARENT_SCOPE) + endif() # CORTEX-M35 if (ARM_CPU MATCHES "^[cC]ortex-[mM]35([^0-9].*)?$") diff --git a/configLib.cmake b/configLib.cmake index 05417795..4129d7e3 100755 --- a/configLib.cmake +++ b/configLib.cmake @@ -8,7 +8,7 @@ endif() include(Toolchain/Tools) -option(OPTIMIZED "Compile for speed" ON) +option(OPTIMIZED "Compile for speed" OFF) option(AUTOVECTORIZE "Prefer autovectorizable code to one using C intrinsics" OFF) enable_language(CXX C ASM) diff --git a/configPlatform.cmake b/configPlatform.cmake index d6ca4e89..866c42ec 100644 --- a/configPlatform.cmake +++ b/configPlatform.cmake @@ -20,6 +20,7 @@ endif() SET(CORE ARMCM7) + include(platform) function(set_platform_core) @@ -61,6 +62,14 @@ function(set_platform_core) SET(CORE ARMCM33 PARENT_SCOPE) endif() + + ################### + # + # Cortex cortex-m55 + # + if (ARM_CPU MATCHES "^[cC]ortex-[mM]55([^0-9].*)?$") + SET(CORE ARMv81MML PARENT_SCOPE) + endif() ################### # diff --git a/gcc.cmake b/gcc.cmake index 77378d13..80949eb1 100644 --- a/gcc.cmake +++ b/gcc.cmake @@ -35,8 +35,8 @@ SET(CMAKE_CXX_OUTPUT_EXTENSION .o) SET(CMAKE_ASM_OUTPUT_EXTENSION .o) # When library defined as STATIC, this line is needed to describe how the .a file must be # create. Some changes to the line may be needed. -SET(CMAKE_C_CREATE_STATIC_LIBRARY " -r -s " ) -SET(CMAKE_CXX_CREATE_STATIC_LIBRARY " -r -s " ) +SET(CMAKE_C_CREATE_STATIC_LIBRARY " -crs " ) +SET(CMAKE_CXX_CREATE_STATIC_LIBRARY " -crs " ) set(GCC ON) # default core