From 2caf8821a4cbd43560d39ea7487e400193da2c96 Mon Sep 17 00:00:00 2001
From: Christophe Favergeon <Christophe.Favergeon@arm.com>
Date: Mon, 20 Jan 2020 14:18:48 +0100
Subject: [PATCH] CMSIS-DSP: Added a script to run all the tests.

Tests are getting too big. This script will help to run all the tests
without having to build an executable containing them all.
---
 Testing/README.md        |  4 ++
 Testing/processResult.py |  6 +++
 Testing/runAllTests.py   | 96 ++++++++++++++++++++++++++++++++++++++++
 3 files changed, 106 insertions(+)
 create mode 100755 Testing/runAllTests.py

diff --git a/Testing/README.md b/Testing/README.md
index 3d03f755..163f19cc 100644
--- a/Testing/README.md
+++ b/Testing/README.md
@@ -378,6 +378,10 @@ Folder Output/BasicMaths should exist. For example, on Windows with ArmDS:
 format with or without -e option.
 
 
+Some cycles are displayed with the test status (passed or failed). **Don't trust** those cycles for a benchmark.
+
+At this point they are only an indication. The timing code will have to be tested and validated.
+
 ### Generate summary statistics
 
 The parsing of the results may have generated some statistics in FullBenchmark folder.
diff --git a/Testing/processResult.py b/Testing/processResult.py
index 65e75f54..4d979c9b 100644
--- a/Testing/processResult.py
+++ b/Testing/processResult.py
@@ -14,6 +14,7 @@ from colorama import init,Fore, Back, Style
 
 init()
 
+
 def errorStr(id):
   if id == 1:
      return("UNKNOWN_ERROR")
@@ -526,6 +527,11 @@ def analyze(root,results,args,trace):
   elif args.m:
      analyseResult(resultPath,root,results,args.e,args.b,trace,MathematicaFormatter())
   else:
+     print("")
+     print(Fore.RED + "The cycles displayed by this script must not be trusted." + Style.RESET_ALL)
+     print(Fore.RED + "They are just an indication. The timing code has not yet been validated." + Style.RESET_ALL)
+     print("")
+
      analyseResult(resultPath,root,results,args.e,args.b,trace,TextFormatter())
 
 parser = argparse.ArgumentParser(description='Parse test description')
diff --git a/Testing/runAllTests.py b/Testing/runAllTests.py
new file mode 100755
index 00000000..653b1173
--- /dev/null
+++ b/Testing/runAllTests.py
@@ -0,0 +1,96 @@
+import os
+import os.path
+import subprocess 
+import colorama
+from colorama import init,Fore, Back, Style
+import argparse
+
+init()
+
+def msg(t):
+    print(Fore.CYAN + t + Style.RESET_ALL)
+
+def processTest(test):
+    subprocess.call(["python","processTests.py","-e",test])
+
+def build(build,fvp,test):
+    result = "results_%s.txt" % test
+    resultPath = os.path.join(build,result)
+
+    current=os.getcwd()
+    try:
+       msg("Build %s" % test)
+       os.chdir(build)
+       subprocess.call(["make"])
+       msg("Run %s" % test)
+       with open(result,"w") as results:
+          subprocess.call([fvp,"-a","Testing"],stdout=results)
+    finally:
+       os.chdir(current)
+
+    msg("Parse result for %s" % test)
+    subprocess.call(["python","processResult.py","-e","-r",resultPath])
+
+def processAndRun(buildfolder,fvp,test):
+    processTest(test)
+    build(buildfolder,fvp,test)
+
+parser = argparse.ArgumentParser(description='Parse test description')
+parser.add_argument('-f', nargs='?',type = str, default="build_m7", help="Build folder")
+parser.add_argument('-v', nargs='?',type = str, default="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe", help="Fast Model")
+
+args = parser.parse_args()
+
+if args.f is not None:
+   BUILDFOLDER=args.f
+else:
+   BUILDFOLDER="build_m7"
+
+if args.v is not None:
+   FVP=args.v
+else:
+   FVP="C:\\Program Files\\ARM\\Development Studio 2019.0\\sw\\models\\bin\\FVP_MPS2_Cortex-M7.exe"
+
+msg("Process test description file")
+subprocess.call(["python", "preprocess.py","-f","desc.txt"])
+
+msg("Generate all missing C files")
+subprocess.call(["python","processTests.py", "-e"])
+
+msg("Statistics Tests")
+processAndRun(BUILDFOLDER,FVP,"StatsTests")
+
+msg("Support Tests")
+processAndRun(BUILDFOLDER,FVP,"SupportTests")
+
+msg("Support Bar Tests F32")
+processAndRun(BUILDFOLDER,FVP,"SupportBarTestsF32")
+
+msg("Basic Tests")
+processAndRun(BUILDFOLDER,FVP,"BasicTests")
+
+msg("Complex Tests")
+processAndRun(BUILDFOLDER,FVP,"ComplexTests")
+
+msg("Fast Maths Tests")
+processAndRun(BUILDFOLDER,FVP,"FastMath")
+
+msg("SVM Tests")
+processAndRun(BUILDFOLDER,FVP,"SVMTests")
+
+msg("Bayes Tests")
+processAndRun(BUILDFOLDER,FVP,"BayesTests")
+
+msg("Distance Tests")
+processAndRun(BUILDFOLDER,FVP,"DistanceTests")
+
+msg("Filtering Tests")
+processAndRun(BUILDFOLDER,FVP,"FilteringTests")
+
+msg("Matrix Tests")
+processAndRun(BUILDFOLDER,FVP,"MatrixTests")
+
+msg("Transform Tests")
+processAndRun(BUILDFOLDER,FVP,"TransformTests")
+
+