CMSIS-DSP: Testing framework

Added support for external trace files for benchmark measurements.

Testing/CMakeLists.txt

@@ -61,6 +61,7 @@ function(writeConfig path)
 endfunction()
 
 option(BENCHMARK "Benchmarking compiled" OFF)
+option(EXTBENCH "Benchmarking with external traces" OFF)
 
 project(Testing)
 
@@ -153,6 +154,10 @@ target_sources(TestingLib PRIVATE GeneratedSource/TestDesc.cpp)
 
 target_sources(FrameworkLib PRIVATE ${FRAMEWORKSRC})
 
+if (EXTBENCH)
+  target_compile_definitions(FrameworkLib PUBLIC EXTBENCH)
+endif()
+
 ### Includes
 target_link_libraries(TestingLib PRIVATE CMSISDSP)
 target_link_libraries(TestingLib PRIVATE CMSISNN)

Testing/FrameworkInclude/Timing.h

@@ -9,4 +9,85 @@ void cycleMeasurementStop();
 
 Testing::cycles_t getCycles();
 
+#ifdef EXTBENCH
+extern unsigned long sectionCounter;
+
+#if   defined ( __CC_ARM )
+    #define dbgInst(imm) __asm volatile{ DBG (imm) }
+#elif defined ( __GNUC__ ) || defined ( __llvm__ )
+    #define dbgInst(imm) __asm volatile("DBG %0\n\t" : :"Ir" ((imm)) )
+#else
+    #error "Unsupported compiler"
+#endif
+#define startSectionNB(num) dbgInst(((num) & 0x7) | 0x8)
+#define stopSectionNB(num)  dbgInst(((num) & 0x7) | 0x0)
+
+static inline void startSection() {
+    switch(sectionCounter & 0x7)
+    {
+      case 0:
+        startSectionNB(0);
+      break;
+      case 1:
+        startSectionNB(1);
+      break;
+      case 2:
+        startSectionNB(2);
+      break;
+      case 3:
+        startSectionNB(3);
+      break;
+      case 4:
+        startSectionNB(4);
+      break;
+      case 5:
+        startSectionNB(5);
+      break;
+      case 6:
+        startSectionNB(6);
+      break;
+      case 7:
+        startSectionNB(7);
+      break;
+      default:
+        startSectionNB(0);
+    }
+}
+
+static inline void stopSection() {
+    switch(sectionCounter & 0x7)
+     {
+      case 0:
+        stopSectionNB(0);
+      break;
+      case 1:
+        stopSectionNB(1);
+      break;
+      case 2:
+        stopSectionNB(2);
+      break;
+      case 3:
+        stopSectionNB(3);
+      break;
+      case 4:
+        stopSectionNB(4);
+      break;
+      case 5:
+        stopSectionNB(5);
+      break;
+      case 6:
+        stopSectionNB(6);
+      break;
+      case 7:
+        stopSectionNB(7);
+      break;
+      default:
+        stopSectionNB(0);
+     }
+     
+     sectionCounter++;
+}
+
+#endif 
+
 #endif

Testing/FrameworkSource/FPGA.cpp

@@ -260,7 +260,11 @@ namespace Client
         }
         else
         {
+#ifdef EXTBENCH
+            printf("S: %ld 0 0 t Y\n",this->currentId);
+#else
             printf("S: %ld 0 0 %u Y\n",this->currentId, cycles);
+#endif
         }
     }
 

Testing/FrameworkSource/IORunner.cpp

@@ -144,8 +144,16 @@ namespace Client
                 s->setUp(m_io->CurrentTestID(),params,m_mgr);
                 // Run the test
                 cycleMeasurementStart();
+#ifdef EXTBENCH
+                startSection();
+#endif
                 (s->*t)();
+#ifdef EXTBENCH
+                stopSection();
+#endif
+#ifndef EXTBENCH
                 cycles=getCycles();
+#endif
                 cycleMeasurementStop();
               } 
               catch(Error &ex)

Testing/FrameworkSource/Semihosting.cpp

@@ -364,7 +364,11 @@ namespace Client
         }
         else
         {
+#ifdef EXTBENCH
+            printf("%ld 0 0 t Y\n",this->currentId);
+#else
             printf("%ld 0 0 %u Y\n",this->currentId,cycles);
+#endif
         }
       }
 

Testing/FrameworkSource/Timing.cpp

@@ -60,6 +60,10 @@ unsigned int startCycles;
 #define ENABLE_DIVIDER 0 
 #endif
 
+#ifdef EXTBENCH
+unsigned long sectionCounter=0;
+#endif 
+
 void initCycleMeasurement()
 {
 #ifdef CORTEXM
@@ -98,6 +102,7 @@ void initCycleMeasurement()
 
 void cycleMeasurementStart()
 {
+#ifndef EXTBENCH
 #ifdef CORTEXM
     SysTick->CTRL = SysTick_CTRL_CLKSOURCE_Msk;
     SysTick->LOAD = SYSTICK_INITIAL_VALUE;
@@ -118,14 +123,18 @@ void cycleMeasurementStart()
     __get_CP(15, 0, value, 9, 13, 0);
     startCycles =  value;
 #endif
+#endif 
+
 }
 
 void cycleMeasurementStop()
 {
+#ifndef EXTBENCH
 #ifdef CORTEXM
     SysTick->CTRL = SysTick_CTRL_CLKSOURCE_Msk;
     SysTick->LOAD = SYSTICK_INITIAL_VALUE;
 #endif
+#endif
 }
 
 Testing::cycles_t getCycles()

Testing/TestScripts/ParseTrace.py

@@ -0,0 +1,36 @@
+import re 
+
+parseRe = re.compile('(.*)\s+([0-9]+):([0-9a-f]+):(.*)')
+
+dbgCnt=0
+
+clk0=0
+clk1=0
+
+def getCycles(t):
+    global dbgCnt
+    global clk0 
+    global clk1
+    while(True):
+      try:
+        line = next(t)
+        if line:
+          m = parseRe.match(line)
+          if m:
+             if (('OP_HINT_DBG_32' in line) or ('DBG' in line)):
+                 curClk = int(m.group(2))
+                 if dbgCnt==0:
+                  clk0 =curClk 
+                 if dbgCnt == 1:
+                  clk1 = curClk
+                 dbgCnt += 1
+                 if dbgCnt == 2:
+                    dbgCnt = 0
+                    return(clk1 - clk0)
+      except StopIteration:
+        dbgCnt = 0
+        return(0)
+
+      
+
+    

Testing/processResult.py

@@ -8,6 +8,8 @@ import TestScripts.CodeGen
 from collections import deque
 import os.path
 import csv
+import TestScripts.ParseTrace
+
 
 def findItem(root,path):
         """ Find a node in a tree
@@ -226,7 +228,13 @@ def writeBenchmark(elem,benchFile,theId,theError,passed,cycles,params,config):
          old=elem.data["testData"]["oldID"]
     benchFile.write("\"%s\",\"%s\",%d,\"%s\",%s,%d,%s\n" % (category,name,theId,old,params,cycles,config))
 
-def analyseResult(root,results,embedded,benchmark,formatter):
+def getCyclesFromTrace(trace):
+  if not trace:
+    return(0)
+  else:
+    return(TestScripts.ParseTrace.getCycles(trace))
+
+def analyseResult(root,results,embedded,benchmark,trace,formatter):
     formatter.start()
     path = []
     state = NORMAL
@@ -321,7 +329,7 @@ def analyseResult(root,results,embedded,benchmark,formatter):
                # In test mode, we are looking for test status.
                # A line starting with S
                # (There may be empty lines or line for data files)
-               passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([YN]).*$'  % prefix
+               passRe = r'^%s([0-9]+)[ ]+([0-9]+)[ ]+([0-9]+)[ ]+([t0-9]+)[ ]+([YN]).*$'  % prefix
                if re.match(passRe,l):
                     # If we have found a test status then we will start again
                     # in normal mode after this.
@@ -338,7 +346,11 @@ def analyseResult(root,results,embedded,benchmark,formatter):
                     theLine=m.group(3)
                     theLine=int(theLine)
       
-                    cycles = int(m.group(4))
+                    maybeCycles = m.group(4)
+                    if maybeCycles == "t":
+                       cycles = getCyclesFromTrace(trace)
+                    else:
+                       cycles = int(maybeCycles)
    
                     status=m.group(5)
                     passed=0
@@ -388,6 +400,14 @@ def analyseResult(root,results,embedded,benchmark,formatter):
     formatter.end()          
 
 
+def analyze(root,results,args,trace):
+  if args.c:
+     analyseResult(root,results,args.e,args.b,trace,CSVFormatter())
+  elif args.m:
+     analyseResult(root,results,args.e,args.b,trace,MathematicaFormatter())
+  else:
+     analyseResult(root,results,args.e,args.b,trace,TextFormatter())
+
 parser = argparse.ArgumentParser(description='Parse test description')
 
 parser.add_argument('-f', nargs='?',type = str, default=None, help="Test description file path")
@@ -400,20 +420,22 @@ parser.add_argument('-o', nargs='?',type = str, default="Output", help="Output d
 
 parser.add_argument('-b', nargs='?',type = str, default="FullBenchmark", help="Full Benchmark dir path")
 parser.add_argument('-m', action='store_true', help="Mathematica output")
+parser.add_argument('-t', nargs='?',type = str, default=None, help="External trace file")
 
 args = parser.parse_args()
 
+
 if args.f is not None:
     p = parse.Parser()
     # Parse the test description file
     root = p.parse(args.f)
-    with open(args.r,"r") as results:
+    if args.t:
-        if args.c:
+       with open(args.t,"r") as trace:
-           analyseResult(root,results,args.e,args.b,CSVFormatter())
+         with open(args.r,"r") as results:
-        elif args.m:
+             analyze(root,results,args,iter(trace))
-           analyseResult(root,results,args.e,args.b,MathematicaFormatter())
+    else:
-        else:
+       with open(args.r,"r") as results:
-           analyseResult(root,results,args.e,args.b,TextFormatter())
+           analyze(root,results,args,None)
     if args.e:
        # In FPGA mode, extract output files from stdout (result file)
        with open(args.r,"r") as results: