CMSIS-DSP: Improvements to test framework

Testing/TestScripts/NewParser.py

@@ -161,7 +161,7 @@ class Parser:
         self.id = 0
 
     def parse(self, filePath):
-        string = Word(alphanums+"_ =+")
+        string = Word(alphanums+"_ =+()")
         ident = Word( alphas+"_", alphanums+"_" )
 
         path = Word(alphanums+"_/.")

Testing/TestScripts/Regression/Commands.py

@@ -330,10 +330,16 @@ class Test:
         return(self._test)
 
     # Process a test from the test description file
-    def processTest(self):
+    def processTest(self,patternConfig):
       if isDebugMode():
+        if patternConfig:
+           completed=subprocess.run([sys.executable,"processTests.py","-p",patternConfig["patterns"],"-d",patternConfig["parameters"],"-e",self.testName(),"1"],timeout=3600)
+        else:
            completed=subprocess.run([sys.executable,"processTests.py","-e",self.testName(),"1"],timeout=3600)
         check(completed)
+      else:
+        if patternConfig:
+           completed=subprocess.run([sys.executable,"processTests.py","-p",patternConfig["patterns"],"-d",patternConfig["parameters"],"-e",self.testName()],timeout=3600)
         else:
            completed=subprocess.run([sys.executable,"processTests.py","-e",self.testName()],timeout=3600)
         check(completed)
@@ -390,9 +396,9 @@ class Test:
         else:
            return(TESTFAILED)
 
-    def runAndProcess(self,compiler,fvp,sim,benchmode,db,regdb,benchid,regid):
+    def runAndProcess(self,patternConfig,compiler,fvp,sim,benchmode,db,regdb,benchid,regid):
         # If we can't parse test description we fail all tests
-        self.processTest()
+        self.processTest(patternConfig)
         # Otherwise if only building or those tests are failing, we continue
         # with other tests
         try:
@@ -432,8 +438,12 @@ def preprocess(desc):
 
 # Generate all missing C code by using all classes in the
 # test description file
-def generateAllCCode():
+def generateAllCCode(patternConfig):
     msg("Generate all missing C files\n")
+    if patternConfig:
+       completed = subprocess.run([sys.executable,"processTests.py",
+        "-p",patternConfig["patterns"],"-d",patternConfig["parameters"],"-e"],timeout=3600)
+    else:
        completed = subprocess.run([sys.executable,"processTests.py", "-e"],timeout=3600)
     check(completed)
 

Testing/TestScripts/doc/Format.py

@@ -84,7 +84,8 @@ class Markdown:
      self._id = self._id - 1 
 
   def visitDocument(self,document):
-      self._output.write("Document generated frun run ids : %s\n" % document.runidHeader)
+      if document.runidHeader:
+         self._output.write("Document generated for run ids : %s\n" % document.runidHeader)
 
   def leaveDocument(self,document):
       pass
@@ -647,6 +648,7 @@ myhist(thehdata%d,"#hi%d");
       else:
          self._output.write("<h1>ECPS Benchmark Summary</h1>\n")
       
+      if document.runidHeader:
          self._output.write("<p>Document generated for run ids : %s</p>\n" % document.runidHeader)
       today = date.today()
       d2 = today.strftime("%B %d, %Y")

Testing/extractDb.py

@@ -56,13 +56,14 @@ parser.add_argument('-g', action='store_true', help="Include graphs in regressio
 
 parser.add_argument('-details', action='store_true', help="Details about runids")
 parser.add_argument('-lastid', action='store_true', help="Get last ID")
-parser.add_argument('-comments', nargs='?',type = str, default="comments.txt", help="Comment section")
+parser.add_argument('-comments', nargs='?',type = str, default="comments", help="Comment folder")
 parser.add_argument('-byd', action='store_true', help="Result oganized by datatype")
 parser.add_argument('-ratio', action='store_true', help="Compute ratios for regression by core instead of cycles")
 parser.add_argument('-ref', nargs='?',type = str, default="M55", help="Reference COREDEF for ratio in db")
 parser.add_argument('-clampval', nargs='?',type = float, default=8.0, help="Clamp for ratio")
 parser.add_argument('-clamp', action='store_true', help="Clamp enabled for ratio")
 parser.add_argument('-keep', nargs='?',type = str, help="Core to keep for ratio")
+parser.add_argument('-disprunid', action='store_true', help="Include runid in html")
 
 # For runid or runid range
 parser.add_argument('others', nargs=argparse.REMAINDER,help="Run ID")
@@ -579,6 +580,26 @@ def convertRowToInt(r):
 
   return(result)
 
+      
+def addSectionComment(section):
+  if os.path.exists(args.comments):
+     fileName=re.sub(r'[() :]','',section.name)
+     path=os.path.join(args.comments,fileName+".html")
+     para=""
+     if os.path.exists(path):
+        commentSection = Section("Comments")
+        section.addSection(commentSection)
+        with open(path,"r") as r:
+            for l in r:
+                if l.strip():
+                  para += l
+                else:
+                  commentSection.addContent(Text(para))
+                  para=""
+        if para:
+           commentSection.addContent(Text(para))
+        
+
 def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals):
     if vals.size != 0:
        ref=pd.DataFrame(vals,columns=cols)
@@ -608,6 +629,7 @@ def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals):
               testSection = Section(name)
               testSection.setTest()
               typeSection.addSection(testSection)
+              addSectionComment(testSection)
 
               maxCyclesSection = Section("Max cycles")
               testSection.addSection(maxCyclesSection)
@@ -655,6 +677,7 @@ def formatTableBy(desc,byname,section,typeSection,testNames,cols,vals):
               testSection = Section(name)
               testSection.setTest()
               typeSection.addSection(testSection)
+              addSectionComment(testSection)
 
               dataForFunc=data.loc[name]
               dataForFunc = dataForFunc.dropna(axis=1)
@@ -860,6 +883,8 @@ def addRatioTable(cols,params,data,section,testNames,byd):
       testSection = Section(name)
       testSection.setTest()
       section.addSection(testSection)
+      addSectionComment(testSection)
+
 
       ratioSection = Section("Ratios")
       testSection.addSection(ratioSection)
@@ -909,9 +934,6 @@ def addRatioTable(cols,params,data,section,testNames,byd):
      
       
       
-      
-  
-
 # Add a report for each table
 def addReportFor(document,benchName):
     nbElems = getNbElemsInBenchCmd(benchName)
@@ -1038,19 +1060,23 @@ Hierarchy("Classical ML",[
   Hierarchy("Bayes"),
   Hierarchy("SVM"),
   Hierarchy("Distance"),
+  Hierarchy("KalmanBenchmarks")
   ]),
 
 ]
 
 processed=[]
 
+
+
 def addComments(document):
   if os.path.exists(args.comments):
      section=Section("Measurement Context")
+     path=os.path.join(args.comments,"comments.txt")
 
      document.addSection(section)
      para=""
-     with open(args.comments,"r") as r:
+     with open(path,"r") as r:
          for l in r:
              if l.strip():
                para += l
@@ -1080,7 +1106,10 @@ def createDoc(document,sections,benchtables):
 
 try:
       benchtables=getBenchTables()
+      if args.disprunid:
          document = Document(runidHeader)
+      else:
+         document = Document(None)
 
       if args.ratio:
          referenceCoreID= getCoreID(args.ref)

Testing/runAllTests.py

@@ -178,6 +178,16 @@ with open(args.i,"r") as f:
 #print(config["IMPLIEDFLAGS"])
 
 
+patternConfig={}
+
+
+
+if "PATTERNS" in config:
+   patternConfig={
+    "patterns":os.path.join(config["PATTERNS"],"Patterns")
+   ,"parameters":os.path.join(config["PATTERNS"],"Parameters")
+   }
+
 
 
 flags = config["FLAGS"]
@@ -265,7 +275,7 @@ def buildAndTest(compiler,theConfig,cmake,sim):
                       if 'SIM' in config:
                         if core in config['SIM']:
                            fvp = config['SIM'][core] 
-                      newTestStatus = test.runAndProcess(compiler,fvp,sim,args.b,args.db,args.regdb,benchID,regID)
+                      newTestStatus = test.runAndProcess(patternConfig,compiler,fvp,sim,args.b,args.db,args.regdb,benchID,regID)
                       testStatusForThisBuild = updateTestStatus(testStatusForThisBuild,newTestStatus)
                       if testStatusForThisBuild != NOTESTFAILED:
                          failedBuild[buildStr] = testStatusForThisBuild
@@ -292,7 +302,7 @@ def buildAndTest(compiler,theConfig,cmake,sim):
 
 if not isDebugMode():
    preprocess(args.f)
-   generateAllCCode()
+   generateAllCCode(patternConfig)
 else:
    msg("Debug Mode\n")