Fixed reports, added -d switchl, better failure checks.

10 files changed, 186 insertions, 129 deletions

diff --git a/blastests.xml b/blastests.xml
new file mode 100644
index 0000000..75f5ba4
--- /dev/null
+++ b/blastests.xml
@@ -0,0 +1,45 @@
+<tests>
+    <test id="reference">
+        <pkg>sci-libs/blas-reference-20120215-r1</pkg>
+        <emergeenv>
+            <var name="FFLAGS">-O3</var>
+        </emergeenv>
+    </test>
+
+<!--
+    <test id="atlas">
+        <pkg>sci-libs/atlas-3.9.67</pkg>
+        <emergeenv>
+            <var name="USE">threads</var>
+        </emergeenv>
+    </test>
+-->
+
+    <test id="eigen">
+        <pkg>dev-cpp/eigen-3.0.5</pkg>
+        <emergeenv>
+            <var name="FFLAGS">-O3</var>
+        </emergeenv>
+    </test>
+
+<!--
+    <test id="openblas">
+        <pkg>sci-libs/openblas-0.1_alpha2</pkg>
+        <emergeenv>
+            <var name="TARGET">NEHALEM</var>
+            <var name="FFLAGS">-O3</var>
+        </emergeenv>
+    </test>
+-->
+
+    <test id="mkl">
+        <pkg>sci-libs/mkl-10.3.7.256</pkg>
+        <skip>mkl32-dynamic</skip>
+        <skip>mkl32-dynamic-openmp</skip>
+        <skip>mkl32-gfortran</skip>
+        <skip>mkl32-gfortran-openmp</skip>
+        <skip>mkl32-intel</skip>
+        <skip>mkl32-intel-openmp</skip>
+    </test>
+
+</tests>
diff --git a/numbench/PortageUtils.py b/numbench/PortageUtils.py
index abc2ce1..1da968d 100644
--- a/numbench/PortageUtils.py
+++ b/numbench/PortageUtils.py
@@ -26,7 +26,7 @@ class InstallException(Exception):
         self.package = package
         self.command = command
         self.logfile = logfile
-        
+
 def _getEnv(root='/', envAdds={}):
     #denv = os.environ.copy()
     denv = {}
@@ -36,40 +36,40 @@ def _getEnv(root='/', envAdds={}):
     if os.environ.has_key('PATH'):
         denv['PATH'] += ':' + os.environ['PATH']
     denv['ROOTPATH'] = denv['PATH']
-    
+
     #LIBRARY_PATH
     denv['LIBRARY_PATH'] = ':'.join([pjoin(root, i) for i in \
       ('usr/lib', 'usr/lib64', 'usr/lib32')])
     if os.environ.has_key('LIBRARY_PATH'):
         denv['LIBRARY_PATH'] += ':' + os.environ['LIBRARY_PATH']
-    
+
     #LD_LIBRARY_PATH
     denv['LD_LIBRARY_PATH'] = ':'.join([pjoin(root, i) for i in \
       ('usr/lib', 'usr/lib64', 'usr/lib32')])
     if os.environ.has_key('LD_LIBRARY_PATH'):
         denv['LD_LIBRARY_PATH'] += ':' + os.environ['LD_LIBRARY_PATH']
-    
+
     #INCLUDE_PATH
     denv['INCLUDE_PATH'] = ':'.join([pjoin(root, i) for i in ('usr/include',)])
     if os.environ.has_key('INCLUDE_PATH'):
         denv['INCLUDE_PATH'] += ':' + os.environ['INCLUDE_PATH']
-        
+
     # Adds
     for k,v in envAdds.items():
         denv[k] = v
-    
+
     return denv
 
 def availablePackages(pattern):
     """Returns a list of packages matching the given pattern.
-    
+
     The packages are returned as (category, package, version, revision) tuple.
     No test for keywords or mask is performed. The function just returns
     every matching pattern in the portage tree and installed overlays.
     """
     return [portage.catpkgsplit(l) \
       for l in cmd.getoutput('equery -q list -po ' + pattern).split()]
-    
+
 def normalize_cpv(cpv):
     if type(cpv) == type(''):
         try:
@@ -82,8 +82,8 @@ def normalize_cpv(cpv):
         return '%s/%s-%s-%s' % cpv
     else:
         return '%s/%s-%s' % cpv[:-1]
-        
-    
+
+
 def getDependencies(package, env={}, split=False):
     pkg = normalize_cpv(package)
     cmd = ['emerge', '--ignore-default-opts', '='+pkg, '-poq']
@@ -103,10 +103,10 @@ def getDependencies(package, env={}, split=False):
 def installDependencies(test):
     # Adjust environment
     denv = _getEnv(test['root'], dict(PKGDIR=test['pkgdir']))
-    
+
     # Retrieve dependencies
     deps = getDependencies(test['package'], denv, False)
-    
+
     for i,d in enumerate(deps):
         logfile = pjoin(test['logdir'], 'emergedep_%i.log' % i)
         installPackage(test, package=d, env=test['dependenv'], logfile=logfile)
@@ -117,20 +117,20 @@ def installDependencies(test):
 def installPackage(test, package=None, env=None, logfile=None):
     # TODO: rewrite docstring
     """Emerge a package in the given root.
-    
+
     package is the package to be emerged. It has to be a tuple
     (category, package, version, revision).
-    
+
     env is a dictionary of KEY:VALUE pairs describing the environment changes
     the package will be emerged with. It is useful to specifiy the compilers and
     compiler flags. It is safe to use ACCEPT_KEYWORDS=* for testing purposes.
-    
+
     root is the directory where the packaged will be emerged. A non-root user
     can use this function, provided he has write access to that directory.
-    
+
     pkgdir is the directory where the binary package will be placed. The user
     has to be able to write in this directory.
-    
+
     The function has no return value and raises an exception in case of building
     or emerging failure. Note: dependencies will NOT be emerged!
     """
@@ -140,43 +140,44 @@ def installPackage(test, package=None, env=None, logfile=None):
         env = test['emergeenv']
     if logfile is None:
         logfile = pjoin(test['logdir'], 'emerge.log')
-    
+
     envAdds = env.copy()
     envAdds['PKGDIR'] = test['pkgdir']
     denv = _getEnv(test['root'], envAdds)
     del envAdds
-    
+
     # Retrieve package string
     pkg = normalize_cpv(package)
-    
+
     # Execute emerge command and log the results
     benchutils.mkdir(dirname(logfile))
     fout = file(logfile, 'w')
     cmd = ['emerge', '--ignore-default-opts', '-OB', '=' + pkg]
     p = sp.Popen(cmd, env=denv, stdout=fout, stderr=sp.STDOUT)
     p.wait()
-    
+
     if p.returncode != 0:
         # In case of error, print the whole emerge command
         raise InstallException(p, ' '.join(cmd), logfile)
-    
+
     fout.write('\n\n' + 80*'#' + '\n\n')
-    
+
     # Unpack package onto root
+    archive = pjoin(test['pkgdir'], pkg+'.tbz2')
     benchutils.mkdir(test['root'])
-    tarcmd = ['tar', 'xjvf', test['archive'], '-C', test['root']]
+    tarcmd = ['tar', 'xjvf', archive, '-C', test['root']]
     fout.write(' '.join(tarcmd) + '\n' + 80*'-' + '\n')
     p = sp.Popen(tarcmd, stdout=fout, stderr=sp.STDOUT)
     p.wait()
     if p.returncode != 0:
         # In case of error, print the whole emerge command
         raise InstallException(pkg, ' '.join(tarcmd), logfile)
-    
+
     # Close, return
     fout.close()
-    
+
 if __name__ == '__main__':
     # Just a test
     from pprint import pprint
-    
+
     pprint(get_dependencies('sci-libs/blas-reference-3.3.1-r1'))
diff --git a/numbench/benchconfig.py b/numbench/benchconfig.py
index 7191892..f4bed38 100644
--- a/numbench/benchconfig.py
+++ b/numbench/benchconfig.py
@@ -18,20 +18,14 @@
 import sys, os, time, subprocess as sp
 from os.path import join as pjoin
 
-try:
-    needsinitialization = not initialized #@UndefinedVariable
-except NameError:
-    needsinitialization = True
-
-
-if needsinitialization:
+if not locals().has_key('initialized'):
     initialized = True
-    
+
     isroot = os.getuid() == 0
-    
+
     modulename = sys.argv[1]
     inputfile = os.path.realpath(sys.argv[2])
-    
+
     # Script directories
     curdir = os.path.abspath('.')
     scriptdir = os.path.dirname(os.path.realpath(__file__))
@@ -39,7 +33,7 @@ if needsinitialization:
         btldir = os.environ['BTLDIR']
     else:
         btldir = '/usr/include/btl'
-    
+
     # Library directory (lib vs. lib32 vs. lib64)
     libdir = sp.Popen \
       ('ABI=$(portageq envvar ABI); echo `portageq envvar LIBDIR_$ABI`', \
@@ -50,38 +44,30 @@ if needsinitialization:
         libdir = '/usr/' + libdir
     while libdir[0] == '/':
         libdir = libdir[1:]
-        
+
+    # Parse arguments
+    passargs = sys.argv[3:]
+    for i,a in enumerate(passargs):
+        if a in ('-d', '--directory'):
+            basedir = passargs[i+1]
+            passargs = passargs[:i] + passargs[i+2:]
+            break
+
     # Storage directories
-    basedirb = pjoin(os.environ['HOME'], '.numbench') \
-               + '/run_' + modulename + '_' + time.strftime('%Y-%m-%d')
-    if os.path.exists(basedirb):
-        n = 1
-        while True:
-            basedir = basedirb + "_%i" % n
-            if not os.path.exists(basedir):
-                break
-            n += 1
-    else:
-        basedir = basedirb
-    del basedirb
-    
+    if not locals().has_key('basedir'):
+        basedirb = pjoin(os.environ['HOME'], '.numbench') \
+                + '/run_' + modulename + '_' + time.strftime('%Y-%m-%d')
+        if os.path.exists(basedirb):
+            n = 1
+            while True:
+                basedir = basedirb + "_%i" % n
+                if not os.path.exists(basedir):
+                    break
+                n += 1
+        else:
+            basedir = basedirb
+        del basedirb
+
     testsdir, rootsdir, pkgsdir, reportdir, logdir = tuple([pjoin(basedir, i) \
       for i in ('tests', 'roots', 'packages', 'report', 'log')])
-    
-    
-#def makedirs():
-#    bu.mkdir(rootsdir)
-#    bu.mkdir(testsdir)
-#    bu.mkdir(pkgsdir)
-#    bu.mkdir(reportdir)
-#    bu.mkdir(logdir)
-#    
-#def purgedirs():
-#    bu.rmdir(rootsdir)
-#    bu.rmdir(testsdir)
-#    bu.rmdir(pkgsdir)
-#    bu.rmdir(pjoin(reportdir, '..'))
-#    bu.rmdir(pjoin(logdir, '..'))
-    
-    
-    
+
diff --git a/numbench/confinput/xmlinput.py b/numbench/confinput/xmlinput.py
index 4d4516c..1d04580 100644
--- a/numbench/confinput/xmlinput.py
+++ b/numbench/confinput/xmlinput.py
@@ -90,18 +90,19 @@ def parseConf(fname):
 
     for t in testNodes:
         tid = t.getAttribute('id')
-        
+
         # Get description
         descr = None
         if len(t.getElementsByTagName('descr')) != 0:
             descr = t.getElementsByTagName('descr')[0].firstChild.data
-        
+
         # Get package
         pkg = portage.catpkgsplit(
           t.getElementsByTagName('pkg')[0].firstChild.data)
         normPkg = pu.normalize_cpv(pkg)
 
         # Skip implementations
+        # TODO: add regexp
         skip = [i.firstChild.data for i in t.getElementsByTagName('skip')]
 
         # Requirements
@@ -114,7 +115,7 @@ def parseConf(fname):
         emergeenv = getEnvFromNode(t, 'emergeenv')
         compileenv = getEnvFromNode(t, 'compileenv')
         runenv = getEnvFromNode(t, 'runenv')
-        
+
         # Adjust PATH
         if runenv.has_key('PATH'):
             runenv['PATH'] += ':' + os.environ['PATH']
@@ -128,12 +129,12 @@ def parseConf(fname):
           normalizedPackage = normPkg,
           skip = skip,
           requires = requires,
-          
+
           dependenv = dependenv,
           emergeenv = emergeenv,
           compileenv = compileenv,
           runenv = runenv,
-          
+
           pkgdir = pjoin(cfg.pkgsdir, tid),
           archive = pjoin(cfg.pkgsdir, tid, normPkg+'.tbz2'),
           root = pjoin(cfg.rootsdir, tid),
diff --git a/numbench/main.py b/numbench/main.py
index 466d6ec..9ba9ae7 100644
--- a/numbench/main.py
+++ b/numbench/main.py
@@ -67,7 +67,7 @@ def loadModule(modulename):
 #    except ImportError as e:
 #        sys.stderr.write('Module ' + modulename + ' not found')
 #        exit(1)
-    
+
     return tmp
 
 
@@ -120,37 +120,37 @@ Print("-------------------------------")
 Print()
 for tname, ttest in cfg.tests.items():
     Print("Test: " + tname)
-    
+
     if ttest['descr'] is not None:
         Print(" - Description: " + ttest['descr'])
-    
+
     Print(" - Package: " + ttest['normalizedPackage'])
-    
+
     if len(ttest['dependenv']) != 0:
         Print(" - Dependencies emerge environment: " + \
           ' '.join([n+'="'+v+'"' for n,v in ttest['dependenv'].items()]))
-    
+
     if len(ttest['emergeenv']) != 0:
         Print(" - Emerge environment: " + \
           ' '.join([n+'="'+v+'"' for n,v in ttest['emergeenv'].items()]))
-    
+
     if len(ttest['compileenv']) != 0:
         Print(" - Suite compile-time environment: " + \
           ' '.join([n+'="'+v+'"' for n,v in ttest['compileenv'].items()]))
-    
+
     if len(ttest['runenv']) != 0:
         Print(" - Suite run-time environment: " + \
           ' '.join([n+'="'+v+'"' for n,v in ttest['runenv'].items()]))
-        
+
     if len(ttest['skip']) != 0:
         Print(" - Skip implementations: " + ' '.join(ttest['skip']))
-        
+
     if len(ttest['requires']) != 0:
         Print(" - Pkg-config requirements substitutions:", '')
         for c_0, c_1 in ttest['requires'].items():
             Print(c_0 + ':' + c_1, '')
         Print()
-        
+
     Print()
 Print(80*'=')
 Print()
@@ -196,6 +196,7 @@ for tn,(name,test) in enumerate(cfg.tests.items(),1):
     Print("Package emerged")
 
     # Find implementations
+    # TODO: check for regexps
     impls = [i for i in mod.getImplementations(test) if not i in test['skip']]
     test['implementations'] = impls
 
@@ -210,7 +211,7 @@ for tn,(name,test) in enumerate(cfg.tests.items(),1):
         test['results'][impl] = mod.runTest(test, impl)
         Print.up()
     # All implementations tested
-    
+
     Print.up()
     print
 
diff --git a/numbench/modules/internal/blasBase.py b/numbench/modules/internal/blasBase.py
index 34f2912..5e691e3 100644
--- a/numbench/modules/internal/blasBase.py
+++ b/numbench/modules/internal/blasBase.py
@@ -31,11 +31,11 @@ defaultTests = ('axpy', 'matrix_vector', 'trisolve_vector', 'matrix_matrix')
 def init(self, args):
     passargs = []
     tests = []
-    
+
     if len(args) == 0:
         self.tests = defaultTests
         return
-    
+
     for i in args:
         if i == '1':
             tests += avail1
@@ -47,7 +47,7 @@ def init(self, args):
             tests += avail3
             continue
         passargs.append(i)
-        
+
     self.tests = btl.selectTests(availableTests, passargs+tests)
 
 
@@ -67,10 +67,10 @@ def runTest(self, test, implementation):
       flags = alt.getFlags(test, self.libname, implementation),
       tests = self.tests
     )
-    
-    return btlBase.runTest(test, btlconfig)
-    
-getTests  btlBase.getTests
+
+    return btlBase.runTest(self, test, btlconfig)
+
+getTests = btlBase.getTests
 reportConf = btlBase.reportConf
-    
+
 reportConf = btlBase.reportConf
diff --git a/numbench/modules/internal/btlBase.py b/numbench/modules/internal/btlBase.py
index 63106b7..3c33202 100644
--- a/numbench/modules/internal/btlBase.py
+++ b/numbench/modules/internal/btlBase.py
@@ -15,26 +15,44 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
+from os.path import join as pjoin, exists
 import numbench.utils.btl as btl
+from numbench.benchprint import Print
 
-def reportConf():
+def reportConf(*args):
     return {'type':'semilogx', 'xlabel':'size', 'ylabel':'MFlops'}
 
-def runTest(test, btlconfig):
-    
+
+def runTest(self, test, btlconfig):
+    # Check if results already exist
+    tmpres = dict( \
+      [(i, pjoin(btlconfig['testdir'], "bench_"+i+"_"+self.libname)) \
+      for i in btlconfig['tests']])
+
+    if all([exists(i) for i in tmpres.values()]):
+        Print("Results exist - skipping run")
+        return tmpres
+
+    # Compile test suite
     ret = btl.compileTest(test, btlconfig)
     if ret != 0:
-        print "Compilation failed with code:", ret
+        Print("Compilation failed with code: %i" % ret)
+        return None
     else:
-        print "Compilation successful"
-    
+        Print("Compilation successful")
+
+    # Run test suite
     ret, result = btl.runTest(test, btlconfig)
     if ret != 0:
-        print "Execution failed with code:", ret
+        Print("Execution failed with code: %i" % ret)
+        Print("The results will be incomplete")
+        # TODO: remove this if possible (return incomplete results)
+        return None
     else:
-        print "Execution successful"
-        
+        Print("Execution successful")
+
     return result
 
+
 def getTests(self):
-    return self.tests
\ No newline at end of file
+    return self.tests
diff --git a/numbench/modules/internal/lapackBase.py b/numbench/modules/internal/lapackBase.py
index c8098fc..e45951d 100644
--- a/numbench/modules/internal/lapackBase.py
+++ b/numbench/modules/internal/lapackBase.py
@@ -49,8 +49,8 @@ def runTest(self, test, implementation):
       flags = alt.getFlags(test, self.libname, implementation),
       tests = self.tests
     )
-    
-    return btlBase.runTest(test, btlconfig)
-    
+
+    return btlBase.runTest(self, test, btlconfig)
+
 getTests  btlBase.getTests
 reportConf = btlBase.reportConf
\ No newline at end of file
diff --git a/numbench/report.py b/numbench/report.py
index da1b5de..f51905a 100644
--- a/numbench/report.py
+++ b/numbench/report.py
@@ -1,13 +1,14 @@
-import cfg
 from os.path import join as pjoin, basename
 import numpy as np
 
+import benchconfig as cfg
+import benchutils as bu
 from htmlreport import HTMLreport
 from testdescr import testdescr
 from benchprint import Print
 
 def saveReport():
-    
+
     # Check whether pyplot is working
     try:
         plt.figure()
@@ -15,10 +16,10 @@ def saveReport():
         Print("Unable to generate plots")
         Print("Please make sure that X is running and $DISPLAY is set")
         return
-    
+
     # Read configuration
-    conf = mod.reportConf()
-    
+    conf = cfg.mod.reportConf()
+
     if conf['type'] == 'plot':
         plotf = plt.plot
     elif conf['type'] == 'semilogx':
@@ -27,44 +28,48 @@ def saveReport():
         plotf = plt.semilogy
     elif conf['type'] == 'loglog':
         plotf = plt.loglog
-    
+
     if conf.has_key('xlabel'):
         xlabel = conf['xlabel']
     else:
         xlabel = ''
-    
+
     if conf.has_key('ylabel'):
         ylabel = conf['ylabel']
     else:
         ylabel = ''
-    
-    
+
+
     # Open HTML file
+    bu.mkdir(cfg.reportdir)
     htmlfname = pjoin(cfg.reportdir, 'index.html')
     html = HTMLreport(htmlfname)
-    
+
     for operation in cfg.mod.getTests():
         plt.figure(figsize=(12,9), dpi=300)
-        
+
         for tid,test in cfg.tests.items():
-            for impl in test['implementations']:
-                x,y = np.loadtxt(test['results'][impl][operation], unpack=True)
-                plotf(x, y, label=tid+'/'+impl, hold=True)
-        
+            if test.has_key('implementations'):
+                for impl in test['implementations']:
+                    if test['results'][impl].has_key(operation):
+                        resultsFile = test['results'][impl][operation]
+                        x,y = np.loadtxt(resultsFile, unpack=True)
+                        plotf(x, y, label=tid+'/'+impl, hold=True)
+
         plt.legend(loc='best')
         plt.xlabel(xlabel)
         plt.ylabel(ylabel)
         plt.grid(True)
-        
+
         fname = pjoin(cfg.reportdir, operation+'.png')
         plt.savefig(fname, format='png', bbox_inches='tight', transparent=True)
         html.addFig(testdescr[operation], image=basename(fname))
-    
+
     # Close HTML file
     html.close()
     Print('HTML report generated: ' + htmlfname)
-    
-    
+
+
 # Initialize module
 try:
     if not locals().has_key('initialized'):
@@ -77,5 +82,5 @@ try:
 except ImportError:
     sys.stderr.write('Error: matplotlib and numpy are needed' + \
       'in order to generate the reports!\n')
-    sys.stderr.write('Continue anyway.\n\n')    
-    with_images = False
\ No newline at end of file
+    sys.stderr.write('Continue anyway.\n\n')
+    with_images = False
diff --git a/numbench/utils/btl.py b/numbench/utils/btl.py
index edb3a18..9865ad5 100644
--- a/numbench/utils/btl.py
+++ b/numbench/utils/btl.py
@@ -201,7 +201,7 @@ def runTest(test, btlconfig):
         
         resfile = errline.split()[-1]
         operation = resfile.split('_', 1)[-1].rsplit('_', 1)[0]
-        result[operation] = resfile
+        result[operation] = pjoin(btlconfig['testdir'], resfile)
         Print(operation + " -> " + resfile)