diff options
author | Andrea Arteaga <andyspiros@gmail.com> | 2012-02-20 19:53:54 +0000 |
---|---|---|
committer | Andrea Arteaga <andyspiros@gmail.com> | 2012-02-20 19:53:54 +0000 |
commit | edbbe2d009d1cc7139d30348694eac3274426306 (patch) | |
tree | e5478c07333b77800eca91cb5a1099fc0b1d7557 | |
parent | Rewritten code... New XML parser, better modules. Not completely tested! (diff) | |
download | auto-numerical-bench-edbbe2d009d1cc7139d30348694eac3274426306.tar.gz auto-numerical-bench-edbbe2d009d1cc7139d30348694eac3274426306.tar.bz2 auto-numerical-bench-edbbe2d009d1cc7139d30348694eac3274426306.zip |
Fixed reports, added -d switchl, better failure checks.
-rw-r--r-- | blastests.xml | 45 | ||||
-rw-r--r-- | numbench/PortageUtils.py | 55 | ||||
-rw-r--r-- | numbench/benchconfig.py | 72 | ||||
-rw-r--r-- | numbench/confinput/xmlinput.py | 11 | ||||
-rw-r--r-- | numbench/main.py | 23 | ||||
-rw-r--r-- | numbench/modules/internal/blasBase.py | 16 | ||||
-rw-r--r-- | numbench/modules/internal/btlBase.py | 38 | ||||
-rw-r--r-- | numbench/modules/internal/lapackBase.py | 6 | ||||
-rw-r--r-- | numbench/report.py | 47 | ||||
-rw-r--r-- | numbench/utils/btl.py | 2 |
10 files changed, 186 insertions, 129 deletions
diff --git a/blastests.xml b/blastests.xml new file mode 100644 index 0000000..75f5ba4 --- /dev/null +++ b/blastests.xml @@ -0,0 +1,45 @@ +<tests> + <test id="reference"> + <pkg>sci-libs/blas-reference-20120215-r1</pkg> + <emergeenv> + <var name="FFLAGS">-O3</var> + </emergeenv> + </test> + +<!-- + <test id="atlas"> + <pkg>sci-libs/atlas-3.9.67</pkg> + <emergeenv> + <var name="USE">threads</var> + </emergeenv> + </test> +--> + + <test id="eigen"> + <pkg>dev-cpp/eigen-3.0.5</pkg> + <emergeenv> + <var name="FFLAGS">-O3</var> + </emergeenv> + </test> + +<!-- + <test id="openblas"> + <pkg>sci-libs/openblas-0.1_alpha2</pkg> + <emergeenv> + <var name="TARGET">NEHALEM</var> + <var name="FFLAGS">-O3</var> + </emergeenv> + </test> +--> + + <test id="mkl"> + <pkg>sci-libs/mkl-10.3.7.256</pkg> + <skip>mkl32-dynamic</skip> + <skip>mkl32-dynamic-openmp</skip> + <skip>mkl32-gfortran</skip> + <skip>mkl32-gfortran-openmp</skip> + <skip>mkl32-intel</skip> + <skip>mkl32-intel-openmp</skip> + </test> + +</tests> diff --git a/numbench/PortageUtils.py b/numbench/PortageUtils.py index abc2ce1..1da968d 100644 --- a/numbench/PortageUtils.py +++ b/numbench/PortageUtils.py @@ -26,7 +26,7 @@ class InstallException(Exception): self.package = package self.command = command self.logfile = logfile - + def _getEnv(root='/', envAdds={}): #denv = os.environ.copy() denv = {} @@ -36,40 +36,40 @@ def _getEnv(root='/', envAdds={}): if os.environ.has_key('PATH'): denv['PATH'] += ':' + os.environ['PATH'] denv['ROOTPATH'] = denv['PATH'] - + #LIBRARY_PATH denv['LIBRARY_PATH'] = ':'.join([pjoin(root, i) for i in \ ('usr/lib', 'usr/lib64', 'usr/lib32')]) if os.environ.has_key('LIBRARY_PATH'): denv['LIBRARY_PATH'] += ':' + os.environ['LIBRARY_PATH'] - + #LD_LIBRARY_PATH denv['LD_LIBRARY_PATH'] = ':'.join([pjoin(root, i) for i in \ ('usr/lib', 'usr/lib64', 'usr/lib32')]) if os.environ.has_key('LD_LIBRARY_PATH'): denv['LD_LIBRARY_PATH'] += ':' + os.environ['LD_LIBRARY_PATH'] - + #INCLUDE_PATH denv['INCLUDE_PATH'] = ':'.join([pjoin(root, i) for i in ('usr/include',)]) if os.environ.has_key('INCLUDE_PATH'): denv['INCLUDE_PATH'] += ':' + os.environ['INCLUDE_PATH'] - + # Adds for k,v in envAdds.items(): denv[k] = v - + return denv def availablePackages(pattern): """Returns a list of packages matching the given pattern. - + The packages are returned as (category, package, version, revision) tuple. No test for keywords or mask is performed. The function just returns every matching pattern in the portage tree and installed overlays. """ return [portage.catpkgsplit(l) \ for l in cmd.getoutput('equery -q list -po ' + pattern).split()] - + def normalize_cpv(cpv): if type(cpv) == type(''): try: @@ -82,8 +82,8 @@ def normalize_cpv(cpv): return '%s/%s-%s-%s' % cpv else: return '%s/%s-%s' % cpv[:-1] - - + + def getDependencies(package, env={}, split=False): pkg = normalize_cpv(package) cmd = ['emerge', '--ignore-default-opts', '='+pkg, '-poq'] @@ -103,10 +103,10 @@ def getDependencies(package, env={}, split=False): def installDependencies(test): # Adjust environment denv = _getEnv(test['root'], dict(PKGDIR=test['pkgdir'])) - + # Retrieve dependencies deps = getDependencies(test['package'], denv, False) - + for i,d in enumerate(deps): logfile = pjoin(test['logdir'], 'emergedep_%i.log' % i) installPackage(test, package=d, env=test['dependenv'], logfile=logfile) @@ -117,20 +117,20 @@ def installDependencies(test): def installPackage(test, package=None, env=None, logfile=None): # TODO: rewrite docstring """Emerge a package in the given root. - + package is the package to be emerged. It has to be a tuple (category, package, version, revision). - + env is a dictionary of KEY:VALUE pairs describing the environment changes the package will be emerged with. It is useful to specifiy the compilers and compiler flags. It is safe to use ACCEPT_KEYWORDS=* for testing purposes. - + root is the directory where the packaged will be emerged. A non-root user can use this function, provided he has write access to that directory. - + pkgdir is the directory where the binary package will be placed. The user has to be able to write in this directory. - + The function has no return value and raises an exception in case of building or emerging failure. Note: dependencies will NOT be emerged! """ @@ -140,43 +140,44 @@ def installPackage(test, package=None, env=None, logfile=None): env = test['emergeenv'] if logfile is None: logfile = pjoin(test['logdir'], 'emerge.log') - + envAdds = env.copy() envAdds['PKGDIR'] = test['pkgdir'] denv = _getEnv(test['root'], envAdds) del envAdds - + # Retrieve package string pkg = normalize_cpv(package) - + # Execute emerge command and log the results benchutils.mkdir(dirname(logfile)) fout = file(logfile, 'w') cmd = ['emerge', '--ignore-default-opts', '-OB', '=' + pkg] p = sp.Popen(cmd, env=denv, stdout=fout, stderr=sp.STDOUT) p.wait() - + if p.returncode != 0: # In case of error, print the whole emerge command raise InstallException(p, ' '.join(cmd), logfile) - + fout.write('\n\n' + 80*'#' + '\n\n') - + # Unpack package onto root + archive = pjoin(test['pkgdir'], pkg+'.tbz2') benchutils.mkdir(test['root']) - tarcmd = ['tar', 'xjvf', test['archive'], '-C', test['root']] + tarcmd = ['tar', 'xjvf', archive, '-C', test['root']] fout.write(' '.join(tarcmd) + '\n' + 80*'-' + '\n') p = sp.Popen(tarcmd, stdout=fout, stderr=sp.STDOUT) p.wait() if p.returncode != 0: # In case of error, print the whole emerge command raise InstallException(pkg, ' '.join(tarcmd), logfile) - + # Close, return fout.close() - + if __name__ == '__main__': # Just a test from pprint import pprint - + pprint(get_dependencies('sci-libs/blas-reference-3.3.1-r1')) diff --git a/numbench/benchconfig.py b/numbench/benchconfig.py index 7191892..f4bed38 100644 --- a/numbench/benchconfig.py +++ b/numbench/benchconfig.py @@ -18,20 +18,14 @@ import sys, os, time, subprocess as sp from os.path import join as pjoin -try: - needsinitialization = not initialized #@UndefinedVariable -except NameError: - needsinitialization = True - - -if needsinitialization: +if not locals().has_key('initialized'): initialized = True - + isroot = os.getuid() == 0 - + modulename = sys.argv[1] inputfile = os.path.realpath(sys.argv[2]) - + # Script directories curdir = os.path.abspath('.') scriptdir = os.path.dirname(os.path.realpath(__file__)) @@ -39,7 +33,7 @@ if needsinitialization: btldir = os.environ['BTLDIR'] else: btldir = '/usr/include/btl' - + # Library directory (lib vs. lib32 vs. lib64) libdir = sp.Popen \ ('ABI=$(portageq envvar ABI); echo `portageq envvar LIBDIR_$ABI`', \ @@ -50,38 +44,30 @@ if needsinitialization: libdir = '/usr/' + libdir while libdir[0] == '/': libdir = libdir[1:] - + + # Parse arguments + passargs = sys.argv[3:] + for i,a in enumerate(passargs): + if a in ('-d', '--directory'): + basedir = passargs[i+1] + passargs = passargs[:i] + passargs[i+2:] + break + # Storage directories - basedirb = pjoin(os.environ['HOME'], '.numbench') \ - + '/run_' + modulename + '_' + time.strftime('%Y-%m-%d') - if os.path.exists(basedirb): - n = 1 - while True: - basedir = basedirb + "_%i" % n - if not os.path.exists(basedir): - break - n += 1 - else: - basedir = basedirb - del basedirb - + if not locals().has_key('basedir'): + basedirb = pjoin(os.environ['HOME'], '.numbench') \ + + '/run_' + modulename + '_' + time.strftime('%Y-%m-%d') + if os.path.exists(basedirb): + n = 1 + while True: + basedir = basedirb + "_%i" % n + if not os.path.exists(basedir): + break + n += 1 + else: + basedir = basedirb + del basedirb + testsdir, rootsdir, pkgsdir, reportdir, logdir = tuple([pjoin(basedir, i) \ for i in ('tests', 'roots', 'packages', 'report', 'log')]) - - -#def makedirs(): -# bu.mkdir(rootsdir) -# bu.mkdir(testsdir) -# bu.mkdir(pkgsdir) -# bu.mkdir(reportdir) -# bu.mkdir(logdir) -# -#def purgedirs(): -# bu.rmdir(rootsdir) -# bu.rmdir(testsdir) -# bu.rmdir(pkgsdir) -# bu.rmdir(pjoin(reportdir, '..')) -# bu.rmdir(pjoin(logdir, '..')) - - - + diff --git a/numbench/confinput/xmlinput.py b/numbench/confinput/xmlinput.py index 4d4516c..1d04580 100644 --- a/numbench/confinput/xmlinput.py +++ b/numbench/confinput/xmlinput.py @@ -90,18 +90,19 @@ def parseConf(fname): for t in testNodes: tid = t.getAttribute('id') - + # Get description descr = None if len(t.getElementsByTagName('descr')) != 0: descr = t.getElementsByTagName('descr')[0].firstChild.data - + # Get package pkg = portage.catpkgsplit( t.getElementsByTagName('pkg')[0].firstChild.data) normPkg = pu.normalize_cpv(pkg) # Skip implementations + # TODO: add regexp skip = [i.firstChild.data for i in t.getElementsByTagName('skip')] # Requirements @@ -114,7 +115,7 @@ def parseConf(fname): emergeenv = getEnvFromNode(t, 'emergeenv') compileenv = getEnvFromNode(t, 'compileenv') runenv = getEnvFromNode(t, 'runenv') - + # Adjust PATH if runenv.has_key('PATH'): runenv['PATH'] += ':' + os.environ['PATH'] @@ -128,12 +129,12 @@ def parseConf(fname): normalizedPackage = normPkg, skip = skip, requires = requires, - + dependenv = dependenv, emergeenv = emergeenv, compileenv = compileenv, runenv = runenv, - + pkgdir = pjoin(cfg.pkgsdir, tid), archive = pjoin(cfg.pkgsdir, tid, normPkg+'.tbz2'), root = pjoin(cfg.rootsdir, tid), diff --git a/numbench/main.py b/numbench/main.py index 466d6ec..9ba9ae7 100644 --- a/numbench/main.py +++ b/numbench/main.py @@ -67,7 +67,7 @@ def loadModule(modulename): # except ImportError as e: # sys.stderr.write('Module ' + modulename + ' not found') # exit(1) - + return tmp @@ -120,37 +120,37 @@ Print("-------------------------------") Print() for tname, ttest in cfg.tests.items(): Print("Test: " + tname) - + if ttest['descr'] is not None: Print(" - Description: " + ttest['descr']) - + Print(" - Package: " + ttest['normalizedPackage']) - + if len(ttest['dependenv']) != 0: Print(" - Dependencies emerge environment: " + \ ' '.join([n+'="'+v+'"' for n,v in ttest['dependenv'].items()])) - + if len(ttest['emergeenv']) != 0: Print(" - Emerge environment: " + \ ' '.join([n+'="'+v+'"' for n,v in ttest['emergeenv'].items()])) - + if len(ttest['compileenv']) != 0: Print(" - Suite compile-time environment: " + \ ' '.join([n+'="'+v+'"' for n,v in ttest['compileenv'].items()])) - + if len(ttest['runenv']) != 0: Print(" - Suite run-time environment: " + \ ' '.join([n+'="'+v+'"' for n,v in ttest['runenv'].items()])) - + if len(ttest['skip']) != 0: Print(" - Skip implementations: " + ' '.join(ttest['skip'])) - + if len(ttest['requires']) != 0: Print(" - Pkg-config requirements substitutions:", '') for c_0, c_1 in ttest['requires'].items(): Print(c_0 + ':' + c_1, '') Print() - + Print() Print(80*'=') Print() @@ -196,6 +196,7 @@ for tn,(name,test) in enumerate(cfg.tests.items(),1): Print("Package emerged") # Find implementations + # TODO: check for regexps impls = [i for i in mod.getImplementations(test) if not i in test['skip']] test['implementations'] = impls @@ -210,7 +211,7 @@ for tn,(name,test) in enumerate(cfg.tests.items(),1): test['results'][impl] = mod.runTest(test, impl) Print.up() # All implementations tested - + Print.up() print diff --git a/numbench/modules/internal/blasBase.py b/numbench/modules/internal/blasBase.py index 34f2912..5e691e3 100644 --- a/numbench/modules/internal/blasBase.py +++ b/numbench/modules/internal/blasBase.py @@ -31,11 +31,11 @@ defaultTests = ('axpy', 'matrix_vector', 'trisolve_vector', 'matrix_matrix') def init(self, args): passargs = [] tests = [] - + if len(args) == 0: self.tests = defaultTests return - + for i in args: if i == '1': tests += avail1 @@ -47,7 +47,7 @@ def init(self, args): tests += avail3 continue passargs.append(i) - + self.tests = btl.selectTests(availableTests, passargs+tests) @@ -67,10 +67,10 @@ def runTest(self, test, implementation): flags = alt.getFlags(test, self.libname, implementation), tests = self.tests ) - - return btlBase.runTest(test, btlconfig) - -getTests btlBase.getTests + + return btlBase.runTest(self, test, btlconfig) + +getTests = btlBase.getTests reportConf = btlBase.reportConf - + reportConf = btlBase.reportConf diff --git a/numbench/modules/internal/btlBase.py b/numbench/modules/internal/btlBase.py index 63106b7..3c33202 100644 --- a/numbench/modules/internal/btlBase.py +++ b/numbench/modules/internal/btlBase.py @@ -15,26 +15,44 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # +from os.path import join as pjoin, exists import numbench.utils.btl as btl +from numbench.benchprint import Print -def reportConf(): +def reportConf(*args): return {'type':'semilogx', 'xlabel':'size', 'ylabel':'MFlops'} -def runTest(test, btlconfig): - + +def runTest(self, test, btlconfig): + # Check if results already exist + tmpres = dict( \ + [(i, pjoin(btlconfig['testdir'], "bench_"+i+"_"+self.libname)) \ + for i in btlconfig['tests']]) + + if all([exists(i) for i in tmpres.values()]): + Print("Results exist - skipping run") + return tmpres + + # Compile test suite ret = btl.compileTest(test, btlconfig) if ret != 0: - print "Compilation failed with code:", ret + Print("Compilation failed with code: %i" % ret) + return None else: - print "Compilation successful" - + Print("Compilation successful") + + # Run test suite ret, result = btl.runTest(test, btlconfig) if ret != 0: - print "Execution failed with code:", ret + Print("Execution failed with code: %i" % ret) + Print("The results will be incomplete") + # TODO: remove this if possible (return incomplete results) + return None else: - print "Execution successful" - + Print("Execution successful") + return result + def getTests(self): - return self.tests
\ No newline at end of file + return self.tests diff --git a/numbench/modules/internal/lapackBase.py b/numbench/modules/internal/lapackBase.py index c8098fc..e45951d 100644 --- a/numbench/modules/internal/lapackBase.py +++ b/numbench/modules/internal/lapackBase.py @@ -49,8 +49,8 @@ def runTest(self, test, implementation): flags = alt.getFlags(test, self.libname, implementation), tests = self.tests ) - - return btlBase.runTest(test, btlconfig) - + + return btlBase.runTest(self, test, btlconfig) + getTests btlBase.getTests reportConf = btlBase.reportConf
\ No newline at end of file diff --git a/numbench/report.py b/numbench/report.py index da1b5de..f51905a 100644 --- a/numbench/report.py +++ b/numbench/report.py @@ -1,13 +1,14 @@ -import cfg from os.path import join as pjoin, basename import numpy as np +import benchconfig as cfg +import benchutils as bu from htmlreport import HTMLreport from testdescr import testdescr from benchprint import Print def saveReport(): - + # Check whether pyplot is working try: plt.figure() @@ -15,10 +16,10 @@ def saveReport(): Print("Unable to generate plots") Print("Please make sure that X is running and $DISPLAY is set") return - + # Read configuration - conf = mod.reportConf() - + conf = cfg.mod.reportConf() + if conf['type'] == 'plot': plotf = plt.plot elif conf['type'] == 'semilogx': @@ -27,44 +28,48 @@ def saveReport(): plotf = plt.semilogy elif conf['type'] == 'loglog': plotf = plt.loglog - + if conf.has_key('xlabel'): xlabel = conf['xlabel'] else: xlabel = '' - + if conf.has_key('ylabel'): ylabel = conf['ylabel'] else: ylabel = '' - - + + # Open HTML file + bu.mkdir(cfg.reportdir) htmlfname = pjoin(cfg.reportdir, 'index.html') html = HTMLreport(htmlfname) - + for operation in cfg.mod.getTests(): plt.figure(figsize=(12,9), dpi=300) - + for tid,test in cfg.tests.items(): - for impl in test['implementations']: - x,y = np.loadtxt(test['results'][impl][operation], unpack=True) - plotf(x, y, label=tid+'/'+impl, hold=True) - + if test.has_key('implementations'): + for impl in test['implementations']: + if test['results'][impl].has_key(operation): + resultsFile = test['results'][impl][operation] + x,y = np.loadtxt(resultsFile, unpack=True) + plotf(x, y, label=tid+'/'+impl, hold=True) + plt.legend(loc='best') plt.xlabel(xlabel) plt.ylabel(ylabel) plt.grid(True) - + fname = pjoin(cfg.reportdir, operation+'.png') plt.savefig(fname, format='png', bbox_inches='tight', transparent=True) html.addFig(testdescr[operation], image=basename(fname)) - + # Close HTML file html.close() Print('HTML report generated: ' + htmlfname) - - + + # Initialize module try: if not locals().has_key('initialized'): @@ -77,5 +82,5 @@ try: except ImportError: sys.stderr.write('Error: matplotlib and numpy are needed' + \ 'in order to generate the reports!\n') - sys.stderr.write('Continue anyway.\n\n') - with_images = False
\ No newline at end of file + sys.stderr.write('Continue anyway.\n\n') + with_images = False diff --git a/numbench/utils/btl.py b/numbench/utils/btl.py index edb3a18..9865ad5 100644 --- a/numbench/utils/btl.py +++ b/numbench/utils/btl.py @@ -201,7 +201,7 @@ def runTest(test, btlconfig): resfile = errline.split()[-1] operation = resfile.split('_', 1)[-1].rsplit('_', 1)[0] - result[operation] = resfile + result[operation] = pjoin(btlconfig['testdir'], resfile) Print(operation + " -> " + resfile) |