diff options
Diffstat (limited to 'demos/python/jlib.py')
| -rw-r--r-- | demos/python/jlib.py | 1355 |
1 files changed, 0 insertions, 1355 deletions
diff --git a/demos/python/jlib.py b/demos/python/jlib.py deleted file mode 100644 index 20506c38..00000000 --- a/demos/python/jlib.py +++ /dev/null @@ -1,1355 +0,0 @@ -from __future__ import print_function - -import codecs -import inspect -import io -import os -import shutil -import subprocess -import sys -import time -import traceback -import threading - - -def place( frame_record): - ''' - Useful debugging function - returns representation of source position of - caller. - ''' - filename = frame_record.filename - line = frame_record.lineno - function = frame_record.function - ret = os.path.split( filename)[1] + ':' + str( line) + ':' + function + ':' - if 0: - tid = str(threading.currentThread()) - ret = '[' + tid + '] ' + ret - return ret - - -def expand_nv( text, caller): - ''' - Returns <text> with special handling of {<expression>} items. - - text: - String containing {<expression>} items. - caller: - If an int, the number of frames to step up when looking for file:line - information or evaluating expressions. - - Otherwise should be a frame record as returned by inspect.stack()[]. - - <expression> is evaluated in <caller>'s context using eval(), and expanded - to <expression> or <expression>=<value>. - - If <expression> ends with '=', this character is removed and we prefix the - result with <expression>=. - - E.g.: - x = 45 - y = 'hello' - expand_nv( 'foo {x} {y=}') - returns: - foo 45 y=hello - - <expression> can also use ':' and '!' to control formatting, like - str.format(). - ''' - if isinstance( caller, int): - frame_record = inspect.stack()[ caller] - else: - frame_record = caller - frame = frame_record.frame - try: - def get_items(): - ''' - Yields (pre, item), where <item> is contents of next {...} or None, - and <pre> is preceding text. - ''' - pos = 0 - pre = '' - while 1: - if pos == len( text): - yield pre, None - break - rest = text[ pos:] - if rest.startswith( '{{') or rest.startswith( '}}'): - pre += rest[0] - pos += 2 - elif text[ pos] == '{': - close = text.find( '}', pos) - if close < 0: - raise Exception( 'After "{" at offset %s, cannot find closing "}". text is: %r' % ( - pos, text)) - yield pre, text[ pos+1 : close] - pre = '' - pos = close + 1 - else: - pre += text[ pos] - pos += 1 - - ret = '' - for pre, item in get_items(): - ret += pre - nv = False - if item: - if item.endswith( '='): - nv = True - item = item[:-1] - expression, tail = split_first_of( item, '!:') - try: - value = eval( expression, frame.f_globals, frame.f_locals) - value_text = ('{0%s}' % tail).format( value) - except Exception as e: - value_text = '{??Failed to evaluate %r in context %s:%s because: %s??}' % ( - expression, - frame_record.filename, - frame_record.lineno, - e, - ) - if nv: - ret += '%s=' % expression - ret += value_text - - return ret - - finally: - del frame - - -class LogPrefixTime: - def __init__( self, date=False, time_=True, elapsed=False): - self.date = date - self.time = time_ - self.elapsed = elapsed - self.t0 = time.time() - def __call__( self): - ret = '' - if self.date: - ret += time.strftime( ' %F') - if self.time: - ret += time.strftime( ' %T') - if self.elapsed: - ret += ' (+%s)' % time_duration( time.time() - self.t0, s_format='%.1f') - if ret: - ret = ret.strip() + ': ' - return ret - -class LogPrefixFileLine: - def __call__( self, caller): - if isinstance( caller, int): - caller = inspect.stack()[ caller] - return place( caller) + ' ' - -class LogPrefixScopes: - ''' - Internal use only. - ''' - def __init__( self): - self.items = [] - def __call__( self): - ret = '' - for item in self.items: - if callable( item): - item = item() - ret += item - return ret - - -class LogPrefixScope: - ''' - Can be used to insert scoped prefix to log output. - ''' - def __init__( self, prefix): - g_log_prefixe_scopes.items.append( prefix) - def __enter__( self): - pass - def __exit__( self, exc_type, exc_value, traceback): - global g_log_prefix - g_log_prefixe_scopes.items.pop() - - -g_log_delta = 0 - -class LogDeltaScope: - ''' - Can be used to temporarily change verbose level of logging. - - E.g to temporarily increase logging: - - with jlib.LogDeltaScope(-1): - ... - ''' - def __init__( self, delta): - self.delta = delta - global g_log_delta - g_log_delta += self.delta - def __enter__( self): - pass - def __exit__( self, exc_type, exc_value, traceback): - global g_log_delta - g_log_delta -= self.delta - -# Special item that can be inserted into <g_log_prefixes> to enable -# temporary addition of text into log prefixes. -# -g_log_prefixe_scopes = LogPrefixScopes() - -# List of items that form prefix for all output from log(). -# -g_log_prefixes = [] - - -def log_text( text=None, caller=1, nv=True): - ''' - Returns log text, prepending all lines with text from g_log_prefixes. - - text: - The text to output. Each line is prepended with prefix text. - caller: - If an int, the number of frames to step up when looking for file:line - information or evaluating expressions. - - Otherwise should be a frame record as returned by inspect.stack()[]. - nv: - If true, we expand {...} in <text> using expand_nv(). - ''' - if isinstance( caller, int): - caller += 1 - prefix = '' - for p in g_log_prefixes: - if callable( p): - if isinstance( p, LogPrefixFileLine): - p = p(caller) - else: - p = p() - prefix += p - - if text is None: - return prefix - - if nv: - text = expand_nv( text, caller) - - if text.endswith( '\n'): - text = text[:-1] - lines = text.split( '\n') - - text = '' - for line in lines: - text += prefix + line + '\n' - return text - - - -s_log_levels_cache = dict() -s_log_levels_items = [] - -def log_levels_find( caller): - if not s_log_levels_items: - return 0 - - tb = traceback.extract_stack( None, 1+caller) - if len(tb) == 0: - return 0 - filename, line, function, text = tb[0] - - key = function, filename, line, - delta = s_log_levels_cache.get( key) - - if delta is None: - # Calculate and populate cache. - delta = 0 - for item_function, item_filename, item_delta in s_log_levels_items: - if item_function and not function.startswith( item_function): - continue - if item_filename and not filename.startswith( item_filename): - continue - delta = item_delta - break - - s_log_levels_cache[ key] = delta - - return delta - - -def log_levels_add( delta, filename_prefix, function_prefix): - ''' - log() calls from locations with filenames starting with <filename_prefix> - and/or function names starting with <function_prefix> will have <delta> - added to their level. - - Use -ve delta to increase verbosity from particular filename or function - prefixes. - ''' - log( 'adding level: {filename_prefix=!r} {function_prefix=!r}') - - # Sort in reverse order so that long functions and filename specs come - # first. - # - s_log_levels_items.append( (function_prefix, filename_prefix, delta)) - s_log_levels_items.sort( reverse=True) - - -def log( text, level=0, caller=1, nv=True, out=None): - ''' - Writes log text, with special handling of {<expression>} items in <text> - similar to python3's f-strings. - - text: - The text to output. - caller: - How many frames to step up to get caller's context when evaluating - file:line information and/or expressions. Or frame record as returned - by inspect.stack()[]. - nv: - If true, we expand {...} in <text> using expand_nv(). - out: - Where to send output. If None we use sys.stdout. - - <expression> is evaluated in our caller's context (<n> stack frames up) - using eval(), and expanded to <expression> or <expression>=<value>. - - If <expression> ends with '=', this character is removed and we prefix the - result with <expression>=. - - E.g.: - x = 45 - y = 'hello' - expand_nv( 'foo {x} {y=}') - returns: - foo 45 y=hello - - <expression> can also use ':' and '!' to control formatting, like - str.format(). - ''' - if out is None: - out = sys.stdout - level += g_log_delta - if isinstance( caller, int): - caller += 1 - level += log_levels_find( caller) - if level <= 0: - text = log_text( text, caller, nv=nv) - out.write( text) - out.flush() - - -def log0( text, caller=1, nv=True, out=None): - ''' - Most verbose log. Same as log(). - ''' - log( text, level=0, caller=caller+1, nv=nv, out=out) - -def log1( text, caller=1, nv=True, out=None): - log( text, level=1, caller=caller+1, nv=nv, out=out) - -def log2( text, caller=1, nv=True, out=None): - log( text, level=2, caller=caller+1, nv=nv, out=out) - -def log3( text, caller=1, nv=True, out=None): - log( text, level=3, caller=caller+1, nv=nv, out=out) - -def log4( text, caller=1, nv=True, out=None): - log( text, level=4, caller=caller+1, nv=nv, out=out) - -def log5( text, caller=1, nv=True, out=None): - ''' - Least verbose log. - ''' - log( text, level=5, caller=caller+1, nv=nv, out=out) - -def logx( text, caller=1, nv=True, out=None): - ''' - Does nothing, useful when commenting out a log(). - ''' - pass - - -def log_levels_add_env( name='JLIB_log_levels'): - ''' - Added log levels encoded in an environmental variable. - ''' - t = os.environ.get( name) - if t: - for ffll in t.split( ','): - ffl, delta = ffll.split( '=', 1) - delta = int( delta) - ffl = ffl.split( ':') - if 0: - pass - elif len( ffl) == 1: - filename = ffl - function = None - elif len( ffl) == 2: - filename, function = ffl - else: - assert 0 - log_levels_add( delta, filename, function) - - -def strpbrk( text, substrings): - ''' - Finds first occurrence of any item in <substrings> in <text>. - - Returns (pos, substring) or (len(text), None) if not found. - ''' - ret_pos = len( text) - ret_substring = None - for substring in substrings: - pos = text.find( substring) - if pos >= 0 and pos < ret_pos: - ret_pos = pos - ret_substring = substring - return ret_pos, ret_substring - - -def split_first_of( text, substrings): - ''' - Returns (pre, post), where <pre> doesn't contain any item in <substrings> - and <post> is empty or starts with an item in <substrings>. - ''' - pos, _ = strpbrk( text, substrings) - return text[ :pos], text[ pos:] - - - -log_levels_add_env() - - -def force_line_buffering(): - ''' - Ensure sys.stdout and sys.stderr are line-buffered. E.g. makes things work - better if output is piped to a file via 'tee'. - - Returns original out,err streams. - ''' - stdout0 = sys.stdout - stderr0 = sys.stderr - sys.stdout = os.fdopen( os.dup( sys.stdout.fileno()), 'w', 1) - sys.stderr = os.fdopen( os.dup( sys.stderr.fileno()), 'w', 1) - return stdout0, stderr0 - - -def exception_info( exception=None, limit=None, out=None, prefix='', oneline=False): - ''' - General replacement for traceback.* functions that print/return information - about exceptions. This function provides a simple way of getting the - functionality provided by these traceback functions: - - traceback.format_exc() - traceback.format_exception() - traceback.print_exc() - traceback.print_exception() - - Returns: - A string containing description of specified exception and backtrace. - - Inclusion of outer frames: - We improve upon traceback.* in that we also include stack frames above - the point at which an exception was caught - frames from the top-level - <module> or thread creation fn to the try..catch block, which makes - backtraces much more useful. - - Google 'sys.exc_info backtrace incomplete' for more details. - - We deliberately leave a slightly curious pair of items in the backtrace - - the point in the try: block that ended up raising an exception, and - the point in the associated except: block from which we were called. - - For clarity, we insert an empty frame in-between these two items, so - that one can easily distinguish the two parts of the backtrace. - - So the backtrace looks like this: - - root (e.g. <module> or /usr/lib/python2.7/threading.py:778:__bootstrap(): - ... - file:line in the except: block where the exception was caught. - ::(): marker - file:line in the try: block. - ... - file:line where the exception was raised. - - The items after the ::(): marker are the usual items that traceback.* - shows for an exception. - - Also the backtraces that are generated are more concise than those provided - by traceback.* - just one line per frame instead of two - and filenames are - output relative to the current directory if applicatble. And one can easily - prefix all lines with a specified string, e.g. to indent the text. - - Returns a string containing backtrace and exception information, and sends - returned string to <out> if specified. - - exception: - None, or a (type, value, traceback) tuple, e.g. from sys.exc_info(). If - None, we call sys.exc_info() and use its return value. - limit: - None or maximum number of stackframes to output. - out: - None or callable taking single <text> parameter or object with a - 'write' member that takes a single <text> parameter. - prefix: - Used to prefix all lines of text. - ''' - if exception is None: - exception = sys.exc_info() - etype, value, tb = exception - - if sys.version_info[0] == 2: - out2 = io.BytesIO() - else: - out2 = io.StringIO() - try: - - frames = [] - - # Get frames above point at which exception was caught - frames - # starting at top-level <module> or thread creation fn, and ending - # at the point in the catch: block from which we were called. - # - # These frames are not included explicitly in sys.exc_info()[2] and are - # also omitted by traceback.* functions, which makes for incomplete - # backtraces that miss much useful information. - # - for f in reversed(inspect.getouterframes(tb.tb_frame)): - ff = f[1], f[2], f[3], f[4][0].strip() - frames.append(ff) - - if 1: - # It's useful to see boundary between upper and lower frames. - frames.append( None) - - # Append frames from point in the try: block that caused the exception - # to be raised, to the point at which the exception was thrown. - # - # [One can get similar information using traceback.extract_tb(tb): - # for f in traceback.extract_tb(tb): - # frames.append(f) - # ] - for f in inspect.getinnerframes(tb): - ff = f[1], f[2], f[3], f[4][0].strip() - frames.append(ff) - - cwd = os.getcwd() + os.sep - if oneline: - if etype and value: - # The 'exception_text' variable below will usually be assigned - # something like '<ExceptionType>: <ExceptionValue>', unless - # there was no explanatory text provided (e.g. "raise Exception()"). - # In this case, str(value) will evaluate to ''. - exception_text = traceback.format_exception_only(etype, value)[0].strip() - filename, line, fnname, text = frames[-1] - if filename.startswith(cwd): - filename = filename[len(cwd):] - if not str(value): - # The exception doesn't have any useful explanatory text - # (for example, maybe it was raised by an expression like - # "assert <expression>" without a subsequent comma). In - # the absence of anything more helpful, print the code that - # raised the exception. - exception_text += ' (%s)' % text - line = '%s%s at %s:%s:%s()' % (prefix, exception_text, filename, line, fnname) - out2.write(line) - else: - out2.write( '%sBacktrace:\n' % prefix) - for frame in frames: - if frame is None: - out2.write( '%s ^except raise:\n' % prefix) - continue - filename, line, fnname, text = frame - if filename.startswith( cwd): - filename = filename[ len(cwd):] - if filename.startswith( './'): - filename = filename[ 2:] - out2.write( '%s %s:%s:%s(): %s\n' % ( - prefix, filename, line, fnname, text)) - - if etype and value: - out2.write( '%sException:\n' % prefix) - lines = traceback.format_exception_only( etype, value) - for line in lines: - out2.write( '%s %s' % ( prefix, line)) - - text = out2.getvalue() - - # Write text to <out> if specified. - out = getattr( out, 'write', out) - if callable( out): - out( text) - return text - - finally: - # clear things to avoid cycles. - exception = None - etype = None - value = None - tb = None - frames = None - - -def number_sep( s): - ''' - Simple number formatter, adds commas in-between thousands. <s> can - be a number or a string. Returns a string. - ''' - if not isinstance( s, str): - s = str( s) - c = s.find( '.') - if c==-1: c = len(s) - end = s.find('e') - if end == -1: end = s.find('E') - if end == -1: end = len(s) - ret = '' - for i in range( end): - ret += s[i] - if i<c-1 and (c-i-1)%3==0: - ret += ',' - elif i>c and i<end-1 and (i-c)%3==0: - ret += ',' - ret += s[end:] - return ret - -assert number_sep(1)=='1' -assert number_sep(12)=='12' -assert number_sep(123)=='123' -assert number_sep(1234)=='1,234' -assert number_sep(12345)=='12,345' -assert number_sep(123456)=='123,456' -assert number_sep(1234567)=='1,234,567' - - -class Stream: - ''' - Base layering abstraction for streams - abstraction for things like - sys.stdout to allow prefixing of all output, e.g. with a timestamp. - ''' - def __init__( self, stream): - self.stream = stream - def write( self, text): - self.stream.write( text) - -class StreamPrefix: - ''' - Prefixes output with a prefix, which can be a string or a callable that - takes no parameters and return a string. - ''' - def __init__( self, stream, prefix): - self.stream = stream - self.at_start = True - if callable(prefix): - self.prefix = prefix - else: - self.prefix = lambda : prefix - - def write( self, text): - if self.at_start: - text = self.prefix() + text - self.at_start = False - append_newline = False - if text.endswith( '\n'): - text = text[:-1] - self.at_start = True - append_newline = True - text = text.replace( '\n', '\n%s' % self.prefix()) - if append_newline: - text += '\n' - self.stream.write( text) - - def flush( self): - self.stream.flush() - - -def debug( text): - if callable(text): - text = text() - print( text) - -debug_periodic_t0 = [0] -def debug_periodic( text, override=0): - interval = 10 - t = time.time() - if t - debug_periodic_t0[0] > interval or override: - debug_periodic_t0[0] = t - debug(text) - - -def time_duration( seconds, verbose=False, s_format='%i'): - ''' - Returns string expressing an interval. - - seconds: - The duration in seconds - verbose: - If true, return like '4 days 1 hour 2 mins 23 secs', otherwise as - '4d3h2m23s'. - s_format: - If specified, use as printf-style format string for seconds. - ''' - x = abs(seconds) - ret = '' - i = 0 - for div, text in [ - ( 60, 'sec'), - ( 60, 'min'), - ( 24, 'hour'), - ( None, 'day'), - ]: - force = ( x == 0 and i == 0) - if div: - remainder = x % div - x = int( x/div) - else: - remainder = x - if not verbose: - text = text[0] - if remainder or force: - if verbose and remainder > 1: - # plural. - text += 's' - if verbose: - text = ' %s ' % text - if i == 0: - remainder = s_format % remainder - ret = '%s%s%s' % ( remainder, text, ret) - i += 1 - ret = ret.strip() - if ret == '': - ret = '0s' - if seconds < 0: - ret = '-%s' % ret - return ret - -assert time_duration( 303333) == '3d12h15m33s' -assert time_duration( 303333.33, s_format='%.1f') == '3d12h15m33.3s' -assert time_duration( 303333, verbose=True) == '3 days 12 hours 15 mins 33 secs' -assert time_duration( 303333.33, verbose=True, s_format='%.1f') == '3 days 12 hours 15 mins 33.3 secs' - -assert time_duration( 0) == '0s' -assert time_duration( 0, verbose=True) == '0 sec' - - -def date_time( t=None): - if t is None: - t = time.time() - return time.strftime( "%F-%T", time.gmtime( t)) - -def stream_prefix_time( stream): - ''' - Returns StreamPrefix that prefixes lines with time and elapsed time. - ''' - t_start = time.time() - def prefix_time(): - return '%s (+%s): ' % ( - time.strftime( '%T'), - time_duration( time.time() - t_start, s_format='0.1f'), - ) - return StreamPrefix( stream, prefix_time) - -def stdout_prefix_time(): - ''' - Changes sys.stdout to prefix time and elapsed time; returns original - sys.stdout. - ''' - ret = sys.stdout - sys.stdout = stream_prefix_time( sys.stdout) - return ret - - -def make_stream( out): - ''' - If <out> already has a .write() member, returns <out>. - - Otherwise a stream-like object with a .write() method that writes to <out>. - - out: - Where output is sent. - If None, output is lost. - Otherwise if an integer, we do: os.write( out, text) - Otherwise if callable, we do: out( text) - Otherwise we assume <out> is python stream or similar already. - ''' - if getattr( out, 'write', None): - return out - class Ret: - def flush(): - pass - ret = Ret() - if out is None: - ret.write = lambda text: None - elif isinstance( out, int): - ret.write = lambda text: os.write( out, text) - elif callable( out): - ret.write = out - else: - ret.write = lambda text: out.write( text) - return ret - - -def system_raw( - command, - out=None, - shell=True, - encoding='latin_1', - errors='strict', - buffer_len=-1, - ): - ''' - Runs command, writing output to <out> which can be an int fd, a python - stream or a Stream object. - - Args: - command: - The command to run. - out: - Where output is sent. - If None, output is lost. - If -1, output is sent to stdout and stderr. - Otherwise if an integer, we do: os.write( out, text) - Otherwise if callable, we do: out( text) - Otherwise we assume <out> is python stream or similar, and do: out.write(text) - shell: - Whether to run command inside a shell (see subprocess.Popen). - encoding: - Sepecify the encoding used to translate the command's output - to characters. - - Note that if <encoding> is None and we are being run by python3, - <out> will be passed bytes, not a string. - - Note that latin_1 will never raise a UnicodeDecodeError. - errors: - How to handle encoding errors; see docs for codecs module for - details. - buffer_len: - The number of bytes we attempt to read at a time. If -1 we read - output one line at a time. - - Returns: - subprocess's <returncode>, i.e. -N means killed by signal N, otherwise - the exit value (e.g. 12 if command terminated with exit(12)). - ''' - if out == -1: - stdin = 0 - stdout = 1 - stderr = 2 - else: - stdin = None - stdout = subprocess.PIPE - stderr = subprocess.STDOUT - child = subprocess.Popen( - command, - shell=shell, - stdin=stdin, - stdout=stdout, - stderr=stderr, - close_fds=True, - #encoding=encoding - only python-3.6+. - ) - - child_out = child.stdout - if encoding: - child_out = codecs.getreader( encoding)( child_out, errors) - - out = make_stream( out) - - if stdout == subprocess.PIPE: - if buffer_len == -1: - for line in child_out: - out.write( line) - else: - while 1: - text = child_out.read( buffer_len) - if not text: - break - out.write( text) - #decode( lambda : os.read( child_out.fileno(), 100), outfn, encoding) - - return child.wait() - -if __name__ == '__main__': - - if os.getenv( 'jtest_py_system_raw_test') == '1': - out = io.StringIO() - system_raw( - 'jtest_py_system_raw_test=2 python jlib.py', - sys.stdout, - encoding='utf-8', - #'latin_1', - errors='replace', - ) - print( repr( out.getvalue())) - - elif os.getenv( 'jtest_py_system_raw_test') == '2': - for i in range(256): - sys.stdout.write( chr(i)) - - -def system( - command, - verbose=None, - raise_errors=True, - out=None, - prefix=None, - rusage=False, - shell=True, - encoding=None, - errors='replace', - buffer_len=-1, - ): - ''' - Runs a command like os.system() or subprocess.*, but with more flexibility. - - We give control over where the command's output is sent, whether to return - the output and/or exit code, and whether to raise an exception if the - command fails. - - We also support the use of /usr/bin/time to gather rusage information. - - command: - The command to run. - verbose: - If true, we output information about the command that we run, and - its result. - - If callable or something with a .write() method, information is - sent to <verbose> itself. Otherwise it is sent to <out> (without - applying <prefix>). - raise_errors: - If true, we raise an exception if the command fails, otherwise we - return the failing error code or zero. - out: - Python stream, fd, callable or Stream instance to which output is - sent. - - If <out> is 'return', we buffer the output and return (e, - <output>). Note that if raise_errors is true, we only return if <e> - is zero. - - If -1, output is sent to stdout and stderr. - prefix: - If not None, should be prefix string or callable used to prefix - all output. [This is for convenience to avoid the need to do - out=StreamPrefix(...).] - rusage: - If true, we run via /usr/bin/time and return rusage string - containing information on execution. <raise_errors> and - out='return' are ignored. - shell: - Passed to underlying subprocess.Popen() call. - encoding: - Sepecify the encoding used to translate the command's output - to characters. Defaults to utf-8. - errors: - How to handle encoding errors; see docs for codecs module - for details. Defaults to 'replace' so we never raise a - UnicodeDecodeError. - buffer_len: - The number of bytes we attempt to read at a time. If -1 we read - output one line at a time. - - Returns: - If <rusage> is true, we return the rusage text. - - Else if raise_errors is true: - If the command failed, we raise an exception. - Else if <out> is 'return' we return the text output from the command. - Else we return None - - Else if <out> is 'return', we return (e, text) where <e> is the - command's exit code and <text> is the output from the command. - - Else we return <e>, the command's exit code. - ''' - if encoding is None: - if sys.version_info[0] == 2: - # python-2 doesn't seem to implement 'replace' properly. - encoding = None - errors = None - else: - encoding = 'utf-8' - errors = 'replace' - - out_original = out - if out is None: - out = sys.stdout - elif out == 'return': - # Store the output ourselves so we can return it. - out = io.StringIO() - else: - out = make_stream( out) - - if verbose: - if getattr( verbose, 'write', None): - pass - elif callable( verbose): - verbose = make_stream( verbose) - else: - verbose = out - - if prefix: - out = StreamPrefix( out, prefix) - - if verbose: - verbose.write( 'running: %s\n' % command) - - if rusage: - command2 = '' - command2 += '/usr/bin/time -o ubt-out -f "D=%D E=%D F=%F I=%I K=%K M=%M O=%O P=%P R=%r S=%S U=%U W=%W X=%X Z=%Z c=%c e=%e k=%k p=%p r=%r s=%s t=%t w=%w x=%x C=%C"' - command2 += ' ' - command2 += command - system_raw( command2, out, shell, encoding, errors, buffer_len=buffer_len) - with open('ubt-out') as f: - rusage_text = f.read() - #print 'have read rusage output: %r' % rusage_text - if rusage_text.startswith( 'Command '): - # Annoyingly, /usr/bin/time appears to write 'Command - # exited with ...' or 'Command terminated by ...' to the - # output file before the rusage info if command doesn't - # exit 0. - nl = rusage_text.find('\n') - rusage_text = rusage_text[ nl+1:] - return rusage_text - else: - e = system_raw( command, out, shell, encoding, errors, buffer_len=buffer_len) - - if verbose: - verbose.write( '[returned e=%s]\n' % e) - - if raise_errors: - if e: - raise Exception( 'command failed: %s' % command) - if out_original == 'return': - return out.getvalue() - return - - if out_original == 'return': - return e, out.getvalue() - return e - -def get_gitfiles( directory, submodules=False): - ''' - Returns list of all files known to git in <directory>; <directory> must be - somewhere within a git checkout. - - Returned names are all relative to <directory>. - - If .git directory, we also create <directory>/jtest-git-files. Otherwise we - assume a this file already exists. - ''' - if os.path.isdir( '%s/.git' % directory): - command = 'cd ' + directory + ' && git ls-files' - if submodules: - command += ' --recurse-submodules' - command += ' > jtest-git-files' - system( command, verbose=sys.stdout) - - with open( '%s/jtest-git-files' % directory, 'r') as f: - text = f.read() - ret = text.split( '\n') - return ret - -def get_git_id_raw( directory): - if not os.path.isdir( '%s/.git' % directory): - return - text = system( - f'cd {directory} && (PAGER= git show --pretty=oneline|head -n 1 && git diff)', - out='return', - ) - return text - -def get_git_id( directory, allow_none=False): - ''' - Returns text where first line is '<git-sha> <commit summary>' and remaining - lines contain output from 'git diff' in <directory>. - - directory: - Root of git checkout. - allow_none: - If true, we return None if <directory> is not a git checkout and - jtest-git-id file does not exist. - ''' - filename = f'{directory}/jtest-git-id' - text = get_git_id_raw( directory) - if text: - with open( filename, 'w') as f: - f.write( text) - elif os.path.isfile( filename): - with open( filename) as f: - text = f.read() - else: - if not allow_none: - raise Exception( f'Not in git checkout, and no file {filename}.') - text = None - return text - -class Args: - ''' - Iterates over argv items. Does getopt-style splitting of args starting with - single '-' character. - ''' - def __init__( self, argv): - self.argv = argv - self.pos = 0 - self.pos_sub = None - def next( self): - while 1: - if self.pos >= len(self.argv): - raise StopIteration() - arg = self.argv[self.pos] - if (not self.pos_sub - and arg.startswith('-') - and not arg.startswith('--') - ): - # Start splitting current arg. - self.pos_sub = 1 - if self.pos_sub and self.pos_sub >= len(arg): - # End of '-' sub-arg. - self.pos += 1 - self.pos_sub = None - continue - if self.pos_sub: - # Return '-' sub-arg. - ret = arg[self.pos_sub] - self.pos_sub += 1 - return f'-{ret}' - # Return normal arg. - self.pos += 1 - return arg - -def update_file( text, filename): - ''' - Writes <text> to <filename>. Does nothing if contents of <filename> are - already <text>. - ''' - try: - with open( filename) as f: - text0 = f.read() - except OSError: - text0 = None - if text == text0: - log( 'Unchanged: ' + filename) - else: - log( 'Updating: ' + filename) - # Write to temp file and rename, to ensure we are atomic. - filename_temp = f'{filename}-jlib-temp' - with open( filename_temp, 'w') as f: - f.write( text) - os.rename( filename_temp, filename) - - -def mtime( filename, default=0): - ''' - Returns mtime of file, or <default> if error - e.g. doesn't exist. - ''' - try: - return os.path.getmtime( filename) - except OSError: - return default - -def get_filenames( paths): - ''' - Yields each file in <paths>, walking any directories. - ''' - if isinstance( paths, str): - paths = (paths,) - for name in paths: - if os.path.isdir( name): - for dirpath, dirnames, filenames in os.walk( name): - for filename in filenames: - path = os.path.join( dirpath, filename) - yield path - else: - yield name - -def remove( path): - ''' - Removes file or directory, without raising exception if it doesn't exist. - - We assert-fail if the path still exists when we return, in case of - permission problems etc. - ''' - try: - os.remove( path) - except Exception: - pass - shutil.rmtree( path, ignore_errors=1) - assert not os.path.exists( path) - - -# Things for figuring out whether files need updating, using mtimes. -# -def newest( names): - ''' - Returns mtime of newest file in <filenames>. Returns 0 if no file exists. - ''' - assert isinstance( names, (list, tuple)) - assert names - ret_t = 0 - ret_name = None - for filename in get_filenames( names): - t = mtime( filename) - if t > ret_t: - ret_t = t - ret_name = filename - return ret_t, ret_name - -def oldest( names): - ''' - Returns mtime of oldest file in <filenames> or 0 if no file exists. - ''' - assert isinstance( names, (list, tuple)) - assert names - ret_t = None - ret_name = None - for filename in get_filenames( names): - t = mtime( filename) - if ret_t is None or t < ret_t: - ret_t = t - ret_name = filename - if ret_t is None: - ret_t = 0 - return ret_t, ret_name - -def update_needed( infiles, outfiles): - ''' - If any file in <infiles> is newer than any file in <outfiles>, returns - string description. Otherwise returns None. - ''' - in_tmax, in_tmax_name = newest( infiles) - out_tmin, out_tmin_name = oldest( outfiles) - if in_tmax > out_tmin: - text = f'{in_tmax_name} is newer than {out_tmin_name}' - return text - -def build( - infiles, - outfiles, - command, - force_rebuild=False, - out=None, - all_reasons=False, - verbose=True, - prefix=None, - ): - ''' - Ensures that <outfiles> are up to date using enhanced makefile-like - determinism of dependencies. - - Rebuilds <outfiles> by running <command> if we determine that any of them - are out of date. - - infiles: - Names of files that are read by <command>. Can be a single filename. If - an item is a directory, we expand to all filenames in the directory's - tree. - outfiles: - Names of files that are written by <command>. Can also be a single - filename. - command: - Command to run. - force_rebuild: - If true, we always re-run the command. - out: - A callable, passed to jlib.system(). If None, we use jlib.log() with - our caller's stack record. - all_reasons: - If true we check all ways for a build being needed, even if we already - know a build is needed; this only affects the diagnostic that we - output. - verbose: - Passed to jlib.system(). - prefix: - Passed to jlib.system(). - - We compare mtimes of <infiles> and <outfiles>, and we also detect changes - to the command itself. - - If any of infiles are newer than any of outfiles, or <command> is - different to contents of commandfile '<outfile[0]>.cmd, then truncates - commandfile and runs <command>. If <command> succeeds we writes <command> - to commandfile. - ''' - if isinstance( infiles, str): - infiles = (infiles,) - if isinstance( outfiles, str): - infiles = (outfiles,) - - if not out: - out_frame_record = inspect.stack()[1] - out = lambda text: log( text, nv=0, caller=out_frame_record) - - command_filename = f'{outfiles[0]}.cmd' - - reasons = [] - - if not reasons or all_reasons: - if force_rebuild: - reasons.append( 'force_rebuild was specified') - - if not reasons or all_reasons: - try: - with open( command_filename) as f: - command0 = f.read() - except Exception: - command0 = None - if command != command0: - if command0: - reasons.append( 'command has changed') - else: - reasons.append( 'no previous command') - - if not reasons or all_reasons: - reason = update_needed( infiles, outfiles) - if reason: - reasons.append( reason) - - if not reasons: - out( 'Already up to date: ' + ' '.join(outfiles)) - return - - if out: - out( 'Rebuilding because %s: %s' % ( - ', and '.join( reasons), - ' '.join(outfiles), - )) - - # Empty <command_filename) while we run the command so that if command - # fails but still creates target(s), then next time we will know target(s) - # are not up to date. - # - with open( command_filename, 'w') as f: - pass - - system( command, out=out, verbose=verbose, prefix=prefix) - - with open( command_filename, 'w') as f: - f.write( command) - - -def link_l_flags( sos): - ''' - Returns flags needed to link with items in <sos>. For each unique item we - use -L with parent directory, and -l with embedded name (without leading - 'lib' or trailing '.co'). - ''' - dirs = set() - names = [] - if isinstance( sos, str): - sos = (sos,) - for so in sos: - dir_ = os.path.dirname( so) - name = os.path.basename( so) - assert name.startswith( 'lib') - assert name.endswith ( '.so') - name = name[3:-3] - dirs.add( dir_) - names.append( name) - ret = '' - # Important to use sorted() here, otherwise ordering from set() is - # arbitrary causing occasional spurious rebuilds by jlib.build(). - for dir_ in sorted(dirs): - ret += f' -L {dir_}' - for name in names: - ret += f' -l {name}' - return ret |