diff options
| -rw-r--r-- | config/simple-deprules.conf | 8 | ||||
| -rw-r--r-- | man/roverlay-deprules.5 | 9 | ||||
| -rw-r--r-- | roverlay/depres/depresolver.py | 15 | ||||
| -rw-r--r-- | roverlay/depres/deprule.py | 12 | ||||
| -rw-r--r-- | roverlay/depres/deptype.py | 6 | ||||
| -rw-r--r-- | roverlay/depres/simpledeprule/console.py | 24 | ||||
| -rw-r--r-- | roverlay/depres/simpledeprule/pool.py | 25 | ||||
| -rw-r--r-- | roverlay/depres/simpledeprule/reader.py | 54 | ||||
| -rw-r--r-- | roverlay/depres/simpledeprule/rulemaker.py | 68 | ||||
| -rw-r--r-- | roverlay/ebuild/depres.py | 6 | ||||
| -rw-r--r-- | roverlay/recipe/easyresolver.py | 6 |
11 files changed, 134 insertions, 99 deletions
diff --git a/config/simple-deprules.conf b/config/simple-deprules.conf index b8fc3b4..3c15bac 100644 --- a/config/simple-deprules.conf +++ b/config/simple-deprules.conf @@ -157,3 +157,11 @@ dev-lang/R { # additional content at the end of the line. #! NOPARSE #! BREAK + +# There's also a keyword to set the dependency type (sys, pkg or all), +# the syntax is '#deptype <type>'. +#deptype sys +#deptype pkg +#deptype all + + diff --git a/man/roverlay-deprules.5 b/man/roverlay-deprules.5 index 2e1ce98..e41beda 100644 --- a/man/roverlay-deprules.5 +++ b/man/roverlay-deprules.5 @@ -65,8 +65,13 @@ where \fIshort_dependency\fR is a package name (\fBDEPEND atom\fR without versio This is the only way to specify selfdeps. .SS "ADDITIONAL SYNTAX" -Comments start with \'#\'. -.PP +Comments start with \'#\', with a few exceptions: +.TP +dependency type +There's a keyword to set the dependency type of all rules following this statement, +\'#deptype <type>\' where type is \'sys\', \'pkg\' or \'all\'. +.TP +stop reading There are two special keywords that both cause reading a rule file to stop, \'#! BREAKPARSE\' and \'#! NOPARSE\'. The line must exactly be one of these keywords (no additional content at the end of the line). diff --git a/roverlay/depres/depresolver.py b/roverlay/depres/depresolver.py index 40d696e..06ba6fc 100644 --- a/roverlay/depres/depresolver.py +++ b/roverlay/depres/depresolver.py @@ -14,7 +14,7 @@ except ImportError: from roverlay import config from roverlay.depres import communication, deptype, events -#from roverlay.depres import simpledeprule +import roverlay.depres.simpledeprule.reader # if false: do not use the "negative" result caching which stores @@ -39,7 +39,7 @@ class DependencyResolver ( object ): 'RESOLVED', 'UNRESOLVABLE' ) - self._jobs = config.get ( "DEPRES.jobcount", 1 ) + self._jobs = config.get ( "DEPRES.jobcount", 0 ) # used to lock the run methods, self._runlock = threading.Lock() @@ -105,11 +105,12 @@ class DependencyResolver ( object ): self._sort() # --- end of _new_rulepools_added (...) --- - #def get_reader ( self ): - # return simpledeprule.reader.SimpleDependencyRuleReader ( - # pool_add=self.static_rule_pools.append, - # when_done=self._new_rulepools_added - # ) + def get_reader ( self ): + return roverlay.depres.simpledeprule.reader.SimpleDependencyRuleReader ( + pool_add=self.static_rule_pools.append, + when_done=self._new_rulepools_added + ) + # --- end of get_reader (...) --- def add_rulepool ( self, rulepool, pool_type=None ): """Adds a (static) rule pool to this resolver. diff --git a/roverlay/depres/deprule.py b/roverlay/depres/deprule.py index 2b5754f..50b921a 100644 --- a/roverlay/depres/deprule.py +++ b/roverlay/depres/deprule.py @@ -28,7 +28,7 @@ class DependencyRule ( object ): class DependencyRulePool ( object ): - def __init__ ( self, name, priority, deptype_mask ): + def __init__ ( self, name, priority, deptype_mask, initial_rules=None ): """Initializes an DependencyRulePool, which basically is a set of dependency rules with methods like "search for x in all rules." @@ -36,7 +36,10 @@ class DependencyRulePool ( object ): * name -- name of this rule pool * priority -- priority of this pool (lower is better) """ - self.rules = list() + if initial_rules is None: + self.rules = list() + else: + self.rules = list ( initial_rules ) self._rule_add = self.rules.append self.name = name self.priority = priority @@ -48,6 +51,11 @@ class DependencyRulePool ( object ): self.rule_weight = 0 # --- end of __init__ (...) --- + def empty ( self ): + """Returns True if this pool has no rules.""" + return len ( self.rules ) == 0 + # --- end of empty (...) --- + def sort ( self ): """Sorts this rule pool and determines its weight which is used to compare rule pools. diff --git a/roverlay/depres/deptype.py b/roverlay/depres/deptype.py index 73d6265..e3938ba 100644 --- a/roverlay/depres/deptype.py +++ b/roverlay/depres/deptype.py @@ -10,10 +10,12 @@ mandatory = 2 external = 4 internal = 8 +_MAX = 15 + ALL = external | internal | mandatory RESOLVE_ALL = external | internal -SYS = internal | mandatory -PKG = external | mandatory +SYS = external | mandatory +PKG = internal | mandatory MANDATORY_TRY = try_other | mandatory diff --git a/roverlay/depres/simpledeprule/console.py b/roverlay/depres/simpledeprule/console.py index 5d84010..2677a44 100644 --- a/roverlay/depres/simpledeprule/console.py +++ b/roverlay/depres/simpledeprule/console.py @@ -4,8 +4,8 @@ import sys import shlex import logging -from roverlay import config -from roverlay.errorqueue import NopErrorQueue +from roverlay import config, util +from roverlay.errorqueue import ErrorQueue from roverlay.depres import deptype from roverlay.depres.depresolver import DependencyResolver from roverlay.depres.channels import EbuildJobChannel @@ -42,14 +42,14 @@ class PackageDirRuleMaker ( object ): if self.fuzzy: for dep in self._scan ( distdir ): yield rules.SimpleFuzzyDependencyRule ( - resolving_package = cat + dep, + resolving_package = util.fix_ebuild_name ( cat + dep ), dep_str = dep, is_selfdep=True ) else: for dep in self._scan ( distdir ): yield rules.SimpleDependencyRule ( - resolving_package = cat + dep, + resolving_package = util.fix_ebuild_name ( cat + dep ), dep_str = dep, is_selfdep=True ) @@ -65,7 +65,7 @@ class DepResConsole ( object ): whitespace = re.compile ( "\s+" ) def __init__ ( self ): - self.err_queue = NopErrorQueue() + self.err_queue = ErrorQueue() self.resolver = DependencyResolver ( err_queue=self.err_queue ) # log everything @@ -125,7 +125,7 @@ class DepResConsole ( object ): # --- end of __init__ (...) --- def _getpool ( self, new=False ): - if new or not self.poolstack: + if not self.poolstack or ( new and not self.poolstack [-1].empty() ): pool = SimpleDependencyRulePool ( "pool" + str ( self.pool_id ), deptype_mask=deptype.RESOLVE_ALL @@ -169,14 +169,14 @@ class DepResConsole ( object ): def cmd_rule_load_from_config ( self, *ignored ): load = config.get_or_fail ( "DEPRES.simple_rules.files" ) - self._getpool ( new=True ).get_reader().read ( load ) + self.resolver.get_reader().read ( load ) # don't write into ^this pool self._getpool ( new=True ) # --- end of cmd_rule_load_from_config (...) --- def cmd_rule_load ( self, argv, line ): if argv: - self._getpool().get_reader().read ( argv ) + self.resolver.get_reader().read ( argv ) else: self.stdout ( "usage is load/li <files or dirs>\n" ) # --- end of cmd_rule_load (...) --- @@ -186,14 +186,14 @@ class DepResConsole ( object ): self.stdout ( "usage is <cmd> <rule>\n" ) elif self._rule_maker.add ( line ): rules = self._rule_maker.done() + pool = self._getpool() self.stdout ( "new rules:\n" ) - for r in rules: + for _deptype, r in rules: self.stdout ( str ( r ) + "\n" ) + pool.rules.append ( r ) self.stdout ( "--- ---\n" ) - pool = self._getpool() - pool.rules.extend ( rules ) pool.sort() self.resolver._reset_unresolvable() @@ -374,7 +374,7 @@ class DepResConsole ( object ): self.stdout ( "Trying to resolve {!r}.\n".format ( dep_list ) ) - channel.add_dependencies ( dep_list ) + channel.add_dependencies ( dep_list, deptype.ALL ) deps = channel.satisfy_request ( close_if_unresolvable=False, preserve_order=True diff --git a/roverlay/depres/simpledeprule/pool.py b/roverlay/depres/simpledeprule/pool.py index 4d7905b..aa5298b 100644 --- a/roverlay/depres/simpledeprule/pool.py +++ b/roverlay/depres/simpledeprule/pool.py @@ -2,14 +2,10 @@ # Copyright 2006-2012 Gentoo Foundation # Distributed under the terms of the GNU General Public License v2 -import logging - from roverlay.depres import deprule -from roverlay.depres.simpledeprule.reader import SimpleDependencyRuleReader from roverlay.depres.simpledeprule.abstractrules import SimpleRule class SimpleDependencyRulePool ( deprule.DependencyRulePool ): - def __init__ ( self, name, priority=70, **kw ): """Initializes a SimpleDependencyRulePool, which is a DependencyRulePool specialized in simple dependency rules; @@ -38,25 +34,6 @@ class SimpleDependencyRulePool ( deprule.DependencyRulePool ): raise Exception ( "bad usage (simple dependency rule expected)." ) # --- end of add (...) --- - def get_reader ( self ): - # trusting in SimpleDependencyRuleReader's correctness, - # let it add rules directly without is-instance checks - return SimpleDependencyRuleReader ( rule_add=self._rule_add ) - # --- end of get_reader (...) --- - - def load_rule_file ( self, filepath ): - """Loads a rule file and adds the read rules to this pool. - - arguments: - * filepath -- file to read - """ - logging.error ( "load_rule_file(***) is deprecated, use get_reader()!" ) - new_rules = SimpleDependencyRuleReader().read_file ( filepath ) - for rule in new_rules: - self.add ( rule ) - - # --- end of load_rule_file (...) --- - def export_rules ( self, fh ): """Exports all rules from this pool into the given file handle. @@ -68,6 +45,4 @@ class SimpleDependencyRulePool ( deprule.DependencyRulePool ): for rule in self.rules: fh.write ( str ( rule ) ) fh.write ( '\n' ) - # --- end of export_rules (...) --- - diff --git a/roverlay/depres/simpledeprule/reader.py b/roverlay/depres/simpledeprule/reader.py index abb16a6..edc04b0 100644 --- a/roverlay/depres/simpledeprule/reader.py +++ b/roverlay/depres/simpledeprule/reader.py @@ -14,7 +14,7 @@ from roverlay.depres.simpledeprule.rulemaker import SimpleRuleMaker class SimpleDependencyRuleReader ( object ): """SimpleDependencyRuleReader is a SimpleRuleMaker frontend for files.""" - def __init__ ( self, rule_add=None ): + def __init__ ( self, pool_add=None, when_done=None ): """ A SimpleDependencyRuleReader reads such rules from a file.""" self.logger = logging.getLogger ( self.__class__.__name__ ) @@ -26,27 +26,39 @@ class SimpleDependencyRuleReader ( object ): self._mimetypes = mimetypes.MimeTypes() self.guess_ftype = self._mimetypes.guess_type - self.rule_add = rule_add + self._pool_add = pool_add + self._when_done = when_done + + self._fcount = 0 + # --- end of __init__ (...) --- def read ( self, files_or_dirs ): - if self.rule_add is None: - raise AssertionError ( "Rule pool expected, but rule_add is None." ) + if self._pool_add is None: + raise AssertionError ( "Resolver, but pool_add is None." ) for k in files_or_dirs: if os.path.isdir ( k ): - self.read_dir ( k ) + # without recursion + for fname in os.listdir ( k ): + f = k + os.sep + fname + if os.path.isfile ( f ): + self.read_file ( f ) else: - self.read_file ( k ) + self._read_file ( k ) + rule_count, pools = self._rmaker.done ( as_pool=True ) + self.logger.debug ( "Read {} rules in {} files.".format ( + rule_count, self._fcount + ) ) + if self._pool_add is not None: + for p in pools: self._pool_add ( p ) - def read_dir ( self, _dir ): - # without recursion - for fname in os.listdir ( _dir ): - f = _dir + os.sep + fname - if os.path.isfile ( f ): - self.read_file ( f ) - # --- end of read_dir (...) --- + if self._when_done is not None: + self._when_done() + else: + return pools + # --- end of read (...) --- def read_file ( self, filepath ): """Reads a file that contains simple dependency rules @@ -59,6 +71,8 @@ class SimpleDependencyRuleReader ( object ): # line number is used for logging lineno = 0 + self._fcount += 1 + try: self.logger.debug ( "Reading simple dependency rule file %{!r}.".format ( filepath ) @@ -101,20 +115,6 @@ class SimpleDependencyRuleReader ( object ): if fh: fh.close() - rules = self._rmaker.done() - - self.logger.info ( - "{}: read {} dependency rules (in {} lines)".format ( - filepath, len ( rules ), lineno - ) - ) - - if self.rule_add is not None: - for rule in rules: - self.rule_add ( rule ) - else: - return rules - except IOError as ioerr: if lineno: self.logger.error ( diff --git a/roverlay/depres/simpledeprule/rulemaker.py b/roverlay/depres/simpledeprule/rulemaker.py index e1ebd6d..d70320f 100644 --- a/roverlay/depres/simpledeprule/rulemaker.py +++ b/roverlay/depres/simpledeprule/rulemaker.py @@ -7,14 +7,12 @@ import logging from roverlay import config -#from roverlay.depres import deptype +from roverlay.depres import deptype from roverlay.depres.simpledeprule import rules from roverlay.depres.simpledeprule.abstractrules import * +from roverlay.depres.simpledeprule.pool import SimpleDependencyRulePool class SimpleRuleMaker ( object ): - -# class RuleTypes ( object ): pass - class RuleKeywords ( object ): def __init__ ( self ): self._default_rule, self._rule_map = rules.get_rule_map() @@ -50,9 +48,12 @@ class SimpleRuleMaker ( object ): self.multiline_stop = '}' self.comment_char = '#' self._kw = self.__class__.RuleKeywords() + # deptype_kw is '#deptype' (this keyword requires comment 'mode') + self.deptype_kw = 'deptype' + self._deptype = deptype.ALL self._next = None + # [ ( deptype, rule ), ... ] self._rules = list() - #self._rules = list() :: ( deptype, rule ) # --- end of __init__ (...) --- def zap ( self ): @@ -64,12 +65,38 @@ class SimpleRuleMaker ( object ): self._rules = list() # --- end of zap (...) --- - def done ( self ): - retrules = self._rules + def done ( self, as_pool=False ): + rule_count = len ( self._rules ) + if as_pool: + poolmap = dict() + for dtype, rule in self._rules: + if dtype not in poolmap: + poolmap [dtype] = SimpleDependencyRulePool ( + name=str ( id ( self ) ), + deptype_mask=dtype + ) + poolmap [dtype].add ( rule ) + ret = ( rule_count, tuple ( poolmap.values() ) ) + else: + ret = self._rules self.zap() - return retrules + return ret # --- end of done (...) --- + def _get_deptype ( self, t ): + if len ( t ) == 0 or t == 'all': + return deptype.ALL + elif t == 'sys': + return deptype.external + elif t == 'pkg': + return deptype.internal + else: + try: + return int ( t ) + except ValueError: + return None + # --- end of _get_deptype (...) --- + def _single_line_rule ( self, dep, dep_str='' ): # single line rule, either selfdep, # e.g. '~zoo' -> fuzzy sci-R/zoo :: zoo @@ -101,7 +128,7 @@ class SimpleRuleMaker ( object ): return False new_rule.done_reading() - self._rules.append ( new_rule ) + self._rules.append ( ( self._deptype, new_rule ) ) return True # --- end of _single_line_rule (...) --- @@ -112,25 +139,38 @@ class SimpleRuleMaker ( object ): if line [0] == self.multiline_stop: # end of a multiline rule, # add rule to rules and set next_rule to None - self._next.done_reading() + self._next [1].done_reading() self._rules.append ( self._next ) self._next = None else: # new resolved str - self._next.add_resolved ( line ) + self._next [1].add_resolved ( line ) return True elif line [0] == self.comment_char: - # comment - # it is intented that multi line rules cannot contain comments + if line [ 1 : 1 + len ( self.deptype_kw ) ] == self.deptype_kw : + # changing deptype ("#deptype <type>") + dtype_str = line [ len ( self.deptype_kw ) + 2 : ].lstrip().lower() + dtype = self._get_deptype ( dtype_str ) + if dtype is not None: + self._deptype = dtype + else: + raise AssertionError ( + "Expected deptype, but got {!r}.".format ( dtype_str ) + ) + # else is a comment, + # it's intented that multi line rules cannot contain comments return True elif len ( line ) > 1 and line [-1] == self.multiline_start: l = line [:-1].rstrip() rule_class, resolving = self._kw.lookup ( l ) - self._next = rule_class ( resolving_package=resolving ) + self._next = ( + self._deptype, + rule_class ( resolving_package=resolving ) + ) return True else: diff --git a/roverlay/ebuild/depres.py b/roverlay/ebuild/depres.py index 5a85bb4..9931093 100644 --- a/roverlay/ebuild/depres.py +++ b/roverlay/ebuild/depres.py @@ -18,17 +18,17 @@ FIELDS_TO_EVAR = { FIELDS = { # "The Depends field gives a comma-separated # list of >>package names<< which this package depends on." - 'Depends' : deptype.SYS, + 'Depends' : deptype.PKG, # "Other dependencies (>>external to the R system<<) # should be listed in the SystemRequirements field" - 'SystemRequirements' : deptype.PKG, + 'SystemRequirements' : deptype.SYS, # "The Imports field lists >>packages<< whose namespaces # are imported from (as specified in the NAMESPACE file) # but which do not need to be attached." 'Imports' : deptype.PKG, # "The Suggests field uses the same syntax as Depends # and lists >>packages<< that are >>not necessarily needed<<." - 'Suggests' : deptype.external, + 'Suggests' : deptype.internal, # "A package that wishes to make use of header files # in other >>packages<< needs to declare them as # a comma-separated list in the field LinkingTo in the DESCRIPTION file." diff --git a/roverlay/recipe/easyresolver.py b/roverlay/recipe/easyresolver.py index 1ed75fa..bf3ff03 100644 --- a/roverlay/recipe/easyresolver.py +++ b/roverlay/recipe/easyresolver.py @@ -11,12 +11,8 @@ def setup ( err_queue ): res.set_logmask ( -1 ) srule_files = config.get ( 'DEPRES.simple_rules.files', None ) - if srule_files: - srule_pool = SimpleDependencyRulePool ( 'default pool', priority=45, deptype_mask=deptype.RESOLVE_ALL, ) - srule_pool.get_reader().read ( srule_files ) - - res.add_rulepool ( srule_pool ) + res.get_reader().read ( srule_files ) unres_file = config.get ( 'LOG.FILE.unresolvable', None ) if unres_file: |