diff options
| -rw-r--r-- | grumpy/vdb/__init__.py | 5 | ||||
| -rw-r--r-- | grumpy/vdb/gnome.py | 58 | ||||
| -rw-r--r-- | grumpy/vdb/pypi.py | 7 | ||||
| -rw-r--r-- | utils/version_check.py | 65 |
4 files changed, 96 insertions, 39 deletions
diff --git a/grumpy/vdb/__init__.py b/grumpy/vdb/__init__.py index e69de29..1a871fa 100644 --- a/grumpy/vdb/__init__.py +++ b/grumpy/vdb/__init__.py @@ -0,0 +1,5 @@ +from .pypi import PyPi +from .gnome import GNOME + +# We need some intelligent lookup mechanism ;) +modules = ['PyPi', 'GNOME'] diff --git a/grumpy/vdb/gnome.py b/grumpy/vdb/gnome.py new file mode 100644 index 0000000..b7bd799 --- /dev/null +++ b/grumpy/vdb/gnome.py @@ -0,0 +1,58 @@ +from lxml.html import fromstring +import string, urllib2 + +class GNOME(object): + # TODO: Shall we use some kind of yaml or json based format instead? + # Or move this data into DB? + pkgs = { + # Package in Portage : Package in GNOME + 'gnome-base/gconf' : 'platform:GConf', + 'gnome-base/orbit' : 'platform:ORBit2', + 'gnome-extra/at-spi' : 'platform:at-spi', + 'dev-libs/atk' : 'platform:atk', + 'media-libs/audiofile' : 'platform:audiofile', + 'media-sound/esound' : 'platform:esound', + 'dev-libs/glib' : 'platform:glib', + 'gnome-base/gnome-mime-data': 'platform:gnome-mime-data', + 'gnome-base/gnome-vfs' : 'platform:gnome-vfs', + 'x11-libs/gtk+' : 'platform:gtk+', + 'dev-util/gtk-doc' : 'platform:gtk-doc', + 'dev-libs/libIDL' : 'platform:libIDL', + 'media-libs/libart_lgpl' : 'platform:libart_lgpl', + 'gnome-base/libbonobo' : 'platform:libbonobo', + 'gnome-base/libbonoboui' : 'platform:libbonoboui', + 'gnome-base/libglade' : 'platform:libglade', + 'gnome-base/libgnome' : 'platform:libgnome', + 'gnome-base/libgnomecanvas' : 'platform:libgnomecanvas', + 'gnome-base/libgnomeui' : 'platform:libgnomeui', + 'x11-libs/pango' : 'platform:pango', + # missing: platform:gnome-vfs-monikers + # + # TODO: desktop, admin, devtools, bindings + } + + # Url for fetching version information + url = 'http://ftp.gnome.org/pub/GNOME' + #teams/releng/2.30.2/versions' + ids = ('platform', 'desktop', 'admin', 'devtools', 'bindings') + + def __init__(self): + pass + + def fetch_and_parse_all(self): + """Download and parse package version information.""" + items = {} + # Read the latest release tarball versions using SHA256SUMS-files + for x in self.ids: + data = urllib2.urlopen('%s/%s/2.30/2.30.2/sources/SHA256SUMS-for-bz2' % \ + (self.url, x)) + for line in data.readlines(): + # Extract tarball names with versions + rawline = line.strip().split(' ')[1][:-len('.tar.bz2')] + pkg, ver = rawline.rsplit('-', 1) + items["%s:%s" % (x, pkg)] = [ver] + data.close() + return items + +if __name__ == '__main__': + print GNOME().fetch_and_parse_all() diff --git a/grumpy/vdb/pypi.py b/grumpy/vdb/pypi.py index 2e27059..eaee00f 100644 --- a/grumpy/vdb/pypi.py +++ b/grumpy/vdb/pypi.py @@ -29,8 +29,8 @@ class PyPi(object): """Download and parse package version information.""" items = {} - #f = urllib.urlopen(self.url) - f = open('utils/pypi.html') + f = urllib.urlopen(self.url) + #f = open('utils/pypi.html') if f: data = fromstring(f.read()).cssselect('table.list')[0] for row in data.getchildren(): @@ -43,3 +43,6 @@ class PyPi(object): items[pkg[0]].append(pkg[1]) f.close() return items + +if __name__ == '__main__': + print PyPi().fetch_and_parse_all() diff --git a/utils/version_check.py b/utils/version_check.py index 20d1caa..eac7dad 100644 --- a/utils/version_check.py +++ b/utils/version_check.py @@ -1,61 +1,52 @@ #! /usr/bin/env python import os, sys from datetime import datetime -from lxml.html import fromstring from optparse import OptionParser +from werkzeug.utils import import_string path = os.path.join(os.path.dirname(__file__), os.path.pardir) sys.path.insert(0, path) del path -from grumpy import app +from grumpy import app, vdb from grumpy.models import db, Package, PkgIssue, Setting from grumpy.utils import compare_version PLUGIN_NAME='ver_bumper::' -def gc_collect(timestamp): - """Remove old QA issues from database returning number of rows deleted.""" - db.session.expire_all() - print ("DEBUG: Deleted %d old issues." % PkgIssue.query \ - .filter_by(plugin=PLUGIN_NAME) \ - .filter(PkgIssue.created_on < timestamp).delete(False)) - -def insert_issues(invalid): - """Insert QA issues into db.""" - if 'maintainer-needed@gentoo.org' in invalid: - h = Developer.query.filter_by(email='maintainer-needed@gentoo.org').first() - for pkg in h.packages: - pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'maintainer-needed')) - invalid.remove('maintainer-needed@gentoo.org') - for dev in invalid: - d = Developer.query.filter_by(email=dev).first() - for pkg in d.packages: - pkg.qaissues.append(PkgIssue(pkg, PLUGIN_NAME, 'unknown-dev', \ - 'Maintainer %s listed in metadata.xml is not as ' \ - 'an active Gentoo developer.' % dev)) - db.session.commit() - if __name__ == '__main__': - from grumpy.vdb.pypi import PyPi - parser = OptionParser(usage="usage: %prog [options] CONFFILE") + parser = OptionParser(usage="usage: %prog [options] CONFFILE module") (opts, args) = parser.parse_args() - if len(args) != 1: - parser.error("provide path to configuration file as first argument") + if len(args) != 2: + parser.error("provide path to configuration file as first argument\n" \ + "package module you want to load as second argument.\n\n" \ + "Supported modules are:\n\t%s" % \ + ('\n\t'.join(vdb.modules))) + sys.exit(1) + # Check whether module is supported + mod = args[1] + if mod not in vdb.modules: + parser.error("Unknown module: %s\n" \ + "Supported modules are:\n\t%s" % \ + (mod, '\n\t'.join(vdb.modules))) sys.exit(1) - # ... - pypi = PyPi() - items = pypi.fetch_and_parse_all() + # Import our module + m = import_string('grumpy.vdb:%s' % mod, True) + if not m: + print "Unknown error occurred: unable to import module %s" % mod + raise RuntimeError + module = m() + items = module.fetch_and_parse_all() # Gather package versions updates = {} missing = [] - for p in pypi.pkgs: - pyp = pypi.pkgs[p] + for p in module.pkgs: + pyp = module.pkgs[p] if pyp not in items.keys(): missing.append([p, pyp]) else: updates[p] = items[pyp] - PLUGIN_NAME = PLUGIN_NAME + pypi.__class__.__name__ + PLUGIN_NAME = PLUGIN_NAME + module.__class__.__name__ with app.test_request_context(): timestamp = datetime.now() app.config.from_pyfile(args[0]) @@ -75,7 +66,7 @@ if __name__ == '__main__': p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \ 'missing-upstream', \ 'No package "%s" found in upstream: %s' % \ - (item, pypi.__class__.__name__))) + (item, module.__class__.__name__))) # Run version checks # TODO: Figure out slotted ebuilds for pkg, ver in updates.iteritems(): @@ -90,12 +81,12 @@ if __name__ == '__main__': p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \ 'version-mismatch', 'Package "%s" has newer version in ' \ 'Portage than on "%s" upstream' % \ - (pkg, pypi.__class__.__name__))) + (pkg, module.__class__.__name__))) elif res == 1: p.qaissues.append(PkgIssue(p, PLUGIN_NAME, \ 'version-bump', 'Package "%s" has newer version "%s" ' 'available in "%s" upstream ' % \ - (pkg, p.versions[0], pypi.__class__.__name__))) + (pkg, p.versions[0], module.__class__.__name__))) db.session.commit() # Update settings and add info about last run.. Setting.query.filter_by(name=PLUGIN_NAME).delete(False) |