#!/usr/bin/env python import os, sys from datetime import datetime from fnmatch import fnmatch from optparse import OptionParser from pkgcore.config import load_config from pkgcore.cache import metadata from pkgcore.ebuild import repository from snakeoil.fileutils import iter_read_bash path = os.path.join(os.path.dirname(__file__), os.path.pardir) sys.path.insert(0, path) del path from grumpy import app from grumpy.models import (db, Category, Developer, Ebuild, Herd, \ Package, Setting) UPDATE_DB_KEY = 'updates_info' def main(path): # pkgcore part to fetch all the ebuild information conf = load_config() eclass_cache = conf.eclass_cache['eclass stack'] cache = metadata.database(readonly=True, location=path) repo = repository.UnconfiguredTree(path, cache=cache, \ eclass_cache=eclass_cache) def get_version(pkg, cpv): """...""" return cpv[len(pkg)+1:-7] def package_move(src, dst): """Rename package and its ebuild in database.""" raise NotImplementedError srccat, srcpkg = src.split('/') dstcat, dstpkg = dst.split('/') print "DEBUG: Handling package move: %s/%s -> %s/%s" % \ (srccat, srcpkg, dstcat, dstpkg) package = Package.query.filter_by(cat=srccat) \ .filter_by(pkg=srcpkg).first() if package: package.rename(dstcat, dstpkg) # Read package move information movedir = os.path.join(path, 'profiles', 'updates') if not os.path.isdir(movedir): print "DEBUG: Missing package moves directory: '%s'" % movedir raise RuntimeError get_key = lambda name: tuple(reversed(name.split('-'))) move_files = sorted(os.listdir(movedir), key=get_key) update_file = move_files[-1] update_path = os.path.join(movedir, update_file) # Fetch existing setting data from database... prev_updates = Setting.query.filter_by(name=UPDATE_DB_KEY).first() if prev_updates: # Check for filename and mtime update_data = prev_updates.data if update_data['file'] == update_file: if update_data['mtime'] == int(os.stat(update_path).st_mtime): # update_file has not been changed, so fall through pass else: # Fetch new move rules for line in iter_read_bash(update_path): line = line.split() if line[0] == 'move' and \ line[1] not in update_data['moves'].keys(): package_move(line[1], line[2]) # Handle new update files else: # Firstly, handle updates from current file... for line in iter_read_bash( \ os.path.join(movedir, update_data['file'])): line = line.split() if line[0] == 'move' and \ line[1] not in update_data['moves'].keys(): package_move(line[1], line[2]) # ...and then move on to newer ones for file in move_files[move_files.index(update_data['file']):]: for line in iter_read_bash(os.path.join(movedir, file)): line = line.split() if line[0] == 'move': package_move(line[1], line[2]) # Parse (again) latest moves file and store its info in database if prev_updates: db.session.delete(prev_updates) db.session.flush() moves = {} for line in iter_read_bash(update_path): line = line.split() if line[0] == 'move': moves[line[1]] = line[2] data = dict(file=update_file, moves=moves, \ mtime=int(os.stat(update_path).st_mtime)) db.session.add(Setting(UPDATE_DB_KEY, data)) db.session.commit() def package_sync(cat, pkg, files, mtime): """Update package information in database.""" # Fetch package from database p = Package.query.filter_by(category=cat,pkg=pkg).first() # Check whether package in database is up-to-date if p and p.mtime == datetime.fromtimestamp(mtime): assert len(p.ebuilds) == len(files) return print "DEBUG: updating package %s/%s" % (cat.name, pkg) # TODO: get rid of str(cat.name) after pkgcore learns about unicode ebuilds = [repo[(str(cat.name), pkg, get_version(pkg, f))] for f in files] # If package exists, remove outdated ebuilds if p: old = [p.ebuilds[e].cpv for e in p.ebuilds] new = [e.cpvstr for e in ebuilds] for ver in [item for item in old if item not in new]: db.session.delete(p.ebuilds[ver]) # Sync package info cat.packages[p.key].sync(ebuilds[0], mtime) # otherwise, create package else: p = cat.packages[ebuilds[0].key] = Package(ebuilds[0], mtime) # Sync new versions of ebuilds for ebuild in ebuilds: if ebuild.cpvstr in p.ebuilds.keys(): p.ebuilds[ebuild.cpvstr].sync(ebuild) else: p.ebuilds[ebuild.cpvstr] = Ebuild(ebuild) db.session.commit() # Compare list of categories in portage vs database old = [c.name for c in Category.query.all()] cats = repo.categories.keys() # Remove old categories for cat in [item for item in old if item not in cats]: c = Category.query.filter_by(name=cat).one() if Package.query.filter_by(category=c).count() > 0: # We shouldn't have anything with this category in db raise RuntimeError db.session.delete(c) # Add new categories for cat in cats: if cat not in old: db.session.add(Category(cat)) db.session.commit() # Traverse portage for cat in cats: catdir = os.path.join(path, cat) # Get list of existing packages in this category c = Category.query.filter_by(name=cat).one() old = [p.pkg for p in Package.query.filter_by(category=c).all()] new = [d for d in os.listdir(catdir) \ if os.path.isdir(os.path.join(catdir, d))] for pkg in [i for i in old if i not in new]: # Handle package deletion print "DEBUG: package has been removed:", pkg package = Package.query.filter_by(category=c, pkg=pkg).one() db.session.delete(package) db.session.commit() for pkg in new: dir = os.path.join(catdir, pkg) files = [f for f in os.listdir(dir) if fnmatch(f, '*.ebuild')] package_sync(c, pkg, files, int(os.stat(dir).st_mtime)) if __name__ == '__main__': parser = OptionParser(usage="usage: %prog [options] CONFFILE") (opts, args) = parser.parse_args() if len(args) != 1: parser.error("provide path to configuration file as first argument") sys.exit(1) with app.test_request_context(): app.config.from_pyfile(args[0]) if 'GRUMPY_PORTAGE_DIR' in app.config.keys(): portagedir = app.config['GRUMPY_PORTAGE_DIR'] if os.path.isdir(portagedir): main(portagedir)