diff options
| -rw-r--r-- | slave/TODO | 6 | ||||
| -rw-r--r-- | slave/autotua/__init__.py | 19 | ||||
| -rw-r--r-- | slave/autotua/chroot/__init__.py | 2 | ||||
| -rw-r--r-- | slave/autotua/const.py | 6 | ||||
| -rw-r--r-- | slave/autotua/sync/__init__.py | 83 | ||||
| -rwxr-xr-x | slave/test_modules.py | 3 |
6 files changed, 80 insertions, 39 deletions
@@ -4,6 +4,6 @@ Do: ----- -- Separate module for running commands -- Atom parsing, jobuild searching, etc etc -- Cleanup autotua/__init__.py:Job() +[ ] Separate module for running commands +[~] Atom parsing, jobuild searching, etc etc +[ ] Auto-detect SCM of source jobtage repo for syncer.Sync().sync() diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py index af16e41..a21934e 100644 --- a/slave/autotua/__init__.py +++ b/slave/autotua/__init__.py @@ -92,19 +92,22 @@ class Job: src_uri = jobuild.get_var('SRC_URI').split() for i in src_uri: job_src.append(fetch.Fetchable(uri=[i])) - for i in job_src: - fetcher.fetch(uri=[i]) + for fetchable in job_src: + fetcher.fetch(fetchable) def fetch(self): # Job metadata stuff ## Get stage3 (if required) + print 'Fetching stage...' fetcher = fetch.Fetcher(const.STAGE_DIR) fetcher.fetch(self.stage) # Sync jobtage tree + print 'Syncing jobtage tree...' sync.Syncer().sync() # Export from local jobtage tree + print 'Exporting jobtage tree...' sync.Syncer(uri=const.JOBTAGE_DIR, destdir=osp.join(self.jobdir, 'jobtage'), - rev=self.jobtagerev, scheme="bzr-export").sync() + rev=self.jobtagerev, scheme="git-export").sync() ## Read config, get portage snapshot if required #self._fetch_portage_snapshot() @@ -112,13 +115,19 @@ class Job: # Create jobuild objects for parsing for atom in self.atoms: self.jobuilds.append(jobuild.Jobuild(self.jobdir, atom)) + print 'Fetch jobuild SRC_URI...' self._fetch_src() + print 'Setup the chroot for usage...' self.chroot.setup() def run(self): print "Rheeet, it's running!~ NOT." + def tidy(self): + print 'Tidying up..' + self.chroot.tidy() + print 'Done. The workdir has not been removed (default)' + def clean(self): - self.chroot.clean() shutil.rmtree(self.jobdir) - os.removedirs(const.WORKDIR, self.maint) + os.removedirs(osp.join(const.WORKDIR, self.maint)) diff --git a/slave/autotua/chroot/__init__.py b/slave/autotua/chroot/__init__.py index 25e5b7d..e4bc4f5 100644 --- a/slave/autotua/chroot/__init__.py +++ b/slave/autotua/chroot/__init__.py @@ -121,7 +121,7 @@ class WorkChroot(object): self._setup_mounts() print "Work Chroot ready." - def clean(self): + def tidy(self): """ Cleanup when done. """ diff --git a/slave/autotua/const.py b/slave/autotua/const.py index dc78584..abb6e2b 100644 --- a/slave/autotua/const.py +++ b/slave/autotua/const.py @@ -13,6 +13,7 @@ Internal Constants import os.path as osp VERBOSE = True +FULL_CLEAN = False AUTOTUA_DIR = osp.dirname(__file__) # FIXME FIXME FIXME: Ugly hack for nao. LOGFILE= '/var/log/autotua-slave.log' TMPDIR = '/var/tmp/autotua' @@ -30,8 +31,7 @@ GENTOO_MIRRORS = [ STAGE_MIRROR_PATH = 'releases/%(gen_arch)s/%(release)s/stages' STAGE_FILENAME = '%(stage)s-%(arch)s-%(release)s.tar.bz2' -#JOBTAGE_URI = 'http://dev.gentooexperimental.org/~bheekling/jobtage' -JOBTAGE_URI = 'file:///home/nirbheek/projects/AutotuA/jobtage' +JOBTAGE_URI = 'git://git.overlays.gentoo.org/proj/jobtage.git' JOBTAGE_DIR = '/var/tmp/autotua/jobtage' # Usually exists here (optional bind mounting) @@ -48,7 +48,7 @@ job_list = [ 'arch': 'i686', 'type': '', 'release': '2008.0_beta2', - 'jobtagerev': '1', + 'jobtagerev': '', #'overlays': ['overlays/bheekling/tag1', 'overlays/bonsaikitten/tag2'] # These are in order of running 'atoms': ['>=bheekling/test-beagle-1.0', '<=bheekling/test-libbeagle-1.0', '=bheekling/build-brasero-1.0'], diff --git a/slave/autotua/sync/__init__.py b/slave/autotua/sync/__init__.py index c724f86..a72125d 100644 --- a/slave/autotua/sync/__init__.py +++ b/slave/autotua/sync/__init__.py @@ -15,77 +15,108 @@ class Syncer(object): Sync stuff """ - def __init__(self, scheme='bzr', uri=const.JOBTAGE_URI, rev='last:1', destdir=const.JOBTAGE_DIR): + def __init__(self, scheme='git', uri=const.JOBTAGE_URI, rev=None, destdir=const.JOBTAGE_DIR): """ Default is to sync the jobtage tree from the default location. - @param destdir: Destination directory + @param destdir: Destination directory; sync *contents* into destdir @scheme destdir: string @param uri: Uri where to sync from @scheme uri: string - @param rev: Revision of SCM (if applicable), default is latest + @param rev: SCM Revision to export. Default is latest @scheme rev: string @param scheme: The URI scheme @scheme scheme: string """ self.destdir = destdir - if not uri.endswith('/'): - uri += '/' + self.scheme = scheme self.command = Command(scheme, uri, rev, destdir) + def _is_repo(self): + """ + Broad classification of repos into three types based on their behaviour + 1. Same commands for init & sync (eg: rsync) + - return 1 (init) + 2. Different init & sync commands (eg: git) + - Detect if self.destdir is a repo + * Return 2 if it is (sync), 3 if not (rmtree && init) + 3. No concept of sync (eg: bzr-export) + - Delete self.destdir before 'sync' + * return 3 (rmtree && init) + """ + if self.scheme in ['rsync']: + return 1 + elif self.scheme in ['git', 'bzr']: + result = subprocess.Popen('cd %s; git-rev-parse' % self.destdir, shell=True) + if result != 0: + return 3 + return 2 + elif self.scheme in ['git-export', 'bzr-export']: + return 3 + def sync(self): """ Sync self.uri contents to self.destdir - self.destdir must not exist if init-ing - Returns None if unsuccessful, returns True otherwise. """ if osp.exists(self.destdir): if not osp.isdir(self.destdir): # FIXME: Custom exceptions raise '"%s" exists and is not a directory' % self.destdir - #print 'Syncing...' - self.command.run('sync') + result = self._is_repo() + if result == '1': + self.command.run('init') + elif result == '2': + self.command.run('sync') + elif result == '3': + raise 'destdir: \"%s\" exists and is not a %s tree' % self.destdir, self.scheme else: if not osp.exists(osp.dirname(self.destdir)): # Create parents os.makedirs(osp.dirname(self.destdir)) - #print 'Initing...' self.command.run('init') class Command(object): """Command to use for each uri scheme and action""" def __init__(self, scheme, uri, rev, destdir): + if not uri.endswith('/'): + uri += '/' + if not rev: + if scheme == 'bzr-export': + rev = 'last:1' + elif scheme == 'git-export': + rev = 'HEAD' self.scheme = scheme self.uri = uri self.rev = rev self.destdir = destdir - def _get_args(self, action): + def _get_args(self, action='init'): if self.scheme == 'bzr': if action == 'init': - return 'bzr branch -r"%s" "%s" "%s"' % (self.rev, self.uri, self.destdir), + return 'bzr branch "%s" "%s"' % (self.uri, self.destdir) elif action == 'sync': - return 'bzr pull --overwrite -r"%s" -d "%s" "%s"' % (self.rev, self.destdir, self.uri), + return 'bzr pull --overwrite "%s" -d "%s"' % (self.uri, self.destdir) elif self.scheme == 'bzr-export': + return 'bzr export -r"%s" "%s" "%s"' % (self.rev, self.destdir, self.uri) + elif self.scheme == 'git': if action == 'init': - return 'bzr export -r"%s" "%s" "%s"' % (self.rev, self.destdir, self.uri), - elif action == 'sync': - return 'bzr export -r"%s" "%s" "%s"' % (self.rev, self.destdir, self.uri), {'rmtree': self.destdir}, + return 'git clone "%s" "%s"' % (self.uri, self.destdir) + if action == 'sync': + return 'cd "%s"; git reset --hard HEAD; git pull "%s"' % (self.destdir, self.uri) + elif self.scheme == 'git-export': + return 'git-archive --prefix="%s/" --remote="%s" "%s" | tar x -C "%s"' % (osp.basename(self.destdir), self.uri, self.rev, osp.dirname(self.destdir)) elif self.scheme == 'rsync': - return 'rsync -a --progress --delete "%s" "%s"' % (self.uri, self.destdir), - print "Unknown scheme: %s" % self.scheme - return None + command = 'rsync -a --delete "%s" "%s"' % (self.uri, self.destdir) + if const.VERBOSE: + command += ' --verbose' + return command + raise "Unknown scheme: %s" % self.scheme def run(self, action): """Run a sync command""" - args = self._get_args(action) - # Uh, yeah, this is ugly. - if len(args) == 2: - params = args[1] - if params.has_key('rmtree'): - shutil.rmtree(params['rmtree']) - subprocess.check_call(args[0], shell=True) + args = self._get_args(action=action) + subprocess.check_call(args, shell=True) diff --git a/slave/test_modules.py b/slave/test_modules.py index 0775407..db3238d 100755 --- a/slave/test_modules.py +++ b/slave/test_modules.py @@ -28,12 +28,13 @@ if 'chroot' in modules: job = autotua.Jobs().getjobs()[0] chroot = autotua.chroot.WorkChroot(job.jobdir, job.stage.filename) chroot.setup() + chroot.tidy() if 'job' in modules: job = autotua.Jobs().getjobs()[0] job.fetch() if os.getuid() == 0: job.prepare() - job.clean() + job.tidy() else: print 'You need to be root to run job.prepare()' |