No matter how hard I try, I can't seem to be able to make atomic commits :(

Here's a load of further changes:
- Move jobtage repo to git; incorporate that change here
  * Massive changes in autotua.sync =p
  * No way I'm sitting and documenting the changes
  * Syncing is still ugly, will document how it works once it's finalised
- Print some more status info in Job() and autotua.sync
- autotua.Job(),autotua.chroot:s/clean/tidy/
- Introduce new variable autotua.const.FULL_CLEAN to remove chroot data (unused atm)
- Update test_modules.py
- Update TODO

6 files changed, 80 insertions, 39 deletions

diff --git a/slave/TODO b/slave/TODO
index 6a593ff..f7a5532 100644
--- a/slave/TODO
+++ b/slave/TODO
@@ -4,6 +4,6 @@
 
 Do:
 -----
-- Separate module for running commands
-- Atom parsing, jobuild searching, etc etc
-- Cleanup autotua/__init__.py:Job()
+[ ] Separate module for running commands
+[~] Atom parsing, jobuild searching, etc etc
+[ ] Auto-detect SCM of source jobtage repo for syncer.Sync().sync()
diff --git a/slave/autotua/__init__.py b/slave/autotua/__init__.py
index af16e41..a21934e 100644
--- a/slave/autotua/__init__.py
+++ b/slave/autotua/__init__.py
@@ -92,19 +92,22 @@ class Job:
             src_uri = jobuild.get_var('SRC_URI').split()
             for i in src_uri:
                 job_src.append(fetch.Fetchable(uri=[i]))
-        for i in job_src:
-            fetcher.fetch(uri=[i])
+        for fetchable in job_src:
+            fetcher.fetch(fetchable)
 
     def fetch(self):
         # Job metadata stuff
         ## Get stage3 (if required)
+        print 'Fetching stage...'
         fetcher = fetch.Fetcher(const.STAGE_DIR)
         fetcher.fetch(self.stage)
         # Sync jobtage tree
+        print 'Syncing jobtage tree...'
         sync.Syncer().sync()
         # Export from local jobtage tree
+        print 'Exporting jobtage tree...'
         sync.Syncer(uri=const.JOBTAGE_DIR, destdir=osp.join(self.jobdir, 'jobtage'),
-                    rev=self.jobtagerev, scheme="bzr-export").sync()
+                    rev=self.jobtagerev, scheme="git-export").sync()
         ## Read config, get portage snapshot if required
         #self._fetch_portage_snapshot()
 
@@ -112,13 +115,19 @@ class Job:
         # Create jobuild objects for parsing
         for atom in self.atoms:
             self.jobuilds.append(jobuild.Jobuild(self.jobdir, atom))
+        print 'Fetch jobuild SRC_URI...'
         self._fetch_src()
+        print 'Setup the chroot for usage...'
         self.chroot.setup()
 
     def run(self):
         print "Rheeet, it's running!~ NOT."
 
+    def tidy(self):
+        print 'Tidying up..'
+        self.chroot.tidy()
+        print 'Done. The workdir has not been removed (default)'
+
     def clean(self):
-        self.chroot.clean()
         shutil.rmtree(self.jobdir)
-        os.removedirs(const.WORKDIR, self.maint)
+        os.removedirs(osp.join(const.WORKDIR, self.maint))
diff --git a/slave/autotua/chroot/__init__.py b/slave/autotua/chroot/__init__.py
index 25e5b7d..e4bc4f5 100644
--- a/slave/autotua/chroot/__init__.py
+++ b/slave/autotua/chroot/__init__.py
@@ -121,7 +121,7 @@ class WorkChroot(object):
         self._setup_mounts()
         print "Work Chroot ready."
 
-    def clean(self):
+    def tidy(self):
         """
         Cleanup when done.
         """
diff --git a/slave/autotua/const.py b/slave/autotua/const.py
index dc78584..abb6e2b 100644
--- a/slave/autotua/const.py
+++ b/slave/autotua/const.py
@@ -13,6 +13,7 @@ Internal Constants
 import os.path as osp
 
 VERBOSE = True
+FULL_CLEAN = False
 AUTOTUA_DIR = osp.dirname(__file__) # FIXME FIXME FIXME: Ugly hack for nao.
 LOGFILE= '/var/log/autotua-slave.log'
 TMPDIR = '/var/tmp/autotua'
@@ -30,8 +31,7 @@ GENTOO_MIRRORS = [
 STAGE_MIRROR_PATH = 'releases/%(gen_arch)s/%(release)s/stages'
 STAGE_FILENAME = '%(stage)s-%(arch)s-%(release)s.tar.bz2'
 
-#JOBTAGE_URI = 'http://dev.gentooexperimental.org/~bheekling/jobtage'
-JOBTAGE_URI = 'file:///home/nirbheek/projects/AutotuA/jobtage'
+JOBTAGE_URI = 'git://git.overlays.gentoo.org/proj/jobtage.git'
 JOBTAGE_DIR = '/var/tmp/autotua/jobtage'
 
 # Usually exists here (optional bind mounting)
@@ -48,7 +48,7 @@ job_list = [
         'arch': 'i686',
         'type': '',
         'release': '2008.0_beta2',
-        'jobtagerev': '1',
+        'jobtagerev': '',
         #'overlays': ['overlays/bheekling/tag1', 'overlays/bonsaikitten/tag2']
         # These are in order of running
         'atoms': ['>=bheekling/test-beagle-1.0', '<=bheekling/test-libbeagle-1.0', '=bheekling/build-brasero-1.0'],
diff --git a/slave/autotua/sync/__init__.py b/slave/autotua/sync/__init__.py
index c724f86..a72125d 100644
--- a/slave/autotua/sync/__init__.py
+++ b/slave/autotua/sync/__init__.py
@@ -15,77 +15,108 @@ class Syncer(object):
     Sync stuff
     """
 
-    def __init__(self, scheme='bzr', uri=const.JOBTAGE_URI, rev='last:1', destdir=const.JOBTAGE_DIR):
+    def __init__(self, scheme='git', uri=const.JOBTAGE_URI, rev=None, destdir=const.JOBTAGE_DIR):
         """
         Default is to sync the jobtage tree from the default location.
 
-        @param destdir: Destination directory
+        @param destdir: Destination directory; sync *contents* into destdir
         @scheme destdir: string
 
         @param uri: Uri where to sync from
         @scheme uri: string
 
-        @param rev: Revision of SCM (if applicable), default is latest
+        @param rev: SCM Revision to export. Default is latest
         @scheme rev: string
 
         @param scheme: The URI scheme
         @scheme scheme: string
         """
         self.destdir = destdir
-        if not uri.endswith('/'):
-            uri += '/'
+        self.scheme = scheme
         self.command = Command(scheme, uri, rev, destdir)
 
+    def _is_repo(self):
+        """
+        Broad classification of repos into three types based on their behaviour
+         1. Same commands for init & sync (eg: rsync)
+            - return 1 (init)
+         2. Different init & sync commands (eg: git)
+            - Detect if self.destdir is a repo
+              * Return 2 if it is (sync), 3 if not (rmtree && init)
+         3. No concept of sync (eg: bzr-export)
+            - Delete self.destdir before 'sync'
+              * return 3 (rmtree && init)
+        """
+        if self.scheme in ['rsync']:
+            return 1
+        elif self.scheme in ['git', 'bzr']:
+            result = subprocess.Popen('cd %s; git-rev-parse' % self.destdir, shell=True)
+            if result != 0:
+                return 3
+            return 2
+        elif self.scheme in ['git-export', 'bzr-export']:
+            return 3
+
     def sync(self):
         """
         Sync self.uri contents to self.destdir 
-        self.destdir must not exist if init-ing
-        Returns None if unsuccessful, returns True otherwise.
         """
         if osp.exists(self.destdir):
             if not osp.isdir(self.destdir):
                 # FIXME: Custom exceptions
                 raise '"%s" exists and is not a directory' % self.destdir
-            #print 'Syncing...'
-            self.command.run('sync')
+            result = self._is_repo()
+            if result == '1':
+                self.command.run('init')
+            elif result == '2':
+                self.command.run('sync')
+            elif result == '3':
+                raise 'destdir: \"%s\" exists and is not a %s tree' % self.destdir, self.scheme
         else:
             if not osp.exists(osp.dirname(self.destdir)):
                 # Create parents
                 os.makedirs(osp.dirname(self.destdir))
-            #print 'Initing...'
             self.command.run('init')
 
 class Command(object):
     """Command to use for each uri scheme and action"""
 
     def __init__(self, scheme, uri, rev, destdir):
+        if not uri.endswith('/'):
+            uri += '/'
+        if not rev:
+            if scheme == 'bzr-export':
+                rev = 'last:1'
+            elif scheme == 'git-export':
+                rev = 'HEAD'
         self.scheme = scheme
         self.uri = uri
         self.rev = rev
         self.destdir = destdir
 
-    def _get_args(self, action):
+    def _get_args(self, action='init'):
         if self.scheme == 'bzr':
             if action == 'init':
-                return 'bzr branch -r"%s" "%s" "%s"' % (self.rev, self.uri, self.destdir),
+                return 'bzr branch "%s" "%s"' % (self.uri, self.destdir)
             elif action == 'sync':
-                return 'bzr pull --overwrite -r"%s" -d "%s" "%s"' % (self.rev, self.destdir, self.uri),
+                return 'bzr pull --overwrite "%s" -d "%s"' % (self.uri, self.destdir)
         elif self.scheme == 'bzr-export':
+            return 'bzr export -r"%s" "%s" "%s"' % (self.rev, self.destdir, self.uri)
+        elif self.scheme == 'git':
             if action == 'init':
-                return 'bzr export -r"%s" "%s" "%s"' % (self.rev, self.destdir, self.uri),
-            elif action == 'sync':
-                return  'bzr export -r"%s" "%s" "%s"' % (self.rev, self.destdir, self.uri), {'rmtree': self.destdir},
+                return 'git clone "%s" "%s"' % (self.uri, self.destdir)
+            if action == 'sync':
+                return 'cd "%s"; git reset --hard HEAD; git pull "%s"' % (self.destdir, self.uri)
+        elif self.scheme == 'git-export':
+            return 'git-archive --prefix="%s/" --remote="%s" "%s" | tar x -C "%s"' % (osp.basename(self.destdir), self.uri, self.rev, osp.dirname(self.destdir))
         elif self.scheme == 'rsync':
-            return 'rsync -a --progress --delete "%s" "%s"' % (self.uri, self.destdir),
-        print "Unknown scheme: %s" % self.scheme
-        return None
+            command = 'rsync -a --delete "%s" "%s"' % (self.uri, self.destdir)
+            if const.VERBOSE:
+                command += ' --verbose'
+            return command
+        raise "Unknown scheme: %s" % self.scheme
 
     def run(self, action):
         """Run a sync command"""
-        args = self._get_args(action)
-        # Uh, yeah, this is ugly.
-        if len(args) == 2:
-            params = args[1]
-            if params.has_key('rmtree'):
-                shutil.rmtree(params['rmtree'])
-        subprocess.check_call(args[0], shell=True)
+        args = self._get_args(action=action)
+        subprocess.check_call(args, shell=True)
diff --git a/slave/test_modules.py b/slave/test_modules.py
index 0775407..db3238d 100755
--- a/slave/test_modules.py
+++ b/slave/test_modules.py
@@ -28,12 +28,13 @@ if 'chroot' in modules:
     job = autotua.Jobs().getjobs()[0]
     chroot = autotua.chroot.WorkChroot(job.jobdir, job.stage.filename)
     chroot.setup()
+    chroot.tidy()
 
 if 'job' in modules:
     job = autotua.Jobs().getjobs()[0]
     job.fetch()
     if os.getuid() == 0:
         job.prepare()
-        job.clean()
+        job.tidy()
     else:
         print 'You need to be root to run job.prepare()'