[g_sorcery/package_db] new DB syncing

5 files changed, 235 insertions, 47 deletions

diff --git a/g_sorcery/backend.py b/g_sorcery/backend.py
index e606348..809ff18 100644
--- a/g_sorcery/backend.py
+++ b/g_sorcery/backend.py
@@ -4,10 +4,10 @@
 """
     backend.py
     ~~~~~~~~~~
-    
+
     base class for backends
-    
-    :copyright: (c) 2013 by Jauhien Piatlicki
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
     :license: GPL-2, see LICENSE for more details.
 """
 
@@ -30,7 +30,7 @@ class Backend(object):
 
     Command format is as follows:
     g-backend [-o overlay_dir] [-r repository] command
-    
+
     where command is one of the following:
     sync
     list
@@ -38,11 +38,11 @@ class Backend(object):
     generate package_name
     generate-tree [-d --digest]
     install package_name [portage flags]
-    
+
     If no overlay directory is given the default one from backend config is used.
     """
-    
-    def __init__(self, package_db_generator_class, 
+
+    def __init__(self, package_db_generator_class,
                  ebuild_g_with_digest_class, ebuild_g_without_digest_class,
                  eclass_g_class, metadata_g_class,
                  package_db_class=PackageDB, sync_db=False):
@@ -74,7 +74,7 @@ class Backend(object):
         p_generate_tree = subparsers.add_parser('generate-tree')
         p_generate_tree.add_argument('-d', '--digest', action='store_true')
         p_generate_tree.set_defaults(func=self.generate_tree)
-        
+
         p_install = subparsers.add_parser('install')
         p_install.add_argument('pkgname')
         p_install.add_argument('pkgmanager_flags', nargs=argparse.REMAINDER)
@@ -150,7 +150,7 @@ class Backend(object):
 
         if repository:
             if not "repositories" in config:
-                self.logger.error("repository " + repository + 
+                self.logger.error("repository " + repository +
                     " specified, but there is no repositories entry in config")
                 return -1
             repositories = config["repositories"]
@@ -161,11 +161,15 @@ class Backend(object):
         else:
             self.logger.error('no repository given\n')
             return -1
-                
+
+        try:
+            sync_method = repository_config["sync_method"]
+        except KeyError:
+            sync_method = "tgz"
         if self.sync_db:
             pkg_db = self.package_db_generator(backend_path, repository,
                             common_config, repository_config, generate=False)
-            pkg_db.sync(repository_config["db_uri"])
+            pkg_db.sync(repository_config["db_uri"], repository_config=repository_config, sync_method=sync_method)
         else:
             pkg_db = self.package_db_generator(backend_path,
                             repository, common_config, repository_config)
@@ -227,7 +231,7 @@ class Backend(object):
         except Exception as e:
             self.logger.error('dependency solving failed: ' + str(e) + '\n')
             return -1
-        
+
         eclasses = []
         for package in dependencies:
             eclasses += pkg_db.get_package_description(package)['eclasses']
@@ -409,17 +413,17 @@ class Backend(object):
             try:
                 versions = package_db.list_package_versions(pkg.category,
                                                         pkg.package)
-                for version in versions:            
+                for version in versions:
                     solved_deps, unsolved_deps = self.solve_dependencies(package_db,
                                     Package(pkg.category, pkg.package, version),
                                     solved_deps, unsolved_deps)
             except InvalidKeyError:
                 # ignore non existing packages
                 continue
-        
+
         solved_deps.add(package)
         unsolved_deps.remove(package)
-        
+
         return (solved_deps, unsolved_deps)
 
 
@@ -449,7 +453,7 @@ class Backend(object):
         for pkgname in pkgnames:
             directory = os.path.join(overlay, pkgname)
             fast_manifest(directory)
-        
+
     def generate_tree(self, args, config, global_config):
         """
         Generate entire overlay.
@@ -509,7 +513,7 @@ class Backend(object):
         with open(os.path.join(overlay, 'metadata', 'layout.conf'), 'w') as f:
             f.write("repo-name = %s\n" % os.path.basename(overlay))
             f.write("masters = %s\n" % masters_overlays)
-        
+
         if args.digest:
             ebuild_g = self.ebuild_g_with_digest_class(pkg_db)
         else:
@@ -566,6 +570,13 @@ class Backend(object):
             self.fast_digest(overlay, pkgnames)
         overlays.write(overlays_info)
 
+        try:
+            clean_db = config["repositories"][args.repository]["clean_db"]
+        except KeyError:
+            clean_db = False
+        if clean_db:
+            pkg_db.clean()
+
     def install(self, args, config, global_config):
         """
         Install a package.
@@ -592,7 +603,7 @@ class Backend(object):
             package_manager_class = package_managers[package_manager]
         package_manager = package_manager_class()
         package_manager.install(args.pkgname, *args.pkgmanager_flags)
-        
+
     def __call__(self, args, config, global_config):
         """
         Execute a command
diff --git a/g_sorcery/git_syncer/__init__.py b/g_sorcery/git_syncer/__init__.py
new file mode 100644
index 0000000..4265cc3
--- /dev/null
+++ b/g_sorcery/git_syncer/__init__.py
@@ -0,0 +1 @@
+#!/usr/bin/env python
diff --git a/g_sorcery/git_syncer/git_syncer.py b/g_sorcery/git_syncer/git_syncer.py
new file mode 100644
index 0000000..0f6c58e
--- /dev/null
+++ b/g_sorcery/git_syncer/git_syncer.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    git_syncer.py
+    ~~~~~~~~~~~~~
+
+    git sync helper
+
+    :copyright: (c) 2015 by Jauhien Piatlicki
+    :license: GPL-2, see LICENSE for more details.
+"""
+
+import os
+
+from g_sorcery.compatibility import TemporaryDirectory
+
+from g_sorcery.exceptions import SyncError
+from g_sorcery.syncer import Syncer, SyncedData, TmpSyncedData
+
+
+class GITSyncer(Syncer):
+    """
+    Class used to sync with git repos.
+    """
+
+    def sync(self, db_uri, repository_config):
+        """
+        Synchronize local directory with remote source.
+
+        Args:
+            db_uri: URI for synchronization with remote source.
+            repository_config: repository config.
+
+        Returns:
+            SyncedData object that gives access to the directory with data.
+        """
+        if self.persistent_datadir is None:
+            tmp_dir = TemporaryDirectory()
+            path = os.path.join(tmp_dir.name, "remote")
+        else:
+            path = self.persistent_datadir
+        try:
+            branch = repository_config["branch"]
+        except KeyError:
+            branch = "master"
+
+        if os.path.exists(path):
+            #TODO: allow changing of remotes/branches
+            self.pull(path)
+        else:
+            self.clone(db_uri, branch, path)
+
+        if self.persistent_datadir is None:
+            return TmpSyncedData(path, tmp_dir)
+        else:
+            return SyncedData(path)
+
+
+    def clone(self, db_uri, branch, path):
+        if os.system("git clone --depth 1 --branch " + branch + " " + db_uri + " " + path):
+            raise SyncError("sync failed (clonning): " + db_uri)
+
+
+    def pull(self, path):
+        if os.system("cd " + path + " && git pull"):
+            raise SyncError("sync failed (pulling): " + path)
diff --git a/g_sorcery/package_db.py b/g_sorcery/package_db.py
index a88474d..8daa665 100644
--- a/g_sorcery/package_db.py
+++ b/g_sorcery/package_db.py
@@ -11,18 +11,18 @@
     :license: GPL-2, see LICENSE for more details.
 """
 
-import glob
 import os
 
 import portage
 
-from .compatibility import basestring, py2k, TemporaryDirectory
+from .compatibility import basestring, py2k
 
 from .db_layout import DBLayout, JSON_FILE_SUFFIX, SUPPORTED_DB_LAYOUTS, SUPPORTED_FILE_FORMATS
 from .exceptions import DBError, DBLayoutError, DBStructureError, InvalidKeyError, SyncError
-from .fileutils import FileJSON, load_remote_file, copy_all, wget
+from .fileutils import FileJSON, load_remote_file, copy_all
 from .g_collections import Package
 from .logger import Logger
+from .syncer import SUPPORTED_SYNCERS
 
 SUPPORTED_DB_STRUCTURES=[0, 1]
 
@@ -137,6 +137,7 @@ class PackageDB(object):
 
 
     def __init__(self, directory,
+                 persistent_datadir = None,
                  preferred_layout_version=1,
                  preferred_db_version=1,
                  preferred_category_format=JSON_FILE_SUFFIX):
@@ -157,6 +158,11 @@ class PackageDB(object):
 
         self.logger = Logger()
         self.directory = os.path.abspath(directory)
+
+        self.persistent_datadir = persistent_datadir
+        if self.persistent_datadir is not None:
+            self.persistent_datadir = os.path.abspath(self.persistent_datadir)
+
         self.preferred_layout_version = preferred_layout_version
         self.preferred_db_version = preferred_db_version
         self.preferred_category_format = preferred_category_format
@@ -176,24 +182,30 @@ class PackageDB(object):
         self.categories = {}
 
 
-    def sync(self, db_uri):
+    def sync(self, db_uri, repository_config = None, sync_method="tgz"):
         """
         Synchronize local database with remote database.
 
         Args:
             db_uri: URI for synchronization with remote database.
-        """
-        real_db_uri = self.get_real_db_uri(db_uri)
-        download_dir = TemporaryDirectory()
-        if wget(real_db_uri, download_dir.name):
-            raise SyncError('sync failed: ' + real_db_uri)
-
-        temp_dir = TemporaryDirectory()
-        for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')):
-            self.logger.info("unpacking " + f_name)
-            os.system("tar -xvzf " + f_name + " -C " + temp_dir.name)
+            repository_config: repository config.
+            sync_method: sync method (tgz or git).
+        """
+        if repository_config is None:
+            repository_config = {}
+
+        try:
+            syncer_cls = SUPPORTED_SYNCERS[sync_method]
+        except KeyError:
+            raise SyncError('unsupported sync method: ' + sync_method)
+        if self.persistent_datadir is not None:
+            remotedb_dir = os.path.join(self.persistent_datadir, 'remote')
+        else:
+            remotedb_dir = None
+        syncer = syncer_cls(remotedb_dir)
+        synced_data = syncer.sync(db_uri, repository_config)
 
-        tempdb_dir = os.path.join(temp_dir.name, os.listdir(temp_dir.name)[0])
+        tempdb_dir = synced_data.get_path()
         tempdb = PackageDB(tempdb_dir)
 
         tempdb.db_layout.check_manifest()
@@ -204,19 +216,7 @@ class PackageDB(object):
 
         self.db_layout.check_manifest()
 
-        del download_dir
-        del temp_dir
-
-
-    def get_real_db_uri(self, db_uri):
-        """
-        Convert self.db_uri to URI where remote database can be
-        fetched from.
-
-        Returns:
-            URI of remote database file.
-        """
-        return db_uri
+        del synced_data
 
 
     def clean(self):
@@ -538,10 +538,12 @@ class DBGenerator(object):
             Package database.
         """
         db_path = os.path.join(directory, repository, "db")
+        persistent_datadir = os.path.join(directory, repository, "persistent")
         pkg_db = self.package_db_class(db_path,
                                        preferred_layout_version=self.preferred_layout_version,
                                        preferred_db_version=self.preferred_db_version,
-                                       preferred_category_format=self.preferred_category_format)
+                                       preferred_category_format=self.preferred_category_format,
+                                       persistent_datadir=persistent_datadir)
 
         config_f = FileJSON(os.path.join(directory, repository),
                             "config.json", [])
diff --git a/g_sorcery/syncer.py b/g_sorcery/syncer.py
new file mode 100644
index 0000000..beeffd4
--- /dev/null
+++ b/g_sorcery/syncer.py
@@ -0,0 +1,107 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+    syncer.py
+    ~~~~~~~~~
+
+    sync helper
+
+    :copyright: (c) 2013-2015 by Jauhien Piatlicki
+    :license: GPL-2, see LICENSE for more details.
+"""
+
+import glob
+import os
+
+from .compatibility import TemporaryDirectory
+
+from .exceptions import SyncError
+from .fileutils import wget
+
+
+class SyncedData(object):
+    """
+    Synced data.
+
+    Directory with sync data is guaranted to exist only as long as this
+    object does.
+    """
+    def __init__(self, directory):
+        self.directory = os.path.abspath(directory)
+
+    def get_path(self):
+        return self.directory
+
+
+class TmpSyncedData(SyncedData):
+    """
+    Synced data that lives in a temporary directory.
+    """
+
+    def __init__(self, directory, tmpdirobj):
+        super(TmpSyncedData, self).__init__(directory)
+        self.tmpdirobj = tmpdirobj
+
+
+class Syncer(object):
+    """
+    Class used to sync data with remote source.
+    """
+
+    def __init__(self, persistent_datadir):
+        self.persistent_datadir = persistent_datadir
+
+    def sync(self, db_uri, repository_config):
+        """
+        Synchronize local directory with remote source.
+
+        Args:
+            db_uri: URI for synchronization with remote source.
+            repository_config: repository config.
+
+        Returns:
+            SyncedData object that gives access to the directory with data.
+        """
+        raise NotImplementedError
+
+
+class TGZSyncer(Syncer):
+    """
+    Class used to download and unpack tarballs.
+    """
+
+    def sync(self, db_uri, repository_config):
+        """
+        Synchronize local directory with remote source.
+
+        Args:
+            db_uri: URI for synchronization with remote source.
+            repository_config: repository config.
+
+        Returns:
+            SyncedData object that gives access to the directory with data.
+        """
+        download_dir = TemporaryDirectory()
+        if wget(db_uri, download_dir.name):
+            raise SyncError('sync failed: ' + db_uri)
+
+        tmp_dir = TemporaryDirectory()
+        for f_name in glob.iglob(os.path.join(download_dir.name, '*.tar.gz')):
+            if os.system("tar -xvzf " + f_name + " -C " + tmp_dir.name):
+                raise SyncError('sync failed (unpacking)')
+
+        tmp_path = os.path.join(tmp_dir.name, os.listdir(tmp_dir.name)[0])
+        del download_dir
+        return TmpSyncedData(tmp_path, tmp_dir)
+
+
+SUPPORTED_SYNCERS = {"tgz": TGZSyncer}
+
+# git_syncer module is optional, we should check if it is installed
+try:
+    from .git_syncer.git_syncer import GITSyncer
+    SUPPORTED_SYNCERS["git"] = GITSyncer
+
+except ImportError as e:
+    pass