fixed the script manage_pkgdb.py and a test case

3 files changed, 101 insertions, 84 deletions

diff --git a/contrib/manage_pkgdb.py b/contrib/manage_pkgdb.py
index 2c46934..f1e0583 100755
--- a/contrib/manage_pkgdb.py
+++ b/contrib/manage_pkgdb.py
@@ -4,9 +4,9 @@
 """
     manage_pkgdb.py
     ~~~~~~~~~~~~~~~
-    
+
     a simple script to update a Git repository with a package database.
-    
+
     :copyright: (c) 2010 by Rafael Goncalves Martins
     :license: GPL-2, see LICENSE for more details.
 """
@@ -27,8 +27,14 @@ import urllib
 
 from contextlib import closing
 
+current_dir = os.path.dirname(os.path.realpath(__file__))
+if os.path.exists(os.path.join(current_dir, '..', 'g_octave')):
+    sys.path.insert(0, os.path.join(current_dir, '..'))
+
+from g_octave import description_tree
+
 class Git:
-    
+
     def __init__(self, repo):
         self._repo = repo
 
@@ -40,16 +46,17 @@ class Git:
 re_tarball = re.compile(r'(([^/]+)-([0-9.]+)\.tar\.gz)$')
 
 class SfUpdates:
-    
+
     # feed url from 'http://sourceforge.net/projects/octave/files/Octave%20Forge%20Packages/Individual%20Package%20Releases/'
     feed_url = u'http://sourceforge.net/api/file/index/project-id/2888/mtime/desc/rss?path=%2FOctave%20Forge%20Packages%2FIndividual%20Package%20Releases'
-    
+
     svnroot_url = u'https://octave.svn.sourceforge.net/svnroot/octave/trunk/octave-forge/'
     categories = [u'main', u'extra', u'language', u'nonfree']
-    
+
     _timestamp = None
-    
+
     def __init__(self, local_dir, repo_dir):
+        os.environ['GOCTAVE_DB'] = repo_dir
         self._local_dir = local_dir
         self._repo_dir = repo_dir
         self.feed = feedparser.parse(self.feed_url)
@@ -63,7 +70,7 @@ class SfUpdates:
                 fp.write(str(self._timestamp))
         except:
             pass
-    
+
     def _load_timestamp(self):
         try:
             with open(os.path.join(self._repo_dir, 'timestamp')) as fp:
@@ -95,66 +102,65 @@ class SfUpdates:
                     'url': entry.link,
                 }
         return entries
-    
+
     def local_files(self):
         # tarball name: {
         #   name,
         #   version,
         #   category
         # }
-        if not os.path.isdir(self._local_dir):
-            return {}
+        db = description_tree.DescriptionTree()
         entries = {}
-        for category in os.listdir(self._local_dir):
-            for tarball in os.listdir(os.path.join(self._local_dir, category)):
-                my_tarball = re_tarball.search(unicode(tarball))
-                if my_tarball is not None:
-                    entries[my_tarball.group(1)] = {
-                        'name': my_tarball.group(2),
-                        'version': my_tarball.group(3),
-                        'category': unicode(category),
-                    }
+        for category in db.pkg_list:
+            for pkg in db.pkg_list[category]:
+                entries['%s-%s.tar.gz' % (pkg['name'], pkg['version'])] = {
+                    'name': pkg['name'],
+                    'version': pkg['version'],
+                    'category': unicode(db.categories[pkg['name']]),
+                }
         return entries
-    
+
     def guess_category(self, pkgname):
         for category in self.categories:
             f = urllib.urlopen(self.svnroot_url + '/' + category + '/' + pkgname + '/DESCRIPTION')
             if f.getcode() == 200:
-                return category            
-    
+                return category
+
     def check_updates(self):
         local_files = self.local_files()
         remote_files = self.remote_files()
         updates = {}
         for remote in remote_files:
-            if remote not in local_files:
-                print('update found: %s; ' % remote, end='')
-                updates[remote] = local_files[remote]
-                category = self.guess_category(remote_files[remote]['name'])
-                if category is None:
-                    remote_name = remote_files[remote]['name'].lower()
-                    for local in local_files:
-                        local_name = local_files[local]['name'].lower()
-                        if remote_name == local_name:
-                            category = local_files[local]['category']
-                            break
-                remote_files[remote]['category'] = category
-                print('category: %s' % category)
-                if self.download(remote, remote_files[remote]) != os.EX_OK:
-                    raise RuntimeError('Failed to download: %s' % remote)
+            print('update found: %s; ' % remote, end='')
+            sys.stdout.flush()
+            updates[remote] = remote_files[remote]
+            category = self.guess_category(remote_files[remote]['name'])
+            if category is None:
+                remote_name = remote_files[remote]['name'].lower()
+                for local in local_files:
+                    local_name = local_files[local]['name'].lower()
+                    if remote_name == local_name:
+                        category = local_files[local]['category']
+                        break
+            remote_files[remote]['category'] = category
+            print('category: %s' % category)
+            if self.download(remote, remote_files[remote]) != os.EX_OK:
+                raise RuntimeError('Failed to download: %s' % remote)
         return updates
-    
+
     def download(self, tarball_name, entry):
-        return subprocess.call([
-            'wget',
-            '--continue',
-            '--output-document', os.path.join(
-                    self._local_dir,
-                    entry['category'],
-                    tarball_name,
-                ),
-            entry['url']
-        ])
+        cat_dir = os.path.join(self._local_dir, entry['category'])
+        file_path = os.path.join(cat_dir, tarball_name)
+        if not os.path.exists(cat_dir):
+            os.makedirs(cat_dir, 0o755)
+        if not os.path.exists(file_path):
+            return subprocess.call([
+                'wget',
+                '--continue',
+                '--output-document', file_path,
+                entry['url']
+            ])
+        return os.EX_OK
 
     def update_package_database(self, local_files, db_dir):
         if not os.path.exists(db_dir):
@@ -192,13 +198,13 @@ class SfUpdates:
 
 
 def main(argv):
-    
+
     parser = optparse.OptionParser(
         usage = '%prog [options] <sources directory> <package database directory>',
         version = 'see g-octave --version',
         description = 'a simple script to update a Git repository with a package database.'
     )
-    
+
     parser.add_option(
         '-c', '--commit',
         action = 'store_true',
@@ -206,7 +212,7 @@ def main(argv):
         default = False,
         help = 'commit the changes to the Git repository'
     )
-    
+
     parser.add_option(
         '-p', '--push',
         action = 'store_true',
@@ -214,9 +220,9 @@ def main(argv):
         default = False,
         help = 'push the changes to the remote Git repository'
     )
-    
+
     options, args = parser.parse_args(argv[1:])
-    
+
     if len(args) != 2:
         print(
             'You need to provide 2 arguments:\n'
@@ -225,7 +231,7 @@ def main(argv):
             file=sys.stderr
         )
         return os.EX_USAGE
-    
+
     print('* Fetching and parsing the Octave-Forge RSS feed ...')
     sf = SfUpdates(args[0], args[1])
     print('* Looking for updates ...')
@@ -244,7 +250,7 @@ def main(argv):
             return os.EX_OK
         return os.EX_SOFTWARE
     return os.EX_OK
-            
+
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))
diff --git a/g_octave/config.py b/g_octave/config.py
index e703d11..0b42c08 100644
--- a/g_octave/config.py
+++ b/g_octave/config.py
@@ -3,10 +3,10 @@
 """
     config.py
     ~~~~~~~~~
-    
+
     This module implements a Python object to handle the configuration
     of g-octave.
-    
+
     :copyright: (c) 2009-2010 by Rafael Goncalves Martins
     :license: GPL-2, see LICENSE for more details.
 """
@@ -27,7 +27,7 @@ else:
     import ConfigParser as configparser
 
 class Config(object):
-    
+
     _defaults = {
         'db': '/var/cache/g-octave',
         'overlay': '/var/lib/g-octave',
@@ -45,12 +45,12 @@ class Config(object):
     _env_namespace = 'GOCTAVE_'
 
     def __init__(self, fetch_phase=False, config_file=None, create_dirs=True):
-        
+
         # Config Parser
         self._config = configparser.ConfigParser(self._defaults)
-        
+
         self._fetch_phase = fetch_phase
-        
+
         parsed_files = self._config.read([
             os.path.join(
                 os.path.dirname(os.path.abspath(__file__)),
@@ -58,41 +58,45 @@ class Config(object):
             ),
             config_file or '/etc/g-octave.cfg',
         ])
-        
+
         if len(parsed_files) == 0:
             raise ConfigException('Configuration file not found.')
-        
+
         _db = self._getattr('db')
         _overlay = self._getattr('overlay')
-        
+
         for dir in [_db, _overlay]:
             if not os.path.exists(dir) and create_dirs:
-                os.makedirs(dir, 0o755)
-        
+                try:
+                    os.makedirs(dir, 0o755)
+                except:
+                    # it's probably safe to ignore that
+                    pass
+
         self._cache = {}
         self._info = {}
-        
+
         if not fetch_phase:
-            
+
             # JSON
             json_file = os.path.join(_db, 'info.json')
             with open(json_file) as fp:
                 self._info = json.load(fp)
-        
+
 
     def __getattr__(self, attr):
-        
+
         if attr in self._defaults:
             return self._getattr(attr)
         elif attr in self._info:
             return self._info[attr]
         else:
             raise ConfigException('Invalid option: %s' % attr)
-    
-    
+
+
     def _getattr(self, attr):
         from_env = os.environ.get(self._env_namespace + attr.upper(), None)
         if from_env is None:
             return self._config.get(self._section_name, attr)
         return from_env
-    
+
diff --git a/tests/test_description.py b/tests/test_description.py
index ea0be68..9016325 100644
--- a/tests/test_description.py
+++ b/tests/test_description.py
@@ -4,28 +4,31 @@
 """
     test_description.py
     ~~~~~~~~~~~~~~~~~~~
-    
+
     test suite for the *g_octave.description* module
-    
+
     :copyright: (c) 2010 by Rafael Goncalves Martins
     :license: GPL-2, see LICENSE for more details.
 """
 
 import os
 import unittest
+import utils
 
 from g_octave import description
 
 
 class TestDescription(unittest.TestCase):
-    
+
     def setUp(self):
+        conf, self._config_file, self._tempdir = utils.create_env()
         self.desc = description.Description(
             os.path.join(
-                os.path.dirname(os.path.abspath(__file__)), 'files', 'DESCRIPTION'
-            )
+                os.path.dirname(os.path.abspath(__file__)), 'files', 'DESCRIPTION',
+            ),
+            conf = conf
         )
-    
+
     def test_re_depends(self):
         depends = [
             ('pkg', ('pkg', None, None)),
@@ -156,7 +159,7 @@ class TestDescription(unittest.TestCase):
                 (match.group(1), match.group(3), match.group(4)),
                 pkgtpl
             )
-    
+
     def test_re_pkg_atom(self):
         depends = [
             ('pkg-1', ('pkg', '1')),
@@ -182,20 +185,24 @@ class TestDescription(unittest.TestCase):
         self.assertEqual(self.desc.description, 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.')
         self.assertEqual(self.desc.categories, 'Category1,Category2, Category3')
         self.assertEqual(self.desc.url, 'http://example.org')
-        
+
         requirements = [
             '>=g-octave/pkg1-4.3.2',
             '<g-octave/pkg2-1.2.3',
             'g-octave/pkg3'
         ]
-        requirements.sort()        
+        requirements.sort()
         self.assertEqual(self.desc.systemrequirements, requirements)
         self.assertEqual(self.desc.buildrequires, ['>g-octave/pkg4-1.0.0'])
-        
+
         self.assertEqual(self.desc.depends, ['>=sci-mathematics/octave-3.0.0'])
         self.assertEqual(self.desc.autoload, 'NO')
         self.assertEqual(self.desc.license, 'GPL version 3 or later')
 
+    def tearDown(self):
+        # removing the temp tree
+        utils.clean_env(self._config_file, self._tempdir)
+
 
 def suite():
     suite = unittest.TestSuite()