- as simple diff

[packages/git-core-slug.git] / git-core-slug-git.patch

diff --git a/git_slug/gitrepo.py b/git_slug/gitrepo.py
index 5234deb..d9f88ee 100644
--- a/git_slug/gitrepo.py
+++ b/git_slug/gitrepo.py
@@ -82,12 +82,21 @@ class GitRepo:
             'refs/notes/*:refs/notes/*'])
 
     def check_remote(self, ref, remote=REMOTE_NAME):
+        localref = EMPTYSHA1
         ref = ref.replace(REFFILE, os.path.join('remotes', remote))
         try:
             with open(os.path.join(self.gdir, ref), 'r') as f:
                 localref = f.readline().strip()
         except IOError:
-            localref = EMPTYSHA1
+            try:
+                with open(os.path.join(self.gdir, 'packed-refs')) as f:
+                    for line in f:
+                        line_data = line.split()
+                        if len(line_data) == 2 and line_data[1] == ref:
+                            localref = line_data[0].strip()
+                            break
+            except IOError:
+                pass
         return localref
 
     def showfile(self, filename, ref="/".join([REMOTE_NAME, "master"])):
diff --git a/git_slug/refsdata.py b/git_slug/refsdata.py
index 4354ac4..67592f8 100644
--- a/git_slug/refsdata.py
+++ b/git_slug/refsdata.py
@@ -16,7 +16,7 @@ class NoMatchedRepos(Exception):
 
 class RemoteRefsData:
     def __init__(self, stream, pattern, dirpattern=('*',)):
-        self.heads = collections.defaultdict(lambda: collections.defaultdict(lambda: EMPTYSHA1))
+        self.heads = collections.defaultdict(self.__dict_var__)
         pats = re.compile('|'.join(fnmatch.translate(os.path.join('refs/heads', p)) for p in pattern))
         dirpat = re.compile('|'.join(fnmatch.translate(p) for p in dirpattern))
         for line in stream.readlines():
@@ -28,6 +28,12 @@ class RemoteRefsData:
         if not self.heads:
             raise NoMatchedRepos
 
+    def __dict_init__(self):
+        return EMPTYSHA1
+
+    def __dict_var__(self):
+        return collections.defaultdict(self.__dict_init__)
+
     def put(self, repo, data):
         for line in data:
             (sha1_old, sha1, ref) = line.split()
diff --git a/slug.py b/slug.py
index 69bd3b9..68f68cd 100755
--- a/slug.py
+++ b/slug.py
@@ -7,26 +7,18 @@ import os
 import shutil
 import subprocess
 import queue
-import threading
-
+import multiprocessing
 import argparse
 
 import signal
 import configparser
 
+from multiprocessing import Pool as WorkerPool
+
 from git_slug.gitconst import GITLOGIN, GITSERVER, GIT_REPO, GIT_REPO_PUSH, REMOTE_NAME, REMOTEREFS
 from git_slug.gitrepo import GitRepo, GitRepoError
 from git_slug.refsdata import GitArchiveRefsData, NoMatchedRepos, RemoteRefsError
 
-class Store():
-    def __init__(self):
-        self.lock = threading.Lock()
-        self.items = []
-
-    def put(self, item):
-        with self.lock:
-            self.items.append(item)
-
 class UnquoteConfig(configparser.ConfigParser):
     def get(self, section, option, **kwargs):
         value = super().get(section, option, **kwargs)
@@ -43,25 +35,15 @@ class DelAppend(argparse._AppendAction):
         item.append(values)
         setattr(namespace, self.dest, item)
 
-class ThreadFetch(threading.Thread):
-    def __init__(self, queue, output, pkgdir, depth=0):
-        threading.Thread.__init__(self)
-        self.queue = queue
-        self.packagesdir = pkgdir
-        self.depth = depth
-        self.output = output
-
-    def run(self):
-        while True:
-            (gitrepo, ref2fetch) = self.queue.get()
-            try:
-                (stdout, stderr) = gitrepo.fetch(ref2fetch, self.depth)
-                if stderr != b'':
-                    print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
-                    self.output.put(gitrepo)
-            except GitRepoError as e:
-                print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
-            self.queue.task_done()
+def cpu_count():
+    try:
+        return multiprocessing.cpu_count()
+    except NotImplementedError:
+        pass
+    return 4
+
+def pool_worker_init():
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 def readconfig(path):
     config = UnquoteConfig(delimiters='=', interpolation=None, strict=False)
@@ -114,38 +96,60 @@ def getrefs(*args):
         sys.exit(2)
     return refs
 
-def fetch_packages(options, return_all=False):
-    fetch_queue = queue.Queue()
-    updated_repos = Store()
-    for i in range(options.jobs):
-        t = ThreadFetch(fetch_queue, updated_repos, options.packagesdir, options.depth)
-        t.setDaemon(True)
-        t.start()
-
-    signal.signal(signal.SIGINT, signal.SIG_DFL)
+def fetch_package(gitrepo, refs_heads, options):
+    ref2fetch = []
+    for ref in refs_heads:
+        if gitrepo.check_remote(ref) != refs_heads[ref]:
+            ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
+    if ref2fetch:
+        ref2fetch.append('refs/notes/*:refs/notes/*')
 
+    try:
+        (stdout, stderr) = gitrepo.fetch(ref2fetch, options.depth)
+        if stderr != b'':
+            print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
+            return gitrepo
+    except GitRepoError as e:
+        print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
+         
+def fetch_packages(options, return_all=False):
     refs = getrefs(options.branch, options.repopattern)
     print('Read remotes data')
+    pkgs_new = []
+    if options.newpkgs:
+        for pkgdir in sorted(refs.heads):
+            gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
+            if not os.path.isdir(gitdir):
+                pkgs_new.append(pkgdir)
+
+        pool = WorkerPool(options.jobs, pool_worker_init)
+        try:
+            pool.starmap(initpackage, zip(pkgs_new, [options] * len(pkgs_new)))
+        except KeyboardInterrupt:
+            pool.terminate()
+        else:
+            pool.close()
+        pool.join()
+
+    args = []
     for pkgdir in sorted(refs.heads):
-        gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
-        if not os.path.isdir(gitdir):
-            if options.newpkgs:
-                gitrepo = initpackage(pkgdir, options)
-            else:
-                continue
-        elif options.omitexisting:
+        if options.omitexisting and pkgdir not in pkgs_new:
             continue
         else:
             gitrepo = GitRepo(os.path.join(options.packagesdir, pkgdir))
-        ref2fetch = []
-        for ref in refs.heads[pkgdir]:
-            if gitrepo.check_remote(ref) != refs.heads[pkgdir][ref]:
-                ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
-        if ref2fetch:
-            ref2fetch.append('refs/notes/*:refs/notes/*')
-            fetch_queue.put((gitrepo, ref2fetch))
+            args.append((gitrepo, refs.heads[pkgdir], options))
 
-    fetch_queue.join()
+    updated_repos = []
+    pool = WorkerPool(options.jobs, pool_worker_init)
+    try:
+        updated_repos = pool.starmap(fetch_package, args)
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
+
+    updated_repos = list(filter(None, updated_repos))
 
     if options.prune:
         refs = getrefs('*')
@@ -158,26 +162,60 @@ def fetch_packages(options, return_all=False):
     if return_all:
         return refs.heads
     else:
-        return updated_repos.items
+        return updated_repos
+
+def checkout_package(repo, options):
+    try:
+        repo.checkout(options.checkout)
+    except GitRepoError as e:
+        print('Problem with checking branch {} in repo {}: {}'.format(options.checkout, repo.gdir, e), file=sys.stderr)
 
 def checkout_packages(options):
     if options.checkout is None:
         options.checkout = "/".join([REMOTE_NAME, options.branch[0]])
     fetch_packages(options)
     refs = getrefs(options.branch, options.repopattern)
+    repos = []
     for pkgdir in sorted(refs.heads):
-        repo = GitRepo(os.path.join(options.packagesdir, pkgdir))
-        try:
-            repo.checkout(options.checkout)
-        except GitRepoError as e:
-            print('Problem with checking branch {} in repo {}: {}'.format(options.checkout, repo.gdir, e), file=sys.stderr)
+        repos.append(GitRepo(os.path.join(options.packagesdir, pkgdir)))
+    pool = WorkerPool(options.jobs)
+    try:
+        pool.starmap(checkout_package, zip(repos, [options] * len(repos)))
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
+
+def clone_package(repo, options):
+    try:
+        repo.checkout('master')
+    except GitRepoError as e:
+        print('Problem with checking branch master in repo {}: {}'.format(repo.gdir, e), file=sys.stderr)
 
 def clone_packages(options):
-    for repo in fetch_packages(options):
-        try:
-            repo.checkout('master')
-        except GitRepoError as e:
-            print('Problem with checking branch master in repo {}: {}'.format(repo.gdir, e), file=sys.stderr)
+    repos = fetch_packages(options)
+    pool = WorkerPool(options.jobs)
+    try:
+        pool.starmap(clone_package, zip(repos, [options] * len(repos)))
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
+
+def pull_package(gitrepo, options):
+    directory = os.path.basename(gitrepo.wtree)
+    try:
+        (out, err) = gitrepo.commandexc(['rev-parse', '-q', '--verify', '@{u}'])
+        sha1 = out.decode().strip()
+        (out, err) = gitrepo.commandexc(['rebase', sha1])
+        for line in out.decode().splitlines():
+            print(directory,":",line)
+    except GitRepoError as e:
+        for line in e.args[0].splitlines():
+            print("{}: {}".format(directory,line))
+        pass
 
 def pull_packages(options):
     repolist = []
@@ -189,19 +227,14 @@ def pull_packages(options):
     else:
         repolist = fetch_packages(options, False)
     print('--------Pulling------------')
-    for gitrepo in repolist:
-        directory = os.path.basename(gitrepo.wtree)
-        try:
-            (out, err) = gitrepo.commandexc(['rev-parse', '-q', '--verify', '@{u}'])
-            sha1 = out.decode().strip()
-            (out, err) = gitrepo.commandexc(['rebase', sha1])
-            for line in out.decode().splitlines():
-                print(directory,":",line)
-        except GitRepoError as e:
-            for line in e.args[0].splitlines():
-                print("{}: {}".format(directory,line))
-            pass
-
+    pool = WorkerPool(options.jobs, pool_worker_init)
+    try:
+        pool.starmap(pull_package, zip(repolist, [options] * len(repolist)))
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
 
 def list_packages(options):
     refs = getrefs(options.branch, options.repopattern)
@@ -213,7 +246,7 @@ common_options.add_argument('-d', '--packagesdir', help='local directory with gi
     default=os.path.expanduser('~/rpm/packages'))
 
 common_fetchoptions = argparse.ArgumentParser(add_help=False, parents=[common_options])
-common_fetchoptions.add_argument('-j', '--jobs', help='number of threads to use', default=4, type=int)
+common_fetchoptions.add_argument('-j', '--jobs', help='number of threads to use', default=cpu_count(), type=int)
 common_fetchoptions.add_argument('repopattern', nargs='*', default = ['*'])
 common_fetchoptions.add_argument('--depth', help='depth of fetch', default=0)
 
@@ -253,10 +286,14 @@ default_options['fetch'] = {'branch': '[*]', 'prune': False, 'newpkgs': False, '
 
 pull = subparsers.add_parser('pull', help='git-pull in all existing repositories', parents=[common_fetchoptions],
         formatter_class=argparse.RawDescriptionHelpFormatter)
-pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=True)
+pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=False)
 pull.add_argument('--noall', help='update local branches only when something has been fetched', dest='updateall', action='store_false', default=True)
+newpkgsopt = pull.add_mutually_exclusive_group()
+newpkgsopt.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side',
+        action='store_true')
+newpkgsopt.add_argument('-nn', '--nonewpkgs', help='do not download new packages', dest='newpkgs', action='store_false')
 pull.set_defaults(func=pull_packages, branch='[*]', prune=False, newpkgs=False, omitexisting=False)
-default_options['pull'] = {'branch': ['*'], 'prune': False, 'newpkgs': False, 'omitexisting': False}
+default_options['pull'] = {'branch': ['*'], 'prune': False, 'omitexisting': False}
 
 checkout =subparsers.add_parser('checkout', help='checkout repositories', parents=[common_fetchoptions],
         formatter_class=argparse.RawDescriptionHelpFormatter)
diff --git a/slug_watch b/slug_watch
index 7b64460..b077de9 100755
--- a/slug_watch
+++ b/slug_watch
@@ -23,6 +23,7 @@ LOCKFILE = 'slug_watch.lock'
 PROJECTS_LIST = 'projects.list'
 PROJECTS_LIST_NEW = PROJECTS_LIST + '.new'
 PROJECTS_LIST_HEAD = PROJECTS_LIST + '.head'
+PROJECTS_LIST_GITWEB = PROJECTS_LIST + ".gitweb"
 REFFILE_NEW = REFFILE + '.new'
 REFREPO_WDIR = 'Refs'
 
@@ -63,10 +64,11 @@ def process_file(pathname):
         print('{} is not an ordinary file'.format(pathname))
         return
 
-    try:
-        shutil.copyfile(PROJECTS_LIST_HEAD, PROJECTS_LIST_NEW)
-    except (OSError, shutil.Error):
-        logger.error('Cannot write {}'.format(PROJECTS_LIST_NEW))
+    if os.path.isfile(PROJECTS_LIST_HEAD):
+        try:
+            shutil.copyfile(PROJECTS_LIST_HEAD, PROJECTS_LIST_NEW)
+        except (OSError, shutil.Error):
+            logger.error('Cannot write {}'.format(PROJECTS_LIST_NEW))
 
     with open(os.path.join(REFREPO_WDIR, REFFILE),'w') as headfile_new, open(pathname, 'r') as newfile, \
             open(PROJECTS_LIST_NEW,'a') as projects:
@@ -82,7 +84,7 @@ def process_file(pathname):
                 if sha1 != EMPTYSHA1:
                     print(sha1, ref, repo, file=headfile_new)
                     if repo != oldtuple[0]:
-                        print(quote_plus('packages/'+repo+'.git', safe='/'), file=projects)
+                        print('packages/'+repo+'.git', file=projects)
                 oldtuple = (repo, ref)
         except ValueError:
             logger.error("Problem with file: {}".format(pathname))
@@ -90,6 +92,10 @@ def process_file(pathname):
         process.wait()
 
     os.rename(PROJECTS_LIST_NEW, PROJECTS_LIST)
+    with open(PROJECTS_LIST, 'r') as projects, open(PROJECTS_LIST_GITWEB, 'w') as output:
+        for line in projects:
+            print(quote_plus(line, safe='/\n'), end='', file=output)
+
     headrepo = GitRepo(REFREPO_WDIR, REFREPO_GDIR)
     headrepo.commitfile(REFFILE, 'Changes by {}'.format(committer))
     os.remove(pathname)