- rel 7; all goodness in one git patch

[packages/git-core-slug.git] / git-core-slug-git.patch

commit 29ab16f193cf3ebccb0c044b98f2ba9be98c3090
Author: Arkadiusz Miśkiewicz <arekm@maven.pl>
Date:   Sun Nov 23 00:32:46 2014 +0100

    Make updateall off by default.
    
    Turn updateall to off by default so 'git pld pull' will only checkout
    packages that were fetched at this session. If you want old behaviour
    use 'git pld pull --all'.

diff --git a/slug.py b/slug.py
index d083cf4..fa8fd89 100755
--- a/slug.py
+++ b/slug.py
@@ -274,7 +274,7 @@ default_options['fetch'] = {'branch': '[*]', 'prune': False, 'newpkgs': False, '
 
 pull = subparsers.add_parser('pull', help='git-pull in all existing repositories', parents=[common_fetchoptions],
         formatter_class=argparse.RawDescriptionHelpFormatter)
-pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=True)
+pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=False)
 pull.add_argument('--noall', help='update local branches only when something has been fetched', dest='updateall', action='store_false', default=True)
 newpkgsopt = pull.add_mutually_exclusive_group()
 newpkgsopt.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side',

commit da9abb0e6d7ef1a1440e7f5ac4ad4dbf5538dc99
Author: Arkadiusz Miśkiewicz <arekm@maven.pl>
Date:   Sun Nov 23 00:28:22 2014 +0100

    Add --newpkgs/--nonewpkgs to pull command.
    
    Allow 'git pld pull --newpkgs' to also fetch and pull new packages. Off
    by default.

diff --git a/slug.py b/slug.py
index da9c050..d083cf4 100755
--- a/slug.py
+++ b/slug.py
@@ -276,8 +276,12 @@ pull = subparsers.add_parser('pull', help='git-pull in all existing repositories
         formatter_class=argparse.RawDescriptionHelpFormatter)
 pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=True)
 pull.add_argument('--noall', help='update local branches only when something has been fetched', dest='updateall', action='store_false', default=True)
+newpkgsopt = pull.add_mutually_exclusive_group()
+newpkgsopt.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side',
+        action='store_true')
+newpkgsopt.add_argument('-nn', '--nonewpkgs', help='do not download new packages', dest='newpkgs', action='store_false')
 pull.set_defaults(func=pull_packages, branch='[*]', prune=False, newpkgs=False, omitexisting=False)
-default_options['pull'] = {'branch': ['*'], 'prune': False, 'newpkgs': False, 'omitexisting': False}
+default_options['pull'] = {'branch': ['*'], 'prune': False, 'omitexisting': False}
 
 checkout =subparsers.add_parser('checkout', help='checkout repositories', parents=[common_fetchoptions],
         formatter_class=argparse.RawDescriptionHelpFormatter)

commit b1096c634ea9b262bd791863d68e2aed3847078d
Author: Arkadiusz Miśkiewicz <arekm@maven.pl>
Date:   Sun Nov 23 00:18:38 2014 +0100

    Parallelize fetching, checking out, cloning.
    
    Parallelize fetching, checking out, cloning using multiprocessing
    module.
    
    By default use number of parallel processes equal to number of
    system CPUs (use old value, 4, as fallback).
    
    Also replace thread based ThreadFetch() with the same multiprocessing
    mechanism as above for consistency.

diff --git a/slug.py b/slug.py
index 69bd3b9..da9c050 100755
--- a/slug.py
+++ b/slug.py
@@ -7,26 +7,18 @@ import os
 import shutil
 import subprocess
 import queue
-import threading
-
+import multiprocessing
 import argparse
 
 import signal
 import configparser
 
+from multiprocessing import Pool as WorkerPool
+
 from git_slug.gitconst import GITLOGIN, GITSERVER, GIT_REPO, GIT_REPO_PUSH, REMOTE_NAME, REMOTEREFS
 from git_slug.gitrepo import GitRepo, GitRepoError
 from git_slug.refsdata import GitArchiveRefsData, NoMatchedRepos, RemoteRefsError
 
-class Store():
-    def __init__(self):
-        self.lock = threading.Lock()
-        self.items = []
-
-    def put(self, item):
-        with self.lock:
-            self.items.append(item)
-
 class UnquoteConfig(configparser.ConfigParser):
     def get(self, section, option, **kwargs):
         value = super().get(section, option, **kwargs)
@@ -43,25 +35,15 @@ class DelAppend(argparse._AppendAction):
         item.append(values)
         setattr(namespace, self.dest, item)
 
-class ThreadFetch(threading.Thread):
-    def __init__(self, queue, output, pkgdir, depth=0):
-        threading.Thread.__init__(self)
-        self.queue = queue
-        self.packagesdir = pkgdir
-        self.depth = depth
-        self.output = output
-
-    def run(self):
-        while True:
-            (gitrepo, ref2fetch) = self.queue.get()
-            try:
-                (stdout, stderr) = gitrepo.fetch(ref2fetch, self.depth)
-                if stderr != b'':
-                    print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
-                    self.output.put(gitrepo)
-            except GitRepoError as e:
-                print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
-            self.queue.task_done()
+def cpu_count():
+    try:
+        return multiprocessing.cpu_count()
+    except NotImplementedError:
+        pass
+    return 4
+
+def pool_worker_init():
+    signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 def readconfig(path):
     config = UnquoteConfig(delimiters='=', interpolation=None, strict=False)
@@ -114,18 +96,19 @@ def getrefs(*args):
         sys.exit(2)
     return refs
 
+def fetch_package(gitrepo, ref2fetch, options):
+    try:
+        (stdout, stderr) = gitrepo.fetch(ref2fetch, options.depth)
+        if stderr != b'':
+            print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
+            return gitrepo
+    except GitRepoError as e:
+        print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
+         
 def fetch_packages(options, return_all=False):
-    fetch_queue = queue.Queue()
-    updated_repos = Store()
-    for i in range(options.jobs):
-        t = ThreadFetch(fetch_queue, updated_repos, options.packagesdir, options.depth)
-        t.setDaemon(True)
-        t.start()
-
-    signal.signal(signal.SIGINT, signal.SIG_DFL)
-
     refs = getrefs(options.branch, options.repopattern)
     print('Read remotes data')
+    args = []
     for pkgdir in sorted(refs.heads):
         gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
         if not os.path.isdir(gitdir):
@@ -143,9 +126,18 @@ def fetch_packages(options, return_all=False):
                 ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
         if ref2fetch:
             ref2fetch.append('refs/notes/*:refs/notes/*')
-            fetch_queue.put((gitrepo, ref2fetch))
+            args.append((gitrepo, ref2fetch, options))
 
-    fetch_queue.join()
+    pool = WorkerPool(options.jobs, pool_worker_init)
+    try:
+        updated_repos = pool.starmap(fetch_package, args)
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
+
+    updated_repos = list(filter(None, updated_repos))
 
     if options.prune:
         refs = getrefs('*')
@@ -158,26 +150,60 @@ def fetch_packages(options, return_all=False):
     if return_all:
         return refs.heads
     else:
-        return updated_repos.items
+        return updated_repos
+
+def checkout_package(repo, options):
+    try:
+        repo.checkout(options.checkout)
+    except GitRepoError as e:
+        print('Problem with checking branch {} in repo {}: {}'.format(options.checkout, repo.gdir, e), file=sys.stderr)
 
 def checkout_packages(options):
     if options.checkout is None:
         options.checkout = "/".join([REMOTE_NAME, options.branch[0]])
     fetch_packages(options)
     refs = getrefs(options.branch, options.repopattern)
+    repos = []
     for pkgdir in sorted(refs.heads):
-        repo = GitRepo(os.path.join(options.packagesdir, pkgdir))
-        try:
-            repo.checkout(options.checkout)
-        except GitRepoError as e:
-            print('Problem with checking branch {} in repo {}: {}'.format(options.checkout, repo.gdir, e), file=sys.stderr)
+        repos.append(GitRepo(os.path.join(options.packagesdir, pkgdir)))
+    pool = WorkerPool(options.jobs)
+    try:
+        pool.starmap(checkout_package, zip(repos, [options] * len(repos)))
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
+
+def clone_package(repo, options):
+    try:
+        repo.checkout('master')
+    except GitRepoError as e:
+        print('Problem with checking branch master in repo {}: {}'.format(repo.gdir, e), file=sys.stderr)
 
 def clone_packages(options):
-    for repo in fetch_packages(options):
-        try:
-            repo.checkout('master')
-        except GitRepoError as e:
-            print('Problem with checking branch master in repo {}: {}'.format(repo.gdir, e), file=sys.stderr)
+    repos = fetch_packages(options)
+    pool = WorkerPool(options.jobs)
+    try:
+        pool.starmap(clone_package, zip(repos, [options] * len(repos)))
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
+
+def pull_package(gitrepo, options):
+    directory = os.path.basename(gitrepo.wtree)
+    try:
+        (out, err) = gitrepo.commandexc(['rev-parse', '-q', '--verify', '@{u}'])
+        sha1 = out.decode().strip()
+        (out, err) = gitrepo.commandexc(['rebase', sha1])
+        for line in out.decode().splitlines():
+            print(directory,":",line)
+    except GitRepoError as e:
+        for line in e.args[0].splitlines():
+            print("{}: {}".format(directory,line))
+        pass
 
 def pull_packages(options):
     repolist = []
@@ -189,19 +215,14 @@ def pull_packages(options):
     else:
         repolist = fetch_packages(options, False)
     print('--------Pulling------------')
-    for gitrepo in repolist:
-        directory = os.path.basename(gitrepo.wtree)
-        try:
-            (out, err) = gitrepo.commandexc(['rev-parse', '-q', '--verify', '@{u}'])
-            sha1 = out.decode().strip()
-            (out, err) = gitrepo.commandexc(['rebase', sha1])
-            for line in out.decode().splitlines():
-                print(directory,":",line)
-        except GitRepoError as e:
-            for line in e.args[0].splitlines():
-                print("{}: {}".format(directory,line))
-            pass
-
+    pool = WorkerPool(options.jobs, pool_worker_init)
+    try:
+        pool.starmap(pull_package, zip(repolist, [options] * len(repolist)))
+    except KeyboardInterrupt:
+        pool.terminate()
+    else:
+        pool.close()
+    pool.join()
 
 def list_packages(options):
     refs = getrefs(options.branch, options.repopattern)
@@ -213,7 +234,7 @@ common_options.add_argument('-d', '--packagesdir', help='local directory with gi
     default=os.path.expanduser('~/rpm/packages'))
 
 common_fetchoptions = argparse.ArgumentParser(add_help=False, parents=[common_options])
-common_fetchoptions.add_argument('-j', '--jobs', help='number of threads to use', default=4, type=int)
+common_fetchoptions.add_argument('-j', '--jobs', help='number of threads to use', default=cpu_count(), type=int)
 common_fetchoptions.add_argument('repopattern', nargs='*', default = ['*'])
 common_fetchoptions.add_argument('--depth', help='depth of fetch', default=0)
 

commit fac30722a98a4d6300822fd3f790ce1fa48e7d83
Author: Arkadiusz Miśkiewicz <arekm@maven.pl>
Date:   Sun Nov 23 00:15:10 2014 +0100

    check_remote(): Add support to packed refs database.
    
    check_remote() did not handle git packed refs database. That made
    fetch_packages() to always fetch packages even if we already had
    them fetched.
    
    Supporting packed refs database fixes this problem.

diff --git a/git_slug/gitrepo.py b/git_slug/gitrepo.py
index 5234deb..d9f88ee 100644
--- a/git_slug/gitrepo.py
+++ b/git_slug/gitrepo.py
@@ -82,12 +82,21 @@ class GitRepo:
             'refs/notes/*:refs/notes/*'])
 
     def check_remote(self, ref, remote=REMOTE_NAME):
+        localref = EMPTYSHA1
         ref = ref.replace(REFFILE, os.path.join('remotes', remote))
         try:
             with open(os.path.join(self.gdir, ref), 'r') as f:
                 localref = f.readline().strip()
         except IOError:
-            localref = EMPTYSHA1
+            try:
+                with open(os.path.join(self.gdir, 'packed-refs')) as f:
+                    for line in f:
+                        line_data = line.split()
+                        if len(line_data) == 2 and line_data[1] == ref:
+                            localref = line_data[0].strip()
+                            break
+            except IOError:
+                pass
         return localref
 
     def showfile(self, filename, ref="/".join([REMOTE_NAME, "master"])):