git-core-slug-git.patch

   1 diff --git a/git_slug/gitrepo.py b/git_slug/gitrepo.py
   2 index 5234deb..d9f88ee 100644
   3 --- a/git_slug/gitrepo.py
   4 +++ b/git_slug/gitrepo.py
   5 @@ -82,12 +82,21 @@ class GitRepo:
   6              'refs/notes/*:refs/notes/*'])
   7
   8      def check_remote(self, ref, remote=REMOTE_NAME):
   9 +        localref = EMPTYSHA1
  10          ref = ref.replace(REFFILE, os.path.join('remotes', remote))
  11          try:
  12              with open(os.path.join(self.gdir, ref), 'r') as f:
  13                  localref = f.readline().strip()
  14          except IOError:
  15 -            localref = EMPTYSHA1
  16 +            try:
  17 +                with open(os.path.join(self.gdir, 'packed-refs')) as f:
  18 +                    for line in f:
  19 +                        line_data = line.split()
  20 +                        if len(line_data) == 2 and line_data[1] == ref:
  21 +                            localref = line_data[0].strip()
  22 +                            break
  23 +            except IOError:
  24 +                pass
  25          return localref
  26
  27      def showfile(self, filename, ref="/".join([REMOTE_NAME, "master"])):
  28 diff --git a/git_slug/refsdata.py b/git_slug/refsdata.py
  29 index 4354ac4..67592f8 100644
  30 --- a/git_slug/refsdata.py
  31 +++ b/git_slug/refsdata.py
  32 @@ -16,7 +16,7 @@ class NoMatchedRepos(Exception):
  33
  34  class RemoteRefsData:
  35      def __init__(self, stream, pattern, dirpattern=('*',)):
  36 -        self.heads = collections.defaultdict(lambda: collections.defaultdict(lambda: EMPTYSHA1))
  37 +        self.heads = collections.defaultdict(self.__dict_var__)
  38          pats = re.compile('|'.join(fnmatch.translate(os.path.join('refs/heads', p)) for p in pattern))
  39          dirpat = re.compile('|'.join(fnmatch.translate(p) for p in dirpattern))
  40          for line in stream.readlines():
  41 @@ -28,6 +28,12 @@ class RemoteRefsData:
  42          if not self.heads:
  43              raise NoMatchedRepos
  44
  45 +    def __dict_init__(self):
  46 +        return EMPTYSHA1
  47 +
  48 +    def __dict_var__(self):
  49 +        return collections.defaultdict(self.__dict_init__)
  50 +
  51      def put(self, repo, data):
  52          for line in data:
  53              (sha1_old, sha1, ref) = line.split()
  54 diff --git a/slug.py b/slug.py
  55 index 69bd3b9..68f68cd 100755
  56 --- a/slug.py
  57 +++ b/slug.py
  58 @@ -7,26 +7,18 @@ import os
  59  import shutil
  60  import subprocess
  61  import queue
  62 -import threading
  63 -
  64 +import multiprocessing
  65  import argparse
  66
  67  import signal
  68  import configparser
  69
  70 +from multiprocessing import Pool as WorkerPool
  71 +
  72  from git_slug.gitconst import GITLOGIN, GITSERVER, GIT_REPO, GIT_REPO_PUSH, REMOTE_NAME, REMOTEREFS
  73  from git_slug.gitrepo import GitRepo, GitRepoError
  74  from git_slug.refsdata import GitArchiveRefsData, NoMatchedRepos, RemoteRefsError
  75
  76 -class Store():
  77 -    def __init__(self):
  78 -        self.lock = threading.Lock()
  79 -        self.items = []
  80 -
  81 -    def put(self, item):
  82 -        with self.lock:
  83 -            self.items.append(item)
  84 -
  85  class UnquoteConfig(configparser.ConfigParser):
  86      def get(self, section, option, **kwargs):
  87          value = super().get(section, option, **kwargs)
  88 @@ -43,25 +35,15 @@ class DelAppend(argparse._AppendAction):
  89          item.append(values)
  90          setattr(namespace, self.dest, item)
  91
  92 -class ThreadFetch(threading.Thread):
  93 -    def __init__(self, queue, output, pkgdir, depth=0):
  94 -        threading.Thread.__init__(self)
  95 -        self.queue = queue
  96 -        self.packagesdir = pkgdir
  97 -        self.depth = depth
  98 -        self.output = output
  99 -
 100 -    def run(self):
 101 -        while True:
 102 -            (gitrepo, ref2fetch) = self.queue.get()
 103 -            try:
 104 -                (stdout, stderr) = gitrepo.fetch(ref2fetch, self.depth)
 105 -                if stderr != b'':
 106 -                    print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
 107 -                    self.output.put(gitrepo)
 108 -            except GitRepoError as e:
 109 -                print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
 110 -            self.queue.task_done()
 111 +def cpu_count():
 112 +    try:
 113 +        return multiprocessing.cpu_count()
 114 +    except NotImplementedError:
 115 +        pass
 116 +    return 4
 117 +
 118 +def pool_worker_init():
 119 +    signal.signal(signal.SIGINT, signal.SIG_IGN)
 120
 121  def readconfig(path):
 122      config = UnquoteConfig(delimiters='=', interpolation=None, strict=False)
 123 @@ -114,38 +96,60 @@ def getrefs(*args):
 124          sys.exit(2)
 125      return refs
 126
 127 -def fetch_packages(options, return_all=False):
 128 -    fetch_queue = queue.Queue()
 129 -    updated_repos = Store()
 130 -    for i in range(options.jobs):
 131 -        t = ThreadFetch(fetch_queue, updated_repos, options.packagesdir, options.depth)
 132 -        t.setDaemon(True)
 133 -        t.start()
 134 -
 135 -    signal.signal(signal.SIGINT, signal.SIG_DFL)
 136 +def fetch_package(gitrepo, refs_heads, options):
 137 +    ref2fetch = []
 138 +    for ref in refs_heads:
 139 +        if gitrepo.check_remote(ref) != refs_heads[ref]:
 140 +            ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
 141 +    if ref2fetch:
 142 +        ref2fetch.append('refs/notes/*:refs/notes/*')
 143
 144 +    try:
 145 +        (stdout, stderr) = gitrepo.fetch(ref2fetch, options.depth)
 146 +        if stderr != b'':
 147 +            print('------', gitrepo.gdir[:-len('.git')], '------\n' + stderr.decode('utf-8'))
 148 +            return gitrepo
 149 +    except GitRepoError as e:
 150 +        print('------', gitrepo.gdir[:-len('.git')], '------\n', e)
 151 +
 152 +def fetch_packages(options, return_all=False):
 153      refs = getrefs(options.branch, options.repopattern)
 154      print('Read remotes data')
 155 +    pkgs_new = []
 156 +    if options.newpkgs:
 157 +        for pkgdir in sorted(refs.heads):
 158 +            gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
 159 +            if not os.path.isdir(gitdir):
 160 +                pkgs_new.append(pkgdir)
 161 +
 162 +        pool = WorkerPool(options.jobs, pool_worker_init)
 163 +        try:
 164 +            pool.starmap(initpackage, zip(pkgs_new, [options] * len(pkgs_new)))
 165 +        except KeyboardInterrupt:
 166 +            pool.terminate()
 167 +        else:
 168 +            pool.close()
 169 +        pool.join()
 170 +
 171 +    args = []
 172      for pkgdir in sorted(refs.heads):
 173 -        gitdir = os.path.join(options.packagesdir, pkgdir, '.git')
 174 -        if not os.path.isdir(gitdir):
 175 -            if options.newpkgs:
 176 -                gitrepo = initpackage(pkgdir, options)
 177 -            else:
 178 -                continue
 179 -        elif options.omitexisting:
 180 +        if options.omitexisting and pkgdir not in pkgs_new:
 181              continue
 182          else:
 183              gitrepo = GitRepo(os.path.join(options.packagesdir, pkgdir))
 184 -        ref2fetch = []
 185 -        for ref in refs.heads[pkgdir]:
 186 -            if gitrepo.check_remote(ref) != refs.heads[pkgdir][ref]:
 187 -                ref2fetch.append('+{}:{}/{}'.format(ref, REMOTEREFS, ref[len('refs/heads/'):]))
 188 -        if ref2fetch:
 189 -            ref2fetch.append('refs/notes/*:refs/notes/*')
 190 -            fetch_queue.put((gitrepo, ref2fetch))
 191 +            args.append((gitrepo, refs.heads[pkgdir], options))
 192
 193 -    fetch_queue.join()
 194 +    updated_repos = []
 195 +    pool = WorkerPool(options.jobs, pool_worker_init)
 196 +    try:
 197 +        updated_repos = pool.starmap(fetch_package, args)
 198 +    except KeyboardInterrupt:
 199 +        pool.terminate()
 200 +    else:
 201 +        pool.close()
 202 +    pool.join()
 203 +
 204 +    updated_repos = list(filter(None, updated_repos))
 205
 206      if options.prune:
 207          refs = getrefs('*')
 208 @@ -158,26 +162,60 @@ def fetch_packages(options, return_all=False):
 209      if return_all:
 210          return refs.heads
 211      else:
 212 -        return updated_repos.items
 213 +        return updated_repos
 214 +
 215 +def checkout_package(repo, options):
 216 +    try:
 217 +        repo.checkout(options.checkout)
 218 +    except GitRepoError as e:
 219 +        print('Problem with checking branch {} in repo {}: {}'.format(options.checkout, repo.gdir, e), file=sys.stderr)
 220
 221  def checkout_packages(options):
 222      if options.checkout is None:
 223          options.checkout = "/".join([REMOTE_NAME, options.branch[0]])
 224      fetch_packages(options)
 225      refs = getrefs(options.branch, options.repopattern)
 226 +    repos = []
 227      for pkgdir in sorted(refs.heads):
 228 -        repo = GitRepo(os.path.join(options.packagesdir, pkgdir))
 229 -        try:
 230 -            repo.checkout(options.checkout)
 231 -        except GitRepoError as e:
 232 -            print('Problem with checking branch {} in repo {}: {}'.format(options.checkout, repo.gdir, e), file=sys.stderr)
 233 +        repos.append(GitRepo(os.path.join(options.packagesdir, pkgdir)))
 234 +    pool = WorkerPool(options.jobs)
 235 +    try:
 236 +        pool.starmap(checkout_package, zip(repos, [options] * len(repos)))
 237 +    except KeyboardInterrupt:
 238 +        pool.terminate()
 239 +    else:
 240 +        pool.close()
 241 +    pool.join()
 242 +
 243 +def clone_package(repo, options):
 244 +    try:
 245 +        repo.checkout('master')
 246 +    except GitRepoError as e:
 247 +        print('Problem with checking branch master in repo {}: {}'.format(repo.gdir, e), file=sys.stderr)
 248
 249  def clone_packages(options):
 250 -    for repo in fetch_packages(options):
 251 -        try:
 252 -            repo.checkout('master')
 253 -        except GitRepoError as e:
 254 -            print('Problem with checking branch master in repo {}: {}'.format(repo.gdir, e), file=sys.stderr)
 255 +    repos = fetch_packages(options)
 256 +    pool = WorkerPool(options.jobs)
 257 +    try:
 258 +        pool.starmap(clone_package, zip(repos, [options] * len(repos)))
 259 +    except KeyboardInterrupt:
 260 +        pool.terminate()
 261 +    else:
 262 +        pool.close()
 263 +    pool.join()
 264 +
 265 +def pull_package(gitrepo, options):
 266 +    directory = os.path.basename(gitrepo.wtree)
 267 +    try:
 268 +        (out, err) = gitrepo.commandexc(['rev-parse', '-q', '--verify', '@{u}'])
 269 +        sha1 = out.decode().strip()
 270 +        (out, err) = gitrepo.commandexc(['rebase', sha1])
 271 +        for line in out.decode().splitlines():
 272 +            print(directory,":",line)
 273 +    except GitRepoError as e:
 274 +        for line in e.args[0].splitlines():
 275 +            print("{}: {}".format(directory,line))
 276 +        pass
 277
 278  def pull_packages(options):
 279      repolist = []
 280 @@ -189,19 +227,14 @@ def pull_packages(options):
 281      else:
 282          repolist = fetch_packages(options, False)
 283      print('--------Pulling------------')
 284 -    for gitrepo in repolist:
 285 -        directory = os.path.basename(gitrepo.wtree)
 286 -        try:
 287 -            (out, err) = gitrepo.commandexc(['rev-parse', '-q', '--verify', '@{u}'])
 288 -            sha1 = out.decode().strip()
 289 -            (out, err) = gitrepo.commandexc(['rebase', sha1])
 290 -            for line in out.decode().splitlines():
 291 -                print(directory,":",line)
 292 -        except GitRepoError as e:
 293 -            for line in e.args[0].splitlines():
 294 -                print("{}: {}".format(directory,line))
 295 -            pass
 296 -
 297 +    pool = WorkerPool(options.jobs, pool_worker_init)
 298 +    try:
 299 +        pool.starmap(pull_package, zip(repolist, [options] * len(repolist)))
 300 +    except KeyboardInterrupt:
 301 +        pool.terminate()
 302 +    else:
 303 +        pool.close()
 304 +    pool.join()
 305
 306  def list_packages(options):
 307      refs = getrefs(options.branch, options.repopattern)
 308 @@ -213,7 +246,7 @@ common_options.add_argument('-d', '--packagesdir', help='local directory with gi
 309      default=os.path.expanduser('~/rpm/packages'))
 310
 311  common_fetchoptions = argparse.ArgumentParser(add_help=False, parents=[common_options])
 312 -common_fetchoptions.add_argument('-j', '--jobs', help='number of threads to use', default=4, type=int)
 313 +common_fetchoptions.add_argument('-j', '--jobs', help='number of threads to use', default=cpu_count(), type=int)
 314  common_fetchoptions.add_argument('repopattern', nargs='*', default = ['*'])
 315  common_fetchoptions.add_argument('--depth', help='depth of fetch', default=0)
 316
 317 @@ -253,10 +286,14 @@ default_options['fetch'] = {'branch': '[*]', 'prune': False, 'newpkgs': False, '
 318
 319  pull = subparsers.add_parser('pull', help='git-pull in all existing repositories', parents=[common_fetchoptions],
 320          formatter_class=argparse.RawDescriptionHelpFormatter)
 321 -pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=True)
 322 +pull.add_argument('--all', help='update local branches in all repositories', dest='updateall', action='store_true', default=False)
 323  pull.add_argument('--noall', help='update local branches only when something has been fetched', dest='updateall', action='store_false', default=True)
 324 +newpkgsopt = pull.add_mutually_exclusive_group()
 325 +newpkgsopt.add_argument('-n', '--newpkgs', help='download packages that do not exist on local side',
 326 +        action='store_true')
 327 +newpkgsopt.add_argument('-nn', '--nonewpkgs', help='do not download new packages', dest='newpkgs', action='store_false')
 328  pull.set_defaults(func=pull_packages, branch='[*]', prune=False, newpkgs=False, omitexisting=False)
 329 -default_options['pull'] = {'branch': ['*'], 'prune': False, 'newpkgs': False, 'omitexisting': False}
 330 +default_options['pull'] = {'branch': ['*'], 'prune': False, 'omitexisting': False}
 331
 332  checkout =subparsers.add_parser('checkout', help='checkout repositories', parents=[common_fetchoptions],
 333          formatter_class=argparse.RawDescriptionHelpFormatter)
 334 diff --git a/slug_watch b/slug_watch
 335 index 7b64460..b077de9 100755
 336 --- a/slug_watch
 337 +++ b/slug_watch
 338 @@ -23,6 +23,7 @@ LOCKFILE = 'slug_watch.lock'
 339  PROJECTS_LIST = 'projects.list'
 340  PROJECTS_LIST_NEW = PROJECTS_LIST + '.new'
 341  PROJECTS_LIST_HEAD = PROJECTS_LIST + '.head'
 342 +PROJECTS_LIST_GITWEB = PROJECTS_LIST + ".gitweb"
 343  REFFILE_NEW = REFFILE + '.new'
 344  REFREPO_WDIR = 'Refs'
 345
 346 @@ -63,10 +64,11 @@ def process_file(pathname):
 347          print('{} is not an ordinary file'.format(pathname))
 348          return
 349
 350 -    try:
 351 -        shutil.copyfile(PROJECTS_LIST_HEAD, PROJECTS_LIST_NEW)
 352 -    except (OSError, shutil.Error):
 353 -        logger.error('Cannot write {}'.format(PROJECTS_LIST_NEW))
 354 +    if os.path.isfile(PROJECTS_LIST_HEAD):
 355 +        try:
 356 +            shutil.copyfile(PROJECTS_LIST_HEAD, PROJECTS_LIST_NEW)
 357 +        except (OSError, shutil.Error):
 358 +            logger.error('Cannot write {}'.format(PROJECTS_LIST_NEW))
 359
 360      with open(os.path.join(REFREPO_WDIR, REFFILE),'w') as headfile_new, open(pathname, 'r') as newfile, \
 361              open(PROJECTS_LIST_NEW,'a') as projects:
 362 @@ -82,7 +84,7 @@ def process_file(pathname):
 363                  if sha1 != EMPTYSHA1:
 364                      print(sha1, ref, repo, file=headfile_new)
 365                      if repo != oldtuple[0]:
 366 -                        print(quote_plus('packages/'+repo+'.git', safe='/'), file=projects)
 367 +                        print('packages/'+repo+'.git', file=projects)
 368                  oldtuple = (repo, ref)
 369          except ValueError:
 370              logger.error("Problem with file: {}".format(pathname))
 371 @@ -90,6 +92,10 @@ def process_file(pathname):
 372          process.wait()
 373
 374      os.rename(PROJECTS_LIST_NEW, PROJECTS_LIST)
 375 +    with open(PROJECTS_LIST, 'r') as projects, open(PROJECTS_LIST_GITWEB, 'w') as output:
 376 +        for line in projects:
 377 +            print(quote_plus(line, safe='/\n'), end='', file=output)
 378 +
 379      headrepo = GitRepo(REFREPO_WDIR, REFREPO_GDIR)
 380      headrepo.commitfile(REFFILE, 'Changes by {}'.format(committer))
 381      os.remove(pathname)