From elliot at rpath.com Fri Jul 9 14:16:06 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 09 Jul 2010 18:16:06 +0000
Subject: mirrorball: add license
Message-ID: <201007091816.o69IG67p022050@scc.eng.rpath.com>
changeset: c01ae04799a8
user: Elliot Peele
date: Tue, 06 Jul 2010 13:29:51 -0400
add license
diff --git a/scripts/buildpackages b/scripts/buildpackages
--- a/scripts/buildpackages
+++ b/scripts/buildpackages
@@ -2,6 +2,17 @@
#
# Copryright (c) 2008-2009 rPath, Inc.
#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
"""
Script for cooking packages with updatebot config.
From elliot at rpath.com Fri Jul 9 14:16:07 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 09 Jul 2010 18:16:07 +0000
Subject: mirrorball: add script for committing rmake jobs
Message-ID: <201007091816.o69IG7Mb022081@scc.eng.rpath.com>
changeset: 194e4e956ac5
user: Elliot Peele
date: Tue, 06 Jul 2010 13:30:02 -0400
add script for committing rmake jobs
diff --git a/scripts/commitjobs b/scripts/commitjobs
new file mode 100755
--- /dev/null
+++ b/scripts/commitjobs
@@ -0,0 +1,33 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2008-2009 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Script for committing a set of rmake jobs one at a time.
+"""
+
+import sys
+import logging
+
+from header import *
+
+log = logging.getLogger('script')
+
+jobIds = sys.argv[2:]
+
+for jobId in jobIds:
+ log.info('starting commit of %s' % jobId)
+ builder.commit(jobId)
+ log.info('commit of %s completed' % jobId)
+
From elliot at rpath.com Fri Jul 9 14:16:07 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 09 Jul 2010 18:16:07 +0000
Subject: mirrorball: pull in askyn from conary and add ability to get a
debugger
Message-ID: <201007091816.o69IG7uS022108@scc.eng.rpath.com>
changeset: 3f51aefb4aab
user: Elliot Peele
date: Fri, 09 Jul 2010 13:56:28 -0400
pull in askyn from conary and add ability to get a debugger
diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py
--- a/updatebot/lib/util.py
+++ b/updatebot/lib/util.py
@@ -24,7 +24,6 @@
import signal
import resource
from conary.lib.util import rmtree
-from conary.conaryclient.cmdline import askYn
from conary.lib.util import convertPackageNameToClassName as _pkgNameToClassName
from rpmutils import rpmvercmp
@@ -189,3 +188,24 @@
def convertPackageNameToClassName(pkgName):
name = _pkgNameToClassName(pkgName)
return name.replace('.', '_')
+
+def askYn(prompt, default=None):
+ while True:
+ try:
+ resp = raw_input(prompt + ' ')
+ except EOFError:
+ return False
+
+ resp = resp.lower()
+ if resp in ('y', 'yes'):
+ return True
+ elif resp in ('n', 'no'):
+ return False
+ elif resp in ('d', 'debug'):
+ epdb.st()
+ elif not resp:
+ return default
+ else:
+ print "Unknown response '%s'." % resp
+
+
From elliot at rpath.com Fri Jul 9 14:16:07 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 09 Jul 2010 18:16:07 +0000
Subject: mirrorball: add support for multiple ordered platforms in
order_promote
Message-ID: <201007091816.o69IG76l022135@scc.eng.rpath.com>
changeset: af72c22a76fe
user: Elliot Peele
date: Fri, 09 Jul 2010 14:11:35 -0400
add support for multiple ordered platforms in order_promote
diff --git a/scripts/order_promote.py b/scripts/order_promote.py
--- a/scripts/order_promote.py
+++ b/scripts/order_promote.py
@@ -31,8 +31,6 @@
confDir = os.path.join(mbdir, 'config', sys.argv[1])
-import rhnmirror
-
from updatebot import log
from updatebot import OrderedBot
from updatebot import UpdateBotConfig
@@ -41,13 +39,32 @@
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
-mcfg = rhnmirror.MirrorConfig()
-mcfg.read(confDir + '/erratarc')
+if cfg.platformName == 'rhel':
+ import rhnmirror
-errata = rhnmirror.Errata(mcfg)
-errata.fetch()
+ mcfg = rhnmirror.MirrorConfig()
+ mcfg.read(confDir + '/erratarc')
-bot = OrderedBot(cfg, errata)
+ errata = rhnmirror.Errata(mcfg)
+ errata.fetch()
+
+ bot = OrderedBot(cfg, errata)
+
+else:
+ bot = OrderedBot(cfg, None)
+
+ if cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
+
+ elif cfg.platformName == 'centos':
+ from errata.centos import AdvisoryManager as Errata
+
+ else:
+ raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+
+ errata = Errata(bot._pkgSource)
+ bot._errata._errata = errata
+
bot.promote()
import epdb; epdb.st()
From elliot at rpath.com Fri Jul 9 14:16:08 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 09 Jul 2010 18:16:08 +0000
Subject: mirrorball: make promote interactive
Message-ID: <201007091816.o69IG891022162@scc.eng.rpath.com>
changeset: f2c40222df90
user: Elliot Peele
date: Fri, 09 Jul 2010 14:13:30 -0400
make promote interactive
diff --git a/scripts/promote b/scripts/promote
--- a/scripts/promote
+++ b/scripts/promote
@@ -15,16 +15,16 @@
from header import *
+import logging
import tempfile
-from updatebot import conaryhelper
-helper = conaryhelper.ConaryHelper(cfg)
+slog = logging.getLogger('script')
-from conary import versions
from conary.deps import deps
-import logging
-slog = logging.getLogger('script')
+from updatebot import conaryhelper
+
+helper = conaryhelper.ConaryHelper(cfg)
#import cProfile
#prof = cProfile.Profile()
@@ -49,7 +49,8 @@
#prof.dump_stats('promote.lsprof')
#prof.print_stats()
-newPkgs = set([ (x[0].split(':')[0], x[1].getSourceVersion(), x[2]) for x in packages])
+newPkgs = set([ (x[0].split(':')[0], x[1].getSourceVersion(), x[2])
+ for x in packages])
pkgMap = {}
for n, v, f in newPkgs:
@@ -68,12 +69,16 @@
continue
slog.info('promoting %s=%s[%s]' % (pkg[0], pkg[1], flv))
-#csFileName = tempfile.mktemp()
-#slog.info('writing changeset to %s' % csFileName)
-#cs.writeToFile(csFileName)
-slog.info('committing')
+if ui.ask('commit?', default=False):
+ slog.info('committing')
+ helper._repos.commitChangeSet(cs)
+elif ui.ask('write to file?', default=True):
+ csFileName = tempfile.mktemp()
+ slog.info('writing changeset to %s' % csFileName)
+ cs.writeToFile(csFileName)
+else:
+ slog.info('not committing')
-helper._repos.commitChangeSet(cs)
-
+# Log a done mostly for timing purposes
slog.info('done')
From elliot at rpath.com Fri Jul 9 14:16:08 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 09 Jul 2010 18:16:08 +0000
Subject: mirrorball: don't blow up when a group has never been promoted to the
target label
Message-ID: <201007091816.o69IG8wr022189@scc.eng.rpath.com>
changeset: 757b19bb7d3a
user: Elliot Peele
date: Fri, 09 Jul 2010 14:14:45 -0400
don't blow up when a group has never been promoted to the target label
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -1084,6 +1084,11 @@
srcSpec = (trvSpec[0], trvSpec[1], None)
srcTrvs = self._conaryhelper.findTrove(srcSpec, getLeaves=False)
+
+ if not srcTrvs:
+ log.warn('no versions of %s found on %s' % (srcSpec[0], srcSpec[1]))
+ return {}
+
filteredSrcTrvs = [ (x, y, None) for x, y, z in srcTrvs ]
assert filteredSrcTrvs
srcMap = self._conaryhelper.getBinaryVersions(filteredSrcTrvs,
From elliot at rpath.com Fri Jul 9 14:16:09 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 09 Jul 2010 18:16:09 +0000
Subject: mirrorball: add group-os to buildreqs as fallback
Message-ID: <201007091816.o69IG9tJ022218@scc.eng.rpath.com>
changeset: 5753167b8cbb
user: Elliot Peele
date: Fri, 09 Jul 2010 14:15:44 -0400
add group-os to buildreqs as fallback
diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage
--- a/scripts/rebuildpackage
+++ b/scripts/rebuildpackage
@@ -170,6 +170,7 @@
'conary=rhel.rpath.com at rpath:rhel-5-devel',
'conary-build=rhel.rpath.com at rpath:rhel-5-devel',
'conary-policy=rhel.rpath.com at rpath:rhel-5-devel',
- 'rpm=rhel.rpath.com at rpath:rhel-5-server-devel', ])
+ 'rpm=rhel.rpath.com at rpath:rhel-5-server-devel',
+ 'group-os=rhel.rpath.com at rpath:rhel-5-server-devel',])
import epdb; epdb.st()
From elliot at rpath.com Tue Jul 13 01:55:25 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 13 Jul 2010 05:55:25 +0000
Subject: mirrorball: Add the repositoryPackage configuration option for
adding packages to the map
Message-ID: <201007130555.o6D5tP5v031358@scc.eng.rpath.com>
changeset: 7e00631e4958
user: Elliot Peele
date: Tue, 13 Jul 2010 01:55:16 -0400
Add the repositoryPackage configuration option for adding packages to the map
that is used for mapping packages to group use flags.
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -195,6 +195,21 @@
return CfgDict.toStrings(self, value, displayOptions)
+class CfgStringFourTuple(CfgString):
+ """
+ Config class to represent a three tuple of strings.
+ """
+
+ def parseString(self, val):
+ splt = val.split(val)
+ if len(splt) != 4:
+ raise ParseError
+ vals = []
+ for val in splt:
+ vals.append(CfgString.parseString(self, splt[0]))
+ return tuple(vals)
+
+
class UpdateBotConfigSection(cfg.ConfigSection):
"""
Config class for updatebot.
@@ -241,6 +256,11 @@
# repositoryName archString
repositoryArch = (CfgDict(CfgString), {})
+ # Add a package to a particular repository. This is useful for adding x86
+ # packages to an x86_64 group. Normally in the form: pkgName conaryVersion
+ # archStr repositoryArch
+ repositoryPackage = (CfgList(CfgStringFourTuple), [])
+
# Associate binaries generated from a nosrc package with a source package
# name if the nosrc package matches a given regular expression.
nosrcFilter = (CfgList(CfgStringFilter), [])
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -401,6 +401,14 @@
if source not in self.useMap:
self.useMap.setdefault(source, set()).add(source[-1])
+ for n, v, a, repoArch in self._cfg.repositoryPackages:
+ specs = [
+ (n, v, a),
+ (n, a),
+ ]
+ for spec in specs:
+ self.useMap.setdefault(spec, set()).add(repoArch)
+
# In the case of SLES 10 we need to combine several source entries in
# the srcPkgMap to create a single unified kernel source package.
if self._cfg.nosrcFilter:
From juphoff at rpath.com Tue Jul 13 10:09:14 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 13 Jul 2010 14:09:14 +0000
Subject: mirrorball: fixup some typoes with the repository Package config item
Message-ID: <201007131409.o6DE9EJ7010179@scc.eng.rpath.com>
changeset: faf9d07d34b3
user: Jeff Uphoff
date: Tue, 13 Jul 2010 02:44:26 -0400
fixup some typoes with the repository Package config item
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -201,12 +201,12 @@
"""
def parseString(self, val):
- splt = val.split(val)
+ splt = val.split()
if len(splt) != 4:
raise ParseError
vals = []
for val in splt:
- vals.append(CfgString.parseString(self, splt[0]))
+ vals.append(CfgString.parseString(self, val))
return tuple(vals)
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -401,7 +401,7 @@
if source not in self.useMap:
self.useMap.setdefault(source, set()).add(source[-1])
- for n, v, a, repoArch in self._cfg.repositoryPackages:
+ for n, v, a, repoArch in self._cfg.repositoryPackage:
specs = [
(n, v, a),
(n, a),
From juphoff at rpath.com Tue Jul 13 10:09:15 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 13 Jul 2010 14:09:15 +0000
Subject: mirrorball: add back ability to restart with a restorefile
Message-ID: <201007131409.o6DE9FbN010209@scc.eng.rpath.com>
changeset: caa92694da48
user: Jeff Uphoff
date: Tue, 13 Jul 2010 09:53:38 -0400
add back ability to restart with a restorefile
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -83,7 +83,6 @@
errata = Errata(bot._pkgSource)
bot._errata._errata = errata
-
-pkgMap = bot.update(fltr=fltr)
+pkgMap = bot.update(fltr=fltr, restoreFile=restoreFile)
import epdb; epdb.st()
From juphoff at rpath.com Tue Jul 13 10:09:15 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 13 Jul 2010 14:09:15 +0000
Subject: mirrorball: All platforms that do not inherit from other
platforms to specify useOldVersions
Message-ID: <201007131409.o6DE9F5A010237@scc.eng.rpath.com>
changeset: b63ee0c3c46b
user: Jeff Uphoff
date: Tue, 13 Jul 2010 09:55:16 -0400
All platforms that do not inherit from other platforms to specify useOldVersions
From juphoff at rpath.com Tue Jul 13 10:09:16 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 13 Jul 2010 14:09:16 +0000
Subject: mirrorball: Allow platforms that do not inherit from other
platforms to specify useOldVersions
Message-ID: <201007131409.o6DE9GeK010264@scc.eng.rpath.com>
changeset: fad523f165dd
user: Jeff Uphoff
date: Tue, 13 Jul 2010 09:59:28 -0400
Allow platforms that do not inherit from other platforms to specify useOldVersions
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -303,7 +303,7 @@
# When deriving from an upstream platform sometimes we don't want
# the latest versions.
oldVersions = self._cfg.useOldVersion.get(updateId, None)
- if self._cfg.platformSearchPath and oldVersions:
+ if oldVersions:
for nvf in oldVersions:
# Lookup all source and binaries that match this binary.
srcMap = self._updater.getSourceVersionMapFromBinaryVersion(
From elliot at rpath.com Tue Jul 13 10:17:52 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 13 Jul 2010 14:17:52 +0000
Subject: mirrorball: Don't require removal packages to be part of the
group model.
Message-ID: <201007131417.o6DEHqcY010548@scc.eng.rpath.com>
changeset: 007a07b03053
user: Elliot Peele
date: Tue, 13 Jul 2010 10:17:35 -0400
Don't require removal packages to be part of the group model.
For some package removals where we have remapped the package name to something
else (eg. kernel-debug -> kernel) this can cause a removal to fail since there
is no package named kernel-debug in the group model.
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -338,7 +338,7 @@
log.info('removing the following packages from the managed '
'group: %s' % ', '.join(requiredRemovals))
for pkg in requiredRemovals:
- group.removePackage(pkg)
+ group.removePackage(pkg, missingOk=True)
if removeObsoleted:
log.info('removing any of obsoleted packages from the managed '
'group: %s' % ', '.join(removeObsoleted))
From elliot at rpath.com Tue Jul 13 10:17:53 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 13 Jul 2010 14:17:53 +0000
Subject: mirrorball: branch merge
Message-ID: <201007131417.o6DEHrCM010579@scc.eng.rpath.com>
changeset: b1e19ab6a6ba
user: Elliot Peele
date: Tue, 13 Jul 2010 10:17:49 -0400
branch merge
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -338,7 +338,7 @@
log.info('removing the following packages from the managed '
'group: %s' % ', '.join(requiredRemovals))
for pkg in requiredRemovals:
- group.removePackage(pkg)
+ group.removePackage(pkg, missingOk=True)
if removeObsoleted:
log.info('removing any of obsoleted packages from the managed '
'group: %s' % ', '.join(removeObsoleted))
From elliot at rpath.com Wed Jul 14 13:30:25 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 14 Jul 2010 17:30:25 +0000
Subject: mirrorball: When using a package map, make sure there are no
packages that have been moved
Message-ID: <201007141730.o6EHUPA2012390@scc.eng.rpath.com>
changeset: 8c770e19831f
user: Elliot Peele
date: Wed, 14 Jul 2010 13:30:11 -0400
When using a package map, make sure there are no packages that have been moved
into the current bucket. If there are new packages in the bucket, build them.
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -286,19 +286,27 @@
allowDowngrades = self._cfg.allowPackageDowngrades.get(updateId, [])
# If recovering from a failure, restore the pkgMap from disk.
+ pkgMap = {}
if restoreFile:
pkgMap = self._restorePackages(restoreFile)
restoreFile = None
+ # Filter out anything that has already been built from the list
+ # of updates.
+ upMap = dict([ (x.name, x) for x in updates ])
+ for n, v, f in pkgMap:
+ if n in upMap:
+ updates.remove(upMap[n])
+
# Update package set.
- else:
+ if updates:
fltr = kwargs.pop('fltr', None)
if fltr:
updates = fltr(updates)
- pkgMap = self._update(*args, updatePkgs=updates,
+ pkgMap.update(self._update(*args, updatePkgs=updates,
expectedRemovals=expectedRemovals,
- allowPackageDowngrades=allowDowngrades, **kwargs)
+ allowPackageDowngrades=allowDowngrades, **kwargs))
# When deriving from an upstream platform sometimes we don't want
# the latest versions.
From elliot at rpath.com Mon Jul 19 15:41:17 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:17 +0000
Subject: mirrorball: Use a dispatcher model for rebuilding packages.
Message-ID: <201007191941.o6JJfHX9018020@scc.eng.rpath.com>
changeset: 67d219e8fde3
user: Elliot Peele
date: Mon, 19 Jul 2010 15:32:29 -0400
Use a dispatcher model for rebuilding packages.
This is useful when rebuilding many package names and versions.
From elliot at rpath.com Mon Jul 19 15:41:18 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:18 +0000
Subject: mirrorball: group rebuild bug fixes and logging improvements
Message-ID: <201007191941.o6JJfI5V018052@scc.eng.rpath.com>
changeset: cf5b13702362
user: Elliot Peele
date: Mon, 19 Jul 2010 15:36:29 -0400
group rebuild bug fixes and logging improvements
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildgroups
@@ -15,6 +15,7 @@
import os
import sys
+import time
import logging
import itertools
@@ -57,16 +58,20 @@
# Make sure to include any packages that were built from the same
# source.
siblingPackages = set()
- for name in updatedPackages:
- log.info('looking up siblings for %s' % name)
- nvfs = self._updater._conaryhelper.findTrove((name, None, None),
- getLeaves=False)
- siblingMap = self._updater._conaryhelper.getSiblingPackages(
- nvfs, allVersions=True)
- for siblings in siblingMap.itervalues():
- siblingPackages |= set([ x[0] for x in siblings ])
+
+ log.info('looking up siblings')
+ req = [ (x, None, None) for x in updatedPackages ]
+ nvfMap = self._updater._conaryhelper.findTroves(req, getLeaves=False)
+ nvfs = [ x for x in itertools.chain(*nvfMap.itervalues()) ]
+
+ siblingMap = self._updater._conaryhelper.getSiblingPackages(nvfs,
+ allVersions=True)
+
+ for siblings in siblingMap.itervalues():
+ siblingPackages |= set([ x[0].split(':')[0] for x in siblings ])
updatedPackages = list(siblingPackages)
+ trvMap = {}
results = []
log.info('done loading siblings')
@@ -101,13 +106,9 @@
else:
nvfs.add((n, v, f))
- # Lookup anything that has an empty flavor.
- req = set([ (x[0], x[1], None) for x in emptyFlavors ])
- found = self._updater._conaryhelper.findTroves(req)
- for n, v, f in itertools.chain(*found.values()):
- nvfs.add((n, v, f))
-
# Lookup anything that is expected to have been rebuilt.
+ log.info('%s: looking up version information for rebuild packages'
+ % version)
for n, v, f in checkUpdates:
upVer = '/'.join([v.branch().label().asString(),
v.trailingRevision().version])
@@ -118,20 +119,35 @@
latest = sorted(binSpecs)[-1]
- log.info('%s: found updated version of %s %s -> %s'
- % (version, n, v, latest[1]))
+ if v != latest[1]:
+ #log.info('%s: found updated version of %s %s -> %s'
+ # % (version, n, v, latest[1]))
- if v != latest[1]:
- nvfs.add(latest)
+ toAdd = set([ x for x in binSpecs if x[1] == latest[1] ])
+ nvfs.update(toAdd)
+ elif f is None:
+ emptyFlavors.add((n, v, f))
else:
nvfs.add((n, v, f))
+ # Lookup anything that has an empty flavor.
+ log.info('%s: looking up version information for empty flavors'
+ % version)
+ req = set([ (x[0], x[1], None) for x in emptyFlavors ])
+ found = self._updater._conaryhelper.findTroves(req)
+
+ for n, v, f in itertools.chain(*found.values()):
+ nvfs.add((n, v, f))
+
# Lookup cloned from info for all versions.
log.info('%s: retrieving target version information' % version)
- targetVersions, failed = self._updater.getTargetVersions(nvfs)
+ targetVersions, failed = self._updater.getTargetVersions(nvfs,
+ logErrors=False)
# Make sure any that failed are in the set of packages that are
# expected to have updates.
+ # NOTE: This normally happens if the group model has already been
+ # remapped to the target label.
assert not [ x for x in failed if x[0] not in updatedPackages ]
# Take the union of all versions on the target label and those that
@@ -174,8 +190,14 @@
res = group.buildmany()
results.append(res)
- import epdb; epdb.st()
+ # Wait for the first results to make sure the group will rebuild
+ # properly.
+ #if not trvMap:
+ # log.info('waiting for first build to complete before continuing')
+ # while not res.isDone:
+ # time.sleep(1)
+ # Check for any built groups.
completed = [ x for x in results if x.isDone ]
if completed:
for res in completed:
@@ -183,7 +205,10 @@
log.info('%s: built troves' % version)
for n, v, f in sorted(bins):
log.info(' %s=%s[%s]' % (n, v, f))
- results.remove(res)
+ results.remove(res)
+ trvMap.update(res.results)
+
+ return trvMap
if __name__ == '__main__':
@@ -219,6 +244,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups(updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', 'kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', 'python-dmidecode', 'python-elementtree', 'python-imaging', 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', 'python-setuptools', 'python-sqlite', 'python-urlgrabber', 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', 'system-config-printer', 'vte', 'wireshark', 'yum', 'yum-metadata-parser',])
+ bot.rebuildgroups(updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', 'kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', 'python-dmidecode', 'python-elementtree', 'python-imaging', 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', 'python-setuptools', 'python-sqlite', 'python-urlgrabber', 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', 'system-config-printer', 'vte', 'wireshark', 'yum', 'yum-metadata-parser', 'mvapich2', 'gnome-doc-utils', 'system-config-lvm', 'booty', 'smartmontools', 'devhelp', 'system-switch-mail', 'gedit', 'system-config-nfs', 'ant', 'kexec-tools', 'inn', 'tetex', 'docbook-style-xsl', 'firstboot', 'system-config-kickstart', 'kdesdk', 'system-config-display', 'switchdesk', 'gimp', 'system-config-samba', 'kdevelop', 'hal', 'frysk', 'redhat-release', 'gettext', 'libglade2', 'system-config-bind', 'system-config-httpd', 'system-config-keyboard', 'system-config-language', 'system-config-kdump', 'libhugetlbfs', 'comps-extras', 'system-config-soundcard', 'systemtap', 'vnc', 'conary-policy', 'system-config-services', 'createrepo', 'blktrace', 'yum-utils', 'system-config-netboot', 'sblim', 'setroubleshoot-plugins', 'xinetd', 'rhn-client-tools', 'libevent', 'mailman', 'system-config-securitylevel', 'selinux-policy', 'hwbrowser', 'bluez-utils', 'system-config-users', 'cman', 'yum-rhn-plugin', 'system-config-boot', 'system-config-network', 'system-config-date', 'ecryptfs-utils', 'tix', 'anaconda', 'system-config-rootpassword', 'dstat'])
import epdb; epdb.st()
From elliot at rpath.com Mon Jul 19 15:41:18 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:18 +0000
Subject: mirrorball: Add an option to not log cloned from lookup failures.
Failures are expected when
Message-ID: <201007191941.o6JJfIT2018079@scc.eng.rpath.com>
changeset: bac202b5c1d0
user: Elliot Peele
date: Mon, 19 Jul 2010 15:37:42 -0400
Add an option to not log cloned from lookup failures. Failures are expected when
inserting new versions of packages during a group rebuild.
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -283,7 +283,7 @@
return binMap
- def getTargetVersions(self, binTrvSpecs):
+ def getTargetVersions(self, binTrvSpecs, logErrors=True):
"""
Given a list of binary trove specs from the devel label return a list of
promoted trove versions.
@@ -306,8 +306,9 @@
for x, y, z in failed
if (x, y.getSourceVersion()) not in seen ]
- for spec in fail:
- log.critical('%s=%s[%s] not found in cloned from map' % spec)
+ if logErrors:
+ for spec in fail:
+ log.critical('%s=%s[%s] not found in cloned from map' % spec)
return targetSpecs, fail
From elliot at rpath.com Mon Jul 19 15:41:18 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:18 +0000
Subject: mirrorball: add support for setproctitle from rmake
Message-ID: <201007191941.o6JJfIMu018107@scc.eng.rpath.com>
changeset: d505bd029b38
user: Elliot Peele
date: Mon, 19 Jul 2010 15:38:05 -0400
add support for setproctitle from rmake
diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py
--- a/updatebot/lib/util.py
+++ b/updatebot/lib/util.py
@@ -23,6 +23,8 @@
import epdb
import signal
import resource
+
+from rmake.lib import osutil
from conary.lib.util import rmtree
from conary.lib.util import convertPackageNameToClassName as _pkgNameToClassName
@@ -208,4 +210,8 @@
else:
print "Unknown response '%s'." % resp
-
+def setproctitle(title):
+ try:
+ osutil.setproctitle('mirrorball %s' % (title,))
+ except:
+ pass
From elliot at rpath.com Mon Jul 19 15:41:19 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:19 +0000
Subject: mirrorball: Only print some pkgsource messages when running in
debug mode
Message-ID: <201007191941.o6JJfJlw018135@scc.eng.rpath.com>
changeset: f7cf1e9a3d56
user: Elliot Peele
date: Mon, 19 Jul 2010 15:38:46 -0400
Only print some pkgsource messages when running in debug mode
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -254,7 +254,7 @@
continue
if key not in self._rpmMap:
- log.warn('found source without binary rpms: %s' % pkg)
+ log.debug('found source without binary rpms: %s' % pkg)
#log.debug(key)
#log.debug([ x for x in self._rpmMap if x[0] == key[0] ])
From elliot at rpath.com Mon Jul 19 15:41:19 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:19 +0000
Subject: mirrorball: improvements to the local group building
infrastructure
Message-ID: <201007191941.o6JJfJg1018163@scc.eng.rpath.com>
changeset: 88b7b9bc4fd2
user: Elliot Peele
date: Mon, 19 Jul 2010 15:39:14 -0400
improvements to the local group building infrastructure
diff --git a/scripts/buildgroups b/scripts/buildlocal
copy from scripts/buildgroups
copy to scripts/buildlocal
--- a/scripts/buildgroups
+++ b/scripts/buildlocal
@@ -9,9 +9,28 @@
from header import *
-builder = build.Builder(cfg, ui, rmakeCfgFn='rmakerc-groups')
+from updatebot.ordered import Bot
-grpTrvMap = builder.build((cfg.topSourceGroup, ))
+from conary.conaryclient import cmdline
+
+bot = Bot(cfg, ui)
+
+results = []
+for trvSpec in sys.argv[2:]:
+ n, v, f = cmdline.parseTroveSpec(trvSpec)
+ group = bot._groupmgr.getGroup(version=v)
+ results.append(group.buildmany())
+else:
+ group = bot._groupmgr.getGroup()
+ results.append(group.buildmany())
+
+import time
+while [ x for x in results if not x.isDone ]:
+ time.sleep(3)
+
+grpTrvMap = {}
+for res in results:
+ grpTrvMap.update(res.results)
print "built:\n"
diff --git a/updatebot/build/common.py b/updatebot/build/common.py
--- a/updatebot/build/common.py
+++ b/updatebot/build/common.py
@@ -24,10 +24,37 @@
from multiprocessing import Process
from multiprocessing.queues import Queue as ProcessQueue
+from updatebot.lib import util
from updatebot.build.constants import MessageTypes
log = logging.getLogger('updatebot.build')
+class LogQueue(object):
+ def __init__(self, logger):
+ self._log = logger
+ self._prefix = None
+
+ def setPrefix(self, prefix):
+ self._prefix = prefix
+
+ def log(self, level, msg):
+ if self._prefix:
+ msg = self._prefix + msg
+ self._log.put((MessageTypes.LOG, msg))
+
+ def info(self, msg):
+ self.log(logging.INFO, msg)
+
+ def warn(self, msg):
+ self.log(logging.WARNING, msg)
+
+ def critical(self, msg):
+ self.log(logging.CRITICAL, msg)
+
+ def error(self, msg):
+ self.log(logging.ERROR, msg)
+
+
class AbstractWorker(object):
"""
Abstract class for all worker nodes.
@@ -38,6 +65,7 @@
def __init__(self, status):
self.status = status
self.workerId = None
+ self.log = LogQueue(status)
def run(self):
"""
@@ -82,6 +110,16 @@
def __init__(self, status):
AbstractWorker.__init__(self, status)
Process.__init__(self)
+ self.processId = '[none set]'
+ self.daemon = True
+
+ def run(self):
+ util.setproctitle(self.processId)
+ self.log.setPrefix('[%s] ' % self.pid)
+ self.log.info('starting %s' % self.processId)
+ AbstractWorker.run(self)
+ self.status.close()
+ self.status.join_thread()
class AbstractStatusMonitor(object):
diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py
--- a/updatebot/build/cvc.py
+++ b/updatebot/build/cvc.py
@@ -18,6 +18,7 @@
import os
import copy
+import time
import logging
import tempfile
@@ -60,7 +61,7 @@
self._client = conaryclient.ConaryClient(self._ccfg)
- def cook(self, troveSpecs, flavorFilter=None, commit=True):
+ def cook(self, troveSpecs, flavorFilter=None, commit=True, callback=None):
"""
Cook a set of trove specs, currently limited to groups.
@params troveSpecs: list of name, version, and flavor tuples.
@@ -73,6 +74,8 @@
@param commit: Optional parameter to control when a changeset is
committed.
@type commit: boolean
+ @param callback: Cook callback to use when building.
+ @type callback: conary.build.cook.CookCallback
"""
# TODO: Look at conary.build.cook.cookCommand for how to setup
@@ -98,7 +101,7 @@
# pulled from conary.cvc
groupCookOptions = cook.GroupCookOptions(
alwaysBumpCount = True,
- errorOnFlavorChange = True,
+ errorOnFlavorChange = False,
shortenFlavors = self._ccfg.shortenGroupFlavors
)
@@ -106,13 +109,16 @@
flavors = set([ x[2] for x in troveSpecs ])
item = (troveSpecs[0][0], troveSpecs[0][1], flavors)
+ if not callback:
+ callback = conarycallbacks.UpdateBotCookCallback()
+
built = cook.cookItem(
self._client.repos,
self._ccfg,
item,
ignoreDeps=True,
logBuild=True,
- callback=conarycallbacks.UpdateBotCookCallback(),
+ callback=callback,
groupOptions=groupCookOptions,
changeSetFile=changeSetFile,
)
@@ -149,9 +155,17 @@
@type callback: conary.callbacks.ChangesetCallback
"""
- return self._client.repos.commitChangeSetFile(
+ log.info('starting commit of %s' % changeSetFile)
+
+ start = time.time()
+ res = self._client.repos.commitChangeSetFile(
changeSetFile, callback=callback)
+ elapsed = time.time() - start
+ log.info('completed commit of %s in %s' % (changeSetFile, elapsed))
+
+ return True
+
def build(self, trvSpec, flavorFilter=None):
"""
Build trove locally.
diff --git a/updatebot/build/jobs.py b/updatebot/build/jobs.py
--- a/updatebot/build/jobs.py
+++ b/updatebot/build/jobs.py
@@ -124,6 +124,7 @@
self._order = []
self._started = False
+ self._done = False
self.daemon = True
@@ -140,7 +141,9 @@
Polling loop to monitor and update job status.
"""
- while not self._jobDone():
+ totalWait = 0
+ committed = []
+ while not self._done:
# Start any jobs that are waiting to be started as long as there
# are available slots.
notStarted = self._getNotStarted()
@@ -175,6 +178,10 @@
# Find jobs that are ready to be committed. Jobs must be committed
# in the order that they were submitted.
for trove in self._order:
+ # continue if this trove has already been committed.
+ if self._jobs[trove][1] in self._completed:
+ continue
+
# Make sure everything has committed in order.
if self._jobs[trove][1] not in self._slotdone:
break
@@ -200,7 +207,8 @@
self._jobs[trove][1] = JobStatus.JOB_COMMITTED
res = self._jobs[trove][2]
res.setStatus('committed')
- res.setResults(results)
+ res.setResults(res.data.buildResults)
+ committed.append(trove)
# Check for commit errors.
for trove, error in self._committer.getErrors():
@@ -209,12 +217,21 @@
self._jobs[trove][2].setError(error)
time.sleep(3)
+ totalWait += 3
+
+ # Only log slot status once a minute
+ if not totalWait % 60:
+ log.info('build slots: %s' % self._slots)
+ log.info('commit slots: %s' % self._commitSlots)
def build(self, troveSpec, flavorFilter=None):
"""
Add one trove spec to the build queue.
"""
+ # Make sure this instance hasn't been marked as done.
+ assert not self._done
+
# Require at least one trove.
if not troveSpec:
return None
@@ -223,14 +240,28 @@
while not self._slots:
time.sleep(3)
- status = self.statusClass(troveSpec)
- status.data.flavorFilter = frozenset(flavorFilter)
+ if troveSpec not in self._jobs:
+ status = self.statusClass(troveSpec)
+ status.data.flavorFilter = frozenset(flavorFilter)
- self._jobs[troveSpec] = [troveSpec, JobStatus.JOB_NOT_STARTED, status]
- self._order.append(troveSpec)
+ self._jobs[troveSpec] = [troveSpec, JobStatus.JOB_NOT_STARTED,
+ status]
+ self._order.append(troveSpec)
+ else:
+ log.warn('already building/built requested trove: %s=%s'
+ % (troveSpec[0], troveSpec[1]))
+ return self._jobs[troveSpec][2]
if not self._started:
self.start()
self._started = True
return status
+
+ def done(self):
+ """
+ Mark this worker as complete. This stops the worker thread. Note that
+ once this is called this instance may no longer be used for building.
+ """
+
+ self._done = True
diff --git a/updatebot/build/local.py b/updatebot/build/local.py
--- a/updatebot/build/local.py
+++ b/updatebot/build/local.py
@@ -16,9 +16,9 @@
Module for cordinating local group builds.
"""
+from updatebot.lib import conarycallbacks
from updatebot.build.constants import WorkerTypes
from updatebot.build.constants import MessageTypes
-
from updatebot.build.common import AbstractStatusMonitor
from updatebot.build.common import AbstractWorkerProcess as AbstractWorker
@@ -35,6 +35,7 @@
self.builder = builder
self.trove = trove
self.workerId = trove
+ self.processId = 'cook %s=%s' % (trove[0], trove[1])
self.flavorFilter = flavorFilter
def work(self):
@@ -42,7 +43,9 @@
Build the specified trove and commit it to the repository.
"""
- res, csfn = self.builder.cvc.cook(self.trove, flavorFilter=self.flavorFilter, commit=False)
+ res, csfn = self.builder.cvc.cook(self.trove,
+ flavorFilter=self.flavorFilter, commit=False,
+ callback=conarycallbacks.UpdateBotCookCallback(log=self.log))
self.status.put((MessageTypes.DATA, (self.trove, res, csfn)))
@@ -60,13 +63,15 @@
self.csfn = csfn
self.trove = trove
self.workerId = trove
+ self.processId = 'commit %s=%s' % (trove[0], trove[1])
def work(self):
"""
Commit the specified changeset.
"""
- results = self.builder.cvc.commitChangeSetFile(self.csfn)
+ results = self.builder.cvc.commitChangeSetFile(self.csfn,
+ callback=conarycallbacks.UpdateBotCookCallback(log=self.log))
self.status.put((MessageTypes.DATA, (self.trove, results)))
From elliot at rpath.com Mon Jul 19 15:41:20 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:20 +0000
Subject: mirrorball: when markremoving groups, make sure to remove all
related group binaries and sources
Message-ID: <201007191941.o6JJfKj7018192@scc.eng.rpath.com>
changeset: 5893ced55c60
user: Elliot Peele
date: Mon, 19 Jul 2010 15:39:53 -0400
when markremoving groups, make sure to remove all related group binaries and sources
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1341,11 +1341,29 @@
resultMap = self._repos.findTroves(self._ccfg.buildLabel, troveSpecs,
getLeaves=not removeAllVersions)
+ query = set()
+ if not removeAllVersions:
+ trvMap = {}
+ for trvLst in resultMap.itervalues():
+ for n, v, f in trvLst:
+ trvMap.setdefault(n, dict()).setdefault(v, set()).add(f)
+
+ # We almost always want the latest versions that were found.
+ for n, vMap in trvMap.iteritems():
+ if len(vMap) > 1:
+ vers = sorted(vMap)
+ latest = vers[-1]
+ else:
+ latest = vMap.keys()[0]
+ for f in vMap[latest]:
+ query.add((n, v, f))
+
# Build trove query list.
- query = set()
- for trv, trvLst in resultMap.iteritems():
- for n, v, f in trvLst:
- query.add((n, v, f))
+ else:
+ query = set()
+ for trv, trvLst in resultMap.iteritems():
+ for n, v, f in trvLst:
+ query.add((n, v, f))
# Get troves from the repository.
troves = self._repos.getTroves(query, withFiles=False)
@@ -1353,9 +1371,6 @@
# Build set of troveSpecs to be removed.
trvSet = set()
for trv in troves:
- # Don't recurse group contents.
- if trv.getName().startswith('group-'):
- continue
# Find all sibling packages.
if removeSiblings:
srcName = trv.troveInfo.sourceName()
@@ -1381,6 +1396,9 @@
for n, v, f in srcLst:
trvSet.add((n, v, f))
+ # Don't recurse group contents.
+ if trv.getName().startswith('group-'):
+ continue
trvSet.update(set([ x for x in
trv.iterTroveList(strongRefs=True) ]))
From elliot at rpath.com Mon Jul 19 15:41:20 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 19:41:20 +0000
Subject: mirrorball: make looking up sibling packages faster
Message-ID: <201007191941.o6JJfKIg018220@scc.eng.rpath.com>
changeset: 553e650faf5c
user: Elliot Peele
date: Mon, 19 Jul 2010 15:40:09 -0400
make looking up sibling packages faster
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1481,19 +1481,19 @@
else:
nvfs = [nvf, ]
- trvs = self._repos.getTroves(nvfs, withFiles=False)
+ srcTroveInfo = self._repos.getTroveInfo(trove._TROVEINFO_TAG_SOURCENAME, nvfs)
# Figure out unique source names and versions for all nvfs in an attempt
# to minimize the number of repository calls.
sources = {}
- for trvSpec, trv in itertools.izip(nvfs, trvs):
- srcName = trv.troveInfo.sourceName()
+ for (n, v, f), frzSrcName in itertools.izip(nvfs, srcTroveInfo):
+ srcName = frzSrcName()
if not srcName:
- srcName = trv.getName()
+ srcName = n.split(':')[0]
- srcVersion = trv.getVersion().getSourceVersion()
+ srcVersion = v.getSourceVersion()
- sources.setdefault((srcName, srcVersion), set()).add(trvSpec)
+ sources.setdefault((srcName, srcVersion), set()).add((n, v, f))
# Map siblings back to nvfs.
siblingMap = {}
From elliot at rpath.com Mon Jul 19 16:42:15 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Jul 2010 20:42:15 +0000
Subject: mirrorball: Use a dispatcher model for rebuilding packages.
Message-ID: <201007192042.o6JKgF6k019380@scc.eng.rpath.com>
changeset: 65a53c881d7b
user: Elliot Peele
date: Mon, 19 Jul 2010 16:41:42 -0400
Use a dispatcher model for rebuilding packages.
This is useful when rebuilding many package names and versions.
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -52,6 +52,7 @@
from updatebot.build.jobs import LocalDispatcher
from updatebot.build.dispatcher import Dispatcher
from updatebot.build.dispatcher import NonCommittalDispatcher
+from updatebot.build.dispatcher import MultiVersionRebuildDispatcher
from updatebot.build.callbacks import StatusOnlyDisplay
log = logging.getLogger('updatebot.build')
@@ -236,7 +237,8 @@
ret = self._formatOutput(trvMap)
return ret
- def rebuild(self, troveSpecs, useLatest=None, additionalResolveTroves=None):
+ def rebuild(self, troveSpecs, useLatest=None, additionalResolveTroves=None,
+ commit=True):
"""
Rebuild a set of troves in the same environment that they were
orignally built in.
@@ -249,7 +251,11 @@
@param additionalResolveTroves: List of additional trove specs to add to
the resolve troves.
@type additionalResolveTroves: list(str, ...)
- @return troveMap: dictionary of troveSpecs to built troves
+ @param commit: Controls waiting for jobs to complete and then committing
+ them one at a time. (default: True)
+ @type commit: boolean
+ @return if commit: troveMap: dictionary of troveSpecs to built troves
+ @return if not commit: list of jobIds
"""
# Set some defaults
@@ -345,6 +351,10 @@
for job in grpByNameVersion(jobs):
jobIds.append(startOne(job))
+ # If not committing jobs, return the list of ids.
+ if not commit:
+ return jobIds
+
# Wait for jobs to complete
dispatcher = NonCommittalDispatcher(self, 0)
dispatcher.watchmany(jobIds)
@@ -357,6 +367,34 @@
return ret
+ def rebuildmany(self, troveSpecs, useLatest=None,
+ additionalResolveTroves=None, commit=True):
+ """
+ Rebuild a set of troves in the same environment that they were
+ orignally built in.
+ @param troveSpecs: set of name, version, flavor tuples
+ @type troveSpecs: set([(name, version, flavor), ..])
+ @param useLatest: A list of package names to use the latest versions of.
+ For instance, you may want to use the latest version
+ of conary to get fixed dependencies.
+ @type useLatest: list(str, ...)
+ @param additionalResolveTroves: List of additional trove specs to add to
+ the resolve troves.
+ @type additionalResolveTroves: list(str, ...)
+ @param commit: Controls waiting for jobs to complete and then committing
+ them one at a time. (default: True)
+ @type commit: boolean
+ @return if commit: troveMap: dictionary of troveSpecs to built troves
+ @return if not commit: list of jobIds
+ """
+
+ dispatcher = MultiVersionRebuildDispatcher(self, 30,
+ useLatest=useLatest,
+ additionalResolveTroves=additionalResolveTroves)
+
+ return dispatcher.buildmany(troveSpecs)
+
+
def start(self, troveSpecs):
"""
Public version of start job that starts a job without monitoring.
diff --git a/updatebot/build/constants.py b/updatebot/build/constants.py
--- a/updatebot/build/constants.py
+++ b/updatebot/build/constants.py
@@ -37,6 +37,7 @@
COMMIT = 2
LOCAL_GROUP_BUILD = 3
LOCAL_CHANGESET_COMMIT = 4
+ REBUILD_START = 5
names = {
START: 'Start',
@@ -44,6 +45,7 @@
COMMIT: 'Commit',
LOCAL_GROUP_BUILD: 'Local Group Build',
LOCAL_CHANGESET_COMMIT: 'Local Changeset Commit',
+ REBUILD_START: 'Rebuild Start',
}
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -27,6 +27,7 @@
from updatebot.build.monitor import JobStarter
from updatebot.build.monitor import JobMonitor
from updatebot.build.monitor import JobCommitter
+from updatebot.build.monitor import JobRebuildStarter
from updatebot.build.constants import JobStatus
@@ -90,6 +91,9 @@
buildjob.JOB_STATE_BUILT,
)
+ _starterClass = JobStarter
+ _monitorClass = JobMonitor
+ _committerClass = JobCommitter
def __init__(self, builder, maxSlots):
AbstractDispatcher.__init__(self, builder, maxSlots)
@@ -100,9 +104,9 @@
self._maxCommitSlots = 2
self._commitSlots = self._maxCommitSlots
- self._starter = JobStarter(self._builder)
- self._monitor = JobMonitor(self._builder._helper.client)
- self._committer = JobCommitter(self._builder)
+ self._starter = self._starterClass(self._builder)
+ self._monitor = self._monitorClass(self._builder._helper.client)
+ self._committer = self._committerClass(self._builder)
def buildmany(self, troveSpecs):
"""
@@ -155,21 +159,14 @@
# submit any jobs that are ready to commit as long as there are
# commit slots
- toCommit = set()
- for jobId, (trove, status, result) in self._jobs.iteritems():
- # don't try to commit anything if there are no free commit
- # slots.
- if not self._commitSlots:
- break
- # batch up all jobs that are ready to be committed
- if status == buildjob.JOB_STATE_BUILT:
+ toCommit = self._getCommitJobs()
+ # commit all available jobs at one time.
+ if toCommit:
+ for jobId in toCommit:
# update status to !BUILT so that we don't try to commit
# this job more than once.
self._jobs[jobId][1] = JobStatus.JOB_COMMITTING
- toCommit.add(jobId)
- # commit all available jobs at one time.
- if toCommit:
self._committer.commitJob(tuple(toCommit))
self._commitSlots -= 1
@@ -223,6 +220,24 @@
return results, self._failures
+ def _getCommitJobs(self):
+ """
+ Get a set of jobIds that are ready to be committed.
+ """
+
+ toCommit = set()
+
+ # don't try to commit anything if there are no free commit slots.
+ if not self._commitSlots:
+ return toCommit
+
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ # batch up all jobs that are ready to be committed
+ if status == buildjob.JOB_STATE_BUILT:
+ toCommit.add(jobId)
+
+ return toCommit
+
class NonCommittalDispatcher(Dispatcher):
"""
@@ -294,3 +309,68 @@
for jobId, (trove, status, result) in self._jobs.iteritems():
if status != buildjob.JOB_STATE_BUILT:
raise JobNotCompleteError(jobId=jobId)
+
+
+class MultiVersionRebuildDispatcher(Dispatcher):
+ """
+ A dispatcher implementation for building many packages with multiple
+ versions of the same package.
+ """
+
+ _starterClass = JobRebuildStarter
+
+ def __init__(self, builder, maxSlots, useLatest=None,
+ additionalResolveTroves=None):
+ Dispatcher.__init__(self, builder, maxSlots)
+
+ self._starter = self._starterClass((builder, useLatest,
+ additionalResolveTroves))
+
+ self._maxCommitSlots = 1
+ self._commitSlots = self._maxCommitSlots
+
+ # Mapping of pkgname to ordered list of trove specs
+ self._pkgs = {}
+
+ def buildmany(self, troveSpecs):
+ """
+ Build as many packages as possible until we run out of slots.
+ """
+
+ for spec in troveSpecs:
+ self._pkgs.setdefault(spec[0], []).append(spec)
+ for name, specLst in self._pkgs.iteritems():
+ self._pkgs[name] = sorted(specLst)
+
+ troveSpecs = sorted(troveSpecs)
+
+ return Dispatcher.buildmany(self, troveSpecs)
+
+ def _getCommitJobs(self):
+ """
+ Get a set of jobIds that are ready to be committed.
+ """
+
+ toCommit = set()
+
+ # don't try to commit anything if there are no free commit slots.
+ if not self._commitSlots:
+ return toCommit
+
+ built = {}
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ # batch up all jobs that are ready to be committed
+ if status == buildjob.JOB_STATE_BUILT:
+ built.setdefault(trove[0], dict())[trove] = jobId
+
+ toCommit = set()
+ for name, jobDict in built.iteritems():
+ # Wait for all versions of a package to build.
+ if len(jobDict) == len(self._pkgs[name]):
+ # Pop off the first trove spec to commit.
+ spec = self._pkgs[name].pop(0)
+
+ jobId = jobDict[spec]
+ toCommit.add(jobId)
+
+ return toCommit
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -50,6 +50,36 @@
self.status.put((MessageTypes.DATA, (self.trove, jobId)))
+class RebuildStartWorker(StartWorker):
+ """
+ Worker thread for starting package rebuild jobs and reporting status.
+ """
+
+ threadType = WorkerTypes.REBUILD_START
+
+ def __init__(self, status, (builder, useLatest,
+ additionalResolveTroves, trove)):
+ StartWorker.__init__(self, status, (builder, trove))
+
+ self.useLatest = useLatest
+ self.additionalResolveTroves = additionalResolveTroves
+
+ def work(self):
+ """
+ Start the specified build and report jobId.
+ """
+
+ jobIds = self.builder.rebuildstart(self.trove, useLatest=self.useLatest,
+ additionalResolveTroves=self.additionalResolveTroves, commit=False)
+
+ if len(jobIds) != 1:
+ self.status.put((MessageTypes.THREAD_ERROR, (self.threadType,
+ self.workerId, 'More jobIds returned than expected while '
+ 'building %s' % self.trove)))
+ else:
+ self.status.put((MessageTypes.DATA, (self.trove, jobIds[0])))
+
+
class MonitorWorker(AbstractWorker):
"""
Worker thread for monitoring jobs and reporting status.
@@ -119,6 +149,15 @@
startJob = AbstractStatusMonitor.addJob
+class JobRebuildStarter(AbstractStatusMonitor):
+ """
+ Abstraction around threaded rebuild starter model.
+ """
+
+ workerClass = RebuildStartWorker
+ startJob = AbstractStatusMonitor.addJob
+
+
class JobMonitor(AbstractStatusMonitor):
"""
Abstraction around threaded monitoring model.
From elliot at rpath.com Wed Jul 21 15:14:35 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:35 +0000
Subject: mirrorball: Mark config based removals as missingok
Message-ID: <201007211914.o6LJEZkM007043@scc.eng.rpath.com>
changeset: c11a8b172fbd
user: Elliot Peele
date: Tue, 20 Jul 2010 10:42:41 -0400
Mark config based removals as missingok
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -398,7 +398,12 @@
if pkgFlv:
group.removePackageFlavor(pkgName, pkgFlv.freeze())
else:
- group.remove(pkgName)
+ # Need to set missingok to True in case this package was
+ # never added to the group in the first place because it is
+ # no longer in the packages group. This tends to happen
+ # during a group rebuild when regenerating the standard
+ # group.
+ group.remove(pkgName, missingOk=True)
# Add requested packages.
for groupName, pkgs in additions.iteritems():
From elliot at rpath.com Wed Jul 21 15:14:35 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:35 +0000
Subject: mirrorball: Split the rebuild dispatcher into a multiversion and
rebuild dispatcher
Message-ID: <201007211914.o6LJEZ41007074@scc.eng.rpath.com>
changeset: 2a4d78a4d55b
user: Elliot Peele
date: Wed, 21 Jul 2010 15:05:02 -0400
Split the rebuild dispatcher into a multiversion and rebuild dispatcher
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -52,7 +52,7 @@
from updatebot.build.jobs import LocalDispatcher
from updatebot.build.dispatcher import Dispatcher
from updatebot.build.dispatcher import NonCommittalDispatcher
-from updatebot.build.dispatcher import MultiVersionRebuildDispatcher
+from updatebot.build.dispatcher import RebuildDispatcher
from updatebot.build.callbacks import StatusOnlyDisplay
log = logging.getLogger('updatebot.build')
@@ -388,8 +388,7 @@
@return if not commit: list of jobIds
"""
- dispatcher = MultiVersionRebuildDispatcher(self, 30,
- useLatest=useLatest,
+ dispatcher = RebuildDispatcher(self, 30, useLatest=useLatest,
additionalResolveTroves=additionalResolveTroves)
return dispatcher.buildmany(troveSpecs)
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -311,20 +311,16 @@
raise JobNotCompleteError(jobId=jobId)
-class MultiVersionRebuildDispatcher(Dispatcher):
+class MultiVersionDispatcher(Dispatcher):
"""
A dispatcher implementation for building many packages with multiple
versions of the same package.
"""
- _starterClass = JobRebuildStarter
-
- def __init__(self, builder, maxSlots, useLatest=None,
- additionalResolveTroves=None):
+ def __init__(self, builder, maxSlots, waitForAllVersions=False):
Dispatcher.__init__(self, builder, maxSlots)
- self._starter = self._starterClass((builder, useLatest,
- additionalResolveTroves))
+ self._waitForAllVersions = waitForAllVersions
self._maxCommitSlots = 1
self._commitSlots = self._maxCommitSlots
@@ -363,10 +359,14 @@
if status == buildjob.JOB_STATE_BUILT:
built.setdefault(trove[0], dict())[trove] = jobId
- toCommit = set()
for name, jobDict in built.iteritems():
- # Wait for all versions of a package to build.
- if len(jobDict) == len(self._pkgs[name]):
+ # Wait for all versions of a package to build.
+ if ((self._waitForAllVersions and
+ len(jobDict) == len(self._pkgs[name])) or
+ # Wait for the first version to be built.
+ (not self._waitForAllVersions and
+ self._pkgs[name] and
+ self._pkgs[name][0] in jobDict)):
# Pop off the first trove spec to commit.
spec = self._pkgs[name].pop(0)
@@ -374,3 +374,19 @@
toCommit.add(jobId)
return toCommit
+
+
+class RebuildDispatcher(MultiVersionDispatcher):
+ """
+ Dispatcher for coordinating massive package rebuilds.
+ """
+
+ _starterClass = JobRebuildStarter
+
+ def __init__(self, builder, maxSlots, useLatest=None,
+ additionalResolveTroves=None):
+ MultiVersionDispatcher.__init__(self, builder, maxSlots,
+ waitForAllVersions=True)
+
+ self._starter = self._starterClass((builder, useLatest,
+ additionalResolveTroves))
From elliot at rpath.com Wed Jul 21 15:14:36 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:36 +0000
Subject: mirrorball: dispatcher for incremental adding of build troves
Message-ID: <201007211914.o6LJEax8007101@scc.eng.rpath.com>
changeset: e032caf88bc2
user: Elliot Peele
date: Wed, 21 Jul 2010 15:07:45 -0400
dispatcher for incremental adding of build troves
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -50,6 +50,7 @@
from updatebot.build.cvc import Cvc
from updatebot.build.jobs import LocalDispatcher
+from updatebot.build.jobs import OrderedCommitDispatcher
from updatebot.build.dispatcher import Dispatcher
from updatebot.build.dispatcher import NonCommittalDispatcher
from updatebot.build.dispatcher import RebuildDispatcher
@@ -154,6 +155,8 @@
self.cvc = Cvc(self._cfg, self._ccfg, self._formatInput,
LocalDispatcher(self, 12))
+ self._asyncDispatcher = OrderedCommitDispatcher(self, 30)
+
def build(self, troveSpecs):
"""
Build a list of troves.
@@ -237,6 +240,18 @@
ret = self._formatOutput(trvMap)
return ret
+ def buildasync(self, troveSpec):
+ """
+ Build troves in much the same way buildmany does, but without blocking
+ the main thread.
+ @param troveSpec: name, version, flavor tuple
+ @type troveSpec: tuple(str, conary.versions.VersionFromString, None)
+ @return status object
+ @rtype updatebot.build.jobs.Status
+ """
+
+ return self._asyncDispatcher.build(troveSpec)
+
def rebuild(self, troveSpecs, useLatest=None, additionalResolveTroves=None,
commit=True):
"""
diff --git a/updatebot/build/constants.py b/updatebot/build/constants.py
--- a/updatebot/build/constants.py
+++ b/updatebot/build/constants.py
@@ -62,3 +62,4 @@
JOB_BUILT = -6
JOB_COMMITTED = -7
JOB_FAILED = -8
+ JOB_STARTING = -9
diff --git a/updatebot/build/jobs.py b/updatebot/build/jobs.py
--- a/updatebot/build/jobs.py
+++ b/updatebot/build/jobs.py
@@ -22,9 +22,14 @@
import logging
from threading import Thread
+from rmake.build import buildjob
+
from updatebot.build.constants import JobStatus
from updatebot.build.dispatcher import AbstractDispatcher
+from updatebot.build.monitor import JobStarter
+from updatebot.build.monitor import JobMonitor
+from updatebot.build.monitor import JobCommitter
from updatebot.build.local import LocalGroupCooker
from updatebot.build.local import LocalChangeSetCommitter
@@ -265,3 +270,273 @@
"""
self._done = True
+
+
+class JobBasedDispatcher(AbstractDispatcher, Thread):
+ """
+ Dispatcher for coordinating builds of multiple troves.
+ """
+
+ statusClass = Status
+
+ _completed = (
+ JobStatus.ERROR_MONITOR_FAILURE,
+ JobStatus.ERROR_COMMITTER_FAILURE,
+ buildjob.JOB_STATE_FAILED,
+ buildjob.JOB_STATE_COMMITTED,
+ )
+
+ _slotdone = (
+ buildjob.JOB_STATE_FAILED,
+ buildjob.JOB_STATE_BUILT,
+ )
+
+ _starterClass = JobStarter
+ _monitorClass = JobMonitor
+ _committerClass = JobCommitter
+
+ def __init__(self, builder, maxSlots):
+ AbstractDispatcher.__init__(self, builder, maxSlots)
+ Thread.__init__(self)
+
+ self._maxStartSlots = 10
+ self._startSlots = self._maxStartSlots
+
+ self._maxCommitSlots = 2
+ self._commitSlots = self._maxCommitSlots
+
+ self._starter = self._starterClass(self._builder)
+ self._monitor = self._monitorClass(self._builder._helper.client)
+ self._committer = self._committerClass(self._builder)
+
+ self._troves = {}
+ self._done = False
+ self._started = False
+
+ self.daemon = True
+
+ def _getNotStarted(self):
+ """
+ Get the list of jobs that have not yet been started.
+ """
+
+ return [ x for x in self._troves.itervalues()
+ if x[1] == JobStatus.JOB_NOT_STARTED ]
+
+ def run(self):
+ """
+ Build as many packages as possible until we run out of slots.
+ """
+
+ while not self._done:
+ # Only create more jobs once the last batch has been started.
+ if self._startSlots == self._maxStartSlots:
+ # fill slots with available troves
+ notStarted = self._getNotStarted()
+ while notStarted and self._slots and self._startSlots:
+ # get trove to work on
+ trove, status, res = notStarted[0]
+
+ # start build job
+ self._starter.startJob(trove)
+ self._troves[trove][1] = JobStatus.JOB_STARTING
+ res.setStatus('starting')
+
+ self._slots -= 1
+ self._startSlots -= 1
+
+ notStarted = self._getNotStarted()
+
+ # get started status
+ for trove, jobId in self._starter.getStatus():
+ self._jobs[jobId] = self._troves[trove]
+ self._jobs[jobId][2].setStatus('started')
+ self._startSlots += 1
+ self._monitor.monitorJob(jobId)
+
+ # process starter errors
+ for trove, error in self._starter.getErrors():
+ self._startSlots += 1
+ self._slots += 1
+ self._failures.append((trove, error))
+ self._troves[trove][2].setStatus('start failed')
+ self._troves[trove][2].setError(error)
+
+ # update job status changes
+ for jobId, status in self._monitor.getStatus():
+ self._jobs[jobId][1] = status
+ # free up the slot once the job is built
+ if status in self._slotdone:
+ self._slots += 1
+
+ if self._slots > self._maxSlots:
+ log.critical('slots is greater than maxSlots')
+
+ res = self._jobs[jobId][2]
+ if status == buildjob.JOB_STATE_FAILED:
+ res.setError('job failed')
+ res.setStatus('job failed')
+ else:
+ res.setStatus('built')
+
+ # submit any jobs that are ready to commit as long as there are
+ # commit slots
+ toCommit = self._getCommitJobs()
+ # commit all available jobs at one time.
+ if toCommit:
+ for jobId in toCommit:
+ # update status to !BUILT so that we don't try to commit
+ # this job more than once.
+ self._jobs[jobId][1] = JobStatus.JOB_COMMITTING
+ self._jobs[jobId][2].setStatus('committing')
+
+ self._committer.commitJob(tuple(toCommit))
+ self._commitSlots -= 1
+
+ # process monitor errors
+ for jobId, error in self._monitor.getErrors():
+ self._slots += 1
+ self._jobs[jobId][1] = JobStatus.ERROR_MONITOR_FAILURE
+ self._jobs[jobId][2].setStatus('monitor failed')
+ self._jobs[jobId][2].setError(error)
+ self._failures.append((jobId, error))
+
+ # check for commit status
+ for jobId, result in self._committer.getStatus():
+ # unbatch commit jobs
+ if not isinstance(jobId, tuple):
+ jobId = (jobId, )
+ for jobId in jobId:
+ self._jobs[jobId][2].setResults(result)
+ self._jobs[jobId][2].setStatus('committed')
+ self._commitSlots += 1
+
+ # process committer errors
+ for jobId, error in self._committer.getErrors():
+ # unbatch commit jobs
+ if not isinstance(jobId, tuple):
+ jobId = (jobId, )
+ for jobId in jobId:
+ self._jobs[jobId][1] = JobStatus.ERROR_COMMITTER_FAILURE
+ self._jobs[jobId][2].setError(error)
+ self._jobs[jobId][2].setStatus('commit failed')
+ self._failures.append((jobId, error))
+ self._commitSlots += 1
+
+ # Flag job as failed so that monitor worker will exit properly.
+ self._builder.setCommitFailed(jobId, reason=str(error))
+
+ # Wait for a bit before polling again.
+ time.sleep(3)
+
+ def _getCommitJobs(self):
+ """
+ Get a set of jobIds that are ready to be committed.
+ """
+
+ toCommit = set()
+
+ # don't try to commit anything if there are no free commit slots.
+ if not self._commitSlots:
+ return toCommit
+
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ # batch up all jobs that are ready to be committed
+ if status == buildjob.JOB_STATE_BUILT:
+ toCommit.add(jobId)
+
+ return toCommit
+
+ def build(self, troveSpec):
+ """
+ Add one trove spec to the build queue.
+ """
+
+ # Make sure this instance hasn't been marked as done.
+ assert not self._done
+
+ # Require at least one trove.
+ if not troveSpec:
+ return None
+
+ # Wait for an available slot.
+ while not self._slots:
+ time.sleep(3)
+
+ if troveSpec not in self._troves:
+ status = self.statusClass(troveSpec)
+
+ self._troves[troveSpec] = [troveSpec, JobStatus.JOB_NOT_STARTED,
+ status]
+ else:
+ log.warn('already building/built requested trove: %s=%s'
+ % (troveSpec[0], troveSpec[1]))
+ return self._jobs[troveSpec][2]
+
+ if not self._started:
+ self.start()
+ self._started = True
+
+ return status
+
+ def done(self):
+ # report failures
+ for job, error in self._failures:
+ log.error('[%s] failed with error: %s' % (job, error))
+
+ results = {}
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ # don't return errors if we intentionaly didn't commit.
+ built = buildjob.JOB_STATE_BUILT
+ if status == built and built in self._completed and not result:
+ continue
+ # log failed jobs
+ if status == buildjob.JOB_STATE_FAILED or not result.isDone:
+ log.info('[%s] failed job: %s' % (jobId, trove))
+ self._failures.append((jobId, status))
+ else:
+ results.update(result.results)
+
+ self._done = True
+ return results, self._failures
+
+
+class OrderedCommitDispatcher(JobBasedDispatcher):
+ """
+ Dispatcher for coordinating trove builds and committing in the order that
+ the builds were submitted.
+ """
+
+ def __init__(self, builder, maxJobs):
+ JobBasedDispatcher.__init__(self, builder, maxJobs)
+
+ self._pkgs = {}
+
+ def build(self, troveSpec):
+ self._pkgs.setdefault(troveSpec[0], []).append(troveSpec)
+ return JobBasedDispatcher.build(self, troveSpec)
+
+ def _getCommitJobs(self):
+ """
+ Get a set of jobIds that are ready to be committed.
+ """
+
+ toCommit = set()
+
+ # don't try to commit anything if there are no free commit slots.
+ if not self._commitSlots:
+ return toCommit
+
+ built = {}
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ # batch up all jobs that are ready to be committed
+ if status == buildjob.JOB_STATE_BUILT:
+ built.setdefault(trove[0], dict())[trove] = jobId
+
+ for name, troveMap in built.iteritems():
+ if self._pkgs[name] and self._pkgs[name][0] in troveMap:
+ spec = self._pkgs[name].pop(0)
+ jobId = troveMap[spec]
+ toCommit.add(jobId)
+
+ return toCommit
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -176,7 +176,7 @@
Build this group alongside other groups.
"""
- return self._mgr.buildGroup(self, multiBuild=True)
+ return self._mgr.buildGroup(self, async=True)
def hasBinaryVersion(self):
"""
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -296,7 +296,7 @@
return self.getGroup(version=newVersion)
@require_write
- def buildGroup(self, group, multiBuild=False):
+ def buildGroup(self, group, multiBuild=False, async=False):
"""
Build the binary version of a given group.
@param group: group model to build.
@@ -304,6 +304,8 @@
@param multiBuild: Optional parameter, defaults to False, control if
builder can build multiple packages at once.
@type mutliBuild: boolean
+ @param async: Use the async builder to build this group.
+ @type async: boolean
@return mapping of built troves, if multiBuild return results object.
@rtype dict(sourceTrv=[binTrv, ..]) or updatebot.build.jobs.Status
"""
@@ -328,10 +330,12 @@
# Create a build job and build groups using cvc.
job = ((self._sourceName, group.conaryVersion, None), )
- if not multiBuild:
+ if multiBuild:
+ results = self._builder.cvc.build(job[0], flavorFilter=use)
+ elif async:
+ results = self._builder.buildasync(job[0])
+ else:
results = self._builder.cvc.cook(job, flavorFilter=use)
- else:
- results = self._builder.cvc.build(job[0], flavorFilter=use)
return results
From elliot at rpath.com Wed Jul 21 15:14:36 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:36 +0000
Subject: mirrorball: Add script for finding point in the repository
history that all packages point
Message-ID: <201007211914.o6LJEapf007128@scc.eng.rpath.com>
changeset: 0b4a1c8ba56d
user: Elliot Peele
date: Wed, 21 Jul 2010 15:08:48 -0400
Add script for finding point in the repository history that all packages point
release are availble.
diff --git a/scripts/rebuildgroups b/scripts/findrelease
copy from scripts/rebuildgroups
copy to scripts/findrelease
--- a/scripts/rebuildgroups
+++ b/scripts/findrelease
@@ -15,9 +15,7 @@
import os
import sys
-import time
import logging
-import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -25,191 +23,68 @@
if 'CONARY_PATH' in os.environ:
sys.path.insert(0, os.environ['CONARY_PATH'])
-
-from conary import versions
-from conary.deps import deps
-
+from updatebot import pkgsource
from updatebot import OrderedBot
log = logging.getLogger('tmplogger')
class Bot(OrderedBot):
- def rebuildgroups(self, updatedPackages=None):
+ def findrelease(self, release):
"""
- Rebuild all groups on the devel label. This requires rewriting the group
- model to point at the target label and readding content.
- @param updatedPackages: List of package names that have been rebuilt and
- thus need to be updated when modifying the
- groups.
- @type updatedPackages: list(str, ...)
+ Find the first point in the update stream that all packages that are
+ included in a release version should be availble in the repository.
"""
- if not updatedPackages:
- updatedPackages = []
+ # load release reference source
+ repoUrl = self._cfg.versionSources[release]
+ relSrc = pkgsource.RpmSource(self._cfg, self._ui)
+ relSrc.loadFromUrl(repoUrl)
+ relSrc.finalize()
+
+ relNevraMap = dict([ (x.getNevra(), x) for x in relSrc.srcPkgMap ])
+ relNameMap = dict([ (x.name, x.getNevra()) for x in relSrc.srcPkgMap ])
+ relNameSet = set(relNameMap.iterkeys())
# load package source
self._pkgSource.load()
- versionMap = {
- 'rhel': {'4': '4.0',
- '5': '5.0', },
- }
+ repo = {}
+ repoNames = set()
- # Make sure to include any packages that were built from the same
- # source.
- siblingPackages = set()
-
- log.info('looking up siblings')
- req = [ (x, None, None) for x in updatedPackages ]
- nvfMap = self._updater._conaryhelper.findTroves(req, getLeaves=False)
- nvfs = [ x for x in itertools.chain(*nvfMap.itervalues()) ]
-
- siblingMap = self._updater._conaryhelper.getSiblingPackages(nvfs,
- allVersions=True)
-
- for siblings in siblingMap.itervalues():
- siblingPackages |= set([ x[0].split(':')[0] for x in siblings ])
- updatedPackages = list(siblingPackages)
-
- trvMap = {}
- results = []
-
- log.info('done loading siblings')
+ found = {}
+ notfound = {}
# Iterate over all upstream group versions that should exist.
for updateId, updates in self._errata.iterByIssueDate(current=-1):
- if updateId == 0:
- prodMap = versionMap[self._cfg.platformName]
- version = prodMap[self._cfg.upstreamProductVersion]
- else:
- version = self._errata.getBucketVersion(updateId)
+ repo.update([ (x.getNevra(), x) for x in updates ])
+ repoNames.update(set([ x.name for x in updates ]))
- log.info('%s: retrieving group model information' % version)
- group = self._groupmgr.getGroup(version=version)
+ version = self._errata.getBucketVersion(updateId)
- # Get all of the nvfs from the group model.
- nvfs = set()
- checkUpdates = set()
- emptyFlavors = set()
- for pkg in group.iterpackages():
- n = str(pkg.name)
- v = versions.ThawVersion(str(pkg.version))
+ log.info('checking: %s(%s)' % (version, updateId))
- f = deps.ThawFlavor(str(pkg.flavor))
- if str(f) == '':
- f = None
+ foundRelease = True
+ names = repoNames & relNameSet
+ for name in names:
+ nevra = relNameMap[name]
+ if nevra not in repo:
+ foundRelease = False
+ notfound.setdefault(updateId, set()).add(nevra)
+ else:
+ found.setdefault(updateId, set()).add(nevra)
- if n in updatedPackages:
- checkUpdates.add((n, v, f))
- elif f is None:
- emptyFlavors.add((n, v, f))
- else:
- nvfs.add((n, v, f))
+ log.info('found: %s, not found: %s'
+ % (len(found.get(updateId, [])),
+ len(notfound.get(updateId, []))))
- # Lookup anything that is expected to have been rebuilt.
- log.info('%s: looking up version information for rebuild packages'
- % version)
- for n, v, f in checkUpdates:
- upVer = '/'.join([v.branch().label().asString(),
- v.trailingRevision().version])
- binSpecs = self._updater._conaryhelper._repos.findTrove(
- v.branch().label(), (n, upVer, None), bestFlavor=False)
+ if foundRelease:
+ log.info('found %s release in %s(%s)' % (release, version,
+ updateId))
+ break
- assert len(binSpecs) > 1
+ import epdb; epdb.st()
- latest = sorted(binSpecs)[-1]
-
- if v != latest[1]:
- #log.info('%s: found updated version of %s %s -> %s'
- # % (version, n, v, latest[1]))
-
- toAdd = set([ x for x in binSpecs if x[1] == latest[1] ])
- nvfs.update(toAdd)
- elif f is None:
- emptyFlavors.add((n, v, f))
- else:
- nvfs.add((n, v, f))
-
- # Lookup anything that has an empty flavor.
- log.info('%s: looking up version information for empty flavors'
- % version)
- req = set([ (x[0], x[1], None) for x in emptyFlavors ])
- found = self._updater._conaryhelper.findTroves(req)
-
- for n, v, f in itertools.chain(*found.values()):
- nvfs.add((n, v, f))
-
- # Lookup cloned from info for all versions.
- log.info('%s: retrieving target version information' % version)
- targetVersions, failed = self._updater.getTargetVersions(nvfs,
- logErrors=False)
-
- # Make sure any that failed are in the set of packages that are
- # expected to have updates.
- # NOTE: This normally happens if the group model has already been
- # remapped to the target label.
- assert not [ x for x in failed if x[0] not in updatedPackages ]
-
- # Take the union of all versions on the target label and those that
- # have not yet been promoted.
- newVersions = set(targetVersions) | set(failed)
-
- # Add all of the new versions.
- log.info('%s: adding remapped versions' % version)
- pkgs = {}
- for n, v, f in newVersions:
- pkgs.setdefault(n, dict()).setdefault(v, set()).add(f)
-
- for n, vMap in pkgs.iteritems():
- assert len(vMap) == 1
- group.removePackage(n)
- for v, flvs in vMap.iteritems():
- group.addPackage(n, v, flvs)
-
- # Now deal with the standard group contents by recreating the
- # standard group from the new group configuration information.
-
- # Remove the existing standard group if there is one.
- log.info('%s: regenerating standard group' % version)
- standardNames = [ n for n, m in group.iteritems()
- if 'standard' in n ]
- assert len(standardNames) == 1
- standardName = standardNames[0]
- group._groups.pop(standardName, None)
-
- # Run through all of the adds and removes for the standard group.
- for uid, ups in self._errata.iterByIssueDate(current=-1):
- if uid > group.errataState:
- break
- self._modifyGroups(uid, group)
-
- log.info('%s: committing contents to latest' % version)
- group = group.commit(copyToLatest=True)
-
- log.info('%s: building group' % version)
- res = group.buildmany()
- results.append(res)
-
- # Wait for the first results to make sure the group will rebuild
- # properly.
- #if not trvMap:
- # log.info('waiting for first build to complete before continuing')
- # while not res.isDone:
- # time.sleep(1)
-
- # Check for any built groups.
- completed = [ x for x in results if x.isDone ]
- if completed:
- for res in completed:
- for src, bins in res.results.iteritems():
- log.info('%s: built troves' % version)
- for n, v, f in sorted(bins):
- log.info(' %s=%s[%s]' % (n, v, f))
- results.remove(res)
- trvMap.update(res.results)
-
- return trvMap
-
+ return updateId
if __name__ == '__main__':
import rmake
@@ -244,6 +119,7 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups(updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', 'kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', 'python-dmidecode', 'python-elementtree', 'python-imaging', 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', 'python-setuptools', 'python-sqlite', 'python-urlgrabber', 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', 'system-config-printer', 'vte', 'wireshark', 'yum', 'yum-metadata-parser', 'mvapich2', 'gnome-doc-utils', 'system-config-lvm', 'booty', 'smartmontools', 'devhelp', 'system-switch-mail', 'gedit', 'system-config-nfs', 'ant', 'kexec-tools', 'inn', 'tetex', 'docbook-style-xsl', 'firstboot', 'system-config-kickstart', 'kdesdk', 'system-config-display', 'switchdesk', 'gimp', 'system-config-samba', 'kdevelop', 'hal', 'frysk', 'redhat-release', 'gettext', 'libglade2', 'system-config-bind', 'system-config-httpd', 'system-config-keyboard', 'system-config-language', 'system-config-kdump', 'libhugetlbfs', 'comps-extras', 'system-config-soundcard', 'systemtap', 'vnc', 'conary-policy', 'system-config-services', 'createrepo', 'blktrace', 'yum-utils', 'system-config-netboot', 'sblim', 'setroubleshoot-plugins', 'xinetd', 'rhn-client-tools', 'libevent', 'mailman', 'system-config-securitylevel', 'selinux-policy', 'hwbrowser', 'bluez-utils', 'system-config-users', 'cman', 'yum-rhn-plugin', 'system-config-boot', 'system-config-network', 'system-config-date', 'ecryptfs-utils', 'tix', 'anaconda', 'system-config-rootpassword', 'dstat'])
+ release = bot.findrelease(sys.argv[2])
+ print release
import epdb; epdb.st()
From elliot at rpath.com Wed Jul 21 15:14:37 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:37 +0000
Subject: mirrorball: add script for building multiple verisons of the same
package
Message-ID: <201007211914.o6LJEbIn007155@scc.eng.rpath.com>
changeset: c708083f7015
user: Elliot Peele
date: Wed, 21 Jul 2010 15:09:26 -0400
add script for building multiple verisons of the same package
diff --git a/scripts/buildmany b/scripts/buildmulti
copy from scripts/buildmany
copy to scripts/buildmulti
--- a/scripts/buildmany
+++ b/scripts/buildmulti
@@ -12,14 +12,21 @@
from updatebot.lib import util
util.setupDebugHandler()
+from conary.conaryclient import cmdline
+
+from updatebot.build.dispatcher import MultiVersionDispatcher as Dispatcher
+
if len(sys.argv) < 3:
usage()
trvs = set()
-label = cfg.topSourceGroup[1]
for pkg in sys.argv[2:]:
- trvs.add((pkg, label, None))
-trvMap, failed = builder.buildmany(trvs)
+ n, v, f = cmdline.parseTroveSpec(pkg)
+ trvs.add((n, v, f))
+
+builder = build.Builder(cfg, ui, rmakeCfgFn='rmakerc-groups')
+disp = Dispatcher(builder, 20, waitForAllVersions=False)
+trvMap, failed = disp.buildmany(trvs)
print "built:\n"
From elliot at rpath.com Wed Jul 21 15:14:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:38 +0000
Subject: mirrorball: switch to using rebuildmany implementation when
rebuilding packages
Message-ID: <201007211914.o6LJEc7E007211@scc.eng.rpath.com>
changeset: c65d76cd1156
user: Elliot Peele
date: Wed, 21 Jul 2010 15:11:50 -0400
switch to using rebuildmany implementation when rebuilding packages
diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage
--- a/scripts/rebuildpackage
+++ b/scripts/rebuildpackage
@@ -61,7 +61,8 @@
if upVer not in latestMap:
latestMap[upVer] = (n, v, f)
- trvMap = self._builder.rebuild(latestMap.values(), useLatest=useLatest,
+ trvMap = self._builder.rebuildmany(latestMap.values(),
+ useLatest=useLatest,
additionalResolveTroves=additionalResolveTroves)
return trvMap
From elliot at rpath.com Wed Jul 21 15:14:37 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:37 +0000
Subject: mirrorball: add various keyword args to make script portable to
platforms other than rhel
Message-ID: <201007211914.o6LJEb6Z007182@scc.eng.rpath.com>
changeset: 77c404f56bcb
user: Elliot Peele
date: Wed, 21 Jul 2010 15:10:46 -0400
add various keyword args to make script portable to platforms other than rhel
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildgroups
@@ -34,7 +34,8 @@
log = logging.getLogger('tmplogger')
class Bot(OrderedBot):
- def rebuildgroups(self, updatedPackages=None):
+ def rebuildgroups(self, updatedPackages=None, resolveTargetVersions=True,
+ readdPackages=True, restart=False):
"""
Rebuild all groups on the devel label. This requires rewriting the group
model to point at the target label and readding content.
@@ -47,6 +48,9 @@
if not updatedPackages:
updatedPackages = []
+ if resolveTargetVersions or updatedPackages:
+ readdPackages = True
+
# load package source
self._pkgSource.load()
@@ -57,24 +61,27 @@
# Make sure to include any packages that were built from the same
# source.
- siblingPackages = set()
+ if updatedPackages:
+ siblingPackages = set()
- log.info('looking up siblings')
- req = [ (x, None, None) for x in updatedPackages ]
- nvfMap = self._updater._conaryhelper.findTroves(req, getLeaves=False)
- nvfs = [ x for x in itertools.chain(*nvfMap.itervalues()) ]
+ log.info('looking up siblings')
+ req = [ (x, None, None) for x in updatedPackages ]
+ nvfMap = self._updater._conaryhelper.findTroves(req, getLeaves=False)
+ nvfs = [ x for x in itertools.chain(*nvfMap.itervalues()) ]
- siblingMap = self._updater._conaryhelper.getSiblingPackages(nvfs,
- allVersions=True)
+ siblingMap = self._updater._conaryhelper.getSiblingPackages(nvfs,
+ allVersions=True)
- for siblings in siblingMap.itervalues():
- siblingPackages |= set([ x[0].split(':')[0] for x in siblings ])
- updatedPackages = list(siblingPackages)
+ for siblings in siblingMap.itervalues():
+ siblingPackages |= set([ x[0].split(':')[0] for x in siblings ])
+ updatedPackages = list(siblingPackages)
+ log.info('done loading siblings')
+
+ seen = {}
trvMap = {}
results = []
-
- log.info('done loading siblings')
+ startReplace = False
# Iterate over all upstream group versions that should exist.
for updateId, updates in self._errata.iterByIssueDate(current=-1):
@@ -84,6 +91,13 @@
else:
version = self._errata.getBucketVersion(updateId)
+ if not restart or updateId > self._groupmgr.latest.errataState:
+ startReplace = True
+ elif startReplace:
+ raise RuntimeError, 'found out of order groups'
+ else:
+ continue
+
log.info('%s: retrieving group model information' % version)
group = self._groupmgr.getGroup(version=version)
@@ -107,8 +121,9 @@
nvfs.add((n, v, f))
# Lookup anything that is expected to have been rebuilt.
- log.info('%s: looking up version information for rebuild packages'
- % version)
+ if checkUpdates:
+ log.info('%s: looking up version information for rebuild '
+ 'packages' % version)
for n, v, f in checkUpdates:
upVer = '/'.join([v.branch().label().asString(),
v.trailingRevision().version])
@@ -140,31 +155,35 @@
nvfs.add((n, v, f))
# Lookup cloned from info for all versions.
- log.info('%s: retrieving target version information' % version)
- targetVersions, failed = self._updater.getTargetVersions(nvfs,
- logErrors=False)
+ if resolveTargetVersions:
+ log.info('%s: retrieving target version information' % version)
+ targetVersions, failed = self._updater.getTargetVersions(nvfs,
+ logErrors=False)
- # Make sure any that failed are in the set of packages that are
- # expected to have updates.
- # NOTE: This normally happens if the group model has already been
- # remapped to the target label.
- assert not [ x for x in failed if x[0] not in updatedPackages ]
+ # Make sure any that failed are in the set of packages that are
+ # expected to have updates.
+ # NOTE: This normally happens if the group model has already
+ # been remapped to the target label.
+ assert not [ x for x in failed if x[0] not in updatedPackages ]
- # Take the union of all versions on the target label and those that
- # have not yet been promoted.
- newVersions = set(targetVersions) | set(failed)
+ # Take the union of all versions on the target label and those
+ # that have not yet been promoted.
+ newVersions = set(targetVersions) | set(failed)
+ else:
+ newVersions = nvfs
# Add all of the new versions.
- log.info('%s: adding remapped versions' % version)
- pkgs = {}
- for n, v, f in newVersions:
- pkgs.setdefault(n, dict()).setdefault(v, set()).add(f)
+ if readdPackages:
+ log.info('%s: adding remapped versions' % version)
+ pkgs = {}
+ for n, v, f in newVersions:
+ pkgs.setdefault(n, dict()).setdefault(v, set()).add(f)
- for n, vMap in pkgs.iteritems():
- assert len(vMap) == 1
- group.removePackage(n)
- for v, flvs in vMap.iteritems():
- group.addPackage(n, v, flvs)
+ for n, vMap in pkgs.iteritems():
+ assert len(vMap) == 1
+ group.removePackage(n)
+ for v, flvs in vMap.iteritems():
+ group.addPackage(n, v, flvs)
# Now deal with the standard group contents by recreating the
# standard group from the new group configuration information.
@@ -192,10 +211,10 @@
# Wait for the first results to make sure the group will rebuild
# properly.
- #if not trvMap:
- # log.info('waiting for first build to complete before continuing')
- # while not res.isDone:
- # time.sleep(1)
+ if not trvMap:
+ log.info('waiting for first build to complete before continuing')
+ while not res.isDone:
+ time.sleep(1)
# Check for any built groups.
completed = [ x for x in results if x.isDone ]
@@ -244,6 +263,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups(updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', 'kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', 'python-dmidecode', 'python-elementtree', 'python-imaging', 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', 'python-setuptools', 'python-sqlite', 'python-urlgrabber', 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', 'system-config-printer', 'vte', 'wireshark', 'yum', 'yum-metadata-parser', 'mvapich2', 'gnome-doc-utils', 'system-config-lvm', 'booty', 'smartmontools', 'devhelp', 'system-switch-mail', 'gedit', 'system-config-nfs', 'ant', 'kexec-tools', 'inn', 'tetex', 'docbook-style-xsl', 'firstboot', 'system-config-kickstart', 'kdesdk', 'system-config-display', 'switchdesk', 'gimp', 'system-config-samba', 'kdevelop', 'hal', 'frysk', 'redhat-release', 'gettext', 'libglade2', 'system-config-bind', 'system-config-httpd', 'system-config-keyboard', 'system-config-language', 'system-config-kdump', 'libhugetlbfs', 'comps-extras', 'system-config-soundcard', 'systemtap', 'vnc', 'conary-policy', 'system-config-services', 'createrepo', 'blktrace', 'yum-utils', 'system-config-netboot', 'sblim', 'setroubleshoot-plugins', 'xinetd', 'rhn-client-tools', 'libevent', 'mailman', 'system-config-securitylevel', 'selinux-policy', 'hwbrowser', 'bluez-utils', 'system-config-users', 'cman', 'yum-rhn-plugin', 'system-config-boot', 'system-config-network', 'system-config-date', 'ecryptfs-utils', 'tix', 'anaconda', 'system-config-rootpassword', 'dstat'])
+ bot.rebuildgroups(restart=True, updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', 'kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', 'python-dmidecode', 'python-elementtree', 'python-imaging', 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', 'python-setuptools', 'python-sqlite', 'python-urlgrabber', 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', 'system-config-printer', 'vte', 'wireshark', 'yum', 'yum-metadata-parser', 'mvapich2', 'gnome-doc-utils', 'system-config-lvm', 'booty', 'smartmontools', 'devhelp', 'system-switch-mail', 'gedit', 'system-config-nfs', 'ant', 'kexec-tools', 'inn', 'tetex', 'docbook-style-xsl', 'firstboot', 'system-config-kickstart', 'kdesdk', 'system-config-display', 'switchdesk', 'gimp', 'system-config-samba', 'kdevelop', 'hal', 'frysk', 'redhat-release', 'gettext', 'libglade2', 'system-config-bind', 'system-config-httpd', 'system-config-keyboard', 'system-config-language', 'system-config-kdump', 'libhugetlbfs', 'comps-extras', 'system-config-soundcard', 'systemtap', 'vnc', 'conary-policy', 'system-config-services', 'createrepo', 'blktrace', 'yum-utils', 'system-config-netboot', 'sblim', 'setroubleshoot-plugins', 'xinetd', 'rhn-client-tools', 'libevent', 'mailman', 'system-config-securitylevel', 'selinux-policy', 'hwbrowser', 'bluez-utils', 'system-config-users', 'cman', 'yum-rhn-plugin', 'system-config-boot', 'system-config-network', 'system-config-date', 'ecryptfs-utils', 'tix', 'anaconda', 'system-config-rootpassword', 'dstat'])
import epdb; epdb.st()
From elliot at rpath.com Wed Jul 21 15:14:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:38 +0000
Subject: mirrorball: fix typo
Message-ID: <201007211914.o6LJEct1007238@scc.eng.rpath.com>
changeset: 2d3768295b02
user: Elliot Peele
date: Wed, 21 Jul 2010 15:12:20 -0400
fix typo
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -69,7 +69,7 @@
Start the specified build and report jobId.
"""
- jobIds = self.builder.rebuildstart(self.trove, useLatest=self.useLatest,
+ jobIds = self.builder.rebuild(self.trove, useLatest=self.useLatest,
additionalResolveTroves=self.additionalResolveTroves, commit=False)
if len(jobIds) != 1:
From elliot at rpath.com Wed Jul 21 15:14:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 21 Jul 2010 19:14:38 +0000
Subject: mirrorball: add script for finding all packages built from the
same source as the specified trove spec
Message-ID: <201007211914.o6LJEctG007265@scc.eng.rpath.com>
changeset: 7abcec5c08af
user: Elliot Peele
date: Wed, 21 Jul 2010 15:12:48 -0400
add script for finding all packages built from the same source as the specified trove spec
diff --git a/scripts/getsiblings b/scripts/getsiblings
new file mode 100755
--- /dev/null
+++ b/scripts/getsiblings
@@ -0,0 +1,41 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+
+import sys
+import itertools
+
+from conary import trove
+from conary import conarycfg
+from conary import conaryclient
+from conary.conaryclient import cmdline
+
+if __name__ == '__main__':
+ cfg = conarycfg.ConaryConfiguration(True)
+ client = conaryclient.ConaryClient(cfg)
+ trvSpec = cmdline.parseTroveSpec(sys.argv[1])
+
+ nvfLst = client.repos.findTrove(cfg.installLabelPath, trvSpec)
+ sources = client.repos.getTroveInfo(trove._TROVEINFO_TAG_SOURCENAME, nvfLst)
+
+ srcreq = set()
+ for (n, v, f), src in itertools.izip(nvfLst, sources):
+ sn = src()
+ sv = v.getSourceVersion()
+
+ srcreq.add((sn, sv))
+
+ nvfmap = {}
+ for sn, sv in srcreq:
+ nvfmap[(sn, sv)] = client.repos.getTrovesBySource(sn, sv)
+
+ for (sn, sv), trvs in nvfmap.iteritems():
+ print '%s=%s' % (sn, sv)
+ vn = {}
+ for n, v, f in trvs:
+ vn.setdefault(v, set()).add(n)
+ for v in reversed(sorted(vn)):
+ print
+ for n in sorted(vn[v]):
+ print ' %s=%s' % (n, v)
From juphoff at rpath.com Wed Jul 21 16:47:34 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Wed, 21 Jul 2010 20:47:34 +0000
Subject: mirrorball: changes to rebuild groups for centos5e
Message-ID: <201007212047.o6LKlYKB008713@scc.eng.rpath.com>
changeset: c83e7ecc8495
user: Jeff Uphoff
date: Wed, 21 Jul 2010 16:47:26 -0400
changes to rebuild groups for centos5e
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildgroups
@@ -263,6 +263,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups(restart=True, updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', 'kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', 'python-dmidecode', 'python-elementtree', 'python-imaging', 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', 'python-setuptools', 'python-sqlite', 'python-urlgrabber', 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', 'system-config-printer', 'vte', 'wireshark', 'yum', 'yum-metadata-parser', 'mvapich2', 'gnome-doc-utils', 'system-config-lvm', 'booty', 'smartmontools', 'devhelp', 'system-switch-mail', 'gedit', 'system-config-nfs', 'ant', 'kexec-tools', 'inn', 'tetex', 'docbook-style-xsl', 'firstboot', 'system-config-kickstart', 'kdesdk', 'system-config-display', 'switchdesk', 'gimp', 'system-config-samba', 'kdevelop', 'hal', 'frysk', 'redhat-release', 'gettext', 'libglade2', 'system-config-bind', 'system-config-httpd', 'system-config-keyboard', 'system-config-language', 'system-config-kdump', 'libhugetlbfs', 'comps-extras', 'system-config-soundcard', 'systemtap', 'vnc', 'conary-policy', 'system-config-services', 'createrepo', 'blktrace', 'yum-utils', 'system-config-netboot', 'sblim', 'setroubleshoot-plugins', 'xinetd', 'rhn-client-tools', 'libevent', 'mailman', 'system-config-securitylevel', 'selinux-policy', 'hwbrowser', 'bluez-utils', 'system-config-users', 'cman', 'yum-rhn-plugin', 'system-config-boot', 'system-config-network', 'system-config-date', 'ecryptfs-utils', 'tix', 'anaconda', 'system-config-rootpassword', 'dstat'])
+ bot.rebuildgroups(restart=False, resolveTargetVersions=False, readdPackages=False)
import epdb; epdb.st()
From elliot at rpath.com Sat Jul 24 11:41:01 2010
From: elliot at rpath.com (Elliot Peele)
Date: Sat, 24 Jul 2010 15:41:01 +0000
Subject: mirrorball: properly handle config based package group modifications
Message-ID: <201007241541.o6OFf1jW019247@scc.eng.rpath.com>
changeset: 526bab2b22f1
user: Elliot Peele
date: Sat, 24 Jul 2010 11:37:54 -0400
properly handle config based package group modifications
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -407,11 +407,6 @@
# Add requested packages.
for groupName, pkgs in additions.iteritems():
- if groupName == self._pkgGroupName:
- log.warn('modifyContents does not support modifying the package'
- 'group, please update your config file')
- continue
-
flavoredPackages = {}
for pkgName, pkgFlv, use in pkgs:
# deffer packages with specifc flavors for later.
@@ -440,12 +435,18 @@
"""
# Get the versions of all packge names.
- pkgs = dict([ (x.name, x) for x in self._groups[self._pkgGroupName] ])
+ pkgs = dict([ (x.name, x) for x in self._groups[self._pkgGroupName]
+ if x.version is not None ])
for group in self:
# skip over package group since it is the version source.
if group.groupName == self._pkgGroupName:
- continue
+ required = [ x for x in group if x.version is None ]
+ for pkg in required:
+ if pkg.name in pkgs:
+ pkg.version = pkgs[pkg.name].version
+ else:
+ raise UnknownPackageFoundInManagedGroupError(what=pkg.name)
# for all other groups iterate over contents and set versions to
# match package group.
From elliot at rpath.com Sat Jul 24 11:41:01 2010
From: elliot at rpath.com (Elliot Peele)
Date: Sat, 24 Jul 2010 15:41:01 +0000
Subject: mirrorball: log information about how many jobs are waiting to be
committed
Message-ID: <201007241541.o6OFf1SQ019277@scc.eng.rpath.com>
changeset: 12e7929f2d01
user: Elliot Peele
date: Sat, 24 Jul 2010 11:38:27 -0400
log information about how many jobs are waiting to be committed
diff --git a/updatebot/build/jobs.py b/updatebot/build/jobs.py
--- a/updatebot/build/jobs.py
+++ b/updatebot/build/jobs.py
@@ -539,4 +539,8 @@
jobId = troveMap[spec]
toCommit.add(jobId)
+ if toCommit:
+ waiting = sum([ len(x) for x in built.itervalues() ])
+ log.info('committing %s of %s' % (len(toCommit), waiting))
+
return toCommit
From elliot at rpath.com Sat Jul 24 11:41:01 2010
From: elliot at rpath.com (Elliot Peele)
Date: Sat, 24 Jul 2010 15:41:01 +0000
Subject: mirrorball: branch merge
Message-ID: <201007241541.o6OFf1Xf019305@scc.eng.rpath.com>
changeset: bcd38ad05d0e
user: Elliot Peele
date: Sat, 24 Jul 2010 11:40:58 -0400
branch merge
diff --git a/updatebot/build/jobs.py b/updatebot/build/jobs.py
--- a/updatebot/build/jobs.py
+++ b/updatebot/build/jobs.py
@@ -539,4 +539,8 @@
jobId = troveMap[spec]
toCommit.add(jobId)
+ if toCommit:
+ waiting = sum([ len(x) for x in built.itervalues() ])
+ log.info('committing %s of %s' % (len(toCommit), waiting))
+
return toCommit
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -407,11 +407,6 @@
# Add requested packages.
for groupName, pkgs in additions.iteritems():
- if groupName == self._pkgGroupName:
- log.warn('modifyContents does not support modifying the package'
- 'group, please update your config file')
- continue
-
flavoredPackages = {}
for pkgName, pkgFlv, use in pkgs:
# deffer packages with specifc flavors for later.
@@ -440,12 +435,18 @@
"""
# Get the versions of all packge names.
- pkgs = dict([ (x.name, x) for x in self._groups[self._pkgGroupName] ])
+ pkgs = dict([ (x.name, x) for x in self._groups[self._pkgGroupName]
+ if x.version is not None ])
for group in self:
# skip over package group since it is the version source.
if group.groupName == self._pkgGroupName:
- continue
+ required = [ x for x in group if x.version is None ]
+ for pkg in required:
+ if pkg.name in pkgs:
+ pkg.version = pkgs[pkg.name].version
+ else:
+ raise UnknownPackageFoundInManagedGroupError(what=pkg.name)
# for all other groups iterate over contents and set versions to
# match package group.
From juphoff at rpath.com Mon Jul 26 15:44:13 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Mon, 26 Jul 2010 19:44:13 +0000
Subject: mirrorball: hacks to rebuildgroups for centos & working around
rpm bug related to path conflicts
Message-ID: <201007261944.o6QJiDqr019530@scc.eng.rpath.com>
changeset: eaae3caa7c86
user: Jeff Uphoff
date: Sat, 24 Jul 2010 18:06:36 -0400
hacks to rebuildgroups for centos & working around rpm bug related to path conflicts
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildgroups
@@ -57,6 +57,7 @@
versionMap = {
'rhel': {'4': '4.0',
'5': '5.0', },
+ 'centos': {'5': '0', },
}
# Make sure to include any packages that were built from the same
@@ -85,6 +86,9 @@
# Iterate over all upstream group versions that should exist.
for updateId, updates in self._errata.iterByIssueDate(current=-1):
+# if updateId == 1192521600:
+# # Stop here so we can restore path-conflict checking.
+# import epdb ; epdb.st()
if updateId == 0:
prodMap = versionMap[self._cfg.platformName]
version = prodMap[self._cfg.upstreamProductVersion]
@@ -101,13 +105,20 @@
log.info('%s: retrieving group model information' % version)
group = self._groupmgr.getGroup(version=version)
+ if version == '0':
+ group.version = '5.0'
+
# Get all of the nvfs from the group model.
nvfs = set()
checkUpdates = set()
emptyFlavors = set()
+ vmap = dict([ (x.name, x.version) for x in group.iterpackages()
+ if x.version is not None ])
for pkg in group.iterpackages():
n = str(pkg.name)
- v = versions.ThawVersion(str(pkg.version))
+ if pkg.version is None:
+ pkg.version = vmap[pkg.name]
+ v = versions.ThawVersion(str(pkg.version))
f = deps.ThawFlavor(str(pkg.flavor))
if str(f) == '':
@@ -242,8 +253,6 @@
from conary.lib import util
sys.excepthook = util.genExcepthook()
- import rhnmirror
-
from updatebot import config
from updatebot import log as logSetup
@@ -256,13 +265,32 @@
cfg = config.UpdateBotConfig()
cfg.read(confDir + '/updatebotrc')
- mcfg = rhnmirror.MirrorConfig()
- mcfg.read(confDir + '/erratarc')
+ if cfg.platformName == 'rhel':
+ import rhnmirror
- errata = rhnmirror.Errata(mcfg)
- errata.fetch()
+ mcfg = rhnmirror.MirrorConfig()
+ mcfg.read(confDir + '/erratarc')
- bot = Bot(cfg, errata)
- bot.rebuildgroups(restart=False, resolveTargetVersions=False, readdPackages=False)
+ errata = rhnmirror.Errata(mcfg)
+
+ bot = Bot(cfg, errata)
+
+ else:
+ bot = Bot(cfg, None)
+
+ if cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
+
+ elif cfg.platformName == 'centos':
+ from errata.centos import AdvisoryManager as Errata
+
+ else:
+ raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+
+ errata = Errata(bot._pkgSource)
+ bot._errata._errata = errata
+
+ bot.rebuildgroups(restart=True, resolveTargetVersions=False, readdPackages=True,
+ updatedPackages=['anaconda', 'yum-cron', ])
import epdb; epdb.st()
From juphoff at rpath.com Wed Jul 28 13:47:39 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Wed, 28 Jul 2010 17:47:39 +0000
Subject: mirrorball: Code example to retry commit of already-built jobs
without rebuilding
Message-ID: <201007281747.o6SHldW1027337@scc.eng.rpath.com>
changeset: 4a4c6ab3ea4f
user: Jeff Uphoff
date: Wed, 28 Jul 2010 13:46:41 -0400
Code example to retry commit of already-built jobs without rebuilding
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -308,6 +308,15 @@
expectedRemovals=expectedRemovals,
allowPackageDowngrades=allowDowngrades, **kwargs))
+ # Comment out the above pkgMap.update() call and
+ # uncomment the below call with list of rmake job id's
+ # in order to commit already-built packages. (Useful in
+ # cases where a large commit has failed and a retry
+ # would save time.) Also, set a breakpoint at end of
+ # enclosing loop so mirrorball won't keep trying this
+ # over and over!
+ #pkgMap.update(self._builder.commit([job1,job2,...]))
+
# When deriving from an upstream platform sometimes we don't want
# the latest versions.
oldVersions = self._cfg.useOldVersion.get(updateId, None)
@@ -402,6 +411,10 @@
log.info('published update %s in %s seconds' % (advTime, totalTime))
count += 1
+ # Set this breakpoint when you're committing a list of
+ # already-built job id's. (See long comment above.)
+ #import epdb ; epdb.st()
+
log.info('update completed')
log.info('applied %s updates in %s seconds'
% (count, time.time() - startime))
From juphoff at rpath.com Thu Jul 29 16:12:35 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Thu, 29 Jul 2010 20:12:35 +0000
Subject: mirrorball: add timestamp slot to _Patch
Message-ID: <201007292012.o6TKCZ9W023054@scc.eng.rpath.com>
changeset: 2ae3571fc7af
user: Jeff Uphoff
date: Thu, 29 Jul 2010 16:12:06 -0400
add timestamp slot to _Patch
diff --git a/repomd/patchxml.py b/repomd/patchxml.py
--- a/repomd/patchxml.py
+++ b/repomd/patchxml.py
@@ -36,7 +36,12 @@
'release', 'requires', 'recommends', 'rebootNeeded',
'licenseToConfirm', 'packageManager', 'category',
'packages', 'provides', 'supplements', 'conflicts',
- 'obsoletes')
+ 'obsoletes', 'timestamp')
+
+ def __init__(self, *args, **kwargs):
+ SlotNode.__init__(self, *args, **kwargs)
+ self.timestamp = self.getAttribute('timestamp')
+
# All attributes are defined in __init__ by iterating over __slots__,
# this confuses pylint.