From elliot at rpath.com Tue Jun 1 12:39:14 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 01 Jun 2010 16:39:14 +0000
Subject: mirrorball: always build the new group rather than rebuilding the old
group
Message-ID: <201006011639.o51GdEPK013710@scc.eng.rpath.com>
changeset: 9dd43a7bd5d0
user: Elliot Peele
date: Tue, 01 Jun 2010 12:37:47 -0400
always build the new group rather than rebuilding the old group
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -176,7 +176,7 @@
self._modifyContents(0, group)
group.errataState = '0'
group.version = '0'
- group.commit()
+ group = group.commit()
group.build()
return pkgMap, failures
@@ -380,7 +380,8 @@
# Build groups.
log.info('setting version %s' % version)
group.version = version
- newGroup = group.save()
+
+ group = group.commit()
grpTrvMap = group.build()
updateSet.update(pkgMap)
@@ -392,8 +393,6 @@
log.info('published update %s in %s seconds' % (advTime, totalTime))
count += 1
- group = newGroup
-
log.info('update completed')
log.info('applied %s updates in %s seconds'
% (count, time.time() - startime))
From elliot at rpath.com Tue Jun 1 12:39:15 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 01 Jun 2010 16:39:15 +0000
Subject: mirrorball: uniq current list of packages in model since a single
name/version pair can
Message-ID: <201006011639.o51GdFS8013741@scc.eng.rpath.com>
changeset: a65e06de7cb2
user: Elliot Peele
date: Tue, 01 Jun 2010 12:39:12 -0400
uniq current list of packages in model since a single name/version pair can
occur more than once.
diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py
--- a/updatebot/groupmgr/sanity.py
+++ b/updatebot/groupmgr/sanity.py
@@ -192,7 +192,9 @@
errors = {}
for name, found in foundTroves.iteritems():
assert name in pkgs
- current = pkgs[name]
+ # Make sure to dedup packages from the model since a name/version
+ # pair can occure more than once.
+ current = sorted(set(pkgs[name]))
if len(current) > len(found):
log.warn('found more packages in the model than in the '
From elliot at rpath.com Wed Jun 2 20:11:37 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:37 +0000
Subject: mirrorball: fix typo
Message-ID: <201006030011.o530Bbj9032385@scc.eng.rpath.com>
changeset: 81c7bd234324
user: Elliot Peele
date: Tue, 01 Jun 2010 15:13:24 -0400
fix typo
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -78,7 +78,7 @@
labels = None
if targetGroup:
- srcLabel = self._cfg.tagetLabel
+ srcLabel = self._cfg.targetLabel
elif parentGroup:
srcName = self._cfg.topParentSourceGroup[0]
srcLabel = self._cfg.topParentSourceGroup[1]
From elliot at rpath.com Wed Jun 2 20:11:37 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:37 +0000
Subject: mirrorball: switch variable names to follow new workflow
Message-ID: <201006030011.o530BbEk032441@scc.eng.rpath.com>
changeset: f8ed456b22c3
user: Elliot Peele
date: Tue, 01 Jun 2010 15:13:56 -0400
switch variable names to follow new workflow
diff --git a/updatebot/groupmgr/single.py b/updatebot/groupmgr/single.py
--- a/updatebot/groupmgr/single.py
+++ b/updatebot/groupmgr/single.py
@@ -52,17 +52,25 @@
def __init__(self, cfg):
self._cfg = cfg
- self._groups = {}
+ self._mgrs = {}
def newGroup(self, name):
"""
- Create a new group instance with the provided name.
+ Create a new group manager instance with the provided name.
"""
- assert name not in self._groups
- group = self._managerClass(name, self._cfg)
- self._groups[name] = group
- return group
+ assert name not in self._mgrs
+ mgr = self._managerClass(name, self._cfg)
+ self._mgrs[name] = mgr
+ return mgr
+
+ def commit(self):
+ """
+ Commit latest group in all managers.
+ """
+
+ for mgr in self._mgrs.itervalues():
+ mgr.latest.commit()
def build(self):
"""
@@ -70,11 +78,11 @@
"""
# Make sure there are groups defined
- assert self._groups
+ assert self._mgrs
pkgMap = {}
- for group in self._groups.itervalues():
- pkgMap.update(group.buildGroup(group.latest))
+ for mgr in self._mgrs.itervalues():
+ pkgMap.update(mgr.latest.build())
return pkgMap
@@ -83,4 +91,4 @@
Add method for checking if a manager has any groups defined.
"""
- return bool(self._groups)
+ return bool(self._mgrs)
From elliot at rpath.com Wed Jun 2 20:11:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:38 +0000
Subject: mirrorball: make sure to commit sources before building
Message-ID: <201006030011.o530Bcho032478@scc.eng.rpath.com>
changeset: a0b893447d30
user: Elliot Peele
date: Tue, 01 Jun 2010 15:14:11 -0400
make sure to commit sources before building
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -598,6 +598,9 @@
log.info('%s: groups already built and promoted' % updateId)
continue
+ log.info('%s: committing group sources' % updateId)
+ mgr.commit()
+
log.info('%s: building groups' % updateId)
trvMap = mgr.build()
From elliot at rpath.com Wed Jun 2 20:11:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:38 +0000
Subject: mirrorball: get errata state from new group model
Message-ID: <201006030011.o530Bc0e032506@scc.eng.rpath.com>
changeset: 99cf44d5fe27
user: Elliot Peele
date: Wed, 02 Jun 2010 14:35:59 -0400
get errata state from new group model
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -254,15 +254,14 @@
# If on a derived platform and the current updateId is greater than
# the parent updateId, stop applying updates.
- if self._cfg.platformSearchPath:
+ if (self._cfg.platformSearchPath and
+ self._parentGroup.latest.errataState < updateId):
# FIXME: This means that if there is an update the the child
# platform that is not included in the parent platform,
# we will not apply the update until there is a later
# update to the parent platform.
- parentState = self._parentGroup.getErrataState()
- if parentState < updateId:
- log.info('reached end of parent platform update stream')
- continue
+ log.info('reached end of parent platform update stream')
+ break
# remove packages from config
removePackages = self._cfg.updateRemovesPackages.get(updateId, [])
From elliot at rpath.com Wed Jun 2 20:11:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:38 +0000
Subject: mirrorball: be more specific about what is a warning and what is
an error
Message-ID: <201006030011.o530BcQ5032533@scc.eng.rpath.com>
changeset: a8e9fdc16828
user: Elliot Peele
date: Wed, 02 Jun 2010 17:05:39 -0400
be more specific about what is a warning and what is an error
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -253,7 +253,7 @@
seen[srpm.name] = srpm
continue
if dups:
- log.warn('found duplicates in %s' % updateId)
+ log.error('found duplicates in %s' % updateId)
errors.setdefault(updateId, []).append(('duplicates', dups))
# Play though entire update history to check for iregularities.
@@ -417,15 +417,15 @@
if obsoleteBinaries:
- log.warn('found obsolete binary packages in %s' % updateId)
+ log.error('found obsolete binary packages in %s' % updateId)
errors.setdefault(updateId, []).append(('obsoleteBinaries',
obsoleteBinaries))
if obsoleteSources:
- log.warn('found obsolete source packages in %s' % updateId)
+ log.error('found obsolete source packages in %s' % updateId)
errors.setdefault(updateId, []).append(('obsoleteSources',
obsoleteSources))
if removedShouldBeReplaced:
- log.warn('found removals for replacements in %s' % updateId)
+ log.error('found removals for replacements in %s' % updateId)
errors.setdefault(updateId, []).append(('removedShouldBeReplaced',
removedShouldBeReplaced))
@@ -436,7 +436,7 @@
one, two = e.why
oneNevra = str(' '.join(one.getNevra()))
twoNevra = str(' '.join(two.getNevra()))
- log.warn('%s %s -revertsTo-> %s' % (updateId,
+ log.error('%s %s -revertsTo-> %s' % (updateId,
oneNevra, twoNevra))
if one in srpmToAdvisory and two in srpmToAdvisory:
log.info('%s -revertsTo-> %s' % (
@@ -453,13 +453,13 @@
updateId, srpmToBucketId[one], errataId))
elif isinstance(e, UpdateReusesPackageError):
# Note that updateObsoletesPackages not yet implemented...
- log.warn('%s %s reused in %s; check for obsoletes?' % (
+ log.error('%s %s reused in %s; check for obsoletes?' % (
updateId, e.pkgNames, e.newspkg))
for name in sorted(set(p.name for p in e.pkgList)):
log.info('? updateObsoletesPackages %s %s' % (
updateId, name))
elif isinstance(e, UpdateRemovesPackageError):
- log.warn('%s %s removed in %s' % (
+ log.error('%s %s removed in %s' % (
updateId, e.pkgNames, e.newspkg))
for name, p in sorted(dict((p.name, p) for p in e.pkgList).items()):
log.info('? updateRemovesPackages %s %s' % (
@@ -471,7 +471,7 @@
previousId = sortedOrder[sortedOrder.index(updateId)-1]
for dupName, dupSet in e[1].iteritems():
dupList = sorted(dupSet)
- log.warn('%s contains duplicate %s %s' %(updateId,
+ log.error('%s contains duplicate %s %s' %(updateId,
dupName, dupList))
for srcPkg in sorted(dupList[1:]):
srcNevra = str(' '.join(srcPkg.getNevra()))
@@ -501,14 +501,14 @@
obsoletedNevra = str(' '.join(obsoletedPkg.getNevra()))
obsoleteName = obsoletedPkg.name
obsoleteNames.add(obsoleteName)
- log.warn('%s %s obsoletes %s (%s)' % (
+ log.error('%s %s obsoletes %s (%s)' % (
updateId, obsoletingPkg,
obsoletedPkg, obsoleteName))
log.info('? keepObsolete %s %s' %
(obsoletingNevra, obsoletedNevra))
log.info('? removeObsoleted %s %s' % (updateId,
obsoleteName))
- log.warn('Not "removeSource %s"; that would remove non-obsoleted %s' %
+ log.error('Not "removeSource %s"; that would remove non-obsoleted %s' %
(srcNevraStr, unremovedStr))
elif e[0] == 'obsoleteSources':
@@ -519,7 +519,7 @@
obsoletingSrcPkgNames = str(' '.join(sorted(set(
x.name for x in obsoletingSrcPkgs))))
pkgList = str(' '.join(repr(x) for x in binPkgs))
- log.warn('%s %s obsolete(s) %s (%s)' % (
+ log.error('%s %s obsolete(s) %s (%s)' % (
updateId, obsoletingSrcPkgNames,
obsoletedPkg, obsoleteName))
log.info('? removeSource %s %s # %s' % (
@@ -532,7 +532,7 @@
log.info(' will remove the following: %s' % pkgList)
elif e[0] == 'removedShouldBeReplaced':
for pkgName in e[1]:
- log.warn('%s removed package %s should be replaced' % (
+ log.error('%s removed package %s should be replaced' % (
updateId, pkgName))
log.info('? updateReplacesPackages %s %s' % (
updateId, pkgName))
@@ -727,10 +727,6 @@
Reschedule a single advisory.
"""
- # Warn when moving advisory into existing bucket.
- if dest in self._order:
- log.warn('inserting %s into pre-existing bucket' % advisory)
-
log.info('rescheduling %s %s -> %s' % (advisory, source, dest))
# Find the srpms that apply to this advisory
@@ -785,10 +781,6 @@
Reschedule an individual srpm to another bucket.
"""
- # Warn when moving srpm into existing bucket.
- if dest in self._order:
- log.warn('inserting %s into pre-existing bucket' % '-'.join(nevra))
-
log.info('rescheduling %s %s -> %s' % (nevra, source, dest))
# Remove specified source nevra from the source bucket
From elliot at rpath.com Wed Jun 2 20:11:39 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:39 +0000
Subject: mirrorball: actually make sure all sourceNames end in :source
Message-ID: <201006030011.o530BdFi032594@scc.eng.rpath.com>
changeset: 5c02979807d0
user: Elliot Peele
date: Wed, 02 Jun 2010 17:51:20 -0400
actually make sure all sourceNames end in :source
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -84,16 +84,15 @@
srcLabel = self._cfg.topParentSourceGroup[1]
labels = self._cfg.platformSearchPath
+ if not srcName.endswith(':source'):
+ srcName = '%s:source' % srcName
+
self._sourceName = srcName
self._sourceLabel = srcLabel
self._searchLabels = labels
- # FIXME: Should figure out a better way to handle package group.
self._pkgGroupName = self._cfg.packageGroupName
- if not srcName.endswith(':source'):
- srcName = '%s:source' % srcName
-
self._readOnly = False
if targetGroup or parentGroup:
self._readOnly = True
From elliot at rpath.com Wed Jun 2 20:11:39 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:39 +0000
Subject: mirrorball: remove redundant :source
Message-ID: <201006030011.o530BdAN032625@scc.eng.rpath.com>
changeset: e6d614c1b2b1
user: Elliot Peele
date: Wed, 02 Jun 2010 17:57:12 -0400
remove redundant :source
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -73,7 +73,7 @@
assert not (parentGroup and targetGroup)
- srcName = '%s:source' % self._cfg.topSourceGroup[0]
+ srcName = self._cfg.topSourceGroup[0]
srcLabel = self._cfg.topSourceGroup[1]
labels = None
From elliot at rpath.com Wed Jun 2 20:11:40 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:40 +0000
Subject: mirrorball: cleanup some FIXMEs
Message-ID: <201006030011.o530Behu032652@scc.eng.rpath.com>
changeset: dd5186b55f30
user: Elliot Peele
date: Wed, 02 Jun 2010 17:57:34 -0400
cleanup some FIXMEs
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -358,14 +358,6 @@
for context, flavor in self._cfg.kernelFlavors:
# Replace flag name to match package
if name != 'kernel':
- # Don't build kernel modules with a .debug flag, that
- # is only for kernels.
- # FIXME: determine what the right thing is here. In SLES
- # many of the kernel modules include debug
- # versions.
- #if flavor.stronglySatisfies(
- # deps.parseFlavor('kernel.debug')):
- # continue
flavor = deps.parseFlavor(
str(flavor).replace('kernel', name))
troves.append((name, version, flavor, context))
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -16,6 +16,8 @@
Module for managing monitors.
"""
+import os
+
from rmake.cmdline import monitor
from updatebot.build.common import AbstractWorker
@@ -68,7 +70,7 @@
Watch the monitor queue and monitor any available jobs.
"""
- # FIXME: This is copied from rmake.cmdlin.monitor for the most part
+ # FIXME: This is copied from rmake.cmdline.monitor for the most part
# because I need to pass extra args to the display class.
uri, tmpPath = monitor._getUri(self.client)
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -971,10 +971,6 @@
trvSpecs = list(binTrvSet)
- # FIXME: figure out why conary does't let you set metadata on
- # source troves.
- #trvSpecs.append(srcTrvSpec)
-
self._conaryhelper.setTroveMetadata(trvSpecs,
license=srcPkg.license,
desc=srcPkg.description,
From elliot at rpath.com Wed Jun 2 20:11:40 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:40 +0000
Subject: mirrorball: parse flavors with spaces correctly
Message-ID: <201006030011.o530Be2D032713@scc.eng.rpath.com>
changeset: 45f487212a28
user: Elliot Peele
date: Wed, 02 Jun 2010 18:50:18 -0400
parse flavors with spaces correctly
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -61,7 +61,8 @@
context = val
flavor = None
else:
- context, flavorStr = splt
+ context = splt[0]
+ flavorStr = ' '.join(splt[1:])
flavor = CfgFlavor.parseString(self, flavorStr)
return context, flavor
except versions.ParseError, e:
From elliot at rpath.com Wed Jun 2 20:11:41 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:41 +0000
Subject: mirrorball: lookup package object correctly
Message-ID: <201006030011.o530BfQd032742@scc.eng.rpath.com>
changeset: 452727126c3a
user: Elliot Peele
date: Wed, 02 Jun 2010 18:50:50 -0400
lookup package object correctly
diff --git a/updatebot/groupmgr/model.py b/updatebot/groupmgr/model.py
--- a/updatebot/groupmgr/model.py
+++ b/updatebot/groupmgr/model.py
@@ -158,10 +158,11 @@
"""
removed = []
- for pkg in self._nameMap[name]:
+ for pkgKey in self._nameMap[name]:
+ pkg = self._data[pkgKey]
if pkg.flavor == frzFlavor:
- self._data.pop(pkg.key)
- removed.append(pkg)
+ self._data.pop(pkgKey)
+ removed.append(pkgKey)
for pkg in removed:
self._nameMap[name].remove(pkg)
From elliot at rpath.com Wed Jun 2 20:11:41 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:41 +0000
Subject: mirrorball: make sure to aways reflect config changes in group flags
Message-ID: <201006030011.o530BfTL000301@scc.eng.rpath.com>
changeset: e21a3d378df4
user: Elliot Peele
date: Wed, 02 Jun 2010 19:28:13 -0400
make sure to aways reflect config changes in group flags
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -186,9 +186,6 @@
# create package group model if it does not exist.
if groupName not in self._groups:
self._groups[groupName] = GroupContentsModel(groupName)
- for key, value in self._cfg.groupContents.get(groupName, []):
- value = value == 'True' and True or False
- setattr(self._groups[groupName], key, value)
self._groups[groupName].add(*args, **kwargs)
@@ -447,6 +444,16 @@
self._sanity.check(self._groups, self.errataState)
+ def _setGroupFlags(self):
+ """
+ Set flags on the group based on the groupContents configuration.
+ """
+
+ for groupName, groupObj in self._groups.iteritems():
+ for key, value in self._cfg.groupContents.get(groupName, []):
+ value = value == 'True' and True or False
+ setattr(groupObj, key, value)
+
@require_write
def finalize(self):
"""
@@ -459,5 +466,8 @@
# Check the sanity of all group models.
self._sanityCheck()
+ # Make sure flags on the group match the config.
+ self._setGroupFlags()
+
# Make as readonly.
self.setReadOnly()
From elliot at rpath.com Wed Jun 2 20:11:41 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:41 +0000
Subject: mirrorball: add group flag for checkPathConflicts
Message-ID: <201006030011.o530BgXd000363@scc.eng.rpath.com>
changeset: 7a1db219d9f0
user: Elliot Peele
date: Wed, 02 Jun 2010 19:28:29 -0400
add group flag for checkPathConflicts
diff --git a/updatebot/groupmgr/helper.py b/updatebot/groupmgr/helper.py
--- a/updatebot/groupmgr/helper.py
+++ b/updatebot/groupmgr/helper.py
@@ -94,32 +94,11 @@
for name, groupObj in model.iteritems():
contentFileName = util.join(recipeDir, groupObj.filename)
contentsModel = GroupContentsModel.thaw(contentFileName,
- (name, groupObj.byDefault, groupObj.depCheck))
+ (name, groupObj.byDefault, groupObj.depCheck,
+ groupObj.checkPathConflicts))
contentsModel.fileName = groupObj.filename
groups[groupObj.name] = contentsModel
- # copy in any group data
- for name, data in self._groupContents.iteritems():
- newGroups = [ x for x in groups.itervalues()
- if x.groupName == name and
- x.fileName == data['filename'] ]
-
- assert len(newGroups) in (0, 1)
-
- byDefault = data['byDefault'] == 'True' and True or False
- depCheck = data['depCheck'] == 'True' and True or False
-
- # load model
- contentsModel = GroupContentsModel.thaw(
- util.join(self._configDir, data['filename']),
- (name, byDefault, depCheck)
- )
-
- # override anything from the repo, unless retriveing a
- # specific version.
- if version is None:
- groups[name] = contentsModel
-
return groups
def setModel(self, pkgName, groups, version=None):
@@ -139,7 +118,8 @@
groupModel.add(name=name,
filename=model.fileName,
byDefault=model.byDefault,
- depCheck=model.depCheck)
+ depCheck=model.depCheck,
+ checkPathConflicts=model.checkPathConflicts,)
self._addFile(recipeDir, model.fileName)
groupModel.freeze(groupFileName)
diff --git a/updatebot/groupmgr/model.py b/updatebot/groupmgr/model.py
--- a/updatebot/groupmgr/model.py
+++ b/updatebot/groupmgr/model.py
@@ -142,11 +142,13 @@
dataClass = XPackageData
elementClass = XPackageItem
- def __init__(self, groupName, byDefault=True, depCheck=True):
+ def __init__(self, groupName, byDefault=True, depCheck=True,
+ checkPathConflicts=False):
AbstractModel.__init__(self)
self.groupName = groupName
self.byDefault = byDefault
self.depCheck = depCheck
+ self.checkPathConflicts = checkPathConflicts
# figure out file name based on group name
name = ''.join([ x.capitalize() for x in self.groupName.split('-') ])
diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py
--- a/updatebot/lib/xobjects.py
+++ b/updatebot/lib/xobjects.py
@@ -235,12 +235,15 @@
filename = str
byDefault = int
depCheck = int
+ checkPathConflicts = int
- def __init__(self, name=None, filename=None, byDefault=True, depCheck=True):
+ def __init__(self, name=None, filename=None, byDefault=True, depCheck=True,
+ checkPathConflicts=False):
self.name = name
self.filename = filename
self.byDefault = byDefault and 1 or 0
self.depCheck = depCheck and 1 or 0
+ self.checkPathConflicts = checkPathConflicts and 1 or 0
@property
def key(self):
From elliot at rpath.com Wed Jun 2 20:11:42 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:42 +0000
Subject: mirrorball: update does not return failures
Message-ID: <201006030011.o530BgDt000390@scc.eng.rpath.com>
changeset: f698edd22a82
user: Elliot Peele
date: Wed, 02 Jun 2010 19:28:51 -0400
update does not return failures
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -84,6 +84,6 @@
bot._errata._errata = errata
-pkgMap, failures = bot.update(fltr=fltr)
+pkgMap = bot.update(fltr=fltr)
import epdb; epdb.st()
From elliot at rpath.com Wed Jun 2 20:11:42 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:42 +0000
Subject: mirrorball: rework for new group model
Message-ID: <201006030011.o530Bgj0000417@scc.eng.rpath.com>
changeset: 4391640121e4
user: Elliot Peele
date: Wed, 02 Jun 2010 19:29:34 -0400
rework for new group model
diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py
--- a/scripts/grpchecker.py
+++ b/scripts/grpchecker.py
@@ -40,6 +40,7 @@
from updatebot import log
from updatebot import groupmgr
+from updatebot import pkgsource
from updatebot import conaryhelper
from updatebot import UpdateBotConfig
@@ -48,14 +49,14 @@
from updatebot.errors import NameVersionConflictsFoundError
from updatebot.errors import ExpectedRemovalValidationFailedError
-import time
-import logging
-
slog = log.addRootLogger()
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
-mgr = groupmgr.GroupManager(cfg)
+pkgSource = pkgsource.PackageSource(cfg)
+pkgSource.load()
+
+mgr = groupmgr.GroupManager(cfg, useMap=pkgSource.useMap)
helper = conaryhelper.ConaryHelper(cfg)
def handleVersionConflicts(group, error):
@@ -97,14 +98,13 @@
return toRemove, toAdd
def checkVersion(ver):
- mgr._sourceVersion = ver
- mgr._checkout()
- mgr._copyVersions()
+ grp = mgr.getGroup(version=ver)
+ grp._copyVersions()
changes = []
try:
- mgr._sanity.check(mgr._groups, mgr.getErrataState())
+ grp._sanityCheck()
except GroupValidationFailedError, e:
for group, error in e.errors:
if isinstance(error, NameVersionConflictsFoundError):
@@ -158,14 +158,13 @@
jobIds = []
removed = {}
for ver, changed in toUpdate:
- mgr._sourceVersion = ver
- mgr._checkout()
+ grp = mgr.getGroup(version=ver)
slog.info('updating %s' % ver)
# Find all names and versions in the group model
nv = dict([ (y.name, versions.ThawVersion(y.version))
- for x, y in mgr._groups[mgr._pkgGroupName].iteritems() ])
+ for x, y in grp._groups[grp._pkgGroupName].iteritems() ])
# Set of packages that should be removed
removals = set([ x for x, y in removed.iteritems() if nv[x] == y ])
@@ -173,13 +172,13 @@
# Remove old removals
for pkg in removals:
slog.info('removing %s' % pkg)
- mgr.remove(pkg)
+ grp.removePackage(pkg)
for toRemove, toAdd in changed:
# Handle removes
for pkg in toRemove:
slog.info('removing %s' % pkg)
- mgr.remove(pkg)
+ grp.removePackage(pkg)
# cache removals
removed[pkg] = nv[pkg]
@@ -187,12 +186,14 @@
# Handle adds
for (n, v), flvs in toAdd.iteritems():
slog.info('adding %s=%s' % (n, v))
- mgr.addPackage(n, v, flvs)
+ grp.addPackage(n, v, flvs)
- mgr._copyVersions()
- mgr._sanity.check(mgr._groups, mgr.getErrataState())
- version = mgr.save(copyToLatest=True)
- jobId = mgr._builder.start(((mgr._sourceName, version, None), ))
+ grp._copyVersions()
+ grp._sanityCheck()
+ mgr._persistGroup(grp)
+
+ newGroup = grp.commit(copyToLatest=True)
+ jobId = mgr._builder.start(((mgr._sourceName, newGroup.conaryVersion, None), ))
jobIds.append(jobId)
for jobId in jobIds:
From elliot at rpath.com Wed Jun 2 20:11:43 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:43 +0000
Subject: mirrorball: groupContents is always a dict
Message-ID: <201006030011.o530BhIc000444@scc.eng.rpath.com>
changeset: ca239d296656
user: Elliot Peele
date: Wed, 02 Jun 2010 19:36:46 -0400
groupContents is always a dict
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -450,7 +450,8 @@
"""
for groupName, groupObj in self._groups.iteritems():
- for key, value in self._cfg.groupContents.get(groupName, []):
+ flags = self._cfg.groupContents.get(groupName, {})
+ for key, value in flags.iteritems():
value = value == 'True' and True or False
setattr(groupObj, key, value)
From elliot at rpath.com Wed Jun 2 20:11:43 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:43 +0000
Subject: mirrorball: ugly hack
Message-ID: <201006030011.o530BhKo000473@scc.eng.rpath.com>
changeset: 40542b182c4b
user: Elliot Peele
date: Wed, 02 Jun 2010 20:09:31 -0400
ugly hack
diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py
--- a/updatebot/groupmgr/sanity.py
+++ b/updatebot/groupmgr/sanity.py
@@ -58,11 +58,14 @@
except NameVersionConflictsFoundError, e:
errors.append((group, e))
- try:
- log.info('checking latest versions')
- self._checkLatestVersion(group)
- except OldVersionsFoundError, e:
- errors.append((group, e))
+ # FIXME: This is a hack, there should be a better way of controlling
+ # what policy runs for a particular group.
+ if 'standard' not in name:
+ try:
+ log.info('checking latest versions')
+ self._checkLatestVersion(group)
+ except OldVersionsFoundError, e:
+ errors.append((group, e))
try:
log.info('checking removals')
From elliot at rpath.com Wed Jun 2 20:11:44 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 03 Jun 2010 00:11:44 +0000
Subject: mirrorball: cleanups
Message-ID: <201006030011.o530BiOV000500@scc.eng.rpath.com>
changeset: b55b30a398d2
user: Elliot Peele
date: Wed, 02 Jun 2010 20:11:33 -0400
cleanups
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -45,472 +45,7 @@
Generate config for standard group contents based on repository history.
"""
- standard = (
- 'aaa_base',
- 'aaa_skel',
- 'acl',
- 'acpid',
- 'ash',
- 'at',
-# 'atk',
-# 'atk-devel',
-# 'atk-doc',
- 'attr',
-# 'audiofile',
-# 'audiofile-devel',
- 'audit',
- 'audit-devel',
- 'audit-libs',
-# 'autoconf',
-# 'autofs',
- 'bash',
-# 'bc',
-# 'bin86',
-# 'bind',
-# 'bind-chrootenv',
-# 'bind-devel',
-# 'bind-doc',
-# 'bind-libs',
-# 'bind-utils',
- 'binutils',
-# 'bison',
- 'blt',
- 'blocxx',
- 'busybox',
- 'bzip2',
-# 'cairo',
-# 'cairo-devel',
-# 'cairo-doc',
-# 'cdrecord',
-# 'cdrecord-devel',
-# 'cifs-mount',
-# 'compat-libstdc++',
-# 'compat-openssl097g',
- 'coreutils',
- 'cpio',
-# 'cpp',
- 'cracklib',
-# 'cracklib-devel',
- 'cron',
-# 'curl',
-# 'curl-devel',
-# 'cvs',
-# 'cvs-doc',
- 'cyrus-sasl',
-# 'cyrus-sasl-crammd5',
-# 'cyrus-sasl-devel',
-# 'cyrus-sasl-digestmd5',
-# 'cyrus-sasl-gssapi',
-# 'cyrus-sasl-otp',
-# 'cyrus-sasl-plain',
-# 'cyrus-sasl-sqlauxprop',
- 'db',
- 'db42',
-# 'db-devel',
-# 'db-utils',
- 'dbus-1',
-# 'dbus-1-devel',
- 'dbus-1-glib',
-# 'dbus-1-gtk',
-# 'dbus-1-java',
-# 'dbus-1-mono',
-# 'dbus-1-python',
-# 'dbus-1-qt3',
-# 'dbus-1-qt3-devel',
-# 'dbus-1-x11',
-# 'dev86',
- 'device-mapper',
-# 'device-mapper-devel',
- 'dhcpcd',
- 'diffutils',
- 'e2fsprogs',
-# 'e2fsprogs-devel',
-# 'eject',
-# 'esound',
-# 'esound-devel',
- 'ethtool',
- 'expat',
-# 'expect',
- 'file',
-# 'file-devel',
- 'filesystem',
- 'fillup',
- 'findutils',
-# 'findutils-locate',
- 'fontconfig',
-# 'fontconfig-devel',
-# 'freetype',
-# 'freetype-tools',
- 'freetype2',
-# 'freetype2-devel',
- 'gawk',
-# 'gawk-doc',
-# 'gcc',
-# 'gcc-c++',
-# 'gcc-fortran',
-# 'gcc-info',
-# 'gcc-java',
-# 'gcc-locale',
-# 'gcc-obj-c++',
-# 'gcc-objc',
-# 'gconf2',
-# 'gconf2-devel',
-# 'gconf2-doc',
-# 'gdb',
- 'gdbm',
-# 'gdbm-devel',
- 'gettext',
-# 'gettext-devel',
- 'glib2',
-# 'glib2-devel',
-# 'glib2-doc',
- 'glibc',
-# 'glibc-debuginfo',
-# 'glibc-devel',
-# 'glibc-html',
-# 'glibc-i18ndata',
-# 'glibc-info',
-# 'glibc-locale',
-# 'glibc-profile',
-# 'glitz',
-# 'glitz-devel',
-# 'gmp',
-# 'gmp-devel',
- 'gnome-filesystem',
-# 'gnome-vfs2',
-# 'gnome-vfs2-devel',
-# 'gnome-vfs2-doc',
-# 'gnuplot',
-# 'gnutls',
-# 'gnutls-devel',
- 'gpg',
-# 'gpg2',
-# 'gpm',
- 'grep',
-# 'groff',
- 'grub',
-# 'gtk2',
-# 'gtk2-devel',
-# 'gtk2-doc',
-# 'gvim',
-# 'gxdview',
- 'gzip',
- 'hal',
-# 'hal-devel',
-# 'hal-gnome',
-# 'hdparm',
- 'hwinfo',
-# 'hwinfo-devel',
- 'info',
- 'insserv',
- 'iproute2',
- 'iptables',
-# 'iptables-devel',
- 'iputils',
-# 'jpeg',
-# 'kbd',
- 'klogd',
- 'krb5',
-# 'krb5-apps-clients',
-# 'krb5-apps-servers',
-# 'krb5-client',
-# 'krb5-devel',
-# 'krb5-server',
-# 'ksh',
-# 'ksh-devel',
- 'less',
- 'libacl',
-# 'libacl-devel',
- 'libaio',
-# 'libaio-devel',
-# 'libapr-util1',
-# 'libapr-util1-devel',
-# 'libapr1',
-# 'libapr1-devel',
-# 'libart_lgpl',
-# 'libart_lgpl-devel',
- 'libattr',
-# 'libattr-devel',
-# 'libbonobo',
-# 'libbonobo-devel',
-# 'libbonobo-doc',
-# 'libbonoboui',
-# 'libbonoboui-devel',
-# 'libbonoboui-doc',
- 'libcap',
-# 'libcap-devel',
- 'libcom_err',
- 'libelf',
- 'libevent',
- 'libgcc',
-# 'libgcj',
-# 'libgcj-devel',
- 'libgcrypt',
-# 'libgcrypt-devel',
-# 'libgfortran',
-# 'libgnome',
-# 'libgnome-devel',
-# 'libgnome-doc',
-# 'libgnomecanvas',
-# 'libgnomecanvas-devel',
-# 'libgnomecanvas-doc',
- 'libgpg-error',
-# 'libgpg-error-devel',
-# 'libgssapi',
-# 'libidn',
-# 'libidn-devel',
-# 'libiniparser',
-# 'libiniparser-devel',
-# 'libjpeg',
-# 'libjpeg-devel',
-# 'libksba',
-# 'libksba-devel',
-# 'libmsrpc',
-# 'libmsrpc-devel',
-# 'libmudflap',
-# 'libnlink',
- 'libnscd',
-# 'libnscd-devel',
-# 'libobjc',
-# 'libopencdk',
-# 'libopencdk-devel',
-# 'libpcap',
-# 'libpng',
-# 'libpng-devel',
-# 'librpcsecgss',
-# 'libsmbclient',
-# 'libsmbclient-devel',
- 'libstdc++',
-# 'libstdc++-devel',
-# 'libstdc++-doc',
- 'libtool',
- 'libusb',
- 'libxcrypt',
-# 'libxcrypt-devel',
- 'libxml2',
-# 'libxml2-devel',
-# 'libxml2-python',
- 'libxslt',
-# 'libxslt-devel',
- 'libzio',
- 'limal',
- 'limal-bootloader',
- 'limal-perl',
- 'logrotate',
-# 'lsof',
- 'lvm2',
-# 'lzo',
-# 'lzo-devel',
- 'm4',
-# 'mailx',
-# 'make',
-# 'man',
-# 'mdadm',
-# 'microcode_ctl',
- 'mingetty',
- 'mkinitrd',
-# 'mkisofs',
- 'mktemp',
-# 'mm',
-# 'mm-devel',
- 'module-init-tools',
-# 'mysql',
-# 'mysql-Max',
-# 'mysql-client',
-# 'mysql-devel',
-# 'mysql-shared',
-# 'nc6',
- 'ncurses',
-# 'ncurses-devel',
-# 'neon',
-# 'net-snmp',
-# 'net-snmp-devel',
- 'net-tools',
- 'netcfg',
-# 'nfs-utils',
-# 'nfsidmap',
-# 'nmap',
-# 'nmap-gtk',
- 'nscd',
- 'openct',
-# 'openct-devel',
- 'openldap2',
-# 'openldap2-back-meta',
-# 'openldap2-back-perl',
- 'openldap2-client',
-# 'openldap2-devel',
- 'opensc',
-# 'opensc-devel',
- 'openslp',
-# 'openslp-devel',
-# 'openslp-server',
- 'openssh',
-# 'openssh-askpass',
- 'openssl',
-# 'openssl-devel',
-# 'openssl-doc',
-# 'orbit2',
-# 'orbit2-devel',
- 'pam',
-# 'pam-devel',
- 'pam-modules',
-# 'pam_krb5',
-# 'pam_smb',
-# 'pango',
-# 'pango-devel',
-# 'pango-doc',
-# 'parted',
-# 'parted-devel',
-# 'patch',
- 'pciutils',
-# 'pciutils-devel',
- 'pciutils-ids',
- 'pcre',
-# 'pcre-devel',
- 'pcsc-lite',
-# 'pcsc-lite-devel',
- 'perl',
-# 'perl-Bit-Vector',
- 'perl-Bootloader',
-# 'perl-Carp-Clan',
-# 'perl-Compress-Zlib',
-# 'perl-DBD-SQLite',
-# 'perl-DBD-mysql',
-# 'perl-DBI',
-# 'perl-Data-ShowTable',
-# 'perl-Date-Calc',
-# 'perl-Digest-SHA1',
-# 'perl-Net-Daemon',
-# 'perl-PlRPC',
-# 'perl-SNMP',
-# 'perl-TermReadKey',
-# 'perl-URI',
-# 'perl-XML-Parser',
-# 'perl-XML-Writer',
- 'perl-gettext',
- 'permissions',
-# 'pinentry',
-# 'pkgconfig',
-# 'pmtools',
- 'popt',
-# 'popt-devel',
-# 'portmap',
-# 'postgresql',
-# 'postgresql-contrib',
-# 'postgresql-devel',
-# 'postgresql-docs',
-# 'postgresql-libs',
-# 'postgresql-server',
- 'procmail',
- 'procps',
- 'psmisc',
- 'pwdutils',
-# 'pwdutils-plugin-audit',
- 'python',
-# 'python-cairo',
- 'python-curses',
-# 'python-demo',
- 'python-devel',
- 'python-gdbm',
-# 'python-gnome',
-# 'python-gtk',
-# 'python-idle',
-# 'python-numeric',
-# 'python-orbit',
-# 'python-pam',
- 'python-tk',
- 'python-xml',
- 'readline',
-# 'readline-devel',
- 'reiserfs',
-# 'resmgr',
- 'rpm',
-# 'rpm-devel',
- 'rpm-python',
-# 'rrdtool',
-# 'rsync',
-# 'samba',
-# 'samba-client',
-# 'samba-krb-printing',
-# 'samba-python',
-# 'samba-vscan',
-# 'samba-winbind',
- 'sed',
- 'sendmail',
-# 'sendmail-devel',
-# 'sensors',
-# 'slang',
-# 'slang-devel',
- 'sles-release',
-# 'smartmontools',
-# 'sqlite',
-# 'sqlite-devel',
-# 'strace',
-# 'sudo',
- 'sysconfig',
- 'sysfsutils',
-# 'syslinux',
- 'syslog-ng',
- 'syslogd',
-# 'sysstat',
-# 'sysstat-isag',
- 'sysvinit',
- 'suse-build-key',
- 'tar',
- 'tcl',
-# 'tcl-devel',
- 'tcpd',
-# 'tcpd-devel',
-# 'tcpdump',
- 'tcsh',
-# 'telnet',
-# 'telnet-server',
- 'termcap',
- 'terminfo',
-# 'texinfo',
- 'timezone',
- 'tk',
-# 'tk-devel',
- 'udev',
-# 'unixODBC',
-# 'unixODBC-devel',
-# 'unzip',
-# 'usbutils',
- 'util-linux',
-# 'uucp',
- 'vim',
- 'wget',
-# 'wireless-tools',
-# 'x11-tools',
- 'xfsprogs',
-# 'xfsprogs-devel',
-# 'xinetd',
-# 'xkeyboard-config',
-# 'xntp',
-# 'xntp-doc',
-# 'xorg-x11',
-# 'xorg-x11-Xnest',
-# 'xorg-x11-Xvfb',
-# 'xorg-x11-Xvnc',
-# 'xorg-x11-devel',
-# 'xorg-x11-doc',
-# 'xorg-x11-fonts-100dpi',
-# 'xorg-x11-fonts-75dpi',
-# 'xorg-x11-fonts-cyrillic',
-# 'xorg-x11-fonts-scalable',
-# 'xorg-x11-fonts-syriac',
- 'xorg-x11-libs',
-# 'xorg-x11-man',
-# 'xorg-x11-sdk',
-# 'xorg-x11-server',
-# 'xorg-x11-server-glx',
-# 'yp-tools',
-# 'ypbind',
-# 'zip',
-# 'zisofs-tools',
- 'zlib',
-# 'zlib-devel',
- )
+ self._pkgSource.load()
log.info('getting latest troves')
troves = self._updater._conaryhelper._getLatestTroves()
@@ -542,16 +77,25 @@
log.info('\t%s' % flv)
group.addPackage(name, version, flavors)
- group.errataState = '0'
- group.version = '0'
+ group.errataState = 0
+ group.version = 'sp3'
- addReq = dict([ ('group-standard', [ (x, None) for x in standard ]), ])
- group.modifyContents(additions=addReq)
+ group._groups.pop('group-standard', None)
- group._groups['group-standard'].depCheck = False
+ removals = set()
+ nevras = dict([ (x.getNevra(), y)
+ for x, y in self._pkgSource.srcPkgMap.iteritems() ])
- group.removePackage('samba-pdb')
- group.removePackage('kiwi-desc-xennetboot')
+ for updateId in range(0, group.errataState + 1):
+ self._modifyGroups(updateId, group)
+
+ for srcNevra in self._cfg.removeSource.get(updateId, ()):
+ removals.update(set([ x.name for x in nevras[srcNevra] ]))
+
+ removals |= set(self._cfg.updateRemovesPackages.get(updateId, ()))
+
+ for name in removals:
+ group.removePackage(name, missingOk=True)
group._copyVersions()
group._sanityCheck()
From elliot at rpath.com Fri Jun 4 13:59:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 04 Jun 2010 17:59:38 +0000
Subject: mirrorball: don't try to load the pkgsource twice
Message-ID: <201006041759.o54Hxc5I027674@scc.eng.rpath.com>
changeset: 6f930bf8cc98
user: Elliot Peele
date: Fri, 04 Jun 2010 13:59:27 -0400
don't try to load the pkgsource twice
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -122,7 +122,6 @@
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
bot = Bot(cfg, None)
- bot._pkgSource.load()
changes = bot.generateInitialGroup()
import epdb; epdb.st()
From elliot at rpath.com Fri Jun 4 16:07:10 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 04 Jun 2010 20:07:10 +0000
Subject: mirrorball: split -32bit rpms into their own sources
Message-ID: <201006042007.o54K7AZb006926@scc.eng.rpath.com>
changeset: a57abce63505
user: Elliot Peele
date: Fri, 04 Jun 2010 16:07:07 -0400
split -32bit rpms into their own sources
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -181,6 +181,13 @@
elif srcParts[-1].endswith('.nosrc.rpm'):
srcRelease = srcParts[-1][:-10]
+ # Change the source rpm for all -32bit packages to avoid having a binary
+ # that only contains a build log.
+ if package.name.endswith('-32bit'):
+ srcName += '-32bit'
+ package.sourcerpm = ('%s-%s-%s.src.rpm'
+ % (srcName, srcVersion, srcRelease))
+
rpmMapKey = (srcName, package.epoch, srcVersion, srcRelease, 'src')
self._rpmMap.setdefault(rpmMapKey, set()).add(package)
From elliot at rpath.com Mon Jun 7 00:38:18 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 07 Jun 2010 04:38:18 +0000
Subject: mirrorball: pull in askYn function
Message-ID: <201006070438.o574cIRw014414@scc.eng.rpath.com>
changeset: f08207272f6e
user: Elliot Peele
date: Sun, 06 Jun 2010 19:13:14 -0400
pull in askYn function
diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py
--- a/updatebot/lib/util.py
+++ b/updatebot/lib/util.py
@@ -24,6 +24,7 @@
import signal
import resource
from conary.lib.util import rmtree
+from conary.conaryclient.cmdline import askYn
from conary.lib.util import convertPackageNameToClassName as _pkgNameToClassName
from rpmutils import rpmvercmp
From elliot at rpath.com Mon Jun 7 00:38:18 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 07 Jun 2010 04:38:18 +0000
Subject: mirrorball: add method for markremoved
Message-ID: <201006070438.o574cII2014445@scc.eng.rpath.com>
changeset: 746ca48c0ea3
user: Elliot Peele
date: Sun, 06 Jun 2010 23:57:27 -0400
add method for markremoved
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -33,6 +33,7 @@
from conary import conaryclient
from conary.lib import log as clog
from conary.conaryclient import mirror
+from conary.repository import changeset
from conary import errors as conaryerrors
from updatebot.lib import util
@@ -1302,3 +1303,86 @@
for x, y in coMap.iteritems() ]))
return coMap
+
+ def markremoved(self, troveSpecs, removeSiblingPackages=False,
+ removeSources=False, removeAllVersions=False):
+ """
+ Remove a list of trove specs from the repository.
+
+ Use this with care, removing troves from the repository is a permanent
+ opperation and can have side affects.
+ @param troveSpecs: list of nvfs
+ @type troveSpecs: list((str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ @param removeSiblingPackages: Optional parameter that controls if
+ packages of the same version built from
+ the same source should also be removed,
+ default: False.
+ @type removeSibilingPackages: boolean
+ @param removeSources: Optional parameter that controls if the source of
+ a given version should be removed, default: False.
+ Removing sources implies removeSiblings.
+ @type removeSources: boolean
+ @param removeAllVersions: Optional prameter that controls if all
+ versions of a given trove should be removed,
+ default: False.
+ @type removeAllVersions: boolean
+ @return removed trove specs and changeset to commit
+ @rtype tuple(list((str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...),
+ conary.changeset.ChangeSet)
+ """
+
+ if removeSources:
+ removeSiblingPackages = True
+
+ log.info('retrieving troves from repository')
+
+ # Resolve any versions to conary versions and flavors.
+ resultMap = self._repos.findTroves(self._ccfg.buildLabel, troveSpecs,
+ getLeaves=removeAllVersions)
+
+ # Build trove query list.
+ query = set()
+ for trv, trvLst in resultMap.iteritems():
+ for n, v, f in trvLst:
+ query.add((n, v, f))
+
+ # Get troves from the repository.
+ troves = self._repos.getTroves(query, withFiles=False)
+
+ # Build set of troveSpecs to be removed.
+ trvSet = set()
+ for trv in troves:
+ # Don't recurse group contents.
+ if trv.getName().startswith('group-'):
+ continue
+ if removeSiblingPackages:
+ srcName = trv.troveInfo.sourceName()
+ srcVersion = trv.getVersion().getSourceVersion()
+ siblings = self._repos.getTrovesBySource(srcName, srcVersion)
+ trvSet.update(set([ x for x in
+ itertools.chain(*self._repos.findTroves(
+ self._ccfg.buildLabel, siblings).values()) ]))
+ if removeSources:
+ srcLst = self._repos.findTrove(self._ccfg.buildLabel,
+ (srcName, srcVersion, None))
+ for n, v, f in srcLst:
+ trvSet.add((n, v, f))
+ trvSet.update(set([ x for x in
+ trv.iterTroveList(strongRefs=True) ]))
+
+ # Add the original requested troves to set of troves to be removed.
+ trvSet.update(query)
+
+ log.info('building removal changeset')
+
+ # Create a changeset of all of the removal specs.
+ cs = changeset.ChangeSet()
+ for n, v, f in trvSet:
+ trv = trove.Trove(n, v, f, type=trove.TROVE_TYPE_REMOVED)
+ trv.computeDigests()
+ trvCs = trv.diff(None, absolute=True)[0]
+ cs.newTrove(trvCs)
+
+ return trvSet, cs
From elliot at rpath.com Mon Jun 7 00:38:19 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 07 Jun 2010 04:38:19 +0000
Subject: mirrorball: add script for markremoving multiple troves at once
Message-ID: <201006070438.o574cJCJ014469@scc.eng.rpath.com>
changeset: 3d646fc014f6
user: Elliot Peele
date: Sun, 06 Jun 2010 23:57:55 -0400
add script for markremoving multiple troves at once
diff --git a/scripts/import b/scripts/markremoved
copy from scripts/import
copy to scripts/markremoved
--- a/scripts/import
+++ b/scripts/markremoved
@@ -1,38 +1,40 @@
#!/usr/bin/python
#
-# Copyright (c) 2008-2009 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
+# Copryright (c) 2008-2009 rPath, Inc.
#
-import os
-import sys
+"""
+Script for cooking packages with updatebot config.
+"""
-sys.path.insert(0, os.path.abspath('../'))
+from header import *
-from conary.lib import util
-sys.excepthook = util.genExcepthook()
+import logging
+log = logging.getLogger('scripts.markremoved')
-from updatebot import bot, config, log
+from updatebot import conaryhelper
+from updatebot.lib.util import askYn
-log.addRootLogger()
-cfg = config.UpdateBotConfig()
-cfg.read(os.path.abspath('../') + '/config/%s/updatebotrc' % sys.argv[1])
-obj = bot.Bot(cfg)
-trvMap = obj.create()
+if len(sys.argv) < 3:
+ usage()
-import epdb ; epdb.st()
+helper = conaryhelper.ConaryHelper(cfg)
-for job in trvMap:
- for source in sorted(job):
- for bin in job[source]:
- if ':' not in bin[0]:
- print '%s=%s[%s]' % bin
+from conary.conaryclient import cmdline
+
+trvSpecs = set([ cmdline.parseTroveSpec(x) for x in sys.argv[2:] ])
+
+removedSpecs, cs = helper.markremoved(trvSpecs, removeSources=True)
+
+log.info('commiting will remove the following trove specs')
+for spec in sorted(removedSpecs):
+ log.info('removing: %s=%s[%s]' % spec)
+
+import epdb; epdb.st()
+
+if askYn('remove troves? (y/N):', default=False):
+ log.info('committing')
+ helper._repos.commitChangeSet(cs)
+ log.info('committed')
+else:
+ log.info('not committed')
From elliot at rpath.com Mon Jun 7 23:54:04 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:04 +0000
Subject: mirrorball: invert getLeaves
Message-ID: <201006080354.o583s474023528@scc.eng.rpath.com>
changeset: 5e17eec43a34
user: Elliot Peele
date: Mon, 07 Jun 2010 10:03:57 -0400
invert getLeaves
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1340,7 +1340,7 @@
# Resolve any versions to conary versions and flavors.
resultMap = self._repos.findTroves(self._ccfg.buildLabel, troveSpecs,
- getLeaves=removeAllVersions)
+ getLeaves=not removeAllVersions)
# Build trove query list.
query = set()
From elliot at rpath.com Mon Jun 7 23:54:05 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:05 +0000
Subject: mirrorball: rename removeSiblingPackages to removeSiblings and
add more comments
Message-ID: <201006080354.o583s501023559@scc.eng.rpath.com>
changeset: b52d58bfd0a7
user: Elliot Peele
date: Mon, 07 Jun 2010 10:47:12 -0400
rename removeSiblingPackages to removeSiblings and add more comments
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1304,7 +1304,7 @@
return coMap
- def markremoved(self, troveSpecs, removeSiblingPackages=False,
+ def markremoved(self, troveSpecs, removeSiblings=False,
removeSources=False, removeAllVersions=False):
"""
Remove a list of trove specs from the repository.
@@ -1314,11 +1314,10 @@
@param troveSpecs: list of nvfs
@type troveSpecs: list((str, conary.versions.VersionFromString,
conary.deps.deps.Flavor), ...)
- @param removeSiblingPackages: Optional parameter that controls if
- packages of the same version built from
- the same source should also be removed,
- default: False.
- @type removeSibilingPackages: boolean
+ @param removeSiblings: Optional parameter that controls if packages of
+ the same version built from the same source
+ should also be removed, default: False.
+ @type removeSibilings: boolean
@param removeSources: Optional parameter that controls if the source of
a given version should be removed, default: False.
Removing sources implies removeSiblings.
@@ -1334,7 +1333,7 @@
"""
if removeSources:
- removeSiblingPackages = True
+ removeSiblings = True
log.info('retrieving troves from repository')
@@ -1357,18 +1356,29 @@
# Don't recurse group contents.
if trv.getName().startswith('group-'):
continue
- if removeSiblingPackages:
+ # Find all sibling packages.
+ if removeSiblings:
srcName = trv.troveInfo.sourceName()
srcVersion = trv.getVersion().getSourceVersion()
siblings = self._repos.getTrovesBySource(srcName, srcVersion)
+
+ # Must lookup versions in the repository since getTrovesBySource
+ # does not include timestamps in the returned version objects
+ # and creating a trove requires versions with timestamps.
trvSet.update(set([ x for x in
itertools.chain(*self._repos.findTroves(
self._ccfg.buildLabel, siblings).values()) ]))
+
+ # Add sources to remmove.
if removeSources:
+ # As mentioned above, must lookup version with timestamp.
srcLst = self._repos.findTrove(self._ccfg.buildLabel,
(srcName, srcVersion, None))
+
+ assert len(set(srcLst)) == 1
for n, v, f in srcLst:
trvSet.add((n, v, f))
+
trvSet.update(set([ x for x in
trv.iterTroveList(strongRefs=True) ]))
From elliot at rpath.com Mon Jun 7 23:54:05 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:05 +0000
Subject: mirrorball: add method for removing packages
Message-ID: <201006080354.o583s5BJ023586@scc.eng.rpath.com>
changeset: e904cecc83c8
user: Elliot Peele
date: Mon, 07 Jun 2010 10:47:35 -0400
add method for removing packages
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -1057,3 +1057,32 @@
srcMap = self._conaryhelper.getBinaryVersions(filteredSrcTrvs,
missingOk=True, labels=[filteredSrcTrvs[0][1].trailingLabel(), ])
return srcMap
+
+ def remove(self, srcPkgs, interactive=True):
+ """
+ Remove all instances of packages from the conary repository that were
+ generated by a given source package.
+ @param srcPkgs: list of source package objects.
+ @type srcPkgs: list(repomd.packagexml._Package, ...)
+ @return list of trove specs that have been removed.
+ @rtype list((str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ """
+
+ trvSpecs = [ ('%s:source' % x.name, x.getConaryVersion(), None)
+ for x in srcPkgs ]
+
+ removeSpecs, cs = self._conaryhelper.markremoved(trvSpecs,
+ removeSources=True, removeSiblings=True, removeAllVersions=False)
+
+ for spec in removeSpecs:
+ log.info('removing: %s=%s[%s]' % spec)
+
+ commit = True
+ if interactive:
+ commit = util.askYn('remove troves? (y/N):', default=False)
+
+ if commit:
+ self._conaryhelper._repos.commitChangeSet(cs)
+
+ return removeSpecs
From elliot at rpath.com Mon Jun 7 23:54:05 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:05 +0000
Subject: mirrorball: add some better logging
Message-ID: <201006080354.o583s5f9023613@scc.eng.rpath.com>
changeset: 077290d20ea7
user: Elliot Peele
date: Mon, 07 Jun 2010 10:50:34 -0400
add some better logging
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -1083,6 +1083,8 @@
commit = util.askYn('remove troves? (y/N):', default=False)
if commit:
+ log.info('committing')
self._conaryhelper._repos.commitChangeSet(cs)
+ log.info('committed')
return removeSpecs
From elliot at rpath.com Mon Jun 7 23:54:06 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:06 +0000
Subject: mirrorball: sources don't have a sourceName, use the trove name
Message-ID: <201006080354.o583s6G6023640@scc.eng.rpath.com>
changeset: fea6c0bff611
user: Elliot Peele
date: Mon, 07 Jun 2010 11:21:59 -0400
sources don't have a sourceName, use the trove name
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1359,6 +1359,8 @@
# Find all sibling packages.
if removeSiblings:
srcName = trv.troveInfo.sourceName()
+ if not srcName:
+ srcName = trv.getName()
srcVersion = trv.getVersion().getSourceVersion()
siblings = self._repos.getTrovesBySource(srcName, srcVersion)
From elliot at rpath.com Mon Jun 7 23:54:06 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:06 +0000
Subject: mirrorball: log removals in sorted order and log the total number of
troves being removed
Message-ID: <201006080354.o583s6F3023667@scc.eng.rpath.com>
changeset: be603ee4ffa3
user: Elliot Peele
date: Mon, 07 Jun 2010 11:24:16 -0400
log removals in sorted order and log the total number of troves being removed
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -1075,9 +1075,11 @@
removeSpecs, cs = self._conaryhelper.markremoved(trvSpecs,
removeSources=True, removeSiblings=True, removeAllVersions=False)
- for spec in removeSpecs:
+ for spec in sorted(removeSpecs):
log.info('removing: %s=%s[%s]' % spec)
+ log.info('total troves to remove: %s' % len(removeSpecs))
+
commit = True
if interactive:
commit = util.askYn('remove troves? (y/N):', default=False)
From elliot at rpath.com Mon Jun 7 23:54:07 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:07 +0000
Subject: mirrorball: add ui callback interface
Message-ID: <201006080354.o583s7GO023696@scc.eng.rpath.com>
changeset: dce1385eb00d
user: Elliot Peele
date: Mon, 07 Jun 2010 12:56:40 -0400
add ui callback interface
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -22,6 +22,7 @@
from updatebot import build
from updatebot import update
+from updatebot import cmdline
from updatebot import pkgsource
from updatebot import advisories
@@ -35,17 +36,18 @@
def __init__(self, cfg):
self._cfg = cfg
- self._patchSourcePopulated = False
+ self._clientcfg = cmdline.clientcfg.UpdateBotClientConfig()
+ self._ui = cmdline.ui.UserInterface(self._clientcfg)
- self._clients = {}
- self._pkgSource = pkgsource.PackageSource(self._cfg)
- self._updater = update.Updater(self._cfg, self._pkgSource)
- self._builder = build.Builder(self._cfg)
+ self._pkgSource = pkgsource.PackageSource(self._cfg, self._ui)
+ self._updater = update.Updater(self._cfg, self._ui, self._pkgSource)
+ self._builder = build.Builder(self._cfg, self._ui)
if not self._cfg.disableAdvisories:
self._advisor = advisories.Advisor(self._cfg, self._pkgSource,
self._cfg.platformName)
+
@staticmethod
def _flattenSetDict(setDict):
"""
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -88,10 +88,13 @@
@param cfg: updateBot configuration object
@type cfg: config.UpdateBotConfig
+ @param ui: command line user interface.
+ @type ui: cmdline.ui.UserInterface
"""
- def __init__(self, cfg, rmakeCfgFn=None):
+ def __init__(self, cfg, ui, rmakeCfgFn=None):
self._cfg = cfg
+ self._ui = ui
self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False)
self._ccfg.read(util.join(self._cfg.configPath, 'conaryrc'))
@@ -364,7 +367,7 @@
# Check if this looks like a kernel module source rpm that wasn't
# handled by the last two checks.
- elif '-kmod' in name:
+ elif '-kmod' in name or '-kmp' in name:
raise UnhandledKernelModule(name=name)
# All other packages.
diff --git a/updatebot/cmdline/clientcfg.py b/updatebot/cmdline/clientcfg.py
--- a/updatebot/cmdline/clientcfg.py
+++ b/updatebot/cmdline/clientcfg.py
@@ -18,8 +18,21 @@
import os
+from conary.lib import cfg
+from conary.lib.cfgtypes import CfgBool
+
from updatebot.config import UpdateBotConfig
+
+class UpdateBotClientConfigSection(cfg.ConfigSection):
+ """
+ Config class for an updatebot command line client.
+ """
+
+ # control to interupt operations based on various conditions
+ interactive = (CfgBool, True)
+
+
class UpdateBotClientConfig(UpdateBotConfig):
"""
Client config object.
diff --git a/updatebot/cmdline/ui.py b/updatebot/cmdline/ui.py
new file mode 100644
--- /dev/null
+++ b/updatebot/cmdline/ui.py
@@ -0,0 +1,71 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Basic client user interface callback for mirrorball.
+"""
+
+from updatebot.lib import util
+
+class AbstractUserInterface(object):
+ """
+ Abstract class to define user interface interface.
+ """
+
+ def ask(self, prompt, default=None):
+ """
+ Simple method for asking yes/no questions.
+ @param prompt: string to prompt the user with.
+ @type prompt: str
+ @param default: Default value if no input is provided. This value will
+ be used if not running in interactive mode.
+ @type default: boolean
+ @return boolean value of the answer.
+ @rtype boolean
+ """
+
+ raise NotImplementedError
+
+
+class UserInterface(object):
+ """
+ Basic user interface class.
+ @param cfg: client config object
+ @type cfg: updatebot.cmdline.clientcfg.UpdateBotClientConfig
+ """
+
+ def __init__(self, cfg):
+ self._cfg = cfg
+
+ def ask(self, prompt, default=None):
+ """
+ Simple method for asking yes/no questions.
+ @param prompt: string to prompt the user with.
+ @type prompt: str
+ @param default: Default value if no input is provided. This value will
+ be used if not running in interactive mode.
+ @type default: boolean
+ @return boolean value of the answer.
+ @rtype boolean
+ """
+
+ if not self._cfg.interactive:
+ return default
+
+ if default is True:
+ prompt += ' [Y/n]:'
+ elif default is False:
+ prompt += ' [y/N]:'
+
+ return util.askYn(prompt, default)
diff --git a/updatebot/pkgsource/__init__.py b/updatebot/pkgsource/__init__.py
--- a/updatebot/pkgsource/__init__.py
+++ b/updatebot/pkgsource/__init__.py
@@ -21,16 +21,16 @@
from updatebot.pkgsource.debsource import DebSource
from updatebot.pkgsource.errors import UnsupportedRepositoryError
-def PackageSource(cfg):
+def PackageSource(cfg, ui):
"""
Method that returns an instance of the appropriate package source
backend based on config data.
"""
if cfg.repositoryFormat == 'apt':
- return DebSource(cfg)
+ return DebSource(cfg, ui)
elif cfg.repositoryFormat == 'yum':
- return YumSource(cfg)
+ return YumSource(cfg, ui)
else:
raise UnsupportedRepositoryError(repo=cfg.repositoryFormat,
supported=['apt', 'yum'])
diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py
--- a/updatebot/pkgsource/common.py
+++ b/updatebot/pkgsource/common.py
@@ -26,8 +26,10 @@
Base class for pkgSources
"""
- def __init__(self, cfg):
+ def __init__(self, cfg, ui):
self._cfg = cfg
+ self._ui = ui
+
self._excludeArch = self._cfg.excludeArch
self._loaded = False
diff --git a/updatebot/pkgsource/debsource.py b/updatebot/pkgsource/debsource.py
--- a/updatebot/pkgsource/debsource.py
+++ b/updatebot/pkgsource/debsource.py
@@ -28,8 +28,8 @@
PackageSource backend for APT repositories.
"""
- def __init__(self, cfg):
- BasePackageSource.__init__(self, cfg)
+ def __init__(self, cfg, ui):
+ BasePackageSource.__init__(self, cfg, ui)
self._binPkgs = set()
self._srcPkgs = set()
diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py
--- a/updatebot/pkgsource/rpmsource.py
+++ b/updatebot/pkgsource/rpmsource.py
@@ -146,8 +146,8 @@
PkgClass = Package
- def __init__(self, cfg, path=None):
- PackageSource.__init__(self, cfg)
+ def __init__(self, cfg, ui, path=None):
+ PackageSource.__init__(self, cfg, ui)
self._path = path
def load(self):
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -44,8 +44,8 @@
PkgClass = repomd.packagexml._Package
- def __init__(self, cfg):
- BasePackageSource.__init__(self, cfg)
+ def __init__(self, cfg, ui):
+ BasePackageSource.__init__(self, cfg, ui)
# {srcTup: srpm}
self._srcMap = dict()
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -42,8 +42,10 @@
Class for finding and updating packages.
"""
- def __init__(self, cfg, pkgSource):
+ def __init__(self, cfg, ui, pkgSource):
self._cfg = cfg
+ self._ui = ui
+
self._pkgSource = pkgSource
self._conaryhelper = conaryhelper.ConaryHelper(self._cfg)
@@ -1082,7 +1084,7 @@
commit = True
if interactive:
- commit = util.askYn('remove troves? (y/N):', default=False)
+ commit = self._ui.ask('remove troves? (y/N):', default=False)
if commit:
log.info('committing')
From elliot at rpath.com Mon Jun 7 23:54:07 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:07 +0000
Subject: mirrorball: hook up more ui bits
Message-ID: <201006080354.o583s71d023723@scc.eng.rpath.com>
changeset: fd9003e95b05
user: Elliot Peele
date: Mon, 07 Jun 2010 13:48:05 -0400
hook up more ui bits
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -22,9 +22,10 @@
from updatebot import build
from updatebot import update
-from updatebot import cmdline
from updatebot import pkgsource
from updatebot import advisories
+from updatebot.cmdline import ui
+from updatebot.cmdline import clientcfg
log = logging.getLogger('updatebot.bot')
@@ -36,8 +37,8 @@
def __init__(self, cfg):
self._cfg = cfg
- self._clientcfg = cmdline.clientcfg.UpdateBotClientConfig()
- self._ui = cmdline.ui.UserInterface(self._clientcfg)
+ self._clientcfg = clientcfg.UpdateBotClientConfig()
+ self._ui = ui.UserInterface(self._clientcfg)
self._pkgSource = pkgsource.PackageSource(self._cfg, self._ui)
self._updater = update.Updater(self._cfg, self._ui, self._pkgSource)
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -36,6 +36,8 @@
Class for managing groups.
@param cfg: updatebot configuration object
@type cfg: updatebot.config.UpdateBotConfig
+ @param ui: updatebot user interface object
+ @type ui: updatebot.cmdline.ui.UserInterface
@param parentGroup: optional argument, if set to True will setup manager to
interact with the parent platform group contents. This
is automatically set to readonly to avoid writing
@@ -59,8 +61,11 @@
_helperClass = GroupHelper
_sanityCheckerClass = GroupSanityChecker
- def __init__(self, cfg, parentGroup=False, targetGroup=False, useMap=None):
+ def __init__(self, cfg, ui, parentGroup=False, targetGroup=False,
+ useMap=None):
+
self._cfg = cfg
+ self._ui = ui
if useMap is None:
self._useMap = {}
@@ -68,7 +73,8 @@
self._useMap = useMap
self._helper = self._helperClass(self._cfg)
- self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups')
+ self._builder = Builder(self._cfg, self._ui,
+ rmakeCfgFn='rmakerc-groups')
self._sanity = self._sanityCheckerClass(self._cfg, self._helper)
assert not (parentGroup and targetGroup)
diff --git a/updatebot/groupmgr/single.py b/updatebot/groupmgr/single.py
--- a/updatebot/groupmgr/single.py
+++ b/updatebot/groupmgr/single.py
@@ -50,8 +50,9 @@
_managerClass = SingleGroupManager
- def __init__(self, cfg):
+ def __init__(self, cfg, ui):
self._cfg = cfg
+ self._ui = ui
self._mgrs = {}
def newGroup(self, name):
@@ -60,7 +61,7 @@
"""
assert name not in self._mgrs
- mgr = self._managerClass(name, self._cfg)
+ mgr = self._managerClass(name, self._cfg, self._ui)
self._mgrs[name] = mgr
return mgr
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -53,11 +53,11 @@
BotSuperClass.__init__(self, cfg)
self._errata = errata.ErrataFilter(self._cfg, self._pkgSource,
errataSource)
- self._groupmgr = groupmgr.GroupManager(self._cfg,
+ self._groupmgr = groupmgr.GroupManager(self._cfg, self._ui,
useMap=self._pkgSource.useMap)
if self._cfg.platformSearchPath:
- self._parentGroup = groupmgr.GroupManager(self._cfg,
+ self._parentGroup = groupmgr.GroupManager(self._cfg, self._ui,
parentGroup=True)
def _addPackages(self, pkgMap, group):
@@ -502,7 +502,8 @@
# Get latest errataState from the targetLabel so that we can fence group
# building based on the target label state.
- targetGroup = groupmgr.GroupManager(self._cfg, targetGroup=True)
+ targetGroup = groupmgr.GroupManager(self._cfg, self._ui,
+ targetGroup=True)
targetErrataState = targetGroup.latest.errataState
log.info('starting errata group processing')
@@ -537,7 +538,7 @@
# Now that we know that the packages that are part of this update
# should be on the target label we can separate things into
# advisories.
- mgr = groupmgr.ErrataGroupManagerSet(self._cfg)
+ mgr = groupmgr.ErrataGroupManagerSet(self._cfg, self._ui)
groupNames = self._errata.getNames(updateId)
for advInfo in self._errata.getUpdateDetail(updateId):
advisory = advInfo['name']
@@ -547,7 +548,7 @@
assert srcPkgs
targetGrp = groupmgr.SingleGroupManager(groupNames[advisory],
- self._cfg, targetGroup=True)
+ self._cfg, self._ui, targetGroup=True)
if targetGrp.hasBinaryVersion():
log.info('%s: found existing version, skipping' % advisory)
From elliot at rpath.com Mon Jun 7 23:54:07 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:07 +0000
Subject: mirrorball: use the correct config section class
Message-ID: <201006080354.o583s7PS023750@scc.eng.rpath.com>
changeset: d00b4d7a506b
user: Elliot Peele
date: Mon, 07 Jun 2010 13:50:23 -0400
use the correct config section class
diff --git a/updatebot/cmdline/clientcfg.py b/updatebot/cmdline/clientcfg.py
--- a/updatebot/cmdline/clientcfg.py
+++ b/updatebot/cmdline/clientcfg.py
@@ -38,6 +38,8 @@
Client config object.
"""
+ _defaultSecitonType = UpdateBotClientConfigSection
+
def __init__(self, readConfigFiles=False, ignoreErrors=False):
UpdateBotConfig.__init__(self)
self._ignoreErrors = ignoreErrors
From elliot at rpath.com Mon Jun 7 23:54:08 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:08 +0000
Subject: mirrorball: more user interface changes
Message-ID: <201006080354.o583s8KK023777@scc.eng.rpath.com>
changeset: e4132f4b09f9
user: Elliot Peele
date: Mon, 07 Jun 2010 14:00:59 -0400
more user interface changes
diff --git a/updatebot/cmdline/clientcfg.py b/updatebot/cmdline/clientcfg.py
--- a/updatebot/cmdline/clientcfg.py
+++ b/updatebot/cmdline/clientcfg.py
@@ -38,7 +38,7 @@
Client config object.
"""
- _defaultSecitonType = UpdateBotClientConfigSection
+ _defaultSectionType = UpdateBotClientConfigSection
def __init__(self, readConfigFiles=False, ignoreErrors=False):
UpdateBotConfig.__init__(self)
diff --git a/updatebot/cmdline/ui.py b/updatebot/cmdline/ui.py
--- a/updatebot/cmdline/ui.py
+++ b/updatebot/cmdline/ui.py
@@ -46,7 +46,7 @@
"""
def __init__(self, cfg):
- self._cfg = cfg
+ self.cfg = cfg
def ask(self, prompt, default=None):
"""
@@ -60,7 +60,7 @@
@rtype boolean
"""
- if not self._cfg.interactive:
+ if not self.cfg.interactive:
return default
if default is True:
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -1060,7 +1060,7 @@
missingOk=True, labels=[filteredSrcTrvs[0][1].trailingLabel(), ])
return srcMap
- def remove(self, srcPkgs, interactive=True):
+ def remove(self, srcPkgs):
"""
Remove all instances of packages from the conary repository that were
generated by a given source package.
@@ -1083,8 +1083,8 @@
log.info('total troves to remove: %s' % len(removeSpecs))
commit = True
- if interactive:
- commit = self._ui.ask('remove troves? (y/N):', default=False)
+ if self._ui.cfg.interactive:
+ commit = self._ui.ask('remove troves?', default=False)
if commit:
log.info('committing')
From elliot at rpath.com Mon Jun 7 23:54:08 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:08 +0000
Subject: mirrorball: fixup scripts for ui bits
Message-ID: <201006080354.o583s8Is023804@scc.eng.rpath.com>
changeset: 5da04ace6b11
user: Elliot Peele
date: Mon, 07 Jun 2010 16:03:29 -0400
fixup scripts for ui bits
diff --git a/scripts/buildgroups b/scripts/buildgroups
--- a/scripts/buildgroups
+++ b/scripts/buildgroups
@@ -9,7 +9,7 @@
from header import *
-builder = build.Builder(cfg, rmakeCfgFn='rmakerc-groups')
+builder = build.Builder(cfg, ui, rmakeCfgFn='rmakerc-groups')
grpTrvMap = builder.build((cfg.topSourceGroup, ))
diff --git a/scripts/compare b/scripts/compare
--- a/scripts/compare
+++ b/scripts/compare
@@ -41,6 +41,8 @@
from updatebot import conaryhelper
from updatebot.lib import util
+from updatebot.cmdline import UserInterface
+
log.addRootLogger()
import logging
@@ -54,8 +56,10 @@
self._cfg = config.UpdateBotConfig()
self._cfg.read(self._cfgPath)
+ self._ui = UserInterface()
+
self._helper = conaryhelper.ConaryHelper(self._cfg)
- self._builder = build.Builder(self._cfg)
+ self._builder = build.Builder(self._cfg, self._ui)
self._checkoutdir = tempfile.mkdtemp(prefix='%s-' % self._cfg.platformName, dir=self._workDir)
self._helper._cacheDir = self._checkoutdir
diff --git a/scripts/header.py b/scripts/header.py
--- a/scripts/header.py
+++ b/scripts/header.py
@@ -46,7 +46,11 @@
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
-builder = build.Builder(cfg)
+from updatebot.cmdline import UserInterface
+
+ui = UserInterface()
+
+builder = build.Builder(cfg, ui)
def displayTrove(nvf):
flavor = ''
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
old mode 100644
new mode 100755
diff --git a/scripts/setmetadata b/scripts/setmetadata
--- a/scripts/setmetadata
+++ b/scripts/setmetadata
@@ -30,6 +30,7 @@
from updatebot import log
from updatebot import config
from updatebot import update
+from udpatebot import cmdline
from updatebot import pkgsource
from updatebot.lib import util
@@ -45,8 +46,10 @@
topGroups = [ (cfg.topGroup[0], cfg.targetLabel, cfg.topGroup[2]),
(cfg.topGroup[0], versions.VersionFromString('/' + cfg.topGroup[1]), cfg.topGroup[2]), ]
+ui = cmdline.UserInterface()
+
pkgSource = pkgsource.PackageSource(cfg)
-updater = update.Updater(cfg, pkgSource)
+updater = update.Updater(cfg, ui, pkgSource)
helper = updater._conaryhelper
pkgSource.load()
From elliot at rpath.com Mon Jun 7 23:54:09 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:09 +0000
Subject: mirrorball: more ui related cleanup
Message-ID: <201006080354.o583s9kM023831@scc.eng.rpath.com>
changeset: 84922479dbd6
user: Elliot Peele
date: Mon, 07 Jun 2010 16:05:27 -0400
more ui related cleanup
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -22,10 +22,9 @@
from updatebot import build
from updatebot import update
+from updatebot import cmdline
from updatebot import pkgsource
from updatebot import advisories
-from updatebot.cmdline import ui
-from updatebot.cmdline import clientcfg
log = logging.getLogger('updatebot.bot')
@@ -37,8 +36,8 @@
def __init__(self, cfg):
self._cfg = cfg
- self._clientcfg = clientcfg.UpdateBotClientConfig()
- self._ui = ui.UserInterface(self._clientcfg)
+ self._clientcfg = cmdline.UpdateBotClientConfig()
+ self._ui = cmdline.UserInterface(self._clientcfg)
self._pkgSource = pkgsource.PackageSource(self._cfg, self._ui)
self._updater = update.Updater(self._cfg, self._ui, self._pkgSource)
diff --git a/updatebot/cmdline/__init__.py b/updatebot/cmdline/__init__.py
--- a/updatebot/cmdline/__init__.py
+++ b/updatebot/cmdline/__init__.py
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2008-2009 rPath, Inc.
+# Copyright (c) 2008-2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
@@ -11,3 +11,6 @@
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
+
+from updatebot.cmdline.ui import UserInterface
+from updatebot.cmdline.clientcfg import UpdateBotClientConfig
diff --git a/updatebot/cmdline/ui.py b/updatebot/cmdline/ui.py
--- a/updatebot/cmdline/ui.py
+++ b/updatebot/cmdline/ui.py
@@ -17,6 +17,7 @@
"""
from updatebot.lib import util
+from updatebot.cmdline import clientcfg
class AbstractUserInterface(object):
"""
@@ -38,15 +39,18 @@
raise NotImplementedError
-class UserInterface(object):
+class UserInterface(AbstractUserInterface):
"""
Basic user interface class.
@param cfg: client config object
@type cfg: updatebot.cmdline.clientcfg.UpdateBotClientConfig
"""
- def __init__(self, cfg):
- self.cfg = cfg
+ def __init__(self, cfg=None):
+ if cfg is None:
+ self.cfg = clientcfg.UpdateBotClientConfig()
+ else:
+ self.cfg = cfg
def ask(self, prompt, default=None):
"""
From elliot at rpath.com Mon Jun 7 23:54:09 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 03:54:09 +0000
Subject: mirrorball: change how we filter contexts to allow for more
control over when contexts are used.
Message-ID: <201006080354.o583s97d023858@scc.eng.rpath.com>
changeset: c30b0c16c6c6
user: Elliot Peele
date: Mon, 07 Jun 2010 23:52:36 -0400
change how we filter contexts to allow for more control over when contexts are used.
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -69,8 +69,7 @@
for (n, v, f), srcPkg in buildSet:
binaryNames = None
if srcPkg:
- binaryNames = tuple([ x.name
- for x in self._pkgSource.srcPkgMap[srcPkg] ])
+ binaryNames = tuple(self._updater.getPackageFileNames(srcPkg))
toBuild.add((n, v, f, binaryNames))
return sorted(toBuild)
@@ -84,7 +83,8 @@
@param recreate - recreate all sources or a specific list of packages
@type recreate - boolean to recreate all sources or a list of specific
package names
- @param toCreate - set of source package objects to create, implies recreate.
+ @param toCreate - set of source package objects to create, implies
+ recreate.
@type toCreate - iterable
"""
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -374,14 +374,13 @@
else:
# Build all packages as x86 and x86_64.
for context, fltr in self._cfg.archContexts:
- # If there is a filter and no binary file names or no files
- # in the binary names match the filter skip this context.
- if (fltr and (not binaryNames or (binaryNames and
- not [ x for x in binaryNames if fltr[1].match(x) ]))):
- continue
- troves.append((name, version, flavor, context))
+ # If this is no filter or if there is a filter and a binary
+ # package matches the filter build in this context.
+ if (not fltr or (fltr and
+ [ x for x in binaryNames if fltr[1].match(x) ])):
+ troves.append((name, version, flavor, context))
- return troves
+ return sorted(set(troves))
@jobInfoExceptionHandler
def _getJob(self, jobId, retry=None):
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -861,6 +861,16 @@
manifest.extend(pkg.files)
return manifest
+ def getPackageFileNames(self, srcPkg):
+ """
+ Get the list of package names with arch flags attached.
+ @param srcPkg: source rpm package object
+ @type srcPkg: repomd.packagexml._Package
+ """
+
+ return [ os.path.basename(x)
+ for x in self._getManifestFromPkgSource(srcPkg) ]
+
def _getMetadataFromPkgSource(self, srcPkg):
"""
Get the data to go into the xml metadata from a srcPkg.
From elliot at rpath.com Tue Jun 8 14:24:48 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 08 Jun 2010 18:24:48 +0000
Subject: mirrorball: fix typo
Message-ID: <201006081824.o58IOmlI019523@scc.eng.rpath.com>
changeset: 1f8091e23062
user: Elliot Peele
date: Tue, 08 Jun 2010 14:24:33 -0400
fix typo
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -173,7 +173,7 @@
# Try to build the group if everything imported.
else:
- self._modifyContents(0, group)
+ self._modifyGroups(0, group)
group.errataState = '0'
group.version = '0'
group = group.commit()
From elliot at rpath.com Thu Jun 10 14:06:02 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 10 Jun 2010 18:06:02 +0000
Subject: mirrorball: instantiate instances correctly when copying package
sources
Message-ID: <201006101806.o5AI62ss027887@scc.eng.rpath.com>
changeset: c87785cf24a3
user: Elliot Peele
date: Wed, 09 Jun 2010 16:21:31 -0400
instantiate instances correctly when copying package sources
diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py
--- a/updatebot/pkgsource/common.py
+++ b/updatebot/pkgsource/common.py
@@ -61,7 +61,7 @@
def __copy__(self):
log.info('copying pkgsource')
cls = self.__class__
- obj = cls(self._cfg)
+ obj = cls(self._cfg, self._ui)
obj.locationMap = copy.copy(self.locationMap)
obj.srcPkgMap = copy.copy(self.srcPkgMap)
obj.binPkgMap = copy.copy(self.binPkgMap)
@@ -73,7 +73,7 @@
def __deepcopy__(self, memo):
log.info('deepcopying pkgsource')
cls = self.__class__
- obj = cls(self._cfg)
+ obj = cls(self._cfg, self._ui)
obj.locationMap = copy.deepcopy(self.locationMap, memo)
obj.srcPkgMap = copy.deepcopy(self.srcPkgMap, memo)
obj.binPkgMap = copy.deepcopy(self.binPkgMap, memo)
From elliot at rpath.com Thu Jun 10 14:06:03 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 10 Jun 2010 18:06:03 +0000
Subject: mirrorball: add ui instances to all scripts
Message-ID: <201006101806.o5AI63ZK027917@scc.eng.rpath.com>
changeset: 2cac4536d772
user: Elliot Peele
date: Thu, 10 Jun 2010 10:39:54 -0400
add ui instances to all scripts
diff --git a/scripts/checksl.py b/scripts/checksl.py
--- a/scripts/checksl.py
+++ b/scripts/checksl.py
@@ -25,11 +25,15 @@
from updatebot import config
from updatebot import pkgsource
from updatebot import log as logger
+from updatebot import cmdline
logger.addRootLogger()
cfg = config.UpdateBotConfig()
cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/scientific/updatebotrc')
-pkgSource = pkgsource.PackageSource(cfg)
+
+ui = cmdline.UserInterface()
+
+pkgSource = pkgsource.PackageSource(cfg, ui)
pkgSource.load()
diff --git a/scripts/depsolver.py b/scripts/depsolver.py
--- a/scripts/depsolver.py
+++ b/scripts/depsolver.py
@@ -30,12 +30,15 @@
log = logging.getLogger('test')
from updatebot import config
+from updatebot import cmdline
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
+ui = cmdline.UserInterface()
+
+pkgSource = pkgsource.PackageSource(cfg, ui)
pkgSource.load()
reqSrcPkgs = set()
diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py
--- a/scripts/grpchecker.py
+++ b/scripts/grpchecker.py
@@ -43,6 +43,7 @@
from updatebot import pkgsource
from updatebot import conaryhelper
from updatebot import UpdateBotConfig
+from updatebot.cmdline import UserInterface
from updatebot.errors import OldVersionsFoundError
from updatebot.errors import GroupValidationFailedError
@@ -53,10 +54,12 @@
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
-pkgSource = pkgsource.PackageSource(cfg)
+ui = UserInterface()
+
+pkgSource = pkgsource.PackageSource(cfg, ui)
pkgSource.load()
-mgr = groupmgr.GroupManager(cfg, useMap=pkgSource.useMap)
+mgr = groupmgr.GroupManager(cfg, ui, useMap=pkgSource.useMap)
helper = conaryhelper.ConaryHelper(cfg)
def handleVersionConflicts(group, error):
diff --git a/scripts/memprof b/scripts/memprof
--- a/scripts/memprof
+++ b/scripts/memprof
@@ -29,12 +29,15 @@
log = logging.getLogger('test')
from updatebot import config
+from updatebot import cmdline
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
+ui = cmdline.UserInterface()
+
+pkgSource = pkgsource.PackageSource(cfg, ui)
from guppy import hpy
hp = hpy()
diff --git a/scripts/pkgsource b/scripts/pkgsource
--- a/scripts/pkgsource
+++ b/scripts/pkgsource
@@ -30,12 +30,15 @@
log = logging.getLogger('test')
from updatebot import config
+from updatebot import cmdline
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
+ui = cmdline.UserInterface()
+
+pkgSource = pkgsource.PackageSource(cfg, ui)
pkgSource.load()
srcs = {}
diff --git a/scripts/setmetadata b/scripts/setmetadata
--- a/scripts/setmetadata
+++ b/scripts/setmetadata
@@ -48,7 +48,7 @@
ui = cmdline.UserInterface()
-pkgSource = pkgsource.PackageSource(cfg)
+pkgSource = pkgsource.PackageSource(cfg, ui)
updater = update.Updater(cfg, ui, pkgSource)
helper = updater._conaryhelper
diff --git a/scripts/sleorder.py b/scripts/sleorder.py
--- a/scripts/sleorder.py
+++ b/scripts/sleorder.py
@@ -19,9 +19,10 @@
confDir = os.path.join(mbdir, 'config', sys.argv[1])
from updatebot import log
+from updatebot import cmdline
+from updatebot import pkgsource
+from updatebot import UpdateBotConfig
from updatebot.ordered import Bot
-from updatebot import UpdateBotConfig
-from updatebot import pkgsource
from errata.sles import AdvisoryManager as Errata
@@ -30,7 +31,9 @@
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
-pkgSource = pkgsource.PackageSource(cfg)
+ui = cmdline.UserInterface()
+
+pkgSource = pkgsource.PackageSource(cfg, ui)
errata = Errata(pkgSource)
errata.fetch()
diff --git a/scripts/verifyrepos b/scripts/verifyrepos
--- a/scripts/verifyrepos
+++ b/scripts/verifyrepos
@@ -32,11 +32,14 @@
import rpmutils
from updatebot import config
+from updatebot import cmdline
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
+ui = cmdline.UserInterface()
+
headers = []
paths = copy.copy(cfg.repositoryPaths)
@@ -61,7 +64,7 @@
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % platform)
cfg.repositoryPaths = [ repoPath, ]
- pkgSource = pkgsource.PackageSource(cfg)
+ pkgSource = pkgsource.PackageSource(cfg, ui)
pkgSource.load()
for location in pkgSource.locationMap:
diff --git a/scripts/virtdepfinder b/scripts/virtdepfinder
--- a/scripts/virtdepfinder
+++ b/scripts/virtdepfinder
@@ -32,16 +32,19 @@
log = logging.getLogger('test')
from updatebot import config
+from updatebot import cmdline
from updatebot.update import Updater
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
+ui = cmdline.UserInterface()
+
+pkgSource = pkgsource.PackageSource(cfg, ui)
pkgSource.load()
-updater = Updater(cfg, pkgSource)
+updater = Updater(cfg, ui, pkgSource)
virtreqs = {}
for srcName in pkgSource.srcNameMap.iterkeys():
From elliot at rpath.com Thu Jun 10 14:06:03 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 10 Jun 2010 18:06:03 +0000
Subject: mirrorball: insert sources iff they do not produce binaries of
the same name
Message-ID: <201006101806.o5AI63W1027945@scc.eng.rpath.com>
changeset: b51b8371d382
user: Elliot Peele
date: Thu, 10 Jun 2010 14:02:23 -0400
insert sources iff they do not produce binaries of the same name
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -343,6 +343,7 @@
log.warn('\t%s' % loc)
if self._repoMap:
+ sourceSet = set()
for binPkg, archSet in self._repoMap.iteritems():
# lookup the source for the binary package
srcPkg = self.binPkgMap[binPkg]
@@ -370,15 +371,18 @@
for flv in possibleFlavors:
trvSpecs = set([
(binPkg.name, conaryVersion, flv),
+ (binPkg.name, flv),
+ ])
+
+ sourceSet.update(set([
# Always include a package with the source name since
# every build will generate a conary package with the
# given build flavor, even if the package only contains
# a buildlog.
(srcPkg.name, conaryVersion, flv),
+ (srcPkg.name, flv),
+ ]))
- (binPkg.name, flv),
- (srcPkg.name, flv),
- ])
for trvSpec in trvSpecs:
useSet = set()
if binPkg.arch == 'noarch':
@@ -393,6 +397,10 @@
if useSet:
self.useMap.setdefault(trvSpec, set()).update(useSet)
+ for source in sourceSet:
+ if source not in self.useMap:
+ self.useMap.setdefault(source, set()).add(source[-1])
+
# In the case of SLES 10 we need to combine several source entries in
# the srcPkgMap to create a single unified kernel source package.
if self._cfg.nosrcFilter:
From elliot at rpath.com Thu Jun 10 14:06:04 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 10 Jun 2010 18:06:04 +0000
Subject: mirrorball: more user interface instance passing
Message-ID: <201006101806.o5AI64Fn027973@scc.eng.rpath.com>
changeset: d357e4c655db
user: Elliot Peele
date: Thu, 10 Jun 2010 14:04:04 -0400
more user interface instance passing
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -48,8 +48,9 @@
Filter data from a given errataSource in chronological order.
"""
- def __init__(self, cfg, pkgSource, errataSource):
+ def __init__(self, cfg, ui, pkgSource, errataSource):
self._cfg = cfg
+ self._ui = ui
self._pkgSource = pkgSource
self._errata = errataSource
@@ -211,7 +212,7 @@
# duplicate updater and pkgsource so as to not change state.
pkgSource = copy.copy(self._pkgSource)
- updater = update.Updater(self._cfg, pkgSource)
+ updater = update.Updater(self._cfg, self._ui, pkgSource)
updater._conaryhelper = _ConaryHelperShim(self._cfg)
if self._cfg.platformSearchPath:
From elliot at rpath.com Thu Jun 10 14:06:04 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 10 Jun 2010 18:06:04 +0000
Subject: mirrorball: more user interface instance passing
Message-ID: <201006101806.o5AI64H7028000@scc.eng.rpath.com>
changeset: 4c1a44be04bf
user: Elliot Peele
date: Thu, 10 Jun 2010 14:04:21 -0400
more user interface instance passing
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -51,7 +51,7 @@
def __init__(self, cfg, errataSource):
BotSuperClass.__init__(self, cfg)
- self._errata = errata.ErrataFilter(self._cfg, self._pkgSource,
+ self._errata = errata.ErrataFilter(self._cfg, self._ui, self._pkgSource,
errataSource)
self._groupmgr = groupmgr.GroupManager(self._cfg, self._ui,
useMap=self._pkgSource.useMap)
From elliot at rpath.com Thu Jun 10 14:06:04 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 10 Jun 2010 18:06:04 +0000
Subject: mirrorball: look at the first item in the list to avoid index
errors for packages that are only built for one arch
Message-ID: <201006101806.o5AI64NU028027@scc.eng.rpath.com>
changeset: 472048f65edf
user: Elliot Peele
date: Thu, 10 Jun 2010 14:05:06 -0400
look at the first item in the list to avoid index errors for packages that are only built for one arch
diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py
--- a/updatebot/groupmgr/sanity.py
+++ b/updatebot/groupmgr/sanity.py
@@ -113,7 +113,7 @@
seen = {}
for (n, v, f), pkgSet in sources.iteritems():
- binVer = list(pkgSet)[1][1]
+ binVer = list(pkgSet)[0][1]
seen.setdefault(n, set()).add(binVer)
binPkgs = {}
From elliot at rpath.com Thu Jun 10 14:06:05 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 10 Jun 2010 18:06:05 +0000
Subject: mirrorball: add use flags to addPackage and removePackage, these
are now required wnen specifying flavors
Message-ID: <201006101806.o5AI65T4028056@scc.eng.rpath.com>
changeset: 22dd17fd3369
user: Elliot Peele
date: Thu, 10 Jun 2010 14:05:35 -0400
add use flags to addPackage and removePackage, these are now required wnen specifying flavors
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -47,7 +47,7 @@
class CfgStringFlavor(CfgFlavor):
"""
- Class for representing a two tuple of a string and an optional flavor.
+ Class for representing a three tuple of a string and an optional flavor.
"""
def parseString(self, val):
@@ -69,6 +69,27 @@
raise ParseError, e
+class CfgStringFlavorUse(CfgStringFlavor):
+ """
+ Class for represnting a three tuple of string and an optional flavor and
+ use flag.
+ """
+
+ def parseString(self, val):
+ """
+ Parse config string.
+ """
+
+ use = None
+ splt = val.split()
+ if len(splt) > 1:
+ use = splt[-1]
+ val = ' '.join(splt[:-1])
+
+ context, flavor = CfgStringFlavor.parseString(self, val)
+ return context, flavor, use
+
+
class CfgStringFilter(CfgRegExp):
"""
Class for parsing (string, regex) tuples.
@@ -439,10 +460,10 @@
useOldVersion = (CfgIntDict(CfgList(CfgTroveSpec)), {})
# Add a package to a specific group
- addPackage = (CfgIntDict(CfgDict(CfgList(CfgStringFlavor))), {})
+ addPackage = (CfgIntDict(CfgDict(CfgList(CfgStringFlavorUse))), {})
# Remove a package from a specific group
- removePackage = (CfgIntDict(CfgDict(CfgList(CfgStringFlavor))), {})
+ removePackage = (CfgIntDict(CfgDict(CfgList(CfgStringFlavorUse))), {})
# Group name for group that contains all packages in a platform.
packageGroupName = (CfgString, 'group-packages')
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -363,9 +363,9 @@
Modify the contents of the group model by adding and/or removing
packages.
@param additions: dictionary of group names to add packages to.
- @type additions: dict(groupName=[(pkgName, frzPkgFlavor), ...])
+ @type additions: dict(groupName=[(pkgName, PkgFlavor, use), ...])
@param removals: dictionary of group names to remove packages from.
- @type additions: dict(groupName=[(pkgName, frzPkgFlavor), ...])
+ @type additions: dict(groupName=[(pkgName, PkgFlavor, use), ...])
"""
if additions is None:
@@ -380,7 +380,7 @@
# Remove requested packages.
for groupName, pkgs in removals.iteritems():
group = self._groups[groupName]
- for pkgName, pkgFlv in pkgs:
+ for pkgName, pkgFlv, use in pkgs:
if pkgFlv:
group.removePackageFlavor(pkgName, pkgFlv.freeze())
else:
@@ -394,10 +394,10 @@
continue
flavoredPackages = {}
- for pkgName, pkgFlv in pkgs:
+ for pkgName, pkgFlv, use in pkgs:
# deffer packages with specifc flavors for later.
if pkgFlv is not None:
- flavoredPackages.setdefault(pkgName, set()).add(pkgFlv)
+ flavoredPackages.setdefault(pkgName, set()).add((pkgFlv, use))
# handle packages where flavor is not specified
else:
@@ -409,9 +409,9 @@
groupName=groupName)
# Add all specifically flavored packages.
- for pkgName, flavors in flavoredPackages.iteritems():
- for flv in flavors:
- self._add(pkgName, version=None, flavor=flv, use=None,
+ for pkgName, flags in flavoredPackages.iteritems():
+ for flv, use in flags:
+ self._add(pkgName, version=None, flavor=flv, use=use,
groupName=groupName)
@require_write
From juphoff at rpath.com Tue Jun 15 16:07:24 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 15 Jun 2010 20:07:24 +0000
Subject: mirrorball: correct set() to add()
Message-ID: <201006152007.o5FK7OOf021005@scc.eng.rpath.com>
changeset: de431e4364c0
user: Jeff Uphoff
date: Tue, 15 Jun 2010 16:05:52 -0400
correct set() to add()
diff --git a/errata/centos.py b/errata/centos.py
--- a/errata/centos.py
+++ b/errata/centos.py
@@ -114,7 +114,7 @@
slices = {}
for srcPkg in self._pkgSource.srcPkgMap:
updateId = slice(int(srcPkg.buildTimestamp))
- slices.setdefault(updateId, set()).set(srcPkg)
+ slices.setdefault(updateId, set()).add(srcPkg)
# find labels
for label in self._pkgSource._clients:
From elliot at rpath.com Wed Jun 16 10:38:37 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 14:38:37 +0000
Subject: mirrorball: add a breakpoint
Message-ID: <201006161438.o5GEcbdJ016701@scc.eng.rpath.com>
changeset: fbf04ebf87ea
user: Elliot Peele
date: Wed, 16 Jun 2010 10:02:09 -0400
add a breakpoint
diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py
--- a/scripts/grpchecker.py
+++ b/scripts/grpchecker.py
@@ -195,6 +195,8 @@
grp._sanityCheck()
mgr._persistGroup(grp)
+ import epdb; epdb.st()
+
newGroup = grp.commit(copyToLatest=True)
jobId = mgr._builder.start(((mgr._sourceName, newGroup.conaryVersion, None), ))
jobIds.append(jobId)
From elliot at rpath.com Wed Jun 16 10:38:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 14:38:38 +0000
Subject: mirrorball: cleanup script
Message-ID: <201006161438.o5GEccQJ016732@scc.eng.rpath.com>
changeset: 863f20912e29
user: Elliot Peele
date: Wed, 16 Jun 2010 10:18:45 -0400
cleanup script
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -16,28 +16,32 @@
import os
import sys
-mirrorballDir = os.path.abspath('../')
-sys.path.insert(0, mirrorballDir)
+if __name__ == '__main__':
+ mirrorballDir = os.path.abspath('../')
+ sys.path.insert(0, mirrorballDir)
-if 'CONARY_PATH' in os.environ:
- sys.path.insert(0, os.environ['CONARY_PATH'])
+ if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
-import rmake
-import conary
-import updatebot
+ import rmake
+ import conary
+ import updatebot
-print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
-print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
-print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
+ print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
+ print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
+ print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
-from conary.lib import util
-sys.excepthook = util.genExcepthook()
+ from conary.lib import util
+ sys.excepthook = util.genExcepthook()
+
+ from updatebot import log as logSetup
+ logSetup.addRootLogger()
import logging
from updatebot import OrderedBot
-log = logging.getLogger('tmplogger')
+log = logging.getLogger('create group')
class Bot(OrderedBot):
def generateInitialGroup(self):
@@ -60,28 +64,41 @@
for flv in flavors:
trvs.setdefault(name, dict()).setdefault(version, set()).add(flv)
+ # Filter out sources, groups, and components; gathering all of the
+ # package versions.
pkgs = set()
for name, vMap in trvs.iteritems():
if name.endswith(':source'):
continue
+ if name.startswith('group-'):
+ continue
name = name.split(':')[0]
for version, flavors in vMap.iteritems():
data = (name, version, tuple(flavors))
pkgs.add(data)
+ # Get the latest group model.
group = self._groupmgr.getGroup()
+ # Remove the existing packages group.
+ group._groups.pop('group-packages', None)
+
+ # Add content to the packages group, which will cause a new model to
+ # be created.
for name, version, flavors in pkgs:
log.info('adding %s=%s' % (name, version))
for flv in flavors:
log.info('\t%s' % flv)
group.addPackage(name, version, flavors)
+ # Set the errata state and version to some defaults.
group.errataState = 0
- group.version = 'sp3'
+ group.version = '0'
+ # Remove the existing standard group if there is one.
group._groups.pop('group-standard', None)
+ # Run through all of the adds and removes for the standard group.
removals = set()
nevras = dict([ (x.getNevra(), y)
for x, y in self._pkgSource.srcPkgMap.iteritems() ])
@@ -94,16 +111,26 @@
removals |= set(self._cfg.updateRemovesPackages.get(updateId, ()))
+ # Remove any packages that would have normally been removed at this
+ # errataState.
for name in removals:
group.removePackage(name, missingOk=True)
+ # Sanity check the group model and write out the current state so that
+ # you can do a local test cook.
group._copyVersions()
group._sanityCheck()
group._mgr._persistGroup(group)
+ # You probably want to do a test cook if your groups here. It would be
+ # nice if mirrorball could just do this for you, but it can't right now.
+ # To run a test cook take a look at group._mgr._helper._checkoutCache to
+ # find the directory where the checkout is and then run cvc cook from
+ # that directory.
import epdb; epdb.st()
- group.commit()
+ # Commit and build the group.
+ group = group.commit()
built = group.build()
import epdb; epdb.st()
@@ -112,16 +139,11 @@
if __name__ == '__main__':
from updatebot import config
- from updatebot import log as logSetup
-
- logSetup.addRootLogger()
-
- log = logging.getLogger('create group')
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
bot = Bot(cfg, None)
- changes = bot.generateInitialGroup()
+ trvMap = bot.generateInitialGroup()
import epdb; epdb.st()
From elliot at rpath.com Wed Jun 16 10:38:38 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 14:38:38 +0000
Subject: mirrorball: initial support for rebuilding packages
Message-ID: <201006161438.o5GEccjh016759@scc.eng.rpath.com>
changeset: 0109835815a9
user: Elliot Peele
date: Wed, 16 Jun 2010 10:38:20 -0400
initial support for rebuilding packages
diff --git a/scripts/rebuildgroups b/scripts/rebuildpackage
copy from scripts/rebuildgroups
copy to scripts/rebuildpackage
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildpackage
@@ -13,45 +13,35 @@
# full details.
#
+import os
+import sys
import logging
-from updatebot import groupmgr
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
from updatebot import OrderedBot
log = logging.getLogger('tmplogger')
class Bot(OrderedBot):
- def rebuildgroups(self):
+ def rebuildpackage(self, name):
"""
- Rebuild all groups on the devel label. This requires rewriting the group
- model to point at the target label and readding content.
"""
- # load package source
- self._pkgSource.load()
+ # Find all versions of name.
+ nvfs = self._updater._conaryhelper.findTrove(
+ ('%s:source' % name, None, None), getLeaves=False)
+ trvSpecs = [ (x[0].split(':')[0], x[1], None) for x in nvfs ]
- for updateId, updates in self._errata.iterByIssueDate(current=-1):
- mgr = groupmgr.GroupManager(self._cfg,
- useMap=self._pkgSource.useMap)
+ trvMap = self._builder.rebuild(trvSpecs)
-
-
- # Find any version exceptions for this update.
- multiVersionExceptions = dict([
- (x[0], x[1]) for x in itertools.chain(
- self._updater.getTargetVersions(itertools.chain(
- *self._getOldVersionExceptions(updateId).itervalues()
- ))[0]
- )
- ])
+ return trvMap
if __name__ == '__main__':
import os
import sys
- mirrorballDir = os.path.abspath('../')
- sys.path.insert(0, mirrorballDir)
-
if 'CONARY_PATH' in os.environ:
sys.path.insert(0, os.environ['CONARY_PATH'])
@@ -73,10 +63,12 @@
logSetup.addRootLogger()
- log = logging.getLogger('grouprebuild')
+ log = logging.getLogger('packagerebuild')
+
+ confDir = mirrorballDir + '/config/%s' % sys.argv[1]
cfg = config.UpdateBotConfig()
- cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+ cfg.read(confDir + '/updatebotrc')
mcfg = rhnmirror.MirrorConfig()
mcfg.read(confDir + '/erratarc')
@@ -85,6 +77,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups()
+ bot.rebuildpackage(sys.argv[2])
import epdb; epdb.st()
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -43,6 +43,7 @@
from updatebot.errors import JobFailedError
from updatebot.errors import CommitFailedError
from updatebot.errors import UnhandledKernelModule
+from updatebot.errors import GroupBuildNotSupportedError
from updatebot.errors import InvalidBuildTroveInputError
from updatebot.errors import FailedToRetrieveChangesetError
from updatebot.errors import ChangesetValidationFailedError
@@ -233,6 +234,105 @@
ret = self._formatOutput(trvMap)
return ret
+ def rebuild(self, troveSpecs):
+ """
+ Rebuild a set of troves in the same environment that they were
+ orignally built in.
+ @param troveSpecs: set of name, version, flavor tuples
+ @type troveSpecs: set([(name, version, flavor), ..])
+ @return troveMap: dictionary of troveSpecs to built troves
+ """
+
+ def grpByNameVersion(jobLst):
+ lst = {}
+ for job in jobLst:
+ lst.setdefault(tuple(job[:2]), set()).add(job)
+ return sorted(lst.values())
+
+ def startOne(job):
+ # Get a new builder so that we don't change the configuration of the
+ # existing builder.
+ cls = self.__class__
+ builder = cls(self._cfg, self._ui)
+
+ # Find the troves that were originally used to build the requested
+ # trove.
+ n, v = list(job)[0][:2]
+ # So that we can find the latest binary built from the closest
+ # versioned source we need to lookup binary versions and then use
+ # that source to get the rest of the binaries. We need to do this
+ # in the case that we are building a source that has been modified
+ # to remove a recipe or something like that.
+ upVer = '/'.join([v.branch().label().asString(),
+ v.trailingRevision().version])
+ binSpecs = self._client.repos.findTrove(v.branch().label(),
+ (n, upVer, None))
+ binTrv = self._client.repos.getTrove(*binSpecs[0], withFiles=False)
+ srcName = binTrv.troveInfo.sourceName()
+ srcVersion = binTrv.getVersion().getSourceVersion()
+
+ specs = self._client.repos.getTrovesBySource(srcName, srcVersion)
+
+ # Find the latest version of each package
+ vMap = {}
+ for n, v, f in specs:
+ n = n.split(':')[0]
+ vMap.setdefault(v, dict()).setdefault(n, set()).add(f)
+
+ latest = sorted(vMap)[-1]
+ trvSpecs = []
+ for n, flvs in vMap[latest].iteritems():
+ for f in flvs:
+ trvSpecs.append((n, latest, f))
+
+ # Get the troves for all binaries built from the given source.
+ troves = self._client.repos.getTroves(trvSpecs, withFiles=False)
+
+ # Take the union of all buildreqs for all flavors of the package.
+ reqs = set()
+ for trv in troves:
+ reqs |= set([ (x.name(), x.version())
+ for x in trv.troveInfo.buildReqs.iter() ])
+
+ # Reconfigure builder to use previous buildrequires as
+ # resolveTroves.
+ resolveTroves = ' '.join([ '%s=%s' % x for x in reqs ])
+ builder._rmakeCfg.resolveTroves = []
+ builder._rmakeCfg.configLine('resolveTroves %s' % resolveTroves)
+
+ # Start the job.
+ jobId = builder._startJob(job)
+
+ return jobId
+
+
+ # Handle empty set.
+ if not troveSpecs:
+ return {}
+
+ jobs = self._formatInput(troveSpecs)
+
+ # Sanity check input to make sure there are no groups.
+ if [ x for x in jobs if x[0].startswith('group-') ]:
+ raise GroupBuildNotSupportedError
+
+ # Start all of the jobs.
+ jobIds = []
+ for job in grpByNameVersion(jobs):
+ jobIds.append(startOne(job))
+
+ # Wait for jobs to complete
+ dispatcher = NonCommittalDispatcher(self, 0)
+ dispatcher.watchmany(jobIds)
+
+ # Commit jobs in order, conary does not do this for you.
+ trvMap = {}
+ for jobId in jobIds:
+ trvMap.update(self._commitJob(jobId))
+ ret = self._formatOutput(trvMap)
+
+ return ret
+
def start(self, troveSpecs):
"""
Public version of start job that starts a job without monitoring.
@@ -411,7 +511,7 @@
# Create rMake job
log.info('Creating build job: %s' % (troveSpecs, ))
- job = self._helper.createBuildJob(troveSpecs)
+ job = self._helper.createBuildJob(list(troveSpecs))
jobId = self._helper.buildJob(job)
log.info('Started jobId: %s' % jobId)
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -260,3 +260,26 @@
res = self._builder.commit(jobIds)
return res, self._failures
+
+ def watchmany(self, jobIds):
+ """
+ Watch a list of jobIds. Blocks until jobs are complete.
+ @param jobIds: list of jobIds
+ @type jobIds: list(int, ...)
+ """
+
+ # Start monitoring each job.
+ for jobId in jobIds:
+ self._jobs[jobId] = [None, JobStatus.JOB_NOT_STARTED, None]
+ self._monitor.monitorJob(jobId)
+
+ # Wait for jobs to complete.
+ while not self._jobDone():
+ # Update job status changes.
+ for jobId, status in self._monitor.getStatus():
+ self._jobs[jobId][1] = status
+
+ # Make sure all jobs are built.
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ if status != buildjob.JOB_STATE_BUILT:
+ raise JobNotCompleteError(jobId=jobId)
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -120,6 +120,16 @@
'define the set of flavors this kernel module should be built in.')
+class GroupBuildNotSupportedError(BuildError):
+ """
+ GroupBuildNotSupportedError, raised when a group is submitted for building
+ through a method that does not support building groups.
+ """
+
+ _param = []
+ _template = 'This build method does not support building groups.'
+
+
class UnhandledUpdateError(UpdateBotError):
"""
UnhandledUpdateError, raised when the bot finds a state that it does not
From elliot at rpath.com Wed Jun 16 10:38:39 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 14:38:39 +0000
Subject: mirrorball: branch merge
Message-ID: <201006161438.o5GEcdM9016786@scc.eng.rpath.com>
changeset: 1de8feb19ac2
user: Elliot Peele
date: Wed, 16 Jun 2010 10:38:36 -0400
branch merge
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -16,28 +16,32 @@
import os
import sys
-mirrorballDir = os.path.abspath('../')
-sys.path.insert(0, mirrorballDir)
+if __name__ == '__main__':
+ mirrorballDir = os.path.abspath('../')
+ sys.path.insert(0, mirrorballDir)
-if 'CONARY_PATH' in os.environ:
- sys.path.insert(0, os.environ['CONARY_PATH'])
+ if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
-import rmake
-import conary
-import updatebot
+ import rmake
+ import conary
+ import updatebot
-print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
-print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
-print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
+ print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
+ print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
+ print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
-from conary.lib import util
-sys.excepthook = util.genExcepthook()
+ from conary.lib import util
+ sys.excepthook = util.genExcepthook()
+
+ from updatebot import log as logSetup
+ logSetup.addRootLogger()
import logging
from updatebot import OrderedBot
-log = logging.getLogger('tmplogger')
+log = logging.getLogger('create group')
class Bot(OrderedBot):
def generateInitialGroup(self):
@@ -60,28 +64,41 @@
for flv in flavors:
trvs.setdefault(name, dict()).setdefault(version, set()).add(flv)
+ # Filter out sources, groups, and components; gathering all of the
+ # package versions.
pkgs = set()
for name, vMap in trvs.iteritems():
if name.endswith(':source'):
continue
+ if name.startswith('group-'):
+ continue
name = name.split(':')[0]
for version, flavors in vMap.iteritems():
data = (name, version, tuple(flavors))
pkgs.add(data)
+ # Get the latest group model.
group = self._groupmgr.getGroup()
+ # Remove the existing packages group.
+ group._groups.pop('group-packages', None)
+
+ # Add content to the packages group, which will cause a new model to
+ # be created.
for name, version, flavors in pkgs:
log.info('adding %s=%s' % (name, version))
for flv in flavors:
log.info('\t%s' % flv)
group.addPackage(name, version, flavors)
+ # Set the errata state and version to some defaults.
group.errataState = 0
- group.version = 'sp3'
+ group.version = '0'
+ # Remove the existing standard group if there is one.
group._groups.pop('group-standard', None)
+ # Run through all of the adds and removes for the standard group.
removals = set()
nevras = dict([ (x.getNevra(), y)
for x, y in self._pkgSource.srcPkgMap.iteritems() ])
@@ -94,16 +111,26 @@
removals |= set(self._cfg.updateRemovesPackages.get(updateId, ()))
+ # Remove any packages that would have normally been removed at this
+ # errataState.
for name in removals:
group.removePackage(name, missingOk=True)
+ # Sanity check the group model and write out the current state so that
+ # you can do a local test cook.
group._copyVersions()
group._sanityCheck()
group._mgr._persistGroup(group)
+ # You probably want to do a test cook if your groups here. It would be
+ # nice if mirrorball could just do this for you, but it can't right now.
+ # To run a test cook take a look at group._mgr._helper._checkoutCache to
+ # find the directory where the checkout is and then run cvc cook from
+ # that directory.
import epdb; epdb.st()
- group.commit()
+ # Commit and build the group.
+ group = group.commit()
built = group.build()
import epdb; epdb.st()
@@ -112,16 +139,11 @@
if __name__ == '__main__':
from updatebot import config
- from updatebot import log as logSetup
-
- logSetup.addRootLogger()
-
- log = logging.getLogger('create group')
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
bot = Bot(cfg, None)
- changes = bot.generateInitialGroup()
+ trvMap = bot.generateInitialGroup()
import epdb; epdb.st()
diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py
--- a/scripts/grpchecker.py
+++ b/scripts/grpchecker.py
@@ -195,6 +195,8 @@
grp._sanityCheck()
mgr._persistGroup(grp)
+ import epdb; epdb.st()
+
newGroup = grp.commit(copyToLatest=True)
jobId = mgr._builder.start(((mgr._sourceName, newGroup.conaryVersion, None), ))
jobIds.append(jobId)
diff --git a/scripts/rebuildgroups b/scripts/rebuildpackage
copy from scripts/rebuildgroups
copy to scripts/rebuildpackage
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildpackage
@@ -13,45 +13,35 @@
# full details.
#
+import os
+import sys
import logging
-from updatebot import groupmgr
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
from updatebot import OrderedBot
log = logging.getLogger('tmplogger')
class Bot(OrderedBot):
- def rebuildgroups(self):
+ def rebuildpackage(self, name):
"""
- Rebuild all groups on the devel label. This requires rewriting the group
- model to point at the target label and readding content.
"""
- # load package source
- self._pkgSource.load()
+ # Find all versions of name.
+ nvfs = self._updater._conaryhelper.findTrove(
+ ('%s:source' % name, None, None), getLeaves=False)
+ trvSpecs = [ (x[0].split(':')[0], x[1], None) for x in nvfs ]
- for updateId, updates in self._errata.iterByIssueDate(current=-1):
- mgr = groupmgr.GroupManager(self._cfg,
- useMap=self._pkgSource.useMap)
+ trvMap = self._builder.rebuild(trvSpecs)
-
-
- # Find any version exceptions for this update.
- multiVersionExceptions = dict([
- (x[0], x[1]) for x in itertools.chain(
- self._updater.getTargetVersions(itertools.chain(
- *self._getOldVersionExceptions(updateId).itervalues()
- ))[0]
- )
- ])
+ return trvMap
if __name__ == '__main__':
import os
import sys
- mirrorballDir = os.path.abspath('../')
- sys.path.insert(0, mirrorballDir)
-
if 'CONARY_PATH' in os.environ:
sys.path.insert(0, os.environ['CONARY_PATH'])
@@ -73,10 +63,12 @@
logSetup.addRootLogger()
- log = logging.getLogger('grouprebuild')
+ log = logging.getLogger('packagerebuild')
+
+ confDir = mirrorballDir + '/config/%s' % sys.argv[1]
cfg = config.UpdateBotConfig()
- cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+ cfg.read(confDir + '/updatebotrc')
mcfg = rhnmirror.MirrorConfig()
mcfg.read(confDir + '/erratarc')
@@ -85,6 +77,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups()
+ bot.rebuildpackage(sys.argv[2])
import epdb; epdb.st()
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -43,6 +43,7 @@
from updatebot.errors import JobFailedError
from updatebot.errors import CommitFailedError
from updatebot.errors import UnhandledKernelModule
+from updatebot.errors import GroupBuildNotSupportedError
from updatebot.errors import InvalidBuildTroveInputError
from updatebot.errors import FailedToRetrieveChangesetError
from updatebot.errors import ChangesetValidationFailedError
@@ -233,6 +234,105 @@
ret = self._formatOutput(trvMap)
return ret
+ def rebuild(self, troveSpecs):
+ """
+ Rebuild a set of troves in the same environment that they were
+ orignally built in.
+ @param troveSpecs: set of name, version, flavor tuples
+ @type troveSpecs: set([(name, version, flavor), ..])
+ @return troveMap: dictionary of troveSpecs to built troves
+ """
+
+ def grpByNameVersion(jobLst):
+ lst = {}
+ for job in jobLst:
+ lst.setdefault(tuple(job[:2]), set()).add(job)
+ return sorted(lst.values())
+
+ def startOne(job):
+ # Get a new builder so that we don't change the configuration of the
+ # existing builder.
+ cls = self.__class__
+ builder = cls(self._cfg, self._ui)
+
+ # Find the troves that were originally used to build the requested
+ # trove.
+ n, v = list(job)[0][:2]
+ # So that we can find the latest binary built from the closest
+ # versioned source we need to lookup binary versions and then use
+ # that source to get the rest of the binaries. We need to do this
+ # in the case that we are building a source that has been modified
+ # to remove a recipe or something like that.
+ upVer = '/'.join([v.branch().label().asString(),
+ v.trailingRevision().version])
+ binSpecs = self._client.repos.findTrove(v.branch().label(),
+ (n, upVer, None))
+ binTrv = self._client.repos.getTrove(*binSpecs[0], withFiles=False)
+ srcName = binTrv.troveInfo.sourceName()
+ srcVersion = binTrv.getVersion().getSourceVersion()
+
+ specs = self._client.repos.getTrovesBySource(srcName, srcVersion)
+
+ # Find the latest version of each package
+ vMap = {}
+ for n, v, f in specs:
+ n = n.split(':')[0]
+ vMap.setdefault(v, dict()).setdefault(n, set()).add(f)
+
+ latest = sorted(vMap)[-1]
+ trvSpecs = []
+ for n, flvs in vMap[latest].iteritems():
+ for f in flvs:
+ trvSpecs.append((n, latest, f))
+
+ # Get the troves for all binaries built from the given source.
+ troves = self._client.repos.getTroves(trvSpecs, withFiles=False)
+
+ # Take the union of all buildreqs for all flavors of the package.
+ reqs = set()
+ for trv in troves:
+ reqs |= set([ (x.name(), x.version())
+ for x in trv.troveInfo.buildReqs.iter() ])
+
+ # Reconfigure builder to use previous buildrequires as
+ # resolveTroves.
+ resolveTroves = ' '.join([ '%s=%s' % x for x in reqs ])
+ builder._rmakeCfg.resolveTroves = []
+ builder._rmakeCfg.configLine('resolveTroves %s' % resolveTroves)
+
+ # Start the job.
+ jobId = builder._startJob(job)
+
+ return jobId
+
+
+ # Handle empty set.
+ if not troveSpecs:
+ return {}
+
+ jobs = self._formatInput(troveSpecs)
+
+ # Sanity check input to make sure there are no groups.
+ if [ x for x in jobs if x[0].startswith('group-') ]:
+ raise GroupBuildNotSupportedError
+
+ # Start all of the jobs.
+ jobIds = []
+ for job in grpByNameVersion(jobs):
+ jobIds.append(startOne(job))
+
+ # Wait for jobs to complete
+ dispatcher = NonCommittalDispatcher(self, 0)
+ dispatcher.watchmany(jobIds)
+
+ # Commit jobs in order, conary does not do this for you.
+ trvMap = {}
+ for jobId in jobIds:
+ trvMap.update(self._commitJob(jobId))
+ ret = self._formatOutput(trvMap)
+
+ return ret
+
def start(self, troveSpecs):
"""
Public version of start job that starts a job without monitoring.
@@ -411,7 +511,7 @@
# Create rMake job
log.info('Creating build job: %s' % (troveSpecs, ))
- job = self._helper.createBuildJob(troveSpecs)
+ job = self._helper.createBuildJob(list(troveSpecs))
jobId = self._helper.buildJob(job)
log.info('Started jobId: %s' % jobId)
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -260,3 +260,26 @@
res = self._builder.commit(jobIds)
return res, self._failures
+
+ def watchmany(self, jobIds):
+ """
+ Watch a list of jobIds. Blocks until jobs are complete.
+ @param jobIds: list of jobIds
+ @type jobIds: list(int, ...)
+ """
+
+ # Start monitoring each job.
+ for jobId in jobIds:
+ self._jobs[jobId] = [None, JobStatus.JOB_NOT_STARTED, None]
+ self._monitor.monitorJob(jobId)
+
+ # Wait for jobs to complete.
+ while not self._jobDone():
+ # Update job status changes.
+ for jobId, status in self._monitor.getStatus():
+ self._jobs[jobId][1] = status
+
+ # Make sure all jobs are built.
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ if status != buildjob.JOB_STATE_BUILT:
+ raise JobNotCompleteError(jobId=jobId)
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -120,6 +120,16 @@
'define the set of flavors this kernel module should be built in.')
+class GroupBuildNotSupportedError(BuildError):
+ """
+ GroupBuildNotSupportedError, raised when a group is submitted for building
+ through a method that does not support building groups.
+ """
+
+ _param = []
+ _template = 'This build method does not support building groups.'
+
+
class UnhandledUpdateError(UpdateBotError):
"""
UnhandledUpdateError, raised when the bot finds a state that it does not
From elliot at rpath.com Wed Jun 16 11:09:50 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 15:09:50 +0000
Subject: mirrorball: add ability to use the latest versions of some troves
Message-ID: <201006161509.o5GF9oOm017539@scc.eng.rpath.com>
changeset: 21f1f7660b75
user: Elliot Peele
date: Wed, 16 Jun 2010 11:04:15 -0400
add ability to use the latest versions of some troves
diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage
--- a/scripts/rebuildpackage
+++ b/scripts/rebuildpackage
@@ -25,8 +25,13 @@
log = logging.getLogger('tmplogger')
class Bot(OrderedBot):
- def rebuildpackage(self, name):
+ def rebuildpackage(self, name, useLatest=None):
"""
+ Rebuild all versions of a given package in order.
+ @param useLatest: A list of package names to use the latest versions of.
+ For instance, you may want to use the latest version
+ of conary to get fixed dependencies.
+ @type useLatest: list(str, ...)
"""
# Find all versions of name.
@@ -34,7 +39,7 @@
('%s:source' % name, None, None), getLeaves=False)
trvSpecs = [ (x[0].split(':')[0], x[1], None) for x in nvfs ]
- trvMap = self._builder.rebuild(trvSpecs)
+ trvMap = self._builder.rebuild(trvSpecs, useLatest=useLatest)
return trvMap
@@ -77,6 +82,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildpackage(sys.argv[2])
+ bot.rebuildpackage(sys.argv[2], useLatest=['conary', 'conary-build', 'conary-policy'])
import epdb; epdb.st()
From elliot at rpath.com Wed Jun 16 11:09:51 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 15:09:51 +0000
Subject: mirrorball: add ability to use the latest versions of some troves
Message-ID: <201006161509.o5GF9pPh017570@scc.eng.rpath.com>
changeset: f029d6daefc5
user: Elliot Peele
date: Wed, 16 Jun 2010 11:04:30 -0400
add ability to use the latest versions of some troves
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -234,12 +234,16 @@
ret = self._formatOutput(trvMap)
return ret
- def rebuild(self, troveSpecs):
+ def rebuild(self, troveSpecs, useLatest=None):
"""
Rebuild a set of troves in the same environment that they were
orignally built in.
@param troveSpecs: set of name, version, flavor tuples
@type troveSpecs: set([(name, version, flavor), ..])
+ @param useLatest: A list of package names to use the latest versions of.
+ For instance, you may want to use the latest version
+ of conary to get fixed dependencies.
+ @type useLatest: list(str, ...)
@return troveMap: dictionary of troveSpecs to built troves
"""
@@ -291,8 +295,12 @@
# Take the union of all buildreqs for all flavors of the package.
reqs = set()
for trv in troves:
- reqs |= set([ (x.name(), x.version())
- for x in trv.troveInfo.buildReqs.iter() ])
+ for req in trv.troveInfo.buildReqs.iter():
+ name = req.name()
+ version = req.version()
+ if useLatest and name.split(':')[0] in useLatest:
+ version = version.branch()
+ reqs.add((name, version))
# Reconfigure builder to use previous buildrequires as
# resolveTroves.
From elliot at rpath.com Wed Jun 16 14:05:49 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 18:05:49 +0000
Subject: mirrorball: add simple interface to clone
Message-ID: <201006161805.o5GI5n3E022521@scc.eng.rpath.com>
changeset: a07bd076cc0a
user: Elliot Peele
date: Wed, 16 Jun 2010 14:05:19 -0400
add simple interface to clone
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1398,3 +1398,49 @@
cs.newTrove(trvCs)
return trvSet, cs
+
+ def clone(self, targetBranch, troveList, writeClonedFromInfo=True,
+ commit=True):
+ """
+ Clone a list of troves to the target branch.
+ @param targetBranch: branch to clone to.
+ @type targetBranch: conary.versions.Branch
+ @param troveList: list of troves to promote.
+ @type troveList: list((str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ..)
+ @param withClonedFromInfo: Optional flag to control the writing of
+ cloned from information in the troveinfo.
+ Defaults to True.
+ @type withClonedFromInfo: boolean
+ @param commit: commit the promote changeset or just return it.
+ @type commit: boolean
+ """
+
+ log.info('creating clone changeset')
+
+ callback = UpdateBotCloneCallback(self._ccfg, 'automated clone',
+ log=log)
+
+ success, cs = self._client.createCloneChangeSet(
+ targetBranch, troveList,
+ updateBuildInfo=writeClonedFromInfo,
+ callback=callback,
+ cloneSources=True)
+
+ log.info('changeset created')
+
+ if not success:
+ raise PromoteFailedError(what=troveList)
+
+ packageList = [ x.getNewNameVersionFlavor()
+ for x in cs.iterNewTroveList() ]
+
+ if not commit:
+ return cs, packageList
+
+ import epdb; epdb.st()
+ log.info('committing changeset')
+ self._repos.commitChangeSet(cs, callback=callback)
+ log.info('changeset committed')
+
+ return packageList
From elliot at rpath.com Wed Jun 16 14:05:49 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 16 Jun 2010 18:05:49 +0000
Subject: mirrorball: add method for removing files
Message-ID: <201006161805.o5GI5ncL022552@scc.eng.rpath.com>
changeset: bbc04c1151f6
user: Elliot Peele
date: Wed, 16 Jun 2010 14:05:47 -0400
add method for removing files
diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage
--- a/scripts/rebuildpackage
+++ b/scripts/rebuildpackage
@@ -25,24 +25,79 @@
log = logging.getLogger('tmplogger')
class Bot(OrderedBot):
+ def _getAllVersions(self, name):
+ """
+ Get all of the versions of a given source.
+ """
+
+ # Find all versions of name.
+ nvfs = self._updater._conaryhelper.findTrove(
+ ('%s:source' % name, None, None), getLeaves=False)
+
+ return sorted(nvfs)
+
def rebuildpackage(self, name, useLatest=None):
"""
Rebuild all versions of a given package in order.
+ @param name: name of the package to rebuild.
+ @type name: str
@param useLatest: A list of package names to use the latest versions of.
For instance, you may want to use the latest version
of conary to get fixed dependencies.
@type useLatest: list(str, ...)
"""
- # Find all versions of name.
- nvfs = self._updater._conaryhelper.findTrove(
- ('%s:source' % name, None, None), getLeaves=False)
- trvSpecs = [ (x[0].split(':')[0], x[1], None) for x in nvfs ]
-
+ trvSpecs = [ (x[0].split(':')[0], x[1], None)
+ for x in self._getAllVersions(name) ]
trvMap = self._builder.rebuild(trvSpecs, useLatest=useLatest)
return trvMap
+ def removeSourceFiles(self, name, keepFiles=None):
+ """
+ Remove all of the files from all source versions of a package that are
+ not listed in keepFiles.
+ @param name: name of the package to rebuild.
+ @type name: str
+ @param keepFiles: Set of files to keep in the source component. By
+ default manifest and CONARY will always been kept.
+ @type keepFiles: set(str, ...)
+ """
+
+ if not keepFiles:
+ keepFiles = set()
+
+ # Always avoid removing the CONARY file and the manifest.
+ keepFiles.add('CONARY')
+ keepFiles.add('manifest')
+
+ helper = self._updater._conaryhelper
+
+ # Get a list of source verisons.
+ trvSpecs = self._getAllVersions(name)
+
+ # Rewrite source versions by cloning old versions forward and then
+ # modifying them.
+ for spec in trvSpecs:
+ # Clone version to tip
+ targetBranch = spec[1].branch()
+ newPkgs = helper.clone(targetBranch, [spec, ], commit=False)
+
+ assert len(newPkgs) == 1
+ n, v, f = newPkgs[0]
+
+ # Edit the source to remove files.
+ checkoutDir = helper._edit(n, version=v)
+ files = set(os.listdir(checkoutDir))
+ removeFiles = files - keepFiles
+ for fn in removeFiles:
+ helper._removeFile(checkoutDir, fn)
+
+ # Commit changes if anything changed.
+ if removeFiles:
+ helper._commit(checkoutDir, 'automated file removal')
+
+
if __name__ == '__main__':
import os
import sys
@@ -81,7 +136,13 @@
errata = rhnmirror.Errata(mcfg)
errata.fetch()
+ pkgNames = sys.argv[2:]
+
bot = Bot(cfg, errata)
- bot.rebuildpackage(sys.argv[2], useLatest=['conary', 'conary-build', 'conary-policy'])
+
+ for pkgName in pkgNames:
+ bot.removeSourceFiles(pkgName)
+ bot.rebuildpackage(pkgName,
+ useLatest=['conary', 'conary-build', 'conary-policy'])
import epdb; epdb.st()
From juphoff at rpath.com Thu Jun 17 09:19:16 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Thu, 17 Jun 2010 13:19:16 +0000
Subject: mirrorball: fix version-detection bug in ordering
Message-ID: <201006171319.o5HDJGo9018181@scc.eng.rpath.com>
changeset: 91c2d6652392
user: Jeff Uphoff
date: Thu, 17 Jun 2010 09:19:13 -0400
fix version-detection bug in ordering
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -419,7 +419,8 @@
# get the correct arch
pkg = [ x for x in self._getLatestOfAvailableArches(pkgs)
- if x.arch == binPkg.arch ][0]
+ if x.arch == binPkg.arch and
+ x.version == binPkg.version ][0]
# Raise an exception if the versions of the packages aren't
# equal or the discovered package comes from a different source.
From elliot at rpath.com Thu Jun 17 12:08:48 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 17 Jun 2010 16:08:48 +0000
Subject: mirrorball: add method for iterating over the packages in a group
Message-ID: <201006171608.o5HG8mI9000379@scc.eng.rpath.com>
changeset: b66f975c6fd0
user: Elliot Peele
date: Wed, 16 Jun 2010 17:14:49 -0400
add method for iterating over the packages in a group
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -143,6 +143,13 @@
return self._groups.iteritems()
+ def iterpackages(self):
+ """
+ Iterate over the set of packages in the package group.
+ """
+
+ return self._groups[self._pkgGroupName]
+
###
# Start of group manager interface
#
From elliot at rpath.com Thu Jun 17 12:08:48 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 17 Jun 2010 16:08:48 +0000
Subject: mirrorball: branch merge
Message-ID: <201006171608.o5HG8mJc000407@scc.eng.rpath.com>
changeset: 7e043d89007b
user: Elliot Peele
date: Thu, 17 Jun 2010 10:21:24 -0400
branch merge
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -143,6 +143,13 @@
return self._groups.iteritems()
+ def iterpackages(self):
+ """
+ Iterate over the set of packages in the package group.
+ """
+
+ return self._groups[self._pkgGroupName]
+
###
# Start of group manager interface
#
From elliot at rpath.com Thu Jun 17 12:08:49 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 17 Jun 2010 16:08:49 +0000
Subject: mirrorball: Raise an exception if a platform is running in the
wrong update mode. All
Message-ID: <201006171608.o5HG8nM7000437@scc.eng.rpath.com>
changeset: 31203d834bf8
user: Elliot Peele
date: Thu, 17 Jun 2010 12:08:43 -0400
Raise an exception if a platform is running in the wrong update mode. All
platforms must now specify updateMode in updatebotrc.
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -26,6 +26,8 @@
from updatebot import pkgsource
from updatebot import advisories
+from updatebot.errors import InvalidUpdateModeError
+
log = logging.getLogger('updatebot.bot')
class Bot(object):
@@ -33,7 +35,11 @@
Top level object for driving update process.
"""
+ _updateMode = 'latest'
+
def __init__(self, cfg):
+ self._validateMode(cfg)
+
self._cfg = cfg
self._clientcfg = cmdline.UpdateBotClientConfig()
@@ -47,6 +53,11 @@
self._advisor = advisories.Advisor(self._cfg, self._pkgSource,
self._cfg.platformName)
+ @classmethod
+ def _validateMode(cls, cfg):
+ if cfg.updateMode != cls._updateMode:
+ raise InvalidUpdateModeError(
+ mode=cfg.updateMode, expected=cls._updateMode)
@staticmethod
def _flattenSetDict(setDict):
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -203,6 +203,10 @@
# R0904 - to many public methods
# pylint: disable-msg=R0904
+ # Mode that updatebot is running in. (possible values ar
+ # 'ordered' and 'latest'.
+ updateMode = CfgString
+
# name of the product to use in advisories
productName = CfgString
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -41,6 +41,18 @@
return '%s(%s)' % (self.__class__, params)
+class InvalidUpdateModeError(UpdateBotError):
+ """
+ InvalidUpdateModeError, raised when trying to use an update mode that is
+ not supported.
+ """
+
+ _params = ['mode', 'expected']
+ _template = ('This platform is not configured with a valid updateMode for '
+ 'the selected workflow. found: "%(mode)s", expected: '
+ '"%(expected)s"')
+
+
class BuildError(UpdateBotError):
"""
BuildError, abstract error for all other build related errors.
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -46,6 +46,8 @@
Implement errata driven create/update interface.
"""
+ _updateMode = 'ordered'
+
_create = BotSuperClass.create
_update = BotSuperClass.update
From elliot at rpath.com Thu Jun 17 12:55:22 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 17 Jun 2010 16:55:22 +0000
Subject: mirrorball: change the way we compare package versions based on
the update method
Message-ID: <201006171655.o5HGtMq6002633@scc.eng.rpath.com>
changeset: 755bdd5eb755
user: Elliot Peele
date: Thu, 17 Jun 2010 12:55:20 -0400
change the way we compare package versions based on the update method
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -415,20 +415,38 @@
# Novell releases updates to only the binary rpms of a package
# that have chnaged. We have to use binaries from the old srpm.
# Get the last version of the pkg and add it to the srcPkgMap.
- pkgs = list(self._pkgSource.binNameMap[binPkg.name])
+ pkgs = sorted([
+ x for x in self._pkgSource.binNameMap[binPkg.name]
+ if x.arch == binPkg.arch ])
- # get the correct arch
- pkg = [ x for x in self._getLatestOfAvailableArches(pkgs)
- if x.arch == binPkg.arch and
- x.version == binPkg.version ][0]
+ # If running in latest mode we really want to compare to the
+ # latest version of this binary, but if we are running in
+ # ordered we really want the "next" version of this binary.
+ pkg = None
+ if self._cfg.updateMode == 'latest':
+ # get the correct arch
+ latestPkgs = self._getLatestOfAvailableArches(pkgs)
+ assert latestPkgs
+ pkg = latestPkgs[0]
+
+ elif self._cfg.updateMode == 'ordered':
+ idx = pkgs.index(binPkg)
+ if len(pkgs) - 1 > idx:
+ pkg = pkgs[idx+1]
+ else:
+ # This means that this package has no newer versions.
+ log.info('no newer version of %s found' % binPkg.name)
+
+ # Get the source that the package was built from for version
+ # comparison since the source and binary can have different
+ # versions.
+ if pkg:
+ src = self._pkgSource.binPkgMap[pkg]
+ else:
+ src = srcPkg
# Raise an exception if the versions of the packages aren't
# equal or the discovered package comes from a different source.
- #
- # Get the source that the package was built from for version
- # comparison since the source and binary can have different
- # versions.
- src = self._pkgSource.binPkgMap[pkg]
if (rpmvercmp(src.epoch, srpm.epoch) != 0 or
rpmvercmp(src.version, srpm.version) != 0 or
# in the suse case we have to ignore release
@@ -437,21 +455,21 @@
# binary does not come from the same source as it used to
src.name != srpm.name):
log.warn('update removes package (%s) %s -> %s'
- % (pkg.name, srcPkg.getNevra(), srpm.getNevra()))
+ % (binPkg.name, srcPkg.getNevra(), srpm.getNevra()))
# allow some packages to be removed.
- if expectedRemovals and pkg.name in expectedRemovals:
+ if expectedRemovals and binPkg.name in expectedRemovals:
log.info('package removal (%s) handled in configuration'
- % pkg.name)
+ % binPkg.name)
continue
- if pkg.getNevra() not in keepRemovedPackages:
- removedPackages.add(pkg)
+ if binPkg.getNevra() not in keepRemovedPackages:
+ removedPackages.add(binPkg)
if not removedPackages:
- reusedPackages.add(pkg)
- log.warn('using old version of package %s' % (pkg, ))
- self._pkgSource.srcPkgMap[srpm].add(pkg)
+ reusedPackages.add(binPkg)
+ log.warn('using old version of package %s' % (binPkg, ))
+ self._pkgSource.srcPkgMap[srpm].add(binPkg)
if removedPackages and not self._cfg.allowRemovedPackages:
pkgList=sorted(removedPackages)
From juphoff at rpath.com Thu Jun 17 14:07:17 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Thu, 17 Jun 2010 18:07:17 +0000
Subject: mirrorball: set group flags so local test cooks will have correct
parameters
Message-ID: <201006171807.o5HI7H4G007382@scc.eng.rpath.com>
changeset: b939dcd43ea2
user: Jeff Uphoff
date: Thu, 17 Jun 2010 14:06:45 -0400
set group flags so local test cooks will have correct parameters
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -120,6 +120,7 @@
# you can do a local test cook.
group._copyVersions()
group._sanityCheck()
+ group._setGroupFlags()
group._mgr._persistGroup(group)
# You probably want to do a test cook if your groups here. It would be
From juphoff at rpath.com Thu Jun 17 14:07:17 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Thu, 17 Jun 2010 18:07:17 +0000
Subject: mirrorball: branch merge
Message-ID: <201006171807.o5HI7Ha3007413@scc.eng.rpath.com>
changeset: c280419bd57d
user: Jeff Uphoff
date: Thu, 17 Jun 2010 14:07:07 -0400
branch merge
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -120,6 +120,7 @@
# you can do a local test cook.
group._copyVersions()
group._sanityCheck()
+ group._setGroupFlags()
group._mgr._persistGroup(group)
# You probably want to do a test cook if your groups here. It would be
From elliot at rpath.com Thu Jun 17 16:57:22 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 17 Jun 2010 20:57:22 +0000
Subject: mirrorball: imported patch a2
Message-ID: <201006172057.o5HKvMD9023554@scc.eng.rpath.com>
changeset: 7ffd9ebfb9ea
user: Elliot Peele
date: Thu, 17 Jun 2010 16:56:03 -0400
imported patch a2
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -467,7 +467,8 @@
removedPackages.add(binPkg)
if not removedPackages:
- reusedPackages.add(binPkg)
+ if binPkg.getNevra() not in keepRemovedPackages:
+ reusedPackages.add(binPkg)
log.warn('using old version of package %s' % (binPkg, ))
self._pkgSource.srcPkgMap[srpm].add(binPkg)
From elliot at rpath.com Fri Jun 25 15:21:39 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 25 Jun 2010 19:21:39 +0000
Subject: mirrorball: add support for rebuilding groups
Message-ID: <201006251921.o5PJLdQI012556@scc.eng.rpath.com>
changeset: 52f1ba168294
user: Elliot Peele
date: Fri, 25 Jun 2010 15:20:37 -0400
add support for rebuilding groups
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildgroups
@@ -13,48 +13,167 @@
# full details.
#
+import os
+import sys
import logging
+import itertools
-from updatebot import groupmgr
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
+
+
+from conary import versions
+from conary.deps import deps
+
from updatebot import OrderedBot
log = logging.getLogger('tmplogger')
class Bot(OrderedBot):
- def rebuildgroups(self):
+ def rebuildgroups(self, updatedPackages=None):
"""
Rebuild all groups on the devel label. This requires rewriting the group
model to point at the target label and readding content.
+ @param updatedPackages: List of package names that have been rebuilt and
+ thus need to be updated when modifying the
+ groups.
+ @type updatedPackages: list(str, ...)
"""
+ if not updatedPackages:
+ updatedPackages = []
+
# load package source
self._pkgSource.load()
+ versionMap = {
+ 'rhel': {'4': '4.0',
+ '5': '5.0', },
+ }
+
+ # Make sure to include any packages that were built from the same
+ # source.
+ siblingPackages = set()
+ for name in updatedPackages:
+ nvfs = self._updater._conaryhelper.findTrove((name, None, None),
+ getLeaves=False)
+ for nvf in nvfs:
+ siblings = self._updater._conaryhelper.getSiblingPackages(
+ nvf, allVersions=True)
+ siblingPackages.update(set([ x[0] for x in siblings ]))
+ updatedPackages = list(siblingPackages)
+
+ # Iterate over all upstream group versions that should exist.
for updateId, updates in self._errata.iterByIssueDate(current=-1):
- mgr = groupmgr.GroupManager(self._cfg,
- useMap=self._pkgSource.useMap)
+ if updateId == 0:
+ prodMap = versionMap[self._cfg.platformName]
+ version = prodMap[self._cfg.upstreamProductVersion]
+ else:
+ version = self._errata.getBucketVersion(updateId)
-
+ log.info('%s: retrieving group model information' % version)
+ group = self._groupmgr.getGroup(version=version)
- # Find any version exceptions for this update.
- multiVersionExceptions = dict([
- (x[0], x[1]) for x in itertools.chain(
- self._updater.getTargetVersions(itertools.chain(
- *self._getOldVersionExceptions(updateId).itervalues()
- ))[0]
- )
- ])
+
+ # Get all of the nvfs from the group model.
+ nvfs = set()
+ checkUpdates = set()
+ emptyFlavors = set()
+ for pkg in group.iterpackages():
+ n = str(pkg.name)
+ v = versions.ThawVersion(str(pkg.version))
+
+ f = deps.ThawFlavor(str(pkg.flavor))
+ if str(f) == '':
+ f = None
+
+ if n in updatedPackages:
+ checkUpdates.add((n, v, f))
+ elif f is None:
+ emptyFlavors.add((n, v, f))
+ else:
+ nvfs.add((n, v, f))
+
+ # Lookup anything that has an empty flavor.
+ req = set([ (x[0], x[1], None) for x in emptyFlavors ])
+ found = self._updater._conaryhelper.findTroves(req)
+ for n, v, f in itertools.chain(*found.values()):
+ nvfs.add((n, v, f))
+
+ # Lookup anything that is expected to have been rebuilt.
+ for n, v, f in checkUpdates:
+ upVer = '/'.join([v.branch().label().asString(),
+ v.trailingRevision().version])
+ binSpecs = self._updater._conaryhelper._repos.findTrove(
+ v.branch().label(), (n, upVer, f))
+
+ assert len(binSpecs) > 1
+ latest = sorted(binSpecs)[-1]
+
+ log.info('%s: found updated version of %s %s -> %s'
+ % (version, n, v, latest[1]))
+
+ if v != latest[1]:
+ nvfs.add(latest)
+ else:
+ nvfs.add((n, v, f))
+
+ # Lookup cloned from info for all versions.
+ log.info('%s: retrieving target version information' % version)
+ targetVersions, failed = self._updater.getTargetVersions(nvfs)
+
+ # Make sure any that failed are in the set of packages that are
+ # expected to have updates.
+ assert not [ x for x in failed if x[0] not in updatedPackages ]
+
+ # Take the union of all versions on the target label and those that
+ # have not yet been promoted.
+ newVersions = set(targetVersions) | set(failed)
+
+ # Add all of the new versions.
+ log.info('%s: adding remapped versions' % version)
+ pkgs = {}
+ for n, v, f in newVersions:
+ pkgs.setdefault(n, dict()).setdefault(v, set()).add(f)
+
+ for n, vMap in pkgs.iteritems():
+ assert len(vMap) == 1
+ group.removePackage(n)
+ for v, flvs in vMap.iteritems():
+ group.addPackage(n, v, flvs)
+
+ # Now deal with the standard group contents by recreating the
+ # standard group from the new group configuration information.
+
+ # Remove the existing standard group if there is one.
+ log.info('%s: regenerating standard group' % version)
+ standardNames = [ n for n, m in group.iteritems()
+ if 'standard' in n ]
+ assert len(standardNames) == 1
+ standardName = standardNames[0]
+ group._groups.pop(standardName, None)
+
+ # Run through all of the adds and removes for the standard group.
+ for updateId in range(0, group.errataState + 1):
+ self._modifyGroups(updateId, group)
+
+ log.info('%s: committing contents to latest' % version)
+ group = group.commit(copyToLatest=True)
+
+ log.info('%s: building group' % version)
+ trvMap = group.build()
+
+ for src, bins in trvMap.iteritems():
+ log.info('%s: built troves' % version)
+ for n, v, f in sorted(bins):
+ log.info(' %s=%s[%s]' % (n, v, f))
+
+
if __name__ == '__main__':
- import os
- import sys
-
- mirrorballDir = os.path.abspath('../')
- sys.path.insert(0, mirrorballDir)
-
- if 'CONARY_PATH' in os.environ:
- sys.path.insert(0, os.environ['CONARY_PATH'])
-
import rmake
import conary
import updatebot
@@ -75,8 +194,10 @@
log = logging.getLogger('grouprebuild')
+ confDir = mirrorballDir + '/config/%s' % sys.argv[1]
+
cfg = config.UpdateBotConfig()
- cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+ cfg.read(confDir + '/updatebotrc')
mcfg = rhnmirror.MirrorConfig()
mcfg.read(confDir + '/erratarc')
@@ -85,6 +206,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups()
+ bot.rebuildgroups(updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', ])
import epdb; epdb.st()
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1438,9 +1438,36 @@
if not commit:
return cs, packageList
- import epdb; epdb.st()
log.info('committing changeset')
self._repos.commitChangeSet(cs, callback=callback)
log.info('changeset committed')
return packageList
+
+ def getSiblingPackages(self, nvf, allVersions=False):
+ """
+ Get a list of packages built from the same source with the same version.
+ @param nvf: name, verison, flavor tuple
+ @type nvf: tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor)
+ @param allVersions: Optional argument, defaults to False; to include all
+ binary versions built from the same source version.
+ @type allVersions: boolean
+ @return list of name, version, flavor tuples
+ @rtype list((str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ """
+
+ trv = self._repos.getTrove(*nvf, withFiles=False)
+
+ srcName = trv.troveInfo.sourceName()
+ if not srcName:
+ srcName = trv.getName()
+
+ srcVersion = trv.getVersion().getSourceVersion()
+ siblings = self._repos.getTrovesBySource(srcName, srcVersion)
+
+ if not allVersions:
+ siblings = [ x for x in siblings if x[1] == nvf[1] ]
+
+ return siblings
From elliot at rpath.com Fri Jun 25 15:21:39 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 25 Jun 2010 19:21:39 +0000
Subject: mirrorball: Implement multiprocess build model
Message-ID: <201006251921.o5PJLdfq012587@scc.eng.rpath.com>
changeset: 106d262da791
user: Elliot Peele
date: Fri, 25 Jun 2010 15:20:38 -0400
Implement multiprocess build model
diff --git a/scripts/creategroup.py b/scripts/grouptest.py
copy from scripts/creategroup.py
copy to scripts/grouptest.py
--- a/scripts/creategroup.py
+++ b/scripts/grouptest.py
@@ -41,7 +41,7 @@
from updatebot import OrderedBot
-log = logging.getLogger('create group')
+log = logging.getLogger('scripts.testgroup')
class Bot(OrderedBot):
def generateInitialGroup(self):
@@ -49,72 +49,16 @@
Generate config for standard group contents based on repository history.
"""
- self._pkgSource.load()
-
- log.info('getting latest troves')
- troves = self._updater._conaryhelper._getLatestTroves()
-
- # combine packages of the same name.
- trvs = {}
- for name, vMap in troves.iteritems():
- if name.endswith(':source'):
- continue
- name = name.split(':')[0]
- for version, flavors in vMap.iteritems():
- for flv in flavors:
- trvs.setdefault(name, dict()).setdefault(version, set()).add(flv)
-
- # Filter out sources, groups, and components; gathering all of the
- # package versions.
- pkgs = set()
- for name, vMap in trvs.iteritems():
- if name.endswith(':source'):
- continue
- if name.startswith('group-'):
- continue
- name = name.split(':')[0]
- for version, flavors in vMap.iteritems():
- data = (name, version, tuple(flavors))
- pkgs.add(data)
+# self._pkgSource.load()
# Get the latest group model.
group = self._groupmgr.getGroup()
- # Remove the existing packages group.
- group._groups.pop('group-packages', None)
+# import epdb; epdb.st()
- # Add content to the packages group, which will cause a new model to
- # be created.
- for name, version, flavors in pkgs:
- log.info('adding %s=%s' % (name, version))
- for flv in flavors:
- log.info('\t%s' % flv)
- group.addPackage(name, version, flavors)
+ trvMap = group.buildmany()
- # Set the errata state and version to some defaults.
- group.errataState = 0
- group.version = '0'
-
- # Remove the existing standard group if there is one.
- group._groups.pop('group-standard', None)
-
- # Run through all of the adds and removes for the standard group.
- removals = set()
- nevras = dict([ (x.getNevra(), y)
- for x, y in self._pkgSource.srcPkgMap.iteritems() ])
-
- for updateId in range(0, group.errataState + 1):
- self._modifyGroups(updateId, group)
-
- for srcNevra in self._cfg.removeSource.get(updateId, ()):
- removals.update(set([ x.name for x in nevras[srcNevra] ]))
-
- removals |= set(self._cfg.updateRemovesPackages.get(updateId, ()))
-
- # Remove any packages that would have normally been removed at this
- # errataState.
- for name in removals:
- group.removePackage(name, missingOk=True)
+ import epdb; epdb.st()
# Sanity check the group model and write out the current state so that
# you can do a local test cook.
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -49,6 +49,7 @@
from updatebot.errors import ChangesetValidationFailedError
from updatebot.build.cvc import Cvc
+from updatebot.build.jobs import LocalDispatcher
from updatebot.build.dispatcher import Dispatcher
from updatebot.build.dispatcher import NonCommittalDispatcher
from updatebot.build.callbacks import StatusOnlyDisplay
@@ -149,7 +150,8 @@
self._helper = helper.rMakeHelper(buildConfig=self._rmakeCfg)
- self.cvc = Cvc(self._cfg, self._ccfg, self._formatInput)
+ self.cvc = Cvc(self._cfg, self._ccfg, self._formatInput,
+ LocalDispatcher(self, 12))
def build(self, troveSpecs):
"""
diff --git a/updatebot/build/common.py b/updatebot/build/common.py
--- a/updatebot/build/common.py
+++ b/updatebot/build/common.py
@@ -17,16 +17,18 @@
"""
import logging
+from Queue import Empty
from Queue import Queue
-from Queue import Empty
from threading import Thread
-from updatebot.build.constants import ThreadTypes
+from multiprocessing import Process
+from multiprocessing.queues import Queue as ProcessQueue
+
from updatebot.build.constants import MessageTypes
log = logging.getLogger('updatebot.build')
-class AbstractWorker(Thread):
+class AbstractWorker(object):
"""
Abstract class for all worker nodes.
"""
@@ -34,8 +36,6 @@
threadType = None
def __init__(self, status):
- Thread.__init__(self)
-
self.status = status
self.workerId = None
@@ -48,7 +48,7 @@
self.work()
except Exception, e:
self.status.put((MessageTypes.THREAD_ERROR,
- (self.threadType, self.workerId, e)))
+ (self.threadType, self.workerId, str(e))))
self.status.put((MessageTypes.THREAD_DONE, self.workerId))
@@ -60,6 +60,30 @@
raise NotImplementedError
+class AbstractWorkerThread(AbstractWorker, Thread):
+ """
+ Abstract class for worker threads.
+ """
+
+ queueClass = Queue
+
+ def __init__(self, status):
+ AbstractWorker.__init__(self, status)
+ Thread.__init__(self)
+
+
+class AbstractWorkerProcess(AbstractWorker, Process):
+ """
+ Abstract class for worker processes.
+ """
+
+ queueClass = ProcessQueue
+
+ def __init__(self, status):
+ AbstractWorker.__init__(self, status)
+ Process.__init__(self)
+
+
class AbstractStatusMonitor(object):
"""
Abstract class for implementing monitoring classes.
@@ -72,7 +96,7 @@
threadArgs = (threadArgs, )
self._threadArgs = threadArgs
- self._status = Queue()
+ self._status = self.workerClass.queueClass()
self._workers = {}
self._errors = []
diff --git a/updatebot/build/constants.py b/updatebot/build/constants.py
--- a/updatebot/build/constants.py
+++ b/updatebot/build/constants.py
@@ -27,7 +27,7 @@
THREAD_ERROR = 3
-class ThreadTypes(object):
+class WorkerTypes(object):
"""
Class for storing thread types.
"""
@@ -35,11 +35,15 @@
START = 0
MONITOR = 1
COMMIT = 2
+ LOCAL_GROUP_BUILD = 3
+ LOCAL_CHANGESET_COMMIT = 4
names = {
START: 'Start',
MONITOR: 'Monitor',
COMMIT: 'Commit',
+ LOCAL_GROUP_BUILD: 'Local Group Build',
+ LOCAL_CHANGESET_COMMIT: 'Local Changeset Commit',
}
@@ -52,3 +56,7 @@
ERROR_MONITOR_FAILURE = -2
ERROR_COMMITTER_FAILURE = -3
JOB_COMMITTING = -4
+ JOB_BUILDING = -5
+ JOB_BUILT = -6
+ JOB_COMMITTED = -7
+ JOB_FAILED = -8
diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py
--- a/updatebot/build/cvc.py
+++ b/updatebot/build/cvc.py
@@ -47,9 +47,10 @@
@type inputFormatter: method
"""
- def __init__(self, cfg, ccfg, inputFormatter):
+ def __init__(self, cfg, ccfg, inputFormatter, dispatcher):
self._cfg = cfg
self._ccfg = copy.deepcopy(ccfg)
+ self._dispatcher = dispatcher
self._formatInput = inputFormatter
# Restet dbPath to the default value for local cooking.
@@ -123,6 +124,22 @@
res = { (troveSpecs[0][0], troveSpecs[0][1], None): results }
return res
+ def build(self, trvSpec, flavorFilter=None):
+ """
+ Build trove locally.
+ @param trvSpec: trove spec to build.
+ @type trvSpec: tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor)
+ @param flavorFilter: Allow caller to filter out the contexts that they
+ want to build. This is mostly used for group
+ building where a given group should not be built
+ for a context.
+ @type flavorFilter: iterable of context names.
+ @return status results instance.
+ """
+
+ return self._dispatcher.build(trvSpec, flavorFilter=flavorFilter)
+
def _filterTroveSpecs(self, troveSpecs, useFlags):
"""
Filter trove specs based on a list of use flags. This is only applicable
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -1,5 +1,5 @@
#
-# Copyright (c) 2009 rPath, Inc.
+# Copyright (c) 2009-2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
@@ -29,11 +29,50 @@
from updatebot.build.monitor import JobCommitter
from updatebot.build.constants import JobStatus
+
from updatebot.errors import JobNotCompleteError
log = logging.getLogger('updatebot.build')
-class Dispatcher(object):
+class AbstractDispatcher(object):
+ """
+ Abstract class for managing builds.
+ """
+
+ _completed = ()
+
+ def __init__(self, builder, maxSlots):
+ self._builder = builder
+ self._maxSlots = maxSlots
+ self._slots = self._maxSlots
+
+ # jobId: (trv, status, commitData)
+ self._jobs = {}
+ self._failures = []
+
+ def _jobDone(self):
+ """
+ Check if all jobs are complete.
+ """
+
+ if not len(self._jobs):
+ return False
+
+ for jobId, (trove, status, result) in self._jobs.iteritems():
+ if status not in self._completed:
+ return False
+ return True
+
+ def _availableFDs(self, setMax=False):
+ """
+ Get 80% of available file descriptors in hopes of not running out.
+ """
+
+ avail = util.getAvailableFileDescriptors(setMax=setMax)
+ return math.floor(0.8 * avail)
+
+
+class Dispatcher(AbstractDispatcher):
"""
Manage building a list of troves in a way that doesn't bring rMake to its
knees.
@@ -53,9 +92,7 @@
def __init__(self, builder, maxSlots):
- self._builder = builder
- self._maxSlots = maxSlots
- self._slots = maxSlots
+ AbstractDispatcher.__init__(self, builder, maxSlots)
self._maxStartSlots = 10
self._startSlots = self._maxStartSlots
@@ -67,11 +104,6 @@
self._monitor = JobMonitor(self._builder._helper.client)
self._committer = JobCommitter(self._builder)
- # jobId: (trv, status, commitData)
- self._jobs = {}
-
- self._failures = []
-
def buildmany(self, troveSpecs):
"""
Build as many packages as possible until we run out of slots.
@@ -100,7 +132,7 @@
self._startSlots -= 1
# get started status
- for jobId, trove in self._starter.getStatus():
+ for trove, jobId in self._starter.getStatus():
self._jobs[jobId] = [trove, JobStatus.JOB_NOT_STARTED, None]
self._startSlots += 1
self._monitor.monitorJob(jobId)
@@ -191,27 +223,6 @@
return results, self._failures
- def _jobDone(self):
- """
- Check if all jobs are complete.
- """
-
- if not len(self._jobs):
- return False
-
- for jobId, (trove, status, result) in self._jobs.iteritems():
- if status not in self._completed:
- return False
- return True
-
- def _availableFDs(self, setMax=False):
- """
- Get 80% of available file descriptors in hopes of not running out.
- """
-
- avail = util.getAvailableFileDescriptors(setMax=setMax)
- return math.floor(0.8 * avail)
-
class NonCommittalDispatcher(Dispatcher):
"""
diff --git a/updatebot/build/jobs.py b/updatebot/build/jobs.py
new file mode 100644
--- /dev/null
+++ b/updatebot/build/jobs.py
@@ -0,0 +1,236 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Implementation of dispatchers and status monitors that is appendable and uses
+temporary job objects to return status, errors and results, rather than blocking
+on everything that has been submitted to finish building.
+"""
+
+import time
+import logging
+from threading import Thread
+
+from updatebot.build.constants import JobStatus
+from updatebot.build.dispatcher import AbstractDispatcher
+
+from updatebot.build.local import LocalGroupCooker
+from updatebot.build.local import LocalChangeSetCommitter
+
+from updatebot.errors import BuildFailedError
+from updatebot.errors import ResultsNotReadyError
+
+log = logging.getLogger('updatebot.build')
+
+class Data(object):
+ """
+ Opaque data object for the dispatcher store attributes of the build.
+ """
+
+class Status(object):
+ """
+ Base container class for job related data.
+ """
+
+ def __init__(self, jobId):
+ self.jobId = jobId
+ self._isDone = False
+
+ self._results = None
+ self._error = None
+ self._status = None
+
+ self.data = Data()
+
+ @property
+ def isDone(self):
+ return self._isDone
+
+ @property
+ def results(self):
+ if self._results is None:
+ raise ResultsNotReadyError(trove=self.jobId)
+ else:
+ return self._results
+
+ def setResults(self, results):
+ """
+ Store the results of the job.
+ """
+
+ self._results = results
+ self._isDone = True
+
+ def setError(self, error):
+ """
+ Store the error and call an error handler.
+ """
+
+ self._error = error
+ self._handleError()
+
+ def _handleError(self):
+ """
+ Handle error conditions.
+ """
+
+ raise BuildFailedError(trove=self.jobId, error=self._error)
+
+ def setStatus(self, statusStr):
+ """
+ Set status information.
+ """
+
+ self._status = statusStr
+ log.info('%s: %s' % (self.jobId, self._status))
+
+
+class LocalDispatcher(AbstractDispatcher, Thread):
+ """
+ Coordinate multiple local builds.
+ """
+
+ statusClass = Status
+
+ _slotdone = (
+ JobStatus.JOB_BUILT,
+ )
+
+ _completed = (
+ JobStatus.JOB_FAILED,
+ JobStatus.JOB_COMMITTED,
+ )
+
+ def __init__(self, builder, maxSlots):
+ AbstractDispatcher.__init__(self, builder, maxSlots)
+ Thread.__init__(self)
+
+ self._maxCommitSlots = 1
+ self._commitSlots = self._maxCommitSlots
+
+ self._cooker = LocalGroupCooker(self._builder)
+ self._committer = LocalChangeSetCommitter(self._builder)
+
+ self._order = []
+ self._started = False
+
+ self.daemon = True
+
+ def _getNotStarted(self):
+ """
+ Get the list of jobs that have not yet been started.
+ """
+
+ return [ x for x in self._jobs.itervalues()
+ if x[1] == JobStatus.JOB_NOT_STARTED ]
+
+ def run(self):
+ """
+ Polling loop to monitor and update job status.
+ """
+
+ while not self._jobDone():
+ # Start any jobs that are waiting to be started as long as there
+ # are available slots.
+ notStarted = self._getNotStarted()
+ while self._slots and notStarted:
+ self._slots -= 1
+ trove, status, res = notStarted[0]
+ self._cooker.buildGroup((trove, res.data.flavorFilter))
+
+ self._jobs[trove][1] = JobStatus.JOB_BUILDING
+ res.setStatus('building')
+
+ notStarted = self._getNotStarted()
+
+ # Check cooker status.
+ for trove, csFileName, result in self._cooker.getStatus():
+ self._slots += 1
+ trove, status, res = self._jobs[trove]
+
+ self._jobs[trove][1] = JobStatus.JOB_BUILT
+ res.setStatus('built')
+
+ res.data.buildResults = result
+ res.data.csFileName = csFileName
+
+ # Check for cooker errors.
+ for trove, error in self._cooker.getErrors():
+ self._slots += 1
+ self._jobs[trove][1] = JobStatus.JOB_FAILED
+ self._jobs[trove][2].setStatus('build failed')
+ self._jobs[trove][2].setError(error)
+
+ # Find jobs that are ready to be committed. Jobs must be committed
+ # in the order that they were submitted.
+ for trove in self._order:
+ # Make sure everything has committed in order.
+ if self._jobs[trove][1] not in self._slotdone:
+ break
+
+ # Make sure there are open slots.
+ if self._commitSlots == 0:
+ break
+
+ # Commit the job.
+ if self._commitSlots > 0:
+ self._commitSlots -= 1
+ self._jobs[trove][2].setStatus('committing')
+ self._jobs[trove][1] = JobStatus.JOB_COMMITTING
+
+ changeSetFile = self._jobs[trove][2].data.csFileName
+ self._committer.commitChangeSet((trove, changeSetFile))
+
+ assert self._commitSlots > -1
+
+ # Check for commit results.
+ for trove, results in self._committer.getStatus():
+ self._commitSlots += 1
+ self._jobs[trove][1] = JobStatus.JOB_COMMITTED
+ res = self._jobs[trove][2]
+ res.setStatus('committed')
+ res.setResults(results)
+
+ # Check for commit errors.
+ for trove, error in self._committer.getErrors():
+ self._jobs[trove][1] = JobStatus.JOB_FAILED
+ self._jobs[trove][2].setStatus('commit failed')
+ self._jobs[trove][2].setError(error)
+
+ time.sleep(3)
+
+ def build(self, troveSpec, flavorFilter=None):
+ """
+ Add one trove spec to the build queue.
+ """
+
+ # Require at least one trove.
+ if not troveSpec:
+ return None
+
+ # Wait for an available slot.
+ while not self._slots:
+ time.sleep(3)
+
+ status = self.statusClass(troveSpec)
+ status.data.flavorFilter = frozenset(flavorFilter)
+
+ self._jobs[troveSpec] = [troveSpec, JobStatus.JOB_NOT_STARTED, status]
+ self._order.append(troveSpec)
+
+ if not self._started:
+ self.start()
+ self._started = True
+
+ return status
diff --git a/updatebot/build/local.py b/updatebot/build/local.py
new file mode 100644
--- /dev/null
+++ b/updatebot/build/local.py
@@ -0,0 +1,89 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Module for cordinating local group builds.
+"""
+
+from updatebot.build.constants import WorkerTypes
+from updatebot.build.constants import MessageTypes
+
+from updatebot.build.common import AbstractStatusMonitor
+from updatebot.build.common import AbstractWorkerProcess as AbstractWorker
+
+class LocalGroupCookWorker(AbstractWorker):
+ """
+ Worker process for doing local cooks of groups.
+ """
+
+ threadType = WorkerTypes.LOCAL_GROUP_BUILD
+
+ def __init__(self, status, (builder, (trove, flavorFilter))):
+ AbstractWorker.__init__(self, status)
+
+ self.builder = builder
+ self.trove = trove
+ self.workerId = trove
+ self.flavorFilter = flavorFilter
+
+ def work(self):
+ """
+ Build the specified trove and commit it to the repository.
+ """
+
+ res, csfn = self.builder.cvc.cook(self.trove, flavorFilter=self.flavorFilter, commit=False)
+ self.status.put((MessageTypes.DATA, (self.trove, res, csfn)))
+
+
+class LocalChangeSetCommitWorker(AbstractWorker):
+ """
+ Worker process for doing local changeset commits.
+ """
+
+ threadType = WorkerTypes.LOCAL_CHANGESET_COMMIT
+
+ def __init__(self, status, (builder, (trove, csfn))):
+ AbstractWorker.__init__(self, status)
+
+ self.builder = builder
+ self.csfn = csfn
+ self.trove = trove
+ self.workerId = trove
+
+ def work(self):
+ """
+ Commit the specified changeset.
+ """
+
+ results = self.builder.cvc.commitChangeSetFile(self.csfn)
+ self.status.put((MessageTypes.DATA, (self.trove, results)))
+
+
+class LocalGroupCooker(AbstractStatusMonitor):
+ """
+ Class for managing group workers.
+ """
+
+ workerClass = LocalGroupCookWorker
+ buildGroup = AbstractStatusMonitor.addJob
+
+
+class LocalChangeSetCommitter(AbstractStatusMonitor):
+ """
+ Class for managing commit workers.
+ """
+
+ workerClass = LocalChangeSetCommitWorker
+ commitChangeSet = AbstractStatusMonitor.addJob
+
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -20,10 +20,10 @@
from rmake.cmdline import monitor
-from updatebot.build.common import AbstractWorker
from updatebot.build.common import AbstractStatusMonitor
+from updatebot.build.common import AbstractWorkerThread as AbstractWorker
-from updatebot.build.constants import ThreadTypes
+from updatebot.build.constants import WorkerTypes
from updatebot.build.constants import MessageTypes
from updatebot.build.callbacks import JobMonitorCallback
@@ -32,7 +32,7 @@
Worker thread for starting jobs and reporting status.
"""
- threadType = ThreadTypes.START
+ threadType = WorkerTypes.START
def __init__(self, status, (builder, trove)):
AbstractWorker.__init__(self, status)
@@ -47,7 +47,7 @@
"""
jobId = self.builder.start(self.trove)
- self.status.put((MessageTypes.DATA, (jobId, self.trove)))
+ self.status.put((MessageTypes.DATA, (self.trove, jobId)))
class MonitorWorker(AbstractWorker):
@@ -55,7 +55,7 @@
Worker thread for monitoring jobs and reporting status.
"""
- threadType = ThreadTypes.MONITOR
+ threadType = WorkerTypes.MONITOR
displayClass = JobMonitorCallback
def __init__(self, status, (rmakeClient, jobId)):
@@ -92,7 +92,7 @@
Worker thread for committing jobs.
"""
- threadType = ThreadTypes.COMMIT
+ threadType = WorkerTypes.COMMIT
def __init__(self, status, (builder, jobId)):
AbstractWorker.__init__(self, status)
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -142,6 +142,31 @@
_template = 'This build method does not support building groups.'
+class LocalBuildError(BuildError):
+ """
+ Abstract class for local errors.
+ """
+
+
+class BuildFailedError(LocalBuildError):
+ """
+ BuildFailedError, raised when a local build fails to build or commit.
+ """
+
+ _params = ['trove', 'error', ]
+ _template = 'Failed to complete build of %(trove)s: %(error)s'
+
+
+class ResultsNotReadyError(LocalBuildError):
+ """
+ ResultsNotReadyError, raised when results are requested from a build status
+ before the job is completed.
+ """
+
+ _params = ['trove', ]
+ _template = 'Job for %(trove)s is not yet complete'
+
+
class UnhandledUpdateError(UpdateBotError):
"""
UnhandledUpdateError, raised when the bot finds a state that it does not
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -171,6 +171,13 @@
return self._mgr.buildGroup(self)
+ def buildmany(self):
+ """
+ Build this group alongside other groups.
+ """
+
+ return self._mgr.buildGroup(self, multiBuild=True)
+
def hasBinaryVersion(self):
"""
Check if this group has a binary version.
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -296,13 +296,16 @@
return self.getGroup(version=newVersion)
@require_write
- def buildGroup(self, group):
+ def buildGroup(self, group, multiBuild=False):
"""
Build the binary version of a given group.
@param group: group model to build.
@type group: updatebot.groupmgr.group.Group
- @return mapping of built troves.
- @rtype dict(sourceTrv=[binTrv, ..])
+ @param multiBuild: Optional parameter, defaults to False, control if
+ builder can build multiple packages at once.
+ @type mutliBuild: boolean
+ @return mapping of built troves, if multiBuild return results object.
+ @rtype dict(sourceTrv=[binTrv, ..]) or updatebot.build.jobs.Status
"""
# Make sure this group has been committed to the repository before
@@ -324,7 +327,12 @@
# Create a build job and build groups using cvc.
job = ((self._sourceName, group.conaryVersion, None), )
- results = self._builder.cvc.cook(job, flavorFilter=use)
+
+ if not multiBuild:
+ results = self._builder.cvc.cook(job, flavorFilter=use)
+ else:
+ results = self._builder.cvc.build(job[0], flavorFilter=use)
+
return results
def getSourceVersions(self):
From elliot at rpath.com Fri Jun 25 15:21:39 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 25 Jun 2010 19:21:39 +0000
Subject: mirrorball: add support for rebuilding packages in the same
environment they were originally built in
Message-ID: <201006251921.o5PJLdRU012614@scc.eng.rpath.com>
changeset: 7809faf7d01f
user: Elliot Peele
date: Fri, 25 Jun 2010 15:20:39 -0400
add support for rebuilding packages in the same environment they were originally built in
diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage
--- a/scripts/rebuildpackage
+++ b/scripts/rebuildpackage
@@ -36,7 +36,8 @@
return sorted(nvfs)
- def rebuildpackage(self, name, useLatest=None):
+ def rebuildpackage(self, name, useLatest=None,
+ additionalResolveTroves=None):
"""
Rebuild all versions of a given package in order.
@param name: name of the package to rebuild.
@@ -45,11 +46,23 @@
For instance, you may want to use the latest version
of conary to get fixed dependencies.
@type useLatest: list(str, ...)
+ @param additionalResolveTroves: List of additional trove specs to add to
+ the resolve troves.
+ @type additionalResolveTroves: list(str, ...)
"""
trvSpecs = [ (x[0].split(':')[0], x[1], None)
for x in self._getAllVersions(name) ]
- trvMap = self._builder.rebuild(trvSpecs, useLatest=useLatest)
+
+ # Build only the latest versions of each sources.
+ latestMap = {}
+ for n, v, f in reversed(sorted(trvSpecs)):
+ upVer = v.trailingRevision().version
+ if upVer not in latestMap:
+ latestMap[upVer] = (n, v, f)
+
+ trvMap = self._builder.rebuild(latestMap.values(), useLatest=useLatest,
+ additionalResolveTroves=additionalResolveTroves)
return trvMap
@@ -79,12 +92,19 @@
# Rewrite source versions by cloning old versions forward and then
# modifying them.
for spec in trvSpecs:
- # Clone version to tip
- targetBranch = spec[1].branch()
- newPkgs = helper.clone(targetBranch, [spec, ], commit=False)
+ # Clone version to tip, only need to clone if there is more than
+ # one version.
+ if len(trvSpecs) > 1:
+ targetBranch = spec[1].branch()
+ cs, newPkgs = helper.clone(targetBranch, [spec, ], commit=False)
- assert len(newPkgs) == 1
- n, v, f = newPkgs[0]
+ assert len(newPkgs) == 1
+ n, v, f = newPkgs[0]
+
+ # Commit the changeset now that it has been validated.
+ helper._repos.commitChangeSet(cs)
+ else:
+ n, v, f = spec
# Edit the source to remove files.
checkoutDir = helper._edit(n, version=v)
@@ -95,7 +115,8 @@
# Commit changes if anything changed.
if removeFiles:
- helper._commit(checkoutDir, 'automated file removal')
+ helper.commit(n, version=v,
+ commitMessage='automated file removal')
if __name__ == '__main__':
@@ -141,8 +162,14 @@
bot = Bot(cfg, errata)
for pkgName in pkgNames:
- bot.removeSourceFiles(pkgName)
+# bot.removeSourceFiles(pkgName)
bot.rebuildpackage(pkgName,
- useLatest=['conary', 'conary-build', 'conary-policy'])
+ useLatest=['conary', 'conary-build', 'conary-policy', 'rpm', ],
+ additionalResolveTroves=[
+ 'libelf-lgpl=rhel.rpath.com at rpath:rhel-5-devel',
+ 'conary=rhel.rpath.com at rpath:rhel-5-devel',
+ 'conary-build=rhel.rpath.com at rpath:rhel-5-devel',
+ 'conary-policy=rhel.rpath.com at rpath:rhel-5-devel',
+ 'rpm=rhel.rpath.com at rpath:rhel-5-server-devel', ])
import epdb; epdb.st()
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -236,7 +236,7 @@
ret = self._formatOutput(trvMap)
return ret
- def rebuild(self, troveSpecs, useLatest=None):
+ def rebuild(self, troveSpecs, useLatest=None, additionalResolveTroves=None):
"""
Rebuild a set of troves in the same environment that they were
orignally built in.
@@ -246,14 +246,24 @@
For instance, you may want to use the latest version
of conary to get fixed dependencies.
@type useLatest: list(str, ...)
+ @param additionalResolveTroves: List of additional trove specs to add to
+ the resolve troves.
+ @type additionalResolveTroves: list(str, ...)
@return troveMap: dictionary of troveSpecs to built troves
"""
+ # Set some defaults
+ if useLatest is None:
+ useLatest = []
+ if additionalResolveTroves is None:
+ additionalResolveTroves = []
+
def grpByNameVersion(jobLst):
lst = {}
for job in jobLst:
lst.setdefault(tuple(job[:2]), set()).add(job)
- return sorted(lst.values())
+
+ return [ lst[x] for x in sorted(lst.keys()) ]
def startOne(job):
# Get a new builder so that we don't change the configuration of the
@@ -307,6 +317,10 @@
# Reconfigure builder to use previous buildrequires as
# resolveTroves.
resolveTroves = ' '.join([ '%s=%s' % x for x in reqs ])
+
+ # Add any additional resolve troves.
+ resolveTroves += ' ' + ' '.join(additionalResolveTroves)
+
builder._rmakeCfg.resolveTroves = []
builder._rmakeCfg.configLine('resolveTroves %s' % resolveTroves)
From elliot at rpath.com Fri Jun 25 15:21:40 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 25 Jun 2010 19:21:40 +0000
Subject: mirrorball: remove break point
Message-ID: <201006251921.o5PJLeTL012641@scc.eng.rpath.com>
changeset: 9d3c6866fe79
user: Elliot Peele
date: Fri, 25 Jun 2010 15:20:39 -0400
remove break point
diff --git a/scripts/markremoved b/scripts/markremoved
--- a/scripts/markremoved
+++ b/scripts/markremoved
@@ -30,8 +30,6 @@
for spec in sorted(removedSpecs):
log.info('removing: %s=%s[%s]' % spec)
-import epdb; epdb.st()
-
if askYn('remove troves? (y/N):', default=False):
log.info('committing')
helper._repos.commitChangeSet(cs)
From elliot at rpath.com Fri Jun 25 15:21:40 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 25 Jun 2010 19:21:40 +0000
Subject: mirrorball: add support for generating changeset files from a
cook
Message-ID: <201006251921.o5PJLeDx012668@scc.eng.rpath.com>
changeset: 2d1da77c973e
user: Elliot Peele
date: Fri, 25 Jun 2010 15:20:39 -0400
add support for generating changeset files from a cook
diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py
--- a/updatebot/build/cvc.py
+++ b/updatebot/build/cvc.py
@@ -16,8 +16,10 @@
An abstraction layer around cvc cook.
"""
+import os
import copy
import logging
+import tempfile
log = logging.getLogger('updatebot.build.cvc')
@@ -58,16 +60,19 @@
self._client = conaryclient.ConaryClient(self._ccfg)
- def cook(self, troveSpecs, flavorFilter=None):
+ def cook(self, troveSpecs, flavorFilter=None, commit=True):
"""
Cook a set of trove specs, currently limited to groups.
@params troveSpecs: list of name, version, and flavor tuples.
@type troveSpecs: [(name, version, flavor), ... ]
- @params flavorFilter: Allow caller to filter out the contexts that they
- want to build. This is mostly used for group
- building where a given group should not be built
- for a context.
+ @param flavorFilter: Allow caller to filter out the contexts that they
+ want to build. This is mostly used for group
+ building where a given group should not be built
+ for a context.
@type flavorFilter: iterable of context names.
+ @param commit: Optional parameter to control when a changeset is
+ committed.
+ @type commit: boolean
"""
# TODO: Look at conary.build.cook.cookCommand for how to setup
@@ -75,6 +80,12 @@
troveSpecs = self._formatInput(troveSpecs)
+ if commit:
+ changeSetFile = None
+ else:
+ fd, changeSetFile = tempfile.mkstemp(prefix='changeset-')
+ os.close(fd)
+
if flavorFilter:
troveSpecs = self._filterTroveSpecs(troveSpecs, flavorFilter)
@@ -103,6 +114,7 @@
logBuild=True,
callback=conarycallbacks.UpdateBotCookCallback(),
groupOptions=groupCookOptions,
+ changeSetFile=changeSetFile,
)
if built is None:
@@ -122,7 +134,23 @@
for x, y, z in components)
res = { (troveSpecs[0][0], troveSpecs[0][1], None): results }
- return res
+
+ if commit:
+ return res
+ else:
+ return changeSetFile, res
+
+ def commitChangeSetFile(self, changeSetFile, callback=None):
+ """
+ Expose commit changeset interface.
+ @param changeSetFile: changeset file name
+ @type changeSetFile: str
+ @param callback: optional commit callback.
+ @type callback: conary.callbacks.ChangesetCallback
+ """
+
+ return self._client.repos.commitChangeSetFile(
+ changeSetFile, callback=callback)
def build(self, trvSpec, flavorFilter=None):
"""
From elliot at rpath.com Fri Jun 25 15:21:41 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 25 Jun 2010 19:21:41 +0000
Subject: mirrorball: utilize the multi process rebuild support when
rebuilding groups
Message-ID: <201006251921.o5PJLfkS012695@scc.eng.rpath.com>
changeset: 5eda0cc537a8
user: Elliot Peele
date: Fri, 25 Jun 2010 15:20:40 -0400
utilize the multi process rebuild support when rebuilding groups
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildgroups
@@ -58,14 +58,19 @@
# source.
siblingPackages = set()
for name in updatedPackages:
+ log.info('looking up siblings for %s' % name)
nvfs = self._updater._conaryhelper.findTrove((name, None, None),
getLeaves=False)
- for nvf in nvfs:
- siblings = self._updater._conaryhelper.getSiblingPackages(
- nvf, allVersions=True)
- siblingPackages.update(set([ x[0] for x in siblings ]))
+ siblingMap = self._updater._conaryhelper.getSiblingPackages(
+ nvfs, allVersions=True)
+ for siblings in siblingMap.itervalues():
+ siblingPackages |= set([ x[0] for x in siblings ])
updatedPackages = list(siblingPackages)
+ results = []
+
+ log.info('done loading siblings')
+
# Iterate over all upstream group versions that should exist.
for updateId, updates in self._errata.iterByIssueDate(current=-1):
if updateId == 0:
@@ -77,7 +82,6 @@
log.info('%s: retrieving group model information' % version)
group = self._groupmgr.getGroup(version=version)
-
# Get all of the nvfs from the group model.
nvfs = set()
checkUpdates = set()
@@ -108,9 +112,10 @@
upVer = '/'.join([v.branch().label().asString(),
v.trailingRevision().version])
binSpecs = self._updater._conaryhelper._repos.findTrove(
- v.branch().label(), (n, upVer, f))
+ v.branch().label(), (n, upVer, None), bestFlavor=False)
assert len(binSpecs) > 1
+
latest = sorted(binSpecs)[-1]
log.info('%s: found updated version of %s %s -> %s'
@@ -157,20 +162,28 @@
group._groups.pop(standardName, None)
# Run through all of the adds and removes for the standard group.
- for updateId in range(0, group.errataState + 1):
- self._modifyGroups(updateId, group)
+ for uid, ups in self._errata.iterByIssueDate(current=-1):
+ if uid > group.errataState:
+ break
+ self._modifyGroups(uid, group)
log.info('%s: committing contents to latest' % version)
group = group.commit(copyToLatest=True)
log.info('%s: building group' % version)
- trvMap = group.build()
+ res = group.buildmany()
+ results.append(res)
- for src, bins in trvMap.iteritems():
- log.info('%s: built troves' % version)
- for n, v, f in sorted(bins):
- log.info(' %s=%s[%s]' % (n, v, f))
+ import epdb; epdb.st()
+ completed = [ x for x in results if x.isDone ]
+ if completed:
+ for res in completed:
+ for src, bins in res.results.iteritems():
+ log.info('%s: built troves' % version)
+ for n, v, f in sorted(bins):
+ log.info(' %s=%s[%s]' % (n, v, f))
+ results.remove(res)
if __name__ == '__main__':
@@ -206,6 +219,6 @@
errata.fetch()
bot = Bot(cfg, errata)
- bot.rebuildgroups(updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', ])
+ bot.rebuildgroups(updatedPackages=['chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', 'xorg-x11-font-utils', 'kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', 'python-dmidecode', 'python-elementtree', 'python-imaging', 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', 'python-setuptools', 'python-sqlite', 'python-urlgrabber', 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', 'system-config-printer', 'vte', 'wireshark', 'yum', 'yum-metadata-parser',])
import epdb; epdb.st()
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1458,16 +1458,37 @@
conary.deps.deps.Flavor), ...)
"""
- trv = self._repos.getTrove(*nvf, withFiles=False)
+ if isinstance(nvf, list):
+ nvfs = nvf
+ else:
+ nvfs = [nvf, ]
- srcName = trv.troveInfo.sourceName()
- if not srcName:
- srcName = trv.getName()
+ trvs = self._repos.getTroves(nvfs, withFiles=False)
- srcVersion = trv.getVersion().getSourceVersion()
- siblings = self._repos.getTrovesBySource(srcName, srcVersion)
+ # Figure out unique source names and versions for all nvfs in an attempt
+ # to minimize the number of repository calls.
+ sources = {}
+ for trvSpec, trv in itertools.izip(nvfs, trvs):
+ srcName = trv.troveInfo.sourceName()
+ if not srcName:
+ srcName = trv.getName()
- if not allVersions:
- siblings = [ x for x in siblings if x[1] == nvf[1] ]
+ srcVersion = trv.getVersion().getSourceVersion()
- return siblings
+ sources.setdefault((srcName, srcVersion), set()).add(trvSpec)
+
+ # Map siblings back to nvfs.
+ siblingMap = {}
+ for (srcName, srcVersion), trvSpecs in sources.iteritems():
+ siblings = self._repos.getTrovesBySource(srcName, srcVersion)
+
+ for trvSpec in trvSpecs:
+ if not allVersions:
+ siblings = [ x for x in siblings if x[1] == trvSpec[1] ]
+
+ siblingMap.setdefault(trvSpec, set()).update(set(siblings))
+
+ if isinstance(nvf, list):
+ return siblingMap
+ else:
+ return siblingMap[nvf]
diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py
--- a/updatebot/groupmgr/sanity.py
+++ b/updatebot/groupmgr/sanity.py
@@ -154,7 +154,7 @@
# FIXME: This should probably be a fully formed version
# as above.
- version = upstreamVersion
+ version = version.branch().label().asString() + '/' + upstreamVersion
flavor = None
# FIXME: At some point we might want to add proper flavor handling,