From elliot at rpath.com Mon Apr 5 16:07:22 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 05 Apr 2010 20:07:22 +0000
Subject: mirrorball: fix typo
Message-ID: <201004052007.o35K7MGl001194@scc.eng.rpath.com>
changeset: 2692a5a27e61
user: Elliot Peele
date: Thu, 01 Apr 2010 15:07:29 -0400
fix typo
don't blow up on sources that haven't been imported
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -400,7 +400,7 @@
# binary does not come from the same source as it used to
self._pkgSource.binPkgMap[pkg].name != srpm.name):
log.warn('update removes package (%s) %s -> %s'
- % (pkg.name, srpm.getNevra(), srcPkg.getNevra()))
+ % (pkg.name, srcPkg.getNevra(), srpm.getNevra()))
# allow some packages to be removed.
if expectedRemovals and pkg.name in expectedRemovals:
@@ -454,6 +454,11 @@
if nvflst:
return None
+ # Source hasn't been imported.
+ if not nvflst:
+ log.warn('source has not been imported: %s' % srpm)
+ return None
+
assert len(nvflst) == 1
n, v, f = nvflst[0]
nvf = (n.split(':')[0], v, None)
From elliot at rpath.com Mon Apr 5 16:07:23 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 05 Apr 2010 20:07:23 +0000
Subject: mirrorball: raise an error when you find packages that don't have an
errata unless there
Message-ID: <201004052007.o35K7Nd8001224@scc.eng.rpath.com>
changeset: 7033d474d2cd
user: Elliot Peele
date: Thu, 01 Apr 2010 15:08:22 -0400
raise an error when you find packages that don't have an errata unless there
is an exception in the config
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -392,6 +392,9 @@
# latest that matches the current rpm version.
useOldVersion = (CfgIntDict(CfgList(CfgTroveSpec)), {})
+ # Allow updates for a given nevra to be published without matching errata.
+ allowMissingErrata = (CfgList(CfgNevra), [])
+
class UpdateBotConfig(cfg.SectionedConfigFile):
"""
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -23,6 +23,7 @@
from updatebot import update
from updatebot import conaryhelper
+from updatebot.errors import MissingErrataError
from updatebot.errors import ErrataPackageNotFoundError
from updatebot.errors import ErrataSourceDataMissingError
from updatebot.errors import PackageNotFoundInBucketError
@@ -521,12 +522,21 @@
# insert packages that did not have errata and were not in the initial
# set of packages (golden bits)
srcMap = {}
+ missing = set()
for pkg in other:
+ if pkg.getNevra() not in self._cfg.allowMissingErrata:
+ missing.add(pkg)
+
src = self._pkgSource.binPkgMap[pkg]
if src not in srcMap:
srcMap[src] = []
srcMap[src].append(pkg)
+ # Raise an error if there are any packages missing an errata that are
+ # now explicitly allowed by the config.
+ if missing:
+ raise MissingErrataError(packages=list(missing))
+
# insert bins by buildstamp
extras = {}
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -592,6 +592,15 @@
_template = ('Can not merge %(source)s into %(target)s due to conflicting '
'package %(package)s')
+class MissingErrataError(ErrataFilterError):
+ """
+ MissingErrataError, raised when packages are discovered without an
+ associated errata.
+ """
+
+ _params = [ 'packages', ]
+ _template = 'The following packages do not have an errata: %(packages)s'
+
class ConfigurationError(UpdateBotError):
"""
Generic exception class for configuration related errors.
From elliot at rpath.com Mon Apr 5 16:07:24 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 05 Apr 2010 20:07:24 +0000
Subject: mirrorball: 1. add handling for package downgrades
Message-ID: <201004052007.o35K7OIQ001253@scc.eng.rpath.com>
changeset: ad9320b4b7fa
user: Elliot Peele
date: Mon, 05 Apr 2010 16:04:51 -0400
1. add handling for package downgrades
2. report errors for all missing sources when validating modified updates
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -142,7 +142,8 @@
return trvMap, failed
- def update(self, force=None, updatePkgs=None, expectedRemovals=None):
+ def update(self, force=None, updatePkgs=None, expectedRemovals=None,
+ allowPackageDowngrades=None):
"""
Update the conary repository from the yum repositories.
@param force: list of packages to update without exception
@@ -151,6 +152,9 @@
@type updatePkgs: iterable of source package objects
@param expectedRemovals: set of packages that are expected to be
removed.
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
@type expectedRemovals: set of package names
"""
@@ -171,7 +175,8 @@
# Get troves to update and send advisories.
toAdvise, toUpdate = self._updater.getUpdates(
updateTroves=updateTroves,
- expectedRemovals=expectedRemovals)
+ expectedRemovals=expectedRemovals,
+ allowPackageDowngrades=allowPackageDowngrades)
# If forcing an update, make sure that all packages are listed in
# toAdvise and toUpdate as needed.
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -108,7 +108,7 @@
return splt
-class CfgObsoletes(CfgString):
+class CfgNevraTuple(CfgString):
"""
Class for parsing obsolete mappings:
@@ -376,7 +376,7 @@
# but CfgSet and CfgTuple do not exist at this point;
# maybe we can add them later.
# keepObsolete
- keepObsolete = (CfgList(CfgObsoletes), [])
+ keepObsolete = (CfgList(CfgNevraTuple), [])
# updateId packageName [packageName ...]
# remove obsoleted packages when other subpackages of the same
@@ -395,6 +395,14 @@
# Allow updates for a given nevra to be published without matching errata.
allowMissingErrata = (CfgList(CfgNevra), [])
+ # Allow updates to have versions that go backwards.
+ # updateId: [ (from srcTrvSpec, to srcTrvSpec), ... ]
+ allowPackageDowngrades = (CfgIntDict(CfgList(CfgNevraTuple)), {})
+
+ # Add a source to a specific updateId. This is used to move updates forward
+ # after allowing an update to downgrade the version.
+ addSource = (CfgIntDict(CfgList(CfgNevra)), {})
+
class UpdateBotConfig(cfg.SectionedConfigFile):
"""
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -261,6 +261,7 @@
parentPackages = []
removals = self._cfg.updateRemovesPackages
replaces = self._cfg.updateReplacesPackages
+ downgraded = self._cfg.allowPackageDowngrades
currentlyRemovedBinaryNevras = set()
foundObsoleteEdges = set()
foundObsoleteSrcs = set()
@@ -270,6 +271,7 @@
expectedReplaces = replaces.get(updateId, [])
explicitSourceRemovals = self._cfg.removeSource.get(updateId, set())
explicitBinaryRemovals = self._cfg.removeObsoleted.get(updateId, set())
+ explicitPackageDowngrades = downgraded.get(updateId, None)
assert len(self._order[updateId])
for srpm in self._order[updateId]:
@@ -278,7 +280,8 @@
# validate updates
try:
assert updater._sanitizeTrove(nvf, srpm,
- expectedRemovals=expectedRemovals + expectedReplaces)
+ expectedRemovals=expectedRemovals + expectedReplaces,
+ allowPackageDowngrades=explicitPackageDowngrades)
except (UpdateGoesBackwardsError,
UpdateRemovesPackageError,
UpdateReusesPackageError), e:
@@ -622,13 +625,23 @@
for source, dest, nevra in self._cfg.reorderSource:
self._reorderSource(source, dest, nevra)
+ # add a source to a specific bucket, used to "promote" newer versions
+ # forward.
+ nevras = dict([ (x.getNevra(), x)
+ for x in self._pkgSource.srcPkgMap ])
+ diffCount = 0
+ for updateId, srcNevras in self._cfg.addSource.iteritems():
+ sources = set(nevras[x] for x in srcNevras)
+ self._order.setdefault(updateId, set()).update(sources)
+ diffCount += len(srcNevras)
+
# Make sure we don't drop any updates
totalPkgs2 = sum([ len(x) for x in self._order.itervalues() ])
pkgs = set()
for pkgSet in self._order.itervalues():
pkgs.update(pkgSet)
- assert len(pkgs) == totalPkgs2
- assert totalPkgs2 == totalPkgs
+ assert len(pkgs) == totalPkgs2 - diffCount
+ assert totalPkgs2 == totalPkgs + diffCount
def _mergeUpdates(self, mergeList):
"""
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -182,6 +182,29 @@
_params = ['pkgname']
_template = 'No checked out version of %(pkgname)s was found.'
+class SourceNotImportedError(UnhandledUpdateError):
+ """
+ SourceNotImportedError, raised when sanity checking an existing source
+ package that can not be found in the conary repository.
+ """
+
+ _params = ['srpm', ]
+ _template = ('Source package (%(srpm)s) has not been imported. This '
+ 'usually means that something in the order stream is in an unexpected '
+ 'state.')
+
+class FoundModifiedNotImportedErrataError(UnhandledUpdateError):
+ """
+ FoundModifiedNotImportedErrataError, raised when an errata is found that
+ should have already been imported and has changed in the upstream data
+ source.
+ """
+
+ _params = ['adivsories', ]
+ _template = ('The following advisories have been modified upstream, but '
+ 'should have already been imported. Check to make sure the '
+ 'modifications are not important: %(advisories)s')
+
class BinariesNotFoundForSourceVersion(UnhandledUpdateError):
"""
BinariesNotFoundForSourceVersion, raised when querying by source name and
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -30,12 +30,14 @@
from updatebot.lib import watchdog
from updatebot.bot import Bot as BotSuperClass
+from updatebot.errors import SourceNotImportedError
from updatebot.errors import UnknownRemoveSourceError
from updatebot.errors import PlatformNotImportedError
from updatebot.errors import TargetVersionNotFoundError
from updatebot.errors import PromoteMissingVersionError
from updatebot.errors import PromoteFlavorMismatchError
from updatebot.errors import PlatformAlreadyImportedError
+from updatebot.errors import FoundModifiedNotImportedErrataError
log = logging.getLogger('updatebot.ordered')
@@ -191,6 +193,9 @@
# Iterate through changed and verify the current conary repository
# contents against any changes.
if changed:
+ notimported = set()
+ expectedDowngrades = [ x for x in
+ itertools.chain(*self._cfg.allowPacakgeDowngrades.values()) ]
log.info('found modified updates, validating repository state')
for advisory, advInfo in changed.iteritems():
log.info('validating %s' % advisory)
@@ -198,7 +203,17 @@
log.info('checking %s' % srpm.name)
# This will raise an exception if any inconsistencies are
# detected.
- self._updater.sanityCheckSource(srpm)
+ try:
+ self._updater.sanityCheckSource(srpm,
+ allowPackageDowngrades=expectedDowngrades)
+ except SourceNotImportedError, e:
+ notimported.add(advisory)
+
+ if notimported:
+ raise FoundModifiedNotImportedErrataError(
+ advisories=notimported)
+
+ import epdb; epdb.st()
log.info('starting update run')
@@ -239,6 +254,8 @@
requiredRemovals = (set(removePackages) |
set(removeReplaced))
+ # Get the list of package that are allowed to be downgraded.
+ allowDowngrades = self._cfg.allowPackageDowngrades.get(updateId, [])
# If recovering from a failure, restore the pkgMap from disk.
if restoreFile:
@@ -248,7 +265,8 @@
# Update package set.
else:
pkgMap = self._update(*args, updatePkgs=updates,
- expectedRemovals=expectedRemovals, **kwargs)
+ expectedRemovals=expectedRemovals,
+ allowPackageDowngrades=allowDowngrades, **kwargs)
# When deriving from an upstream platform sometimes we don't want
# the latest versions.
@@ -320,6 +338,9 @@
# Make sure built troves are part of the group.
self._addPackages(pkgMap)
+ if updateId == 1270008000:
+ import epdb; epdb.st()
+
# Get timestamp version.
version = self._errata.getBucketVersion(updateId)
if not version:
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -27,6 +27,7 @@
from updatebot import conaryhelper
from updatebot.errors import GroupNotFound
from updatebot.errors import NoManifestFoundError
+from updatebot.errors import SourceNotImportedError
from updatebot.errors import OldVersionNotFoundError
from updatebot.errors import UpdateGoesBackwardsError
from updatebot.errors import UpdateRemovesPackageError
@@ -45,7 +46,8 @@
self._conaryhelper = conaryhelper.ConaryHelper(self._cfg)
- def getUpdates(self, updateTroves=None, expectedRemovals=None):
+ def getUpdates(self, updateTroves=None, expectedRemovals=None,
+ allowPackageDowngrades=None):
"""
Find all packages that need updates and/or advisories from a top level
binary group.
@@ -54,6 +56,9 @@
@param expectedRemovals: set of package names that are expected to be
removed.
@type expectedRemovals: set of package names
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
@return list of packages to send advisories for and list of packages
to update
"""
@@ -73,7 +78,8 @@
for nvf, srpm in updateTroves:
# Will raise exception if any errors are found, halting execution.
if self._sanitizeTrove(nvf, srpm,
- expectedRemovals=expectedRemovals):
+ expectedRemovals=expectedRemovals,
+ allowPackageDowngrades=allowPackageDowngrades):
toUpdate.append((nvf, srpm))
toAdvise.append((nvf, srpm))
@@ -313,7 +319,8 @@
srpms.sort(util.packagevercmp)
return srpms[-1]
- def _sanitizeTrove(self, nvf, srpm, expectedRemovals=None):
+ def _sanitizeTrove(self, nvf, srpm, expectedRemovals=None,
+ allowPackageDowngrades=None):
"""
Verifies the package update to make sure it looks correct and is a
case that the bot knows how to handle.
@@ -330,6 +337,9 @@
@param expectedRemovals: set of package names that are expected to be
removed.
@type expectedRemovals: set of package names
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
@return needsUpdate boolean
@raises UpdateGoesBackwardsError
@raises UpdateRemovesPackageError
@@ -341,6 +351,9 @@
removedPackages = set()
reusedPackages = set()
+ if allowPackageDowngrades is None:
+ allowPackageDowngrades = ()
+
try:
manifest = self._conaryhelper.getManifest(nvf[0], version=nvf[1])
except NoManifestFoundError, e:
@@ -374,9 +387,14 @@
# make sure new package is actually newer
if util.packagevercmp(srpm, srcPkg) == -1:
- log.warn('version goes backwards %s -> %s' %
- (srcPkg.getNevra(), srpm.getNevra()))
- raise UpdateGoesBackwardsError(why=(srcPkg, srpm))
+ srcTuple = (srcPkg.getNevra(), srpm.getNevra())
+ log.warn('version goes backwards %s -> %s' % srcTuple)
+ if srcTuple in allowPackageDowngrades:
+ log.info('found version downgrade exception in '
+ 'configuration')
+ needsUpdate = True
+ else:
+ raise UpdateGoesBackwardsError(why=(srcPkg, srpm))
# make sure we aren't trying to remove a package
if ((binPkg.name, binPkg.arch) not in newNames and
@@ -433,12 +451,15 @@
return needsUpdate
- def sanityCheckSource(self, srpm):
+ def sanityCheckSource(self, srpm, allowPackageDowngrades=None):
"""
Look up the matching source version in the conary repository and verify
that the manifest matches the package list in the package source.
@param srpm: src pacakge object
@type srpm: repomd.packagexml._Package
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
"""
srcQuery = ('%s:source' % srpm.name, srpm.getConaryVersion(), None)
@@ -456,14 +477,15 @@
# Source hasn't been imported.
if not nvflst:
- log.warn('source has not been imported: %s' % srpm)
- return None
+ log.error('source has not been imported: %s' % srpm)
+ raise SourceNotImportedError(srpm=srpm)
assert len(nvflst) == 1
n, v, f = nvflst[0]
nvf = (n.split(':')[0], v, None)
- needsUpdate = self._sanitizeTrove(nvf, srpm)
+ needsUpdate = self._sanitizeTrove(nvf, srpm,
+ allowPackageDowngrades=allowPackageDowngrades)
# If anything has chnaged raise an error.
if needsUpdate:
From elliot at rpath.com Mon Apr 5 16:07:25 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 05 Apr 2010 20:07:25 +0000
Subject: mirrorball: adapt to new group sanity checking interface
Message-ID: <201004052007.o35K7PNB001276@scc.eng.rpath.com>
changeset: ecc8dd0af73a
user: Elliot Peele
date: Mon, 05 Apr 2010 16:07:19 -0400
adapt to new group sanity checking interface
diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py
--- a/scripts/grpchecker.py
+++ b/scripts/grpchecker.py
@@ -104,7 +104,7 @@
changes = []
try:
- mgr._validateGroups()
+ mgr._sanity.check(mgr._groups, mgr.getErrataState())
except GroupValidationFailedError, e:
for group, error in e.errors:
if isinstance(error, NameVersionConflictsFoundError):
@@ -190,7 +190,7 @@
mgr.addPackage(n, v, flvs)
mgr._copyVersions()
- mgr._validateGroups()
+ mgr._sanity.check(mgr._groups, mgr.getErrataState())
version = mgr.save(copyToLatest=True)
jobId = mgr._builder.start(((mgr._sourceName, version, None), ))
jobIds.append(jobId)
From elliot at rpath.com Tue Apr 6 09:39:33 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 06 Apr 2010 13:39:33 +0000
Subject: mirrorball: fix typos and remove breakpoints
Message-ID: <201004061339.o36DdXOs020277@scc.eng.rpath.com>
changeset: 191b6ab98fc7
user: Elliot Peele
date: Tue, 06 Apr 2010 09:39:31 -0400
fix typos and remove breakpoints
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -200,7 +200,7 @@
source.
"""
- _params = ['adivsories', ]
+ _params = ['advisories', ]
_template = ('The following advisories have been modified upstream, but '
'should have already been imported. Check to make sure the '
'modifications are not important: %(advisories)s')
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -195,7 +195,9 @@
if changed:
notimported = set()
expectedDowngrades = [ x for x in
- itertools.chain(*self._cfg.allowPacakgeDowngrades.values()) ]
+ itertools.chain(*self._cfg.allowPackageDowngrades.values()) ]
+ sourceExceptions = dict((x[2], x[1])
+ for x in self._cfg.reorderAdvisory)
log.info('found modified updates, validating repository state')
for advisory, advInfo in changed.iteritems():
log.info('validating %s' % advisory)
@@ -207,14 +209,16 @@
self._updater.sanityCheckSource(srpm,
allowPackageDowngrades=expectedDowngrades)
except SourceNotImportedError, e:
+ if (advisory in sourceExceptions and
+ sourceExceptions[advisory] > current):
+ log.info('found exception for advisory')
+ continue
notimported.add(advisory)
if notimported:
raise FoundModifiedNotImportedErrataError(
advisories=notimported)
- import epdb; epdb.st()
-
log.info('starting update run')
count = 0
@@ -338,9 +342,6 @@
# Make sure built troves are part of the group.
self._addPackages(pkgMap)
- if updateId == 1270008000:
- import epdb; epdb.st()
-
# Get timestamp version.
version = self._errata.getBucketVersion(updateId)
if not version:
From elliot at rpath.com Tue Apr 20 18:21:21 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:21 +0000
Subject: mirrorball: add missing s
Message-ID: <201004202221.o3KMLLOw019353@scc.eng.rpath.com>
changeset: 702af828296c
user: Elliot Peele
date: Wed, 07 Apr 2010 10:13:27 -0400
add missing s
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -592,7 +592,7 @@
"""
_params = ['count', ]
- _template = ('Could not find %(count) sources for matching binary '
+ _template = ('Could not find %(count)s sources for matching binary '
'packages. This generally means that there is a binary package with a '
'source of a different name and a source can not be found with a '
'matching source name, version, and release.')
From elliot at rpath.com Tue Apr 20 18:21:22 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:22 +0000
Subject: mirrorball: dedup sources when populating the srcPkgMap and favor
sources with correct
Message-ID: <201004202221.o3KMLNZv019384@scc.eng.rpath.com>
changeset: 3401022ea970
user: Elliot Peele
date: Wed, 14 Apr 2010 15:55:24 -0400
dedup sources when populating the srcPkgMap and favor sources with correct
file names that match what the factory is expecting
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -135,10 +135,7 @@
@type package: repomd.packagexml._Package
"""
- if package.name not in self.srcNameMap:
- self.srcNameMap[package.name] = set()
- self.srcNameMap[package.name].add(package)
-
+ self.srcNameMap.setdefault(package.name, set()).add(package)
self.locationMap[package.location] = package
# In case the a synthesized source ever turns into real source add the
@@ -168,10 +165,9 @@
srcRelease = srcParts[-1][:-8] # remove '.src.rpm'
elif srcParts[-1].endswith('.nosrc.rpm'):
srcRelease = srcParts[-1][:-10]
+
rpmMapKey = (srcName, package.epoch, srcVersion, srcRelease, 'src')
- if rpmMapKey not in self._rpmMap:
- self._rpmMap[rpmMapKey] = set()
- self._rpmMap[rpmMapKey].add(package)
+ self._rpmMap.setdefault(rpmMapKey, set()).add(package)
# The normal case of "obsoletes foo < version" (or with
# "requires foo", though that normally also follows the
@@ -239,6 +235,27 @@
self.srcPkgMap[pkg] = self._rpmMap[key]
self.srcPkgMap[pkg].add(pkg)
+
+ # Remove any duplicate sources, favoring sources with the source
+ # version in the file name.
+ # FIXME: This doesn't really work for nosrc rpms
+ sources = sorted([ (os.path.basename(x.location), x)
+ for x in self.srcPkgMap[pkg] if x.arch in ('src', 'nosrc') ])
+ if len(sources) > 1:
+
+ log.info('filtering multiple sources')
+ primary = None
+ for fn, src in sources:
+ if fn == '%s-%s-%s.%s.rpm' % (src.name, src.version, src.release, src.arch):
+ log.info('found primary sources %s' % src)
+ primary = src
+ break
+
+ if primary:
+ for fn, src in sources:
+ if src is not primary:
+ self.srcPkgMap[pkg].remove(src)
+
toDelete.add(key)
for binPkg in self.srcPkgMap[pkg]:
From elliot at rpath.com Tue Apr 20 18:21:23 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:23 +0000
Subject: mirrorball: do what we can to remove sources without the version
in the file name
Message-ID: <201004202221.o3KMLNZp019411@scc.eng.rpath.com>
changeset: 32d3612aabf5
user: Elliot Peele
date: Thu, 15 Apr 2010 10:51:57 -0400
do what we can to remove sources without the version in the file name
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -135,7 +135,15 @@
@type package: repomd.packagexml._Package
"""
- self.srcNameMap.setdefault(package.name, set()).add(package)
+ other = package
+ if package in self._srcPkgs:
+ other = [ x for x in self._srcPkgs if x == package ][0]
+ self.srcNameMap[package.name].remove(other)
+ self._srcPkgs.remove(other)
+ if package.getFileName() == os.path.basename(package.location):
+ other = package
+
+ self.srcNameMap.setdefault(package.name, set()).add(other)
self.locationMap[package.location] = package
# In case the a synthesized source ever turns into real source add the
@@ -145,7 +153,7 @@
if baseLoc not in self.locationMap:
self.locationMap[baseLoc] = package
- self._srcPkgs.add(package)
+ self._srcPkgs.add(other)
self._srcMap[(package.name, package.epoch, package.version,
package.release, package.arch)] = package
@@ -242,12 +250,9 @@
sources = sorted([ (os.path.basename(x.location), x)
for x in self.srcPkgMap[pkg] if x.arch in ('src', 'nosrc') ])
if len(sources) > 1:
-
- log.info('filtering multiple sources')
primary = None
for fn, src in sources:
if fn == '%s-%s-%s.%s.rpm' % (src.name, src.version, src.release, src.arch):
- log.info('found primary sources %s' % src)
primary = src
break
From elliot at rpath.com Tue Apr 20 18:21:23 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:23 +0000
Subject: mirrorball: add method for returning the expected filename of the rpm
Message-ID: <201004202221.o3KMLNrJ019438@scc.eng.rpath.com>
changeset: 4fabf84985aa
user: Elliot Peele
date: Thu, 15 Apr 2010 10:52:40 -0400
add method for returning the expected filename of the rpm
diff --git a/repomd/packagexml.py b/repomd/packagexml.py
--- a/repomd/packagexml.py
+++ b/repomd/packagexml.py
@@ -180,6 +180,14 @@
ver = "_".join(nameVerRelease.split("-")[-2:])
return ver
+ def getFileName(self):
+ """
+ Returns the expected package file name.
+ """
+
+ return '%s-%s-%s.%s.rpm' % (self.name, self.version,
+ self.release, self.arch)
+
class _RpmEntry(SlotNode):
"""
From elliot at rpath.com Tue Apr 20 18:21:24 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:24 +0000
Subject: mirrorball: switch to better commit message when promoting
Message-ID: <201004202221.o3KMLOmv019465@scc.eng.rpath.com>
changeset: 0d040ab925de
user: Elliot Peele
date: Thu, 15 Apr 2010 10:53:35 -0400
switch to better commit message when promoting
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1049,7 +1049,8 @@
if version == latestVer:
trvLst.append((name, version, flavor))
- callback = UpdateBotCloneCallback(self._ccfg, 'test', log=log)
+ callback = UpdateBotCloneCallback(self._ccfg, 'automated promote',
+ log=log)
success, cs = self._client.createSiblingCloneChangeSet(
labelMap,
From elliot at rpath.com Tue Apr 20 18:21:25 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:25 +0000
Subject: mirrorball: use the correct part of the trove tuple for the name
Message-ID: <201004202221.o3KMLP0v019494@scc.eng.rpath.com>
changeset: 8cbcd1887010
user: Elliot Peele
date: Thu, 15 Apr 2010 14:12:17 -0400
use the correct part of the trove tuple for the name
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -91,7 +91,7 @@
assert len(trvs)
- self._sourceName = self._cfg.topParentSourceGroup
+ self._sourceName = self._cfg.topParentSourceGroup[0]
self._sourceVersion = trvs[0][1]
self._readonly = True
From elliot at rpath.com Tue Apr 20 18:21:25 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:25 +0000
Subject: mirrorball: filter group flavors based on package use flags
Message-ID: <201004202221.o3KMLQQY019521@scc.eng.rpath.com>
changeset: 751211b70feb
user: Elliot Peele
date: Thu, 15 Apr 2010 16:35:37 -0400
filter group flavors based on package use flags
diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py
--- a/updatebot/build/cvc.py
+++ b/updatebot/build/cvc.py
@@ -23,8 +23,9 @@
from conary import versions
from conary import conarycfg
+from conary.deps import deps
+from conary.build import cook
from conary import conaryclient
-from conary.build import cook
from updatebot.lib import conarycallbacks
from updatebot.errors import LocalCookFailedError
@@ -56,11 +57,16 @@
self._client = conaryclient.ConaryClient(self._ccfg)
- def cook(self, troveSpecs):
+ def cook(self, troveSpecs, flavorFilter=None):
"""
Cook a set of trove specs, currently limited to groups.
@params troveSpecs: list of name, version, and flavor tuples.
@type troveSpecs: [(name, version, flavor), ... ]
+ @params flavorFilter: Allow caller to filter out the contexts that they
+ want to build. This is mostly used for group
+ building where a given group should not be built
+ for a context.
+ @type flavorFilter: iterable of context names.
"""
# TODO: Look at conary.build.cook.cookCommand for how to setup
@@ -68,6 +74,9 @@
troveSpecs = self._formatInput(troveSpecs)
+ if flavorFilter:
+ troveSpecs = self._filterTroveSpecs(troveSpecs, flavorFilter)
+
# make sure all troves are groups
assert not [ x for x in troveSpecs if not x[0].startswith('group-') ]
@@ -113,3 +122,30 @@
res = { (troveSpecs[0][0], troveSpecs[0][1], None): results }
return res
+
+ def _filterTroveSpecs(self, troveSpecs, useFlags):
+ """
+ Filter trove specs based on a list of use flags. This is only applicable
+ to groups.
+ @param troveSpecs: iterable of nvf tuples.
+ @type troveSpecs: list(tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ @param useFlags: iterable of valid use flags (x86 and x86_64)
+ @type useFlags: list(str, ...)
+ @return modified list of trove specs
+ @rtype list(tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ """
+
+ useMap = {
+ 'x86': deps.parseFlavor('is: x86'),
+ 'x86_64': deps.parseFlavor('is: x86_64'),
+ }
+
+ specs = set()
+ for n, v, f in troveSpecs:
+ for flag in useFlags:
+ if f.satisfies(useMap[flag]):
+ specs.add((n, v, f))
+
+ return list(specs)
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -221,7 +221,8 @@
"""
groupTroves = self.getBuildJob()
- built = self._builder.cvc.cook(groupTroves)
+ useFlags = self._getUseFlags()
+ built = self._builder.cvc.cook(groupTroves, flavorFilter=useFlags)
return built
@checkout
@@ -408,6 +409,20 @@
# Unknown state.
raise UnhandledPackageAdditionError(name=name)
+ def _getUseFlags(self):
+ """
+ Get the set of use flags used across all managed groups.
+ """
+
+ use = set()
+ for group in self._groups.itervalues():
+ for name, pkg in group.iteritems():
+ if pkg.use:
+ use.add(pkg.use)
+ else:
+ use.update(set(['x86', 'x86_64']))
+ return use
+
@checkout
def setVersion(self, version):
"""
From elliot at rpath.com Tue Apr 20 18:21:26 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:26 +0000
Subject: mirrorball: pick the latest version when there is more than one
Message-ID: <201004202221.o3KMLQKj019548@scc.eng.rpath.com>
changeset: 6e3de0294394
user: Elliot Peele
date: Fri, 16 Apr 2010 15:47:40 -0400
pick the latest version when there is more than one
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -604,9 +604,15 @@
# Handle the case where a package has been rebuilt for some
# reason, but we need to use the old version of the package.
pkgName = n.split(':')[0]
- if len(vMap) > 1 and pkgName in exceptions:
- vMap = dict((x, y) for x, y in vMap.iteritems()
- if x == exceptions[pkgName])
+ if len(vMap) > 1:
+ if pkgName in exceptions:
+ log.info('using old version of %s' % n)
+ vMap = dict((x, y) for x, y in vMap.iteritems()
+ if x == exceptions[pkgName])
+ else:
+ log.info('found multiple versions of %s, using latest' % n)
+ v = sorted(vMap)[-1]
+ vMap = { v: vMap[v], }
assert len(vMap) == 1
From elliot at rpath.com Tue Apr 20 18:21:27 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:27 +0000
Subject: mirrorball: include the dvds in our mirrors
Message-ID: <201004202221.o3KMLRG1019575@scc.eng.rpath.com>
changeset: 39f8a07ce263
user: Elliot Peele
date: Fri, 16 Apr 2010 15:48:05 -0400
include the dvds in our mirrors
diff --git a/scripts/sync-centos.sh b/scripts/sync-centos.sh
--- a/scripts/sync-centos.sh
+++ b/scripts/sync-centos.sh
@@ -13,7 +13,7 @@
# full details.
#
-SOURCE=rsync://mirrors.us.kernel.org/CentOS-nodvd
+SOURCE=rsync://mirrors.us.kernel.org/CentOS-incdvd
DEST=/l/CentOS/
date
From elliot at rpath.com Tue Apr 20 18:21:27 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:27 +0000
Subject: mirrorball: add some new scripts
Message-ID: <201004202221.o3KMLSYF019602@scc.eng.rpath.com>
changeset: da549fc7f200
user: Elliot Peele
date: Fri, 16 Apr 2010 15:48:32 -0400
add some new scripts
diff --git a/scripts/grouptoxml.py b/scripts/creategroup.py
copy from scripts/grouptoxml.py
copy to scripts/creategroup.py
--- a/scripts/grouptoxml.py
+++ b/scripts/creategroup.py
@@ -10,395 +10,554 @@
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
+# full details.
#
-"""
-Script to used to generate xml model for standard rhel groups.
-"""
-
-rhel4corePackages = (
- ('kernel', 'kernel.smp,!kernel.largesmp,!kernel.hugemem,!xen,!domU,!dom0'),
- 'acl',
- 'ash',
- 'attr',
- 'audit',
- 'audit-libs',
- 'authconfig',
- 'basesystem',
- 'bash',
- 'beecrypt',
- 'bzip2',
- 'bzip2-libs',
- 'checkpolicy',
- 'chkconfig',
- 'coreutils',
- 'cpio',
- 'cracklib',
- 'cracklib-dicts',
- 'crontabs',
- 'cyrus-sasl',
- 'cyrus-sasl-md5',
- 'db4',
- 'dbus',
- 'dbus-glib',
- 'device-mapper',
- 'dhclient',
- 'diffutils',
- 'dmraid',
- 'e2fsprogs',
- 'ed',
- 'elfutils-libelf',
- 'ethtool',
- 'expat',
- 'file',
- 'filesystem',
- 'findutils',
- 'fontconfig',
- 'freetype',
- 'gawk',
- 'gdbm',
- 'glib2',
- 'glibc',
- 'glibc-common',
- 'gmp',
- 'grep',
- 'grub',
- 'gzip',
- 'hal',
- 'hdparm',
- 'hesiod',
- 'hotplug',
- 'hwdata',
- 'indexhtml',
- 'info',
- 'initscripts',
- 'iproute',
- 'iptables',
- 'iputils',
- 'kbd',
- 'krb5-libs',
- 'kudzu',
- 'less',
- 'libacl',
- 'libattr',
- 'libcap',
- 'libgcc',
- 'libgcrypt',
- 'libgpg-error',
- 'libselinux',
- 'libsepol',
- 'libstdc++',
- 'libtermcap',
- 'libuser',
- 'libxml2',
- 'libxml2-python',
- 'libxslt',
- 'libxslt-python',
- 'logrotate',
- 'lvm2',
- 'MAKEDEV',
- 'mdadm',
- 'mingetty',
- 'mkinitrd',
- 'mktemp',
- 'module-init-tools',
- 'ncurses',
- 'net-tools',
- 'newt',
- 'nscd',
- 'openldap',
- 'openssl',
- 'pam',
- 'parted',
- 'passwd',
- 'pciutils',
- 'pcre',
- 'perl',
- 'perl-Filter',
- 'policycoreutils',
- 'popt',
- 'prelink',
- 'procmail',
- 'procps',
- 'psmisc',
- 'python',
- 'pyxf86config',
- 'readline',
- 'redhat-logos',
- 'redhat-release',
- 'rhpl',
- 'rootfiles',
- 'rpm',
- 'rpmdb-redhat',
- 'rpm-libs',
- 'sed',
- 'selinux-doc',
- 'selinux-policy-targeted',
- 'sendmail',
- 'setools',
- 'setserial',
- 'setup',
- 'shadow-utils',
- 'slang',
- 'sysklogd',
- 'system-config-mouse',
- 'SysVinit',
- 'tar',
- 'tcp_wrappers',
- 'termcap',
- 'tmpwatch',
- 'tzdata',
- 'udev',
- 'usbutils',
- 'usermode',
- 'util-linux',
- 'vim-minimal',
- 'vixie-cron',
- 'wireless-tools',
- 'xorg-x11-libs',
- 'xorg-x11-Mesa-libGL',
- 'zlib',
-)
-
-
-rhel5corePackages = (
- ('kernel', 'kernel.smp,!kernel.debug,!kernel.pae,!xen,!domU,!dom0'),
- 'acl',
- 'alchemist',
- 'atk',
- 'attr',
- 'audit',
- 'audit-libs',
- 'audit-libs-python',
- 'authconfig',
- 'basesystem',
- 'bash',
- 'beecrypt',
- 'bzip2',
- 'bzip2-libs',
- 'cairo',
- 'checkpolicy',
- 'chkconfig',
- 'coreutils',
- 'cpio',
- 'cracklib',
- 'cracklib-dicts',
- 'crontabs',
- 'cryptsetup-luks',
- 'cups-libs',
- 'cyrus-sasl',
- 'cyrus-sasl-lib',
- 'cyrus-sasl-md5',
- 'db4',
- 'dbus',
- 'dbus-glib',
- 'Deployment_Guide-en-US',
- 'device-mapper',
- 'device-mapper-multipath',
- 'dhclient',
- 'diffutils',
- 'dmidecode',
- 'dmraid',
- 'e2fsprogs',
- 'e2fsprogs-libs',
- 'ed',
- 'elfutils-libelf',
- 'ethtool',
- 'expat',
- 'file',
- 'filesystem',
- 'findutils',
- 'fontconfig',
- 'freetype',
- 'gawk',
- 'gdbm',
- 'glib2',
- 'glibc',
- 'glibc-common',
- 'gmp',
- 'gnutls',
- 'grep',
- 'grub',
- 'gtk2',
- 'gzip',
- 'hal',
- 'hdparm',
- 'hesiod',
- 'hicolor-icon-theme',
- 'hwdata',
- 'info',
- 'initscripts',
- 'iproute',
- 'iptables',
- 'iputils',
- 'kbd',
- 'kernel-headers',
- 'keyutils-libs',
- 'kpartx',
- 'krb5-libs',
- 'kudzu',
- 'less',
- 'libacl',
- 'libattr',
- 'libcap',
- 'libgcc',
- 'libgcrypt',
- 'libgpg-error',
- 'libhugetlbfs',
- 'libhugetlbfs-lib',
- 'libjpeg',
- 'libpng',
- 'libselinux',
- 'libselinux-python',
- 'libsemanage',
- 'libsepol',
- 'libstdc++',
- 'libsysfs',
- 'libtermcap',
- 'libtiff',
- 'libusb',
- 'libuser',
- 'libvolume_id',
- 'libX11',
- 'libXau',
- 'libXcursor',
- 'libXdmcp',
- 'libXext',
- 'libXfixes',
- 'libXft',
- 'libXi',
- 'libXinerama',
- 'libxml2',
- 'libxml2-python',
- 'libXrandr',
- 'libXrender',
- 'libxslt',
- 'libxslt-python',
- 'logrotate',
- 'lvm2',
- 'MAKEDEV',
- 'mcstrans',
- 'mdadm',
- 'mingetty',
- 'mkinitrd',
- 'mktemp',
- 'module-init-tools',
- 'nash',
- 'ncurses',
- 'net-tools',
- 'newt',
- 'nscd',
- 'openldap',
- 'openssl',
- 'pam',
- 'pango',
- 'parted',
- 'passwd',
- 'pciutils',
- 'pcre',
- 'pm-utils',
- 'policycoreutils',
- 'popt',
- 'prelink',
- 'procmail',
- 'procps',
- 'psmisc',
- 'pycairo',
- 'pygobject2',
- 'pygtk2',
- 'python',
- 'python-numeric',
- 'pyxf86config',
- 'readline',
- 'redhat-logos',
- 'redhat-release',
- 'redhat-release-notes',
- 'rhpl',
- 'rootfiles',
- 'rpm',
- 'rpm-libs',
- 'sed',
- 'selinux-policy',
- 'selinux-policy-targeted',
- 'sendmail',
- 'setools',
- 'setserial',
- 'setup',
- 'shadow-utils',
- 'slang',
- 'sqlite',
- 'sysfsutils',
- 'sysklogd',
- 'SysVinit',
- 'tar',
- 'tcl',
- 'tcp_wrappers',
- 'termcap',
- 'tmpwatch',
- 'tzdata',
- 'udev',
- 'usbutils',
- 'usermode',
- 'util-linux',
- 'vim-minimal',
- 'vixie-cron',
- 'wireless-tools',
- 'xorg-x11-filesystem',
- 'zlib',
-)
-
import os
import sys
-import time
-import logging
-sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
-sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py')
-sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib')
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
+
+import rmake
+import conary
+import updatebot
+
+print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
+print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
+print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
from conary.lib import util
sys.excepthook = util.genExcepthook()
-from conary.deps import deps
+import logging
-mbdir = os.path.abspath('../')
-sys.path.insert(0, mbdir)
+from updatebot import groupmgr
+from updatebot import OrderedBot
+from updatebot.groupmgr.model import GroupContentsModel
-from updatebot import log
-from updatebot import groupmgr
+log = logging.getLogger('tmplogger')
-slog = log.addRootLogger()
+class Bot(OrderedBot):
+ def generateInitialGroup(self):
+ """
+ Generate config for standard group contents based on repository history.
+ """
-def toxml(pkgList, toFile):
- groupName = 'group-rhel-standard'
- byDefault = True
- depCheck = True
+ standard = (
+ 'aaa_base',
+ 'aaa_skel',
+ 'acl',
+ 'acpid',
+ 'ash',
+ 'at',
+# 'atk',
+# 'atk-devel',
+# 'atk-doc',
+ 'attr',
+# 'audiofile',
+# 'audiofile-devel',
+ 'audit',
+ 'audit-devel',
+ 'audit-libs',
+# 'autoconf',
+# 'autofs',
+ 'bash',
+# 'bc',
+# 'bin86',
+# 'bind',
+# 'bind-chrootenv',
+# 'bind-devel',
+# 'bind-doc',
+# 'bind-libs',
+# 'bind-utils',
+ 'binutils',
+# 'bison',
+ 'blt',
+ 'blocxx',
+ 'busybox',
+ 'bzip2',
+# 'cairo',
+# 'cairo-devel',
+# 'cairo-doc',
+# 'cdrecord',
+# 'cdrecord-devel',
+# 'cifs-mount',
+# 'compat-libstdc++',
+# 'compat-openssl097g',
+ 'coreutils',
+ 'cpio',
+# 'cpp',
+ 'cracklib',
+# 'cracklib-devel',
+ 'cron',
+# 'curl',
+# 'curl-devel',
+# 'cvs',
+# 'cvs-doc',
+ 'cyrus-sasl',
+# 'cyrus-sasl-crammd5',
+# 'cyrus-sasl-devel',
+# 'cyrus-sasl-digestmd5',
+# 'cyrus-sasl-gssapi',
+# 'cyrus-sasl-otp',
+# 'cyrus-sasl-plain',
+# 'cyrus-sasl-sqlauxprop',
+ 'db',
+ 'db42',
+# 'db-devel',
+# 'db-utils',
+ 'dbus-1',
+# 'dbus-1-devel',
+ 'dbus-1-glib',
+# 'dbus-1-gtk',
+# 'dbus-1-java',
+# 'dbus-1-mono',
+# 'dbus-1-python',
+# 'dbus-1-qt3',
+# 'dbus-1-qt3-devel',
+# 'dbus-1-x11',
+# 'dev86',
+ 'device-mapper',
+# 'device-mapper-devel',
+ 'dhcpcd',
+ 'diffutils',
+ 'e2fsprogs',
+# 'e2fsprogs-devel',
+# 'eject',
+# 'esound',
+# 'esound-devel',
+ 'ethtool',
+ 'expat',
+# 'expect',
+ 'file',
+# 'file-devel',
+ 'filesystem',
+ 'fillup',
+ 'findutils',
+# 'findutils-locate',
+ 'fontconfig',
+# 'fontconfig-devel',
+# 'freetype',
+# 'freetype-tools',
+ 'freetype2',
+# 'freetype2-devel',
+ 'gawk',
+# 'gawk-doc',
+# 'gcc',
+# 'gcc-c++',
+# 'gcc-fortran',
+# 'gcc-info',
+# 'gcc-java',
+# 'gcc-locale',
+# 'gcc-obj-c++',
+# 'gcc-objc',
+# 'gconf2',
+# 'gconf2-devel',
+# 'gconf2-doc',
+# 'gdb',
+ 'gdbm',
+# 'gdbm-devel',
+ 'gettext',
+# 'gettext-devel',
+ 'glib2',
+# 'glib2-devel',
+# 'glib2-doc',
+ 'glibc',
+# 'glibc-debuginfo',
+# 'glibc-devel',
+# 'glibc-html',
+# 'glibc-i18ndata',
+# 'glibc-info',
+# 'glibc-locale',
+# 'glibc-profile',
+# 'glitz',
+# 'glitz-devel',
+# 'gmp',
+# 'gmp-devel',
+ 'gnome-filesystem',
+# 'gnome-vfs2',
+# 'gnome-vfs2-devel',
+# 'gnome-vfs2-doc',
+# 'gnuplot',
+# 'gnutls',
+# 'gnutls-devel',
+ 'gpg',
+# 'gpg2',
+# 'gpm',
+ 'grep',
+# 'groff',
+ 'grub',
+# 'gtk2',
+# 'gtk2-devel',
+# 'gtk2-doc',
+# 'gvim',
+# 'gxdview',
+ 'gzip',
+ 'hal',
+# 'hal-devel',
+# 'hal-gnome',
+# 'hdparm',
+ 'hwinfo',
+# 'hwinfo-devel',
+ 'info',
+ 'insserv',
+ 'iproute2',
+ 'iptables',
+# 'iptables-devel',
+ 'iputils',
+# 'jpeg',
+# 'kbd',
+ 'klogd',
+ 'krb5',
+# 'krb5-apps-clients',
+# 'krb5-apps-servers',
+# 'krb5-client',
+# 'krb5-devel',
+# 'krb5-server',
+# 'ksh',
+# 'ksh-devel',
+ 'less',
+ 'libacl',
+# 'libacl-devel',
+ 'libaio',
+# 'libaio-devel',
+# 'libapr-util1',
+# 'libapr-util1-devel',
+# 'libapr1',
+# 'libapr1-devel',
+# 'libart_lgpl',
+# 'libart_lgpl-devel',
+ 'libattr',
+# 'libattr-devel',
+# 'libbonobo',
+# 'libbonobo-devel',
+# 'libbonobo-doc',
+# 'libbonoboui',
+# 'libbonoboui-devel',
+# 'libbonoboui-doc',
+ 'libcap',
+# 'libcap-devel',
+ 'libcom_err',
+ 'libelf',
+ 'libevent',
+ 'libgcc',
+# 'libgcj',
+# 'libgcj-devel',
+ 'libgcrypt',
+# 'libgcrypt-devel',
+# 'libgfortran',
+# 'libgnome',
+# 'libgnome-devel',
+# 'libgnome-doc',
+# 'libgnomecanvas',
+# 'libgnomecanvas-devel',
+# 'libgnomecanvas-doc',
+ 'libgpg-error',
+# 'libgpg-error-devel',
+# 'libgssapi',
+# 'libidn',
+# 'libidn-devel',
+# 'libiniparser',
+# 'libiniparser-devel',
+# 'libjpeg',
+# 'libjpeg-devel',
+# 'libksba',
+# 'libksba-devel',
+# 'libmsrpc',
+# 'libmsrpc-devel',
+# 'libmudflap',
+# 'libnlink',
+ 'libnscd',
+# 'libnscd-devel',
+# 'libobjc',
+# 'libopencdk',
+# 'libopencdk-devel',
+# 'libpcap',
+# 'libpng',
+# 'libpng-devel',
+# 'librpcsecgss',
+# 'libsmbclient',
+# 'libsmbclient-devel',
+ 'libstdc++',
+# 'libstdc++-devel',
+# 'libstdc++-doc',
+ 'libtool',
+ 'libusb',
+ 'libxcrypt',
+# 'libxcrypt-devel',
+ 'libxml2',
+# 'libxml2-devel',
+# 'libxml2-python',
+ 'libxslt',
+# 'libxslt-devel',
+ 'libzio',
+ 'limal',
+ 'limal-bootloader',
+ 'limal-perl',
+ 'logrotate',
+# 'lsof',
+ 'lvm2',
+# 'lzo',
+# 'lzo-devel',
+ 'm4',
+# 'mailx',
+# 'make',
+# 'man',
+# 'mdadm',
+# 'microcode_ctl',
+ 'mingetty',
+ 'mkinitrd',
+# 'mkisofs',
+ 'mktemp',
+# 'mm',
+# 'mm-devel',
+ 'module-init-tools',
+# 'mysql',
+# 'mysql-Max',
+# 'mysql-client',
+# 'mysql-devel',
+# 'mysql-shared',
+# 'nc6',
+ 'ncurses',
+# 'ncurses-devel',
+# 'neon',
+# 'net-snmp',
+# 'net-snmp-devel',
+ 'net-tools',
+ 'netcfg',
+# 'nfs-utils',
+# 'nfsidmap',
+# 'nmap',
+# 'nmap-gtk',
+ 'nscd',
+ 'openct',
+# 'openct-devel',
+ 'openldap2',
+# 'openldap2-back-meta',
+# 'openldap2-back-perl',
+ 'openldap2-client',
+# 'openldap2-devel',
+ 'opensc',
+# 'opensc-devel',
+ 'openslp',
+# 'openslp-devel',
+# 'openslp-server',
+ 'openssh',
+# 'openssh-askpass',
+ 'openssl',
+# 'openssl-devel',
+# 'openssl-doc',
+# 'orbit2',
+# 'orbit2-devel',
+ 'pam',
+# 'pam-devel',
+ 'pam-modules',
+# 'pam_krb5',
+# 'pam_smb',
+# 'pango',
+# 'pango-devel',
+# 'pango-doc',
+# 'parted',
+# 'parted-devel',
+# 'patch',
+ 'pciutils',
+# 'pciutils-devel',
+ 'pciutils-ids',
+ 'pcre',
+# 'pcre-devel',
+ 'pcsc-lite',
+# 'pcsc-lite-devel',
+ 'perl',
+# 'perl-Bit-Vector',
+ 'perl-Bootloader',
+# 'perl-Carp-Clan',
+# 'perl-Compress-Zlib',
+# 'perl-DBD-SQLite',
+# 'perl-DBD-mysql',
+# 'perl-DBI',
+# 'perl-Data-ShowTable',
+# 'perl-Date-Calc',
+# 'perl-Digest-SHA1',
+# 'perl-Net-Daemon',
+# 'perl-PlRPC',
+# 'perl-SNMP',
+# 'perl-TermReadKey',
+# 'perl-URI',
+# 'perl-XML-Parser',
+# 'perl-XML-Writer',
+ 'perl-gettext',
+ 'permissions',
+# 'pinentry',
+# 'pkgconfig',
+# 'pmtools',
+ 'popt',
+# 'popt-devel',
+# 'portmap',
+# 'postgresql',
+# 'postgresql-contrib',
+# 'postgresql-devel',
+# 'postgresql-docs',
+# 'postgresql-libs',
+# 'postgresql-server',
+ 'procmail',
+ 'procps',
+ 'psmisc',
+ 'pwdutils',
+# 'pwdutils-plugin-audit',
+ 'python',
+# 'python-cairo',
+ 'python-curses',
+# 'python-demo',
+ 'python-devel',
+ 'python-gdbm',
+# 'python-gnome',
+# 'python-gtk',
+# 'python-idle',
+# 'python-numeric',
+# 'python-orbit',
+# 'python-pam',
+ 'python-tk',
+ 'python-xml',
+ 'readline',
+# 'readline-devel',
+ 'reiserfs',
+# 'resmgr',
+ 'rpm',
+# 'rpm-devel',
+ 'rpm-python',
+# 'rrdtool',
+# 'rsync',
+# 'samba',
+# 'samba-client',
+# 'samba-krb-printing',
+# 'samba-python',
+# 'samba-vscan',
+# 'samba-winbind',
+ 'sed',
+ 'sendmail',
+# 'sendmail-devel',
+# 'sensors',
+# 'slang',
+# 'slang-devel',
+ 'sles-release',
+# 'smartmontools',
+# 'sqlite',
+# 'sqlite-devel',
+# 'strace',
+# 'sudo',
+ 'sysconfig',
+ 'sysfsutils',
+# 'syslinux',
+ 'syslog-ng',
+ 'syslogd',
+# 'sysstat',
+# 'sysstat-isag',
+ 'sysvinit',
+ 'suse-build-key',
+ 'tar',
+ 'tcl',
+# 'tcl-devel',
+ 'tcpd',
+# 'tcpd-devel',
+# 'tcpdump',
+ 'tcsh',
+# 'telnet',
+# 'telnet-server',
+ 'termcap',
+ 'terminfo',
+# 'texinfo',
+ 'timezone',
+ 'tk',
+# 'tk-devel',
+ 'udev',
+# 'unixODBC',
+# 'unixODBC-devel',
+# 'unzip',
+# 'usbutils',
+ 'util-linux',
+# 'uucp',
+ 'vim',
+ 'wget',
+# 'wireless-tools',
+# 'x11-tools',
+ 'xfsprogs',
+# 'xfsprogs-devel',
+# 'xinetd',
+# 'xkeyboard-config',
+# 'xntp',
+# 'xntp-doc',
+# 'xorg-x11',
+# 'xorg-x11-Xnest',
+# 'xorg-x11-Xvfb',
+# 'xorg-x11-Xvnc',
+# 'xorg-x11-devel',
+# 'xorg-x11-doc',
+# 'xorg-x11-fonts-100dpi',
+# 'xorg-x11-fonts-75dpi',
+# 'xorg-x11-fonts-cyrillic',
+# 'xorg-x11-fonts-scalable',
+# 'xorg-x11-fonts-syriac',
+ 'xorg-x11-libs',
+# 'xorg-x11-man',
+# 'xorg-x11-sdk',
+# 'xorg-x11-server',
+# 'xorg-x11-server-glx',
+# 'yp-tools',
+# 'ypbind',
+# 'zip',
+# 'zisofs-tools',
+ 'zlib',
+# 'zlib-devel',
+ )
- contents = groupmgr.GroupContentsModel(groupName,
- byDefault=byDefault,
- depCheck=depCheck)
+ troves = self._updater._conaryhelper._getLatestTroves()
- for pkg in pkgList:
- slog.info('adding %s' % (pkg, ))
- if type(pkg) == tuple:
- pkg, flavor = pkg
- contents.add(pkg, flavor=deps.parseFlavor(flavor))
- else:
- contents.add(pkg)
+ pkgs = set()
+ for name, vMap in troves.iteritems():
+ if name.endswith(':source'):
+ continue
+ name = name.split(':')[0]
+ for version, flavors in vMap.iteritems():
+ pkgs.add((name, version, tuple(flavors)))
- contents.freeze(toFile)
+ for name, version, flavors in pkgs:
+ log.info('adding %s=%s' % (name, version))
+ for flv in flavors:
+ log.info('\t%s' % flv)
+ self._groupmgr.addPackage(name, version, flavors)
+ self._groupmgr.setErrataState(0)
+ self._groupmgr.setVersion('0')
+
+ pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
+# pkgGroup.depCheck = False
+
+ contents = GroupContentsModel('group-standard', depCheck=True)
+ self._groupmgr._groups['group-standard'] = contents
+
+ for pkgName in standard:
+ for key in pkgGroup._nameMap[pkgName]:
+ contents._addItem(pkgGroup._data[key])
+
+ built = self._groupmgr.build()
+ return built
if __name__ == '__main__':
- platforms = {
- 'rhel4': rhel4corePackages,
- 'rhel5': rhel5corePackages,
- }
+ from updatebot import config
+ from updatebot import log as logSetup
- platform = sys.argv[1]
- assert platform in platforms
+ logSetup.addRootLogger()
- toFile = sys.argv[2]
+ log = logging.getLogger('create group')
- pkgs = platforms[platform]
- toxml(pkgs, toFile)
+ cfg = config.UpdateBotConfig()
+ cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+
+ bot = Bot(cfg, None)
+ changes = bot.generateInitialGroup()
+
+ import epdb; epdb.st()
diff --git a/scripts/depsolver.py b/scripts/depsolver.py
new file mode 100755
--- /dev/null
+++ b/scripts/depsolver.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2008-2009 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+import os
+import sys
+
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+from conary.lib import util
+sys.excepthook = util.genExcepthook()
+
+import copy
+import logging
+import updatebot.log
+
+updatebot.log.addRootLogger()
+log = logging.getLogger('test')
+
+from updatebot import config
+from updatebot import pkgsource
+
+cfg = config.UpdateBotConfig()
+cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
+
+pkgSource = pkgsource.PackageSource(cfg)
+pkgSource.load()
+
+reqSrcPkgs = set()
+for pkgName in cfg.package:
+ bins = sorted(pkgSource.binNameMap.get(pkgName, []))
+ if not bins:
+ continue
+ reqSrcPkgs.add(pkgSource.binPkgMap[bins[-1]])
+
+reqBinPkgs = set()
+for srcPkg in reqSrcPkgs:
+ reqBinPkgs.update(set([ x for x in pkgSource.srcPkgMap[srcPkg]
+ if x.arch not in ('nosrc', 'src') ]))
+
+requires = {}
+provides = {}
+log.info('loading package requires/provides')
+for bin, src in pkgSource.binPkgMap.iteritems():
+ if bin.arch in ('nosrc', 'src'):
+ continue
+ for format in bin.format:
+ if format.getName() == 'rpm:provides':
+ bins = set([ x.name for x in pkgSource.srcPkgMap[src]
+ if x.arch not in ('nosrc', 'src') ])
+ for child in format.iterChildren():
+ provides.setdefault(child.name, set()).update(bins)
+ if format.getName() == 'rpm:requires':
+ for child in format.iterChildren():
+ requires.setdefault(bin.name, set()).add(child.name)
+
+
+solved = set()
+working = set([ x.name for x in reqBinPkgs ])
+
+log.info('resolving deps')
+while working:
+ pkg = working.pop()
+ log.info('resolving %s' % pkg)
+ for req in requires[pkg]:
+ if req.startswith('rpmlib'):
+ continue
+ if req not in provides:
+ log.warn('requirement not found: %s' % req)
+ continue
+ for provPkg in provides[req]:
+ if provPkg not in solved and provPkg != pkg:
+ working.add(provPkg)
+
+ solved.add(pkg)
+ log.info('solved: %s, working set: %s' % (len(solved), len(working)))
+
+needed = set()
+for pkgName in solved:
+ bins = sorted(pkgSource.binNameMap[pkgName])
+ src = pkgSource.binPkgMap[bins[-1]]
+ if src in reqSrcPkgs:
+ continue
+ pkgs = sorted([ x for x in pkgSource.srcPkgMap[src]
+ if x.arch not in ('nosrc', 'src') ])
+ needed.add(pkgs[0])
+
+import epdb; epdb.st()
+
+
From elliot at rpath.com Tue Apr 20 18:21:28 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:28 +0000
Subject: mirrorball: filter exceptions down to a name:version dictionary
Message-ID: <201004202221.o3KMLTsw019623@scc.eng.rpath.com>
changeset: 089936b371ab
user: Elliot Peele
date: Mon, 19 Apr 2010 11:20:29 -0400
filter exceptions down to a name:version dictionary
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -429,7 +429,8 @@
# Find excepted promote packages.
srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages(
bucket)
- exceptions = self._getOldVersionExceptions(updateId)
+ exceptions = dict([ (x[0], x[1]) for x in itertools.chain(
+ *self._getOldVersionExceptions(updateId).itervalues()) ])
# These are the binary trove specs that we expect to be promoted.
expected = self._filterBinPkgSet(
From elliot at rpath.com Tue Apr 20 18:21:30 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:30 +0000
Subject: mirrorball: select latest cloned from
Message-ID: <201004202221.o3KMLUsB019656@scc.eng.rpath.com>
changeset: 2ee8321f0c7d
user: Elliot Peele
date: Tue, 20 Apr 2010 11:10:23 -0400
select latest cloned from
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -371,8 +371,8 @@
ret = {}
for f, t in cfMap.iteritems():
- assert len(cfMap[f]) == 1
- ret[f] = list(t)[0]
+ assert len(cfMap[f]) >= 1
+ ret[f] = sorted(t)[-1]
self._cache.labelClonedFromCache[label] = ret
return ret
From elliot at rpath.com Tue Apr 20 18:21:30 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:30 +0000
Subject: mirrorball: allow package sources to not change if a source was
added manually to a given
Message-ID: <201004202221.o3KMLVCk019683@scc.eng.rpath.com>
changeset: 27b26135f7be
user: Elliot Peele
date: Tue, 20 Apr 2010 18:20:23 -0400
allow package sources to not change if a source was added manually to a given
updateId
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -279,9 +279,17 @@
# validate updates
try:
- assert updater._sanitizeTrove(nvf, srpm,
+ toUpdate = updater._sanitizeTrove(nvf, srpm,
expectedRemovals=expectedRemovals + expectedReplaces,
allowPackageDowngrades=explicitPackageDowngrades)
+
+ # If a source was manually added to this updateId it may
+ # have already been part of another update, which would
+ # cause the manifest not to change.
+ if (srpm.getNevra() not in
+ self._cfg.addSource.get(updateId, [])):
+ assert toUpdate
+
except (UpdateGoesBackwardsError,
UpdateRemovesPackageError,
UpdateReusesPackageError), e:
From elliot at rpath.com Tue Apr 20 18:21:32 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Apr 2010 22:21:32 +0000
Subject: mirrorball: branch merge
Message-ID: <201004202221.o3KMLWkf019710@scc.eng.rpath.com>
changeset: b5c62432c502
user: Elliot Peele
date: Tue, 20 Apr 2010 18:21:17 -0400
branch merge
diff --git a/repomd/packagexml.py b/repomd/packagexml.py
--- a/repomd/packagexml.py
+++ b/repomd/packagexml.py
@@ -180,6 +180,14 @@
ver = "_".join(nameVerRelease.split("-")[-2:])
return ver
+ def getFileName(self):
+ """
+ Returns the expected package file name.
+ """
+
+ return '%s-%s-%s.%s.rpm' % (self.name, self.version,
+ self.release, self.arch)
+
class _RpmEntry(SlotNode):
"""
diff --git a/scripts/grouptoxml.py b/scripts/creategroup.py
copy from scripts/grouptoxml.py
copy to scripts/creategroup.py
--- a/scripts/grouptoxml.py
+++ b/scripts/creategroup.py
@@ -10,395 +10,554 @@
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
+# full details.
#
-"""
-Script to used to generate xml model for standard rhel groups.
-"""
-
-rhel4corePackages = (
- ('kernel', 'kernel.smp,!kernel.largesmp,!kernel.hugemem,!xen,!domU,!dom0'),
- 'acl',
- 'ash',
- 'attr',
- 'audit',
- 'audit-libs',
- 'authconfig',
- 'basesystem',
- 'bash',
- 'beecrypt',
- 'bzip2',
- 'bzip2-libs',
- 'checkpolicy',
- 'chkconfig',
- 'coreutils',
- 'cpio',
- 'cracklib',
- 'cracklib-dicts',
- 'crontabs',
- 'cyrus-sasl',
- 'cyrus-sasl-md5',
- 'db4',
- 'dbus',
- 'dbus-glib',
- 'device-mapper',
- 'dhclient',
- 'diffutils',
- 'dmraid',
- 'e2fsprogs',
- 'ed',
- 'elfutils-libelf',
- 'ethtool',
- 'expat',
- 'file',
- 'filesystem',
- 'findutils',
- 'fontconfig',
- 'freetype',
- 'gawk',
- 'gdbm',
- 'glib2',
- 'glibc',
- 'glibc-common',
- 'gmp',
- 'grep',
- 'grub',
- 'gzip',
- 'hal',
- 'hdparm',
- 'hesiod',
- 'hotplug',
- 'hwdata',
- 'indexhtml',
- 'info',
- 'initscripts',
- 'iproute',
- 'iptables',
- 'iputils',
- 'kbd',
- 'krb5-libs',
- 'kudzu',
- 'less',
- 'libacl',
- 'libattr',
- 'libcap',
- 'libgcc',
- 'libgcrypt',
- 'libgpg-error',
- 'libselinux',
- 'libsepol',
- 'libstdc++',
- 'libtermcap',
- 'libuser',
- 'libxml2',
- 'libxml2-python',
- 'libxslt',
- 'libxslt-python',
- 'logrotate',
- 'lvm2',
- 'MAKEDEV',
- 'mdadm',
- 'mingetty',
- 'mkinitrd',
- 'mktemp',
- 'module-init-tools',
- 'ncurses',
- 'net-tools',
- 'newt',
- 'nscd',
- 'openldap',
- 'openssl',
- 'pam',
- 'parted',
- 'passwd',
- 'pciutils',
- 'pcre',
- 'perl',
- 'perl-Filter',
- 'policycoreutils',
- 'popt',
- 'prelink',
- 'procmail',
- 'procps',
- 'psmisc',
- 'python',
- 'pyxf86config',
- 'readline',
- 'redhat-logos',
- 'redhat-release',
- 'rhpl',
- 'rootfiles',
- 'rpm',
- 'rpmdb-redhat',
- 'rpm-libs',
- 'sed',
- 'selinux-doc',
- 'selinux-policy-targeted',
- 'sendmail',
- 'setools',
- 'setserial',
- 'setup',
- 'shadow-utils',
- 'slang',
- 'sysklogd',
- 'system-config-mouse',
- 'SysVinit',
- 'tar',
- 'tcp_wrappers',
- 'termcap',
- 'tmpwatch',
- 'tzdata',
- 'udev',
- 'usbutils',
- 'usermode',
- 'util-linux',
- 'vim-minimal',
- 'vixie-cron',
- 'wireless-tools',
- 'xorg-x11-libs',
- 'xorg-x11-Mesa-libGL',
- 'zlib',
-)
-
-
-rhel5corePackages = (
- ('kernel', 'kernel.smp,!kernel.debug,!kernel.pae,!xen,!domU,!dom0'),
- 'acl',
- 'alchemist',
- 'atk',
- 'attr',
- 'audit',
- 'audit-libs',
- 'audit-libs-python',
- 'authconfig',
- 'basesystem',
- 'bash',
- 'beecrypt',
- 'bzip2',
- 'bzip2-libs',
- 'cairo',
- 'checkpolicy',
- 'chkconfig',
- 'coreutils',
- 'cpio',
- 'cracklib',
- 'cracklib-dicts',
- 'crontabs',
- 'cryptsetup-luks',
- 'cups-libs',
- 'cyrus-sasl',
- 'cyrus-sasl-lib',
- 'cyrus-sasl-md5',
- 'db4',
- 'dbus',
- 'dbus-glib',
- 'Deployment_Guide-en-US',
- 'device-mapper',
- 'device-mapper-multipath',
- 'dhclient',
- 'diffutils',
- 'dmidecode',
- 'dmraid',
- 'e2fsprogs',
- 'e2fsprogs-libs',
- 'ed',
- 'elfutils-libelf',
- 'ethtool',
- 'expat',
- 'file',
- 'filesystem',
- 'findutils',
- 'fontconfig',
- 'freetype',
- 'gawk',
- 'gdbm',
- 'glib2',
- 'glibc',
- 'glibc-common',
- 'gmp',
- 'gnutls',
- 'grep',
- 'grub',
- 'gtk2',
- 'gzip',
- 'hal',
- 'hdparm',
- 'hesiod',
- 'hicolor-icon-theme',
- 'hwdata',
- 'info',
- 'initscripts',
- 'iproute',
- 'iptables',
- 'iputils',
- 'kbd',
- 'kernel-headers',
- 'keyutils-libs',
- 'kpartx',
- 'krb5-libs',
- 'kudzu',
- 'less',
- 'libacl',
- 'libattr',
- 'libcap',
- 'libgcc',
- 'libgcrypt',
- 'libgpg-error',
- 'libhugetlbfs',
- 'libhugetlbfs-lib',
- 'libjpeg',
- 'libpng',
- 'libselinux',
- 'libselinux-python',
- 'libsemanage',
- 'libsepol',
- 'libstdc++',
- 'libsysfs',
- 'libtermcap',
- 'libtiff',
- 'libusb',
- 'libuser',
- 'libvolume_id',
- 'libX11',
- 'libXau',
- 'libXcursor',
- 'libXdmcp',
- 'libXext',
- 'libXfixes',
- 'libXft',
- 'libXi',
- 'libXinerama',
- 'libxml2',
- 'libxml2-python',
- 'libXrandr',
- 'libXrender',
- 'libxslt',
- 'libxslt-python',
- 'logrotate',
- 'lvm2',
- 'MAKEDEV',
- 'mcstrans',
- 'mdadm',
- 'mingetty',
- 'mkinitrd',
- 'mktemp',
- 'module-init-tools',
- 'nash',
- 'ncurses',
- 'net-tools',
- 'newt',
- 'nscd',
- 'openldap',
- 'openssl',
- 'pam',
- 'pango',
- 'parted',
- 'passwd',
- 'pciutils',
- 'pcre',
- 'pm-utils',
- 'policycoreutils',
- 'popt',
- 'prelink',
- 'procmail',
- 'procps',
- 'psmisc',
- 'pycairo',
- 'pygobject2',
- 'pygtk2',
- 'python',
- 'python-numeric',
- 'pyxf86config',
- 'readline',
- 'redhat-logos',
- 'redhat-release',
- 'redhat-release-notes',
- 'rhpl',
- 'rootfiles',
- 'rpm',
- 'rpm-libs',
- 'sed',
- 'selinux-policy',
- 'selinux-policy-targeted',
- 'sendmail',
- 'setools',
- 'setserial',
- 'setup',
- 'shadow-utils',
- 'slang',
- 'sqlite',
- 'sysfsutils',
- 'sysklogd',
- 'SysVinit',
- 'tar',
- 'tcl',
- 'tcp_wrappers',
- 'termcap',
- 'tmpwatch',
- 'tzdata',
- 'udev',
- 'usbutils',
- 'usermode',
- 'util-linux',
- 'vim-minimal',
- 'vixie-cron',
- 'wireless-tools',
- 'xorg-x11-filesystem',
- 'zlib',
-)
-
import os
import sys
-import time
-import logging
-sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
-sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py')
-sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib')
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
+
+import rmake
+import conary
+import updatebot
+
+print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
+print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
+print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
from conary.lib import util
sys.excepthook = util.genExcepthook()
-from conary.deps import deps
+import logging
-mbdir = os.path.abspath('../')
-sys.path.insert(0, mbdir)
+from updatebot import groupmgr
+from updatebot import OrderedBot
+from updatebot.groupmgr.model import GroupContentsModel
-from updatebot import log
-from updatebot import groupmgr
+log = logging.getLogger('tmplogger')
-slog = log.addRootLogger()
+class Bot(OrderedBot):
+ def generateInitialGroup(self):
+ """
+ Generate config for standard group contents based on repository history.
+ """
-def toxml(pkgList, toFile):
- groupName = 'group-rhel-standard'
- byDefault = True
- depCheck = True
+ standard = (
+ 'aaa_base',
+ 'aaa_skel',
+ 'acl',
+ 'acpid',
+ 'ash',
+ 'at',
+# 'atk',
+# 'atk-devel',
+# 'atk-doc',
+ 'attr',
+# 'audiofile',
+# 'audiofile-devel',
+ 'audit',
+ 'audit-devel',
+ 'audit-libs',
+# 'autoconf',
+# 'autofs',
+ 'bash',
+# 'bc',
+# 'bin86',
+# 'bind',
+# 'bind-chrootenv',
+# 'bind-devel',
+# 'bind-doc',
+# 'bind-libs',
+# 'bind-utils',
+ 'binutils',
+# 'bison',
+ 'blt',
+ 'blocxx',
+ 'busybox',
+ 'bzip2',
+# 'cairo',
+# 'cairo-devel',
+# 'cairo-doc',
+# 'cdrecord',
+# 'cdrecord-devel',
+# 'cifs-mount',
+# 'compat-libstdc++',
+# 'compat-openssl097g',
+ 'coreutils',
+ 'cpio',
+# 'cpp',
+ 'cracklib',
+# 'cracklib-devel',
+ 'cron',
+# 'curl',
+# 'curl-devel',
+# 'cvs',
+# 'cvs-doc',
+ 'cyrus-sasl',
+# 'cyrus-sasl-crammd5',
+# 'cyrus-sasl-devel',
+# 'cyrus-sasl-digestmd5',
+# 'cyrus-sasl-gssapi',
+# 'cyrus-sasl-otp',
+# 'cyrus-sasl-plain',
+# 'cyrus-sasl-sqlauxprop',
+ 'db',
+ 'db42',
+# 'db-devel',
+# 'db-utils',
+ 'dbus-1',
+# 'dbus-1-devel',
+ 'dbus-1-glib',
+# 'dbus-1-gtk',
+# 'dbus-1-java',
+# 'dbus-1-mono',
+# 'dbus-1-python',
+# 'dbus-1-qt3',
+# 'dbus-1-qt3-devel',
+# 'dbus-1-x11',
+# 'dev86',
+ 'device-mapper',
+# 'device-mapper-devel',
+ 'dhcpcd',
+ 'diffutils',
+ 'e2fsprogs',
+# 'e2fsprogs-devel',
+# 'eject',
+# 'esound',
+# 'esound-devel',
+ 'ethtool',
+ 'expat',
+# 'expect',
+ 'file',
+# 'file-devel',
+ 'filesystem',
+ 'fillup',
+ 'findutils',
+# 'findutils-locate',
+ 'fontconfig',
+# 'fontconfig-devel',
+# 'freetype',
+# 'freetype-tools',
+ 'freetype2',
+# 'freetype2-devel',
+ 'gawk',
+# 'gawk-doc',
+# 'gcc',
+# 'gcc-c++',
+# 'gcc-fortran',
+# 'gcc-info',
+# 'gcc-java',
+# 'gcc-locale',
+# 'gcc-obj-c++',
+# 'gcc-objc',
+# 'gconf2',
+# 'gconf2-devel',
+# 'gconf2-doc',
+# 'gdb',
+ 'gdbm',
+# 'gdbm-devel',
+ 'gettext',
+# 'gettext-devel',
+ 'glib2',
+# 'glib2-devel',
+# 'glib2-doc',
+ 'glibc',
+# 'glibc-debuginfo',
+# 'glibc-devel',
+# 'glibc-html',
+# 'glibc-i18ndata',
+# 'glibc-info',
+# 'glibc-locale',
+# 'glibc-profile',
+# 'glitz',
+# 'glitz-devel',
+# 'gmp',
+# 'gmp-devel',
+ 'gnome-filesystem',
+# 'gnome-vfs2',
+# 'gnome-vfs2-devel',
+# 'gnome-vfs2-doc',
+# 'gnuplot',
+# 'gnutls',
+# 'gnutls-devel',
+ 'gpg',
+# 'gpg2',
+# 'gpm',
+ 'grep',
+# 'groff',
+ 'grub',
+# 'gtk2',
+# 'gtk2-devel',
+# 'gtk2-doc',
+# 'gvim',
+# 'gxdview',
+ 'gzip',
+ 'hal',
+# 'hal-devel',
+# 'hal-gnome',
+# 'hdparm',
+ 'hwinfo',
+# 'hwinfo-devel',
+ 'info',
+ 'insserv',
+ 'iproute2',
+ 'iptables',
+# 'iptables-devel',
+ 'iputils',
+# 'jpeg',
+# 'kbd',
+ 'klogd',
+ 'krb5',
+# 'krb5-apps-clients',
+# 'krb5-apps-servers',
+# 'krb5-client',
+# 'krb5-devel',
+# 'krb5-server',
+# 'ksh',
+# 'ksh-devel',
+ 'less',
+ 'libacl',
+# 'libacl-devel',
+ 'libaio',
+# 'libaio-devel',
+# 'libapr-util1',
+# 'libapr-util1-devel',
+# 'libapr1',
+# 'libapr1-devel',
+# 'libart_lgpl',
+# 'libart_lgpl-devel',
+ 'libattr',
+# 'libattr-devel',
+# 'libbonobo',
+# 'libbonobo-devel',
+# 'libbonobo-doc',
+# 'libbonoboui',
+# 'libbonoboui-devel',
+# 'libbonoboui-doc',
+ 'libcap',
+# 'libcap-devel',
+ 'libcom_err',
+ 'libelf',
+ 'libevent',
+ 'libgcc',
+# 'libgcj',
+# 'libgcj-devel',
+ 'libgcrypt',
+# 'libgcrypt-devel',
+# 'libgfortran',
+# 'libgnome',
+# 'libgnome-devel',
+# 'libgnome-doc',
+# 'libgnomecanvas',
+# 'libgnomecanvas-devel',
+# 'libgnomecanvas-doc',
+ 'libgpg-error',
+# 'libgpg-error-devel',
+# 'libgssapi',
+# 'libidn',
+# 'libidn-devel',
+# 'libiniparser',
+# 'libiniparser-devel',
+# 'libjpeg',
+# 'libjpeg-devel',
+# 'libksba',
+# 'libksba-devel',
+# 'libmsrpc',
+# 'libmsrpc-devel',
+# 'libmudflap',
+# 'libnlink',
+ 'libnscd',
+# 'libnscd-devel',
+# 'libobjc',
+# 'libopencdk',
+# 'libopencdk-devel',
+# 'libpcap',
+# 'libpng',
+# 'libpng-devel',
+# 'librpcsecgss',
+# 'libsmbclient',
+# 'libsmbclient-devel',
+ 'libstdc++',
+# 'libstdc++-devel',
+# 'libstdc++-doc',
+ 'libtool',
+ 'libusb',
+ 'libxcrypt',
+# 'libxcrypt-devel',
+ 'libxml2',
+# 'libxml2-devel',
+# 'libxml2-python',
+ 'libxslt',
+# 'libxslt-devel',
+ 'libzio',
+ 'limal',
+ 'limal-bootloader',
+ 'limal-perl',
+ 'logrotate',
+# 'lsof',
+ 'lvm2',
+# 'lzo',
+# 'lzo-devel',
+ 'm4',
+# 'mailx',
+# 'make',
+# 'man',
+# 'mdadm',
+# 'microcode_ctl',
+ 'mingetty',
+ 'mkinitrd',
+# 'mkisofs',
+ 'mktemp',
+# 'mm',
+# 'mm-devel',
+ 'module-init-tools',
+# 'mysql',
+# 'mysql-Max',
+# 'mysql-client',
+# 'mysql-devel',
+# 'mysql-shared',
+# 'nc6',
+ 'ncurses',
+# 'ncurses-devel',
+# 'neon',
+# 'net-snmp',
+# 'net-snmp-devel',
+ 'net-tools',
+ 'netcfg',
+# 'nfs-utils',
+# 'nfsidmap',
+# 'nmap',
+# 'nmap-gtk',
+ 'nscd',
+ 'openct',
+# 'openct-devel',
+ 'openldap2',
+# 'openldap2-back-meta',
+# 'openldap2-back-perl',
+ 'openldap2-client',
+# 'openldap2-devel',
+ 'opensc',
+# 'opensc-devel',
+ 'openslp',
+# 'openslp-devel',
+# 'openslp-server',
+ 'openssh',
+# 'openssh-askpass',
+ 'openssl',
+# 'openssl-devel',
+# 'openssl-doc',
+# 'orbit2',
+# 'orbit2-devel',
+ 'pam',
+# 'pam-devel',
+ 'pam-modules',
+# 'pam_krb5',
+# 'pam_smb',
+# 'pango',
+# 'pango-devel',
+# 'pango-doc',
+# 'parted',
+# 'parted-devel',
+# 'patch',
+ 'pciutils',
+# 'pciutils-devel',
+ 'pciutils-ids',
+ 'pcre',
+# 'pcre-devel',
+ 'pcsc-lite',
+# 'pcsc-lite-devel',
+ 'perl',
+# 'perl-Bit-Vector',
+ 'perl-Bootloader',
+# 'perl-Carp-Clan',
+# 'perl-Compress-Zlib',
+# 'perl-DBD-SQLite',
+# 'perl-DBD-mysql',
+# 'perl-DBI',
+# 'perl-Data-ShowTable',
+# 'perl-Date-Calc',
+# 'perl-Digest-SHA1',
+# 'perl-Net-Daemon',
+# 'perl-PlRPC',
+# 'perl-SNMP',
+# 'perl-TermReadKey',
+# 'perl-URI',
+# 'perl-XML-Parser',
+# 'perl-XML-Writer',
+ 'perl-gettext',
+ 'permissions',
+# 'pinentry',
+# 'pkgconfig',
+# 'pmtools',
+ 'popt',
+# 'popt-devel',
+# 'portmap',
+# 'postgresql',
+# 'postgresql-contrib',
+# 'postgresql-devel',
+# 'postgresql-docs',
+# 'postgresql-libs',
+# 'postgresql-server',
+ 'procmail',
+ 'procps',
+ 'psmisc',
+ 'pwdutils',
+# 'pwdutils-plugin-audit',
+ 'python',
+# 'python-cairo',
+ 'python-curses',
+# 'python-demo',
+ 'python-devel',
+ 'python-gdbm',
+# 'python-gnome',
+# 'python-gtk',
+# 'python-idle',
+# 'python-numeric',
+# 'python-orbit',
+# 'python-pam',
+ 'python-tk',
+ 'python-xml',
+ 'readline',
+# 'readline-devel',
+ 'reiserfs',
+# 'resmgr',
+ 'rpm',
+# 'rpm-devel',
+ 'rpm-python',
+# 'rrdtool',
+# 'rsync',
+# 'samba',
+# 'samba-client',
+# 'samba-krb-printing',
+# 'samba-python',
+# 'samba-vscan',
+# 'samba-winbind',
+ 'sed',
+ 'sendmail',
+# 'sendmail-devel',
+# 'sensors',
+# 'slang',
+# 'slang-devel',
+ 'sles-release',
+# 'smartmontools',
+# 'sqlite',
+# 'sqlite-devel',
+# 'strace',
+# 'sudo',
+ 'sysconfig',
+ 'sysfsutils',
+# 'syslinux',
+ 'syslog-ng',
+ 'syslogd',
+# 'sysstat',
+# 'sysstat-isag',
+ 'sysvinit',
+ 'suse-build-key',
+ 'tar',
+ 'tcl',
+# 'tcl-devel',
+ 'tcpd',
+# 'tcpd-devel',
+# 'tcpdump',
+ 'tcsh',
+# 'telnet',
+# 'telnet-server',
+ 'termcap',
+ 'terminfo',
+# 'texinfo',
+ 'timezone',
+ 'tk',
+# 'tk-devel',
+ 'udev',
+# 'unixODBC',
+# 'unixODBC-devel',
+# 'unzip',
+# 'usbutils',
+ 'util-linux',
+# 'uucp',
+ 'vim',
+ 'wget',
+# 'wireless-tools',
+# 'x11-tools',
+ 'xfsprogs',
+# 'xfsprogs-devel',
+# 'xinetd',
+# 'xkeyboard-config',
+# 'xntp',
+# 'xntp-doc',
+# 'xorg-x11',
+# 'xorg-x11-Xnest',
+# 'xorg-x11-Xvfb',
+# 'xorg-x11-Xvnc',
+# 'xorg-x11-devel',
+# 'xorg-x11-doc',
+# 'xorg-x11-fonts-100dpi',
+# 'xorg-x11-fonts-75dpi',
+# 'xorg-x11-fonts-cyrillic',
+# 'xorg-x11-fonts-scalable',
+# 'xorg-x11-fonts-syriac',
+ 'xorg-x11-libs',
+# 'xorg-x11-man',
+# 'xorg-x11-sdk',
+# 'xorg-x11-server',
+# 'xorg-x11-server-glx',
+# 'yp-tools',
+# 'ypbind',
+# 'zip',
+# 'zisofs-tools',
+ 'zlib',
+# 'zlib-devel',
+ )
- contents = groupmgr.GroupContentsModel(groupName,
- byDefault=byDefault,
- depCheck=depCheck)
+ troves = self._updater._conaryhelper._getLatestTroves()
- for pkg in pkgList:
- slog.info('adding %s' % (pkg, ))
- if type(pkg) == tuple:
- pkg, flavor = pkg
- contents.add(pkg, flavor=deps.parseFlavor(flavor))
- else:
- contents.add(pkg)
+ pkgs = set()
+ for name, vMap in troves.iteritems():
+ if name.endswith(':source'):
+ continue
+ name = name.split(':')[0]
+ for version, flavors in vMap.iteritems():
+ pkgs.add((name, version, tuple(flavors)))
- contents.freeze(toFile)
+ for name, version, flavors in pkgs:
+ log.info('adding %s=%s' % (name, version))
+ for flv in flavors:
+ log.info('\t%s' % flv)
+ self._groupmgr.addPackage(name, version, flavors)
+ self._groupmgr.setErrataState(0)
+ self._groupmgr.setVersion('0')
+
+ pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
+# pkgGroup.depCheck = False
+
+ contents = GroupContentsModel('group-standard', depCheck=True)
+ self._groupmgr._groups['group-standard'] = contents
+
+ for pkgName in standard:
+ for key in pkgGroup._nameMap[pkgName]:
+ contents._addItem(pkgGroup._data[key])
+
+ built = self._groupmgr.build()
+ return built
if __name__ == '__main__':
- platforms = {
- 'rhel4': rhel4corePackages,
- 'rhel5': rhel5corePackages,
- }
+ from updatebot import config
+ from updatebot import log as logSetup
- platform = sys.argv[1]
- assert platform in platforms
+ logSetup.addRootLogger()
- toFile = sys.argv[2]
+ log = logging.getLogger('create group')
- pkgs = platforms[platform]
- toxml(pkgs, toFile)
+ cfg = config.UpdateBotConfig()
+ cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+
+ bot = Bot(cfg, None)
+ changes = bot.generateInitialGroup()
+
+ import epdb; epdb.st()
diff --git a/scripts/depsolver.py b/scripts/depsolver.py
new file mode 100755
--- /dev/null
+++ b/scripts/depsolver.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2008-2009 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+import os
+import sys
+
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+from conary.lib import util
+sys.excepthook = util.genExcepthook()
+
+import copy
+import logging
+import updatebot.log
+
+updatebot.log.addRootLogger()
+log = logging.getLogger('test')
+
+from updatebot import config
+from updatebot import pkgsource
+
+cfg = config.UpdateBotConfig()
+cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
+
+pkgSource = pkgsource.PackageSource(cfg)
+pkgSource.load()
+
+reqSrcPkgs = set()
+for pkgName in cfg.package:
+ bins = sorted(pkgSource.binNameMap.get(pkgName, []))
+ if not bins:
+ continue
+ reqSrcPkgs.add(pkgSource.binPkgMap[bins[-1]])
+
+reqBinPkgs = set()
+for srcPkg in reqSrcPkgs:
+ reqBinPkgs.update(set([ x for x in pkgSource.srcPkgMap[srcPkg]
+ if x.arch not in ('nosrc', 'src') ]))
+
+requires = {}
+provides = {}
+log.info('loading package requires/provides')
+for bin, src in pkgSource.binPkgMap.iteritems():
+ if bin.arch in ('nosrc', 'src'):
+ continue
+ for format in bin.format:
+ if format.getName() == 'rpm:provides':
+ bins = set([ x.name for x in pkgSource.srcPkgMap[src]
+ if x.arch not in ('nosrc', 'src') ])
+ for child in format.iterChildren():
+ provides.setdefault(child.name, set()).update(bins)
+ if format.getName() == 'rpm:requires':
+ for child in format.iterChildren():
+ requires.setdefault(bin.name, set()).add(child.name)
+
+
+solved = set()
+working = set([ x.name for x in reqBinPkgs ])
+
+log.info('resolving deps')
+while working:
+ pkg = working.pop()
+ log.info('resolving %s' % pkg)
+ for req in requires[pkg]:
+ if req.startswith('rpmlib'):
+ continue
+ if req not in provides:
+ log.warn('requirement not found: %s' % req)
+ continue
+ for provPkg in provides[req]:
+ if provPkg not in solved and provPkg != pkg:
+ working.add(provPkg)
+
+ solved.add(pkg)
+ log.info('solved: %s, working set: %s' % (len(solved), len(working)))
+
+needed = set()
+for pkgName in solved:
+ bins = sorted(pkgSource.binNameMap[pkgName])
+ src = pkgSource.binPkgMap[bins[-1]]
+ if src in reqSrcPkgs:
+ continue
+ pkgs = sorted([ x for x in pkgSource.srcPkgMap[src]
+ if x.arch not in ('nosrc', 'src') ])
+ needed.add(pkgs[0])
+
+import epdb; epdb.st()
+
+
diff --git a/scripts/sync-centos.sh b/scripts/sync-centos.sh
--- a/scripts/sync-centos.sh
+++ b/scripts/sync-centos.sh
@@ -13,7 +13,7 @@
# full details.
#
-SOURCE=rsync://mirrors.us.kernel.org/CentOS-nodvd
+SOURCE=rsync://mirrors.us.kernel.org/CentOS-incdvd
DEST=/l/CentOS/
date
diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py
--- a/updatebot/build/cvc.py
+++ b/updatebot/build/cvc.py
@@ -23,8 +23,9 @@
from conary import versions
from conary import conarycfg
+from conary.deps import deps
+from conary.build import cook
from conary import conaryclient
-from conary.build import cook
from updatebot.lib import conarycallbacks
from updatebot.errors import LocalCookFailedError
@@ -56,11 +57,16 @@
self._client = conaryclient.ConaryClient(self._ccfg)
- def cook(self, troveSpecs):
+ def cook(self, troveSpecs, flavorFilter=None):
"""
Cook a set of trove specs, currently limited to groups.
@params troveSpecs: list of name, version, and flavor tuples.
@type troveSpecs: [(name, version, flavor), ... ]
+ @params flavorFilter: Allow caller to filter out the contexts that they
+ want to build. This is mostly used for group
+ building where a given group should not be built
+ for a context.
+ @type flavorFilter: iterable of context names.
"""
# TODO: Look at conary.build.cook.cookCommand for how to setup
@@ -68,6 +74,9 @@
troveSpecs = self._formatInput(troveSpecs)
+ if flavorFilter:
+ troveSpecs = self._filterTroveSpecs(troveSpecs, flavorFilter)
+
# make sure all troves are groups
assert not [ x for x in troveSpecs if not x[0].startswith('group-') ]
@@ -113,3 +122,30 @@
res = { (troveSpecs[0][0], troveSpecs[0][1], None): results }
return res
+
+ def _filterTroveSpecs(self, troveSpecs, useFlags):
+ """
+ Filter trove specs based on a list of use flags. This is only applicable
+ to groups.
+ @param troveSpecs: iterable of nvf tuples.
+ @type troveSpecs: list(tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ @param useFlags: iterable of valid use flags (x86 and x86_64)
+ @type useFlags: list(str, ...)
+ @return modified list of trove specs
+ @rtype list(tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ """
+
+ useMap = {
+ 'x86': deps.parseFlavor('is: x86'),
+ 'x86_64': deps.parseFlavor('is: x86_64'),
+ }
+
+ specs = set()
+ for n, v, f in troveSpecs:
+ for flag in useFlags:
+ if f.satisfies(useMap[flag]):
+ specs.add((n, v, f))
+
+ return list(specs)
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -371,8 +371,8 @@
ret = {}
for f, t in cfMap.iteritems():
- assert len(cfMap[f]) == 1
- ret[f] = list(t)[0]
+ assert len(cfMap[f]) >= 1
+ ret[f] = sorted(t)[-1]
self._cache.labelClonedFromCache[label] = ret
return ret
@@ -1049,7 +1049,8 @@
if version == latestVer:
trvLst.append((name, version, flavor))
- callback = UpdateBotCloneCallback(self._ccfg, 'test', log=log)
+ callback = UpdateBotCloneCallback(self._ccfg, 'automated promote',
+ log=log)
success, cs = self._client.createSiblingCloneChangeSet(
labelMap,
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -279,9 +279,17 @@
# validate updates
try:
- assert updater._sanitizeTrove(nvf, srpm,
+ toUpdate = updater._sanitizeTrove(nvf, srpm,
expectedRemovals=expectedRemovals + expectedReplaces,
allowPackageDowngrades=explicitPackageDowngrades)
+
+ # If a source was manually added to this updateId it may
+ # have already been part of another update, which would
+ # cause the manifest not to change.
+ if (srpm.getNevra() not in
+ self._cfg.addSource.get(updateId, [])):
+ assert toUpdate
+
except (UpdateGoesBackwardsError,
UpdateRemovesPackageError,
UpdateReusesPackageError), e:
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -592,7 +592,7 @@
"""
_params = ['count', ]
- _template = ('Could not find %(count) sources for matching binary '
+ _template = ('Could not find %(count)s sources for matching binary '
'packages. This generally means that there is a binary package with a '
'source of a different name and a source can not be found with a '
'matching source name, version, and release.')
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -91,7 +91,7 @@
assert len(trvs)
- self._sourceName = self._cfg.topParentSourceGroup
+ self._sourceName = self._cfg.topParentSourceGroup[0]
self._sourceVersion = trvs[0][1]
self._readonly = True
@@ -221,7 +221,8 @@
"""
groupTroves = self.getBuildJob()
- built = self._builder.cvc.cook(groupTroves)
+ useFlags = self._getUseFlags()
+ built = self._builder.cvc.cook(groupTroves, flavorFilter=useFlags)
return built
@checkout
@@ -408,6 +409,20 @@
# Unknown state.
raise UnhandledPackageAdditionError(name=name)
+ def _getUseFlags(self):
+ """
+ Get the set of use flags used across all managed groups.
+ """
+
+ use = set()
+ for group in self._groups.itervalues():
+ for name, pkg in group.iteritems():
+ if pkg.use:
+ use.add(pkg.use)
+ else:
+ use.update(set(['x86', 'x86_64']))
+ return use
+
@checkout
def setVersion(self, version):
"""
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -429,7 +429,8 @@
# Find excepted promote packages.
srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages(
bucket)
- exceptions = self._getOldVersionExceptions(updateId)
+ exceptions = dict([ (x[0], x[1]) for x in itertools.chain(
+ *self._getOldVersionExceptions(updateId).itervalues()) ])
# These are the binary trove specs that we expect to be promoted.
expected = self._filterBinPkgSet(
@@ -604,9 +605,15 @@
# Handle the case where a package has been rebuilt for some
# reason, but we need to use the old version of the package.
pkgName = n.split(':')[0]
- if len(vMap) > 1 and pkgName in exceptions:
- vMap = dict((x, y) for x, y in vMap.iteritems()
- if x == exceptions[pkgName])
+ if len(vMap) > 1:
+ if pkgName in exceptions:
+ log.info('using old version of %s' % n)
+ vMap = dict((x, y) for x, y in vMap.iteritems()
+ if x == exceptions[pkgName])
+ else:
+ log.info('found multiple versions of %s, using latest' % n)
+ v = sorted(vMap)[-1]
+ vMap = { v: vMap[v], }
assert len(vMap) == 1
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -135,10 +135,15 @@
@type package: repomd.packagexml._Package
"""
- if package.name not in self.srcNameMap:
- self.srcNameMap[package.name] = set()
- self.srcNameMap[package.name].add(package)
+ other = package
+ if package in self._srcPkgs:
+ other = [ x for x in self._srcPkgs if x == package ][0]
+ self.srcNameMap[package.name].remove(other)
+ self._srcPkgs.remove(other)
+ if package.getFileName() == os.path.basename(package.location):
+ other = package
+ self.srcNameMap.setdefault(package.name, set()).add(other)
self.locationMap[package.location] = package
# In case the a synthesized source ever turns into real source add the
@@ -148,7 +153,7 @@
if baseLoc not in self.locationMap:
self.locationMap[baseLoc] = package
- self._srcPkgs.add(package)
+ self._srcPkgs.add(other)
self._srcMap[(package.name, package.epoch, package.version,
package.release, package.arch)] = package
@@ -168,10 +173,9 @@
srcRelease = srcParts[-1][:-8] # remove '.src.rpm'
elif srcParts[-1].endswith('.nosrc.rpm'):
srcRelease = srcParts[-1][:-10]
+
rpmMapKey = (srcName, package.epoch, srcVersion, srcRelease, 'src')
- if rpmMapKey not in self._rpmMap:
- self._rpmMap[rpmMapKey] = set()
- self._rpmMap[rpmMapKey].add(package)
+ self._rpmMap.setdefault(rpmMapKey, set()).add(package)
# The normal case of "obsoletes foo < version" (or with
# "requires foo", though that normally also follows the
@@ -239,6 +243,24 @@
self.srcPkgMap[pkg] = self._rpmMap[key]
self.srcPkgMap[pkg].add(pkg)
+
+ # Remove any duplicate sources, favoring sources with the source
+ # version in the file name.
+ # FIXME: This doesn't really work for nosrc rpms
+ sources = sorted([ (os.path.basename(x.location), x)
+ for x in self.srcPkgMap[pkg] if x.arch in ('src', 'nosrc') ])
+ if len(sources) > 1:
+ primary = None
+ for fn, src in sources:
+ if fn == '%s-%s-%s.%s.rpm' % (src.name, src.version, src.release, src.arch):
+ primary = src
+ break
+
+ if primary:
+ for fn, src in sources:
+ if src is not primary:
+ self.srcPkgMap[pkg].remove(src)
+
toDelete.add(key)
for binPkg in self.srcPkgMap[pkg]:
From agrimm at rpath.com Tue Apr 20 15:28:19 2010
From: agrimm at rpath.com (agrimm at rpath.com)
Date: Tue, 20 Apr 2010 19:28:19 +0000
Subject: mirrorball: add allowReusedPackages option
Message-ID: <201004201928.o3KJSJMl011665@scc.eng.rpath.com>
changeset: 8790917afd91
user: Andy Grimm
date: Tue, 20 Apr 2010 15:28:13 -0400
add allowReusedPackages option
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -399,6 +399,10 @@
# updateId: [ (from srcTrvSpec, to srcTrvSpec), ... ]
allowPackageDowngrades = (CfgIntDict(CfgList(CfgNevraTuple)), {})
+ # Allow updates which don't include all binary packages corresponding
+ # to a given source.
+ allowReusedPackages = (CfgBool, False)
+
# Add a source to a specific updateId. This is used to move updates forward
# after allowing an update to downgrade the version.
addSource = (CfgIntDict(CfgList(CfgNevra)), {})
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -430,8 +430,8 @@
if not removedPackages:
reusedPackages.add(pkg)
- #log.warn('using old version of package %s' % (pkg, ))
- #self._pkgSource.srcPkgMap[srpm].add(pkg)
+ log.warn('using old version of package %s' % (pkg, ))
+ self._pkgSource.srcPkgMap[srpm].add(pkg)
if removedPackages:
pkgList=sorted(removedPackages)
@@ -441,7 +441,7 @@
oldNevra=str(' '.join(srcPkg.getNevra())),
newNevra=str(' '.join(srpm.getNevra())))
- if reusedPackages:
+ if reusedPackages and not self._cfg.allowReusedPackages:
pkgList=sorted(reusedPackages)
raise UpdateReusesPackageError(pkgList=pkgList,
pkgNames=' '.join([str(x) for x in pkgList]),
From elliot at rpath.com Thu Apr 29 01:19:53 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 29 Apr 2010 05:19:53 +0000
Subject: mirrorball: If configured to do so, write out a version file to
the source checkout.
Message-ID: <201004290519.o3T5JrQE005689@scc.eng.rpath.com>
changeset: 8c57bda99470
user: Elliot Peele
date: Wed, 28 Apr 2010 22:46:03 -0400
If configured to do so, write out a version file to the source checkout.
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -308,6 +308,10 @@
# package format.
writePackageMetadata = (CfgBool, False)
+ # Write version information to the source trove, generated from the source
+ # version and revision.
+ writePackageVersion = (CfgBool, False)
+
# If sources are not available pkgSource will attempt to build artificial
# source information if this is set to True.
synthesizeSources = (CfgBool, False)
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -720,6 +720,10 @@
manifest = self._getManifestFromPkgSource(srcPkg)
self._conaryhelper.setManifest(nvf[0], manifest)
+ if self._cfg.writePackageVersion:
+ self._conaryhelper.setVersion(nvf[0], '%s_%s'
+ % (srcPkg.version, srcPkg.release))
+
# FIXME: This is apt specific for now. Once repomd has been rewritten
# to use something other than rpath-xmllib we should be able to
# convert this to xobj.
From elliot at rpath.com Thu Apr 29 18:39:27 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 29 Apr 2010 22:39:27 +0000
Subject: mirrorball: add option to ignore 32bit packages in SLES based
platforms. This is only an option to preseve backwards compatibility
with older sles imports.
Message-ID: <201004292239.o3TMdRVM017234@scc.eng.rpath.com>
changeset: bf40336131e6
user: Elliot Peele
date: Thu, 29 Apr 2010 18:37:35 -0400
add option to ignore 32bit packages in SLES based platforms. This is only an option to preseve backwards compatibility with older sles imports.
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -187,6 +187,10 @@
# Paths based off of the repositoryUrl to get to individual repositories.
repositoryPaths = (CfgList(CfgString), ['/'])
+ # Ignore packages with "32bit" in the name. This is intened for use with
+ # SLES based platforms.
+ ignore32bitPackages = (CfgBool, False)
+
# Data source for determining platform version information, only used for
# group versioning.
versionSources = (CfgDict(CfgString), {})
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -103,7 +103,7 @@
for pkg in client.getPackageDetail():
# ignore the 32-bit compatibility libs - we will
# simply use the 32-bit components from the repository
- if '32bit' in pkg.name:
+ if self._cfg.ignore32bitPackages and '32bit' in pkg.name:
continue
# Don't use all arches.
From elliot at rpath.com Thu Apr 29 18:39:27 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 29 Apr 2010 22:39:27 +0000
Subject: mirrorball: the beginings of a common module for advisory processing
Message-ID: <201004292239.o3TMdRRP017267@scc.eng.rpath.com>
changeset: f2aa65d1bedc
user: Elliot Peele
date: Thu, 29 Apr 2010 18:38:06 -0400
the beginings of a common module for advisory processing
diff --git a/errata/__init__.py b/errata/__init__.py
new file mode 100644
--- /dev/null
+++ b/errata/__init__.py
@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
diff --git a/errata/common.py b/errata/common.py
new file mode 100644
--- /dev/null
+++ b/errata/common.py
@@ -0,0 +1,125 @@
+#
+# Copyright (c) 2009 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+This module is here as an example of the data model and interfaces that
+mirrorball is looking for when importing and updating platforms in advisory
+order. These are not meant as superclasses for any sort of implementation. The
+variables and methods that are defined below must be available.
+"""
+
+def reqfetch(func):
+ """
+ Decorator to make sure manager data is loaded before a method is called.
+ """
+
+ def wrap(self, *args, **kwargs):
+ if not self._fetched:
+ self.fetch()
+ return func(self, *args, **kwargs)
+ return wrap
+
+
+class Package(object):
+ """
+ Class to represent a package.
+
+ @param channels: List of channel objects that this package can be found in.
+ @type channels: list(Repository, ...)
+ """
+
+ def __init__(self, channels):
+ for ch in channels:
+ assert isinstance(ch, Channel)
+
+ self.channels = channels
+
+ def getNevra(self):
+ """
+ Returns a tuple of (name, epoch, version, release, arch) for
+ this package.
+ """
+
+ raise NotImplemetedError
+
+
+class Channel(object):
+ """
+ Class to represent a repository.
+
+ @param label: Unique key for the name of a repository.
+ @type label: str
+ """
+
+ def __init__(self, label):
+ self.label = label
+
+
+class Advisory(object):
+ """
+ Class to represent an errata or advisory.
+
+ @param issue_date: Date the advisory was issued in the following format.
+ (format characters are defined in the python time
+ module). '%Y-%m-%d %H:%M:%S'
+ @type issue_date: str
+ @param packages: List of package objects.
+ @type packages: list(Package, ...)
+ @param advisory: Unique key for the advisory
+ @type advisory: str
+ @param synopsis: Brief description of the advisory.
+ @type synopsis: str
+ """
+
+ def __init__(self, advisory, synopsis, issue_date, packages):
+ self.advisory = advisory
+ self.synposis = synopsis
+ self.issue_date = issue_date
+
+ for pkg in packages:
+ assert isinstance(pkg, Package)
+
+ self.packages = packages
+
+
+class AdvisoryManager(object):
+ """
+ Class to provide an interface for accessing advisory information for a
+ platform that can then be matched up to a package source.
+ """
+
+ def getRepositories(self):
+ """
+ Returns a list of repository labels that have been fetched.
+ """
+
+ raise NotImplementedError
+
+ def iterByIssueDate(self):
+ """
+ Yields Errata objects by the issue date of the errata.
+ """
+
+ raise NotImplementedError
+
+ def fetch(self):
+ """
+ Retrieve all required advisory data.
+
+ This is probably going to cache any data, probably in a database, that
+ is being fetched from the internet somewhere so that we don't cause
+ excesive load for anyone's servers.
+ """
+
+ raise NotImplementedError
diff --git a/errata/sles.py b/errata/sles.py
new file mode 100644
--- /dev/null
+++ b/errata/sles.py
@@ -0,0 +1,94 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Generate update information based on the patch detail in SuSE repositories.
+"""
+
+import logging
+
+from errata import common
+
+log = logging.getLogger('errata')
+
+class Package(common.Package):
+ """
+ Class to represent a package.
+ """
+
+ def getNevra(self):
+ """
+ Returns a tuple of (name, epoch, version, release, arch) for
+ this package.
+ """
+
+class Repository(common.Repository):
+ """
+ Class to represent a repository.
+ """
+
+
+class Advisory(common.Advisory):
+ """
+ Class to represent an errata or advisory.
+ """
+
+
+class AdvisoryManager(common.AdvisoryManager):
+ def __init__(self, pkgSource):
+ self._pkgSource = pkgSource
+
+ slef._fetched = False
+ self._patches = set()
+
+ @common.reqfetch
+ def getRepositories(self):
+ """
+ Returns a list of repository labels that have been fetched.
+ """
+
+ @common.reqfetch
+ def iterByIssueDate(self):
+ """
+ Yields Errata objects by the issue date of the errata.
+ """
+
+ return []
+
+ def fetch(self):
+ """
+ Retrieve all required advisory data.
+
+ This is probably going to cache any data, probably in a database, that
+ is being fetched from the internet somewhere so that we don't cause
+ excesive load for anyone's servers.
+ """
+
+ self._fetched = True
+
+ def _fetchPatches(self):
+ """
+ Fetch all patch data from the package source.
+ """
+
+ # make sure the pkg source is loaded.
+ self._pkgSource.load()
+
+ # now get the patch data
+ patches = set()
+ for path, client in self._pkgSource.getClients().iteritems():
+ log.info('loading patches for %s' % path)
+ patches.update(set(client.getPatchDetail()))
+
+ return patches