From juphoff at rpath.com Thu Aug 5 10:59:03 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 05 Aug 2010 14:59:03 +0000 Subject: mirrorball: compare packages on arch before checksum in case of different-arch packages with same content Message-ID: <201008051459.o75Ex3fE014610@scc.eng.rpath.com> changeset: a34f3796ac6f user: Jeff Uphoff date: Thu, 05 Aug 2010 10:48:04 -0400 compare packages on arch before checksum in case of different-arch packages with same content diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -155,6 +155,13 @@ if pkgcmp != 0: return pkgcmp + # Compare arch before checksum to catch cases of multiple + # arch-specific packages that happen to have same content + # (e.g. SLES xorg-x11-fonts packages). + archcmp = cmp(self.arch, other.arch) + if archcmp != 0: + return archcmp + # Compare checksum only for equality, otherwise sorting will result in # checksum ordering. if (self.checksum and other.checksum and From juphoff at rpath.com Thu Aug 5 10:59:03 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 05 Aug 2010 14:59:03 +0000 Subject: mirrorball: update copyright Message-ID: <201008051459.o75Ex31f014635@scc.eng.rpath.com> changeset: dfc71cc34ee1 user: Jeff Uphoff date: Thu, 05 Aug 2010 10:49:39 -0400 update copyright diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this From juphoff at rpath.com Thu Aug 5 10:59:03 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 05 Aug 2010 14:59:03 +0000 Subject: mirrorball: sync timestamps across arches for same-named patches, add logging of same Message-ID: <201008051459.o75Ex3uS014668@scc.eng.rpath.com> changeset: b0bbbfd108fb user: Jeff Uphoff date: Thu, 05 Aug 2010 10:51:34 -0400 sync timestamps across arches for same-named patches, add logging of same diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -1,5 +1,5 @@ # -# Copryright (c) 2008-2009 rPath, Inc. +# Copryright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -16,6 +16,10 @@ Module for parsing patch-*.xml files from the repository metadata. """ +import logging + +log = logging.getLogger('repomd') + __all__ = ('PatchXml', ) from rpath_xmllib import api1 as xmllib @@ -40,9 +44,10 @@ def __init__(self, *args, **kwargs): SlotNode.__init__(self, *args, **kwargs) + # Need access to this so it can be modified when syncing a + # patch's timestamp across architectures. self.timestamp = self.getAttribute('timestamp') - # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. # W0201 - Attribute $foo defined outside __init__ @@ -110,6 +115,14 @@ if desccmp != 0: return desccmp + if self.timestamp != other.timestamp: + maxtime = max(self.timestamp, other.timestamp) + log.info('syncing timestamps (%s %s) ' % (self.timestamp, + other.timestamp) + + 'for %s-%s to %s' % (self.name, self.version, maxtime)) + self.timestamp = other.timestamp = maxtime + # Don't return here--they're now equal. + for pkg in other.packages: if pkg not in self.packages: self.packages.append(pkg) From juphoff at rpath.com Thu Aug 5 10:59:04 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 05 Aug 2010 14:59:04 +0000 Subject: mirrorball: don't assert on multiple same-name/(no)arch packages generated from same srpm Message-ID: <201008051459.o75Ex4AC014695@scc.eng.rpath.com> changeset: 86407c94b7ca user: Jeff Uphoff date: Thu, 05 Aug 2010 10:54:07 -0400 don't assert on multiple same-name/(no)arch packages generated from same srpm diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -593,7 +593,7 @@ for src, bins in srcMap.iteritems(): # Pull out any package sets that look like they are incomplete. - if len(bins) != len(self._pkgSource.srcPkgMap[src]) - 1: + if len(bins) != len(set([ (x.name, x.arch) for x in self._pkgSource.srcPkgMap[src] ])) - 1: extras[src] = bins continue From juphoff at rpath.com Thu Aug 5 10:59:04 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 05 Aug 2010 14:59:04 +0000 Subject: mirrorball: modify to match centos ordering script Message-ID: <201008051459.o75Ex4xk014722@scc.eng.rpath.com> changeset: 54024d642547 user: Jeff Uphoff date: Thu, 05 Aug 2010 10:55:04 -0400 modify to match centos ordering script diff --git a/scripts/sleorder.py b/scripts/sleorder.py --- a/scripts/sleorder.py +++ b/scripts/sleorder.py @@ -3,12 +3,8 @@ import os import sys import time -import tempfile sys.path.insert(0, os.environ['HOME'] + '/hg/conary') -sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-xmllib') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-capsule-indexer') from conary.lib import util sys.excepthook = util.genExcepthook() @@ -19,10 +15,8 @@ confDir = os.path.join(mbdir, 'config', sys.argv[1]) from updatebot import log -from updatebot import cmdline -from updatebot import pkgsource +from updatebot.ordered import Bot from updatebot import UpdateBotConfig -from updatebot.ordered import Bot from errata.sles import AdvisoryManager as Errata @@ -31,14 +25,12 @@ cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) -ui = cmdline.UserInterface() +bot = Bot(cfg, None) +errata = Errata(bot._pkgSource) +bot._errata._errata = errata -pkgSource = pkgsource.PackageSource(cfg, ui) - -errata = Errata(pkgSource) errata.fetch() -bot = Bot(cfg, errata) bot._pkgSource.load() bot._errata._orderErrata() From juphoff at rpath.com Thu Aug 5 13:10:05 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 05 Aug 2010 17:10:05 +0000 Subject: mirrorball: encapsulated sles 10sp3 patch ordering (checkpoint) Message-ID: <201008051710.o75HA5DW016206@scc.eng.rpath.com> changeset: fed478e9a21b user: Jeff Uphoff date: Thu, 05 Aug 2010 13:09:19 -0400 encapsulated sles 10sp3 patch ordering (checkpoint) diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -16,47 +16,26 @@ Generate update information based on the patch detail in SuSE repositories. """ +import time import logging from errata import common +from errata.common import Nevra +from errata.common import Package +from errata.common import Channel +from errata.common import Advisory log = logging.getLogger('errata') -class Package(common.Package): - """ - Class to represent a package. - """ - - def getNevra(self): - """ - Returns a tuple of (name, epoch, version, release, arch) for - this package. - """ - -class Channel(common.Channel): - """ - Class to represent a repository. - """ - - -class Advisory(common.Advisory): - """ - Class to represent an errata or advisory. - """ - - class AdvisoryManager(common.AdvisoryManager): def __init__(self, pkgSource): + common.AdvisoryManager.__init__(self) + self._pkgSource = pkgSource - self._fetched = False - self._patches = set() - - @common.reqfetch - def getRepositories(self): - """ - Returns a list of repository labels that have been fetched. - """ + self._channels = {} + self._advOrder = {} + self._advisories = set() @common.reqfetch def iterByIssueDate(self): @@ -64,7 +43,10 @@ Yields Errata objects by the issue date of the errata. """ - return [] + # FIXME: this work here? (cribbed from centos.py) + for updateId in sorted(self._advOrder): + for adv in self._advOrder[updateId]: + yield adv def fetch(self): """ @@ -75,7 +57,9 @@ excesive load for anyone's servers. """ + self._order() self._fetched = True + return self._advisories @common.reqfetch def getChannels(self): @@ -84,15 +68,16 @@ @return list of indexed channel names """ - return self._pkgSource._clients.keys() - + return self._channels.keys() def cleanup(self): """ Free all cached results. """ - self._patches = set() + self._channels = {} + self._advOrder = {} + self._advisories = set() def getModifiedErrata(self, updateId): """ @@ -102,18 +87,128 @@ return [] - def _fetchPatches(self): + def _order(self): """ Fetch all patch data from the package source. """ + def bin_timestamp(ts): + """ + Convert a time stamp into the desired time slice. + """ + + # convert to current day + return int(time.mktime(time.strptime(time.strftime('%Y%m%d', + time.gmtime(ts)), '%Y%m%d'))) + + def slice(patches): + """ + Build a dictionary of binned (sliced) timestamps and patches. + """ + + slices = {} + + for patch in patches: + # Bin by day: + updateId = bin_timestamp(int(patch.timestamp)) + # ...or uncomment the below line to disable binning: + #updateId = int(patch.timestamp) + slices.setdefault(updateId, + set()).add(patch.getAttribute('patchid')) + return slices + + def getChannel(pkg): + for label, channel in self._channels.iteritems(): + if label in pkg.location: + return channel + raise RuntimeError , 'unable to find channel for %s' % pkg.location + # make sure the pkg source is loaded. self._pkgSource.load() - # now get the patch data + # Each client value is a SLES release/update repository object. + # self._pkgSource._clients.values()[...] + + # now get the patch data... patches = set() + for path, client in self._pkgSource.getClients().iteritems(): - log.info('loading patches for %s' % path) - patches.update(set(client.getPatchDetail())) + log.info('loading patches for path %s' % path) + for patch in client.getPatchDetail(): + for pkg in patch.packages: + pkg.location = path + '/' + pkg.location + patches.add(patch) - return patches + # ...and (time-)slice it up + slices = slice(patches) + + for label in self._pkgSource._clients: + self._channels[label] = Channel(label) + + patchmap = {} + + for timestamp, patchids in slices.iteritems(): + for patchid in patchids: + patchmap.setdefault('-'.join(patchid.split('-')[1:]), + set()).add(timestamp) + + # This requires no more than two timestamps per patchid; + # one each for slesp3 and sdkp3 (bails out otherwise): + # + # Pondering how this can be consolidated with above code to + # reduce iterating... + # + # Also needs tightening up! + # + for patchid, timestamps in patchmap.iteritems(): + if len(timestamps) > 1: + # Untested beyond 2. + assert(len(timestamps) == 2) + # FIXME: refactor this monster. + splitpatch = [ (patch.timestamp, patch.getAttribute('patchid'), + set(patch.packages), patch) for patch in + patches if '-'.join(patch.getAttribute('patchid').split('-')[1:]) == patchid ] + if splitpatch[0][2].issubset(splitpatch[1][2]): + log.info('syncing timestamps (%s %s) ' % ( + splitpatch[0][3].timestamp, + splitpatch[1][3].timestamp) + + 'across repositories for %s & %s ' % ( + splitpatch[0][1], splitpatch[1][1]) + + 'to superset timestamp %s' % splitpatch[1][3].timestamp) + splitpatch[0][3].timestamp = splitpatch[1][3].timestamp + elif splitpatch[1][2].issubset(splitpatch[0][2]): + log.info('syncing timestamps (%s %s) ' % ( + splitpatch[0][3].timestamp, + splitpatch[1][3].timestamp) + + 'across repositories for %s & %s ' % ( + splitpatch[0][1], splitpatch[1][1]) + + 'to superset timestamp %s' % splitpatch[0][3].timestamp) + splitpatch[1][3].timestamp = splitpatch[0][3].timestamp + # So far this has only been tested in pure-subset cases. + else: + raise RuntimeError , 'neither %s nor %s is a subset of the other' % (splitpatch[0][1], splitpatch[1][1]) + + advPkgMap = {} + nevras = {} + packages = {} + + for patch in patches: + advisory = patch.getAttribute('patchid') + issue_date = time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime(int(patch.timestamp))) + + for binPkg in patch.packages: + nevra = binPkg.getNevra() + nevraObj = nevras.setdefault(nevra, Nevra(*nevra)) + channelObj = getChannel(binPkg) + package = Package(channelObj, nevraObj) + packageObj = packages.setdefault(package, package) + advPkgMap.setdefault(advisory, set()).add(packageObj) + + log.info('creating advisory: %s' % advisory) + adv = Advisory(advisory, patch.summary, issue_date, + advPkgMap[advisory]) + self._advisories.add(adv) + self._advOrder.setdefault(int(patch.timestamp), set()).add(adv) + + import epdb ; epdb.st() From juphoff at rpath.com Thu Aug 5 14:38:56 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 05 Aug 2010 18:38:56 +0000 Subject: mirrorball: minor tidy of patch timestamp syncing Message-ID: <201008051838.o75Icutv017462@scc.eng.rpath.com> changeset: 24bd50619954 user: Jeff Uphoff date: Thu, 05 Aug 2010 14:38:50 -0400 minor tidy of patch timestamp syncing diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -165,28 +165,28 @@ # Untested beyond 2. assert(len(timestamps) == 2) # FIXME: refactor this monster. - splitpatch = [ (patch.timestamp, patch.getAttribute('patchid'), + splitpatch = [ (patch.getAttribute('patchid'), set(patch.packages), patch) for patch in patches if '-'.join(patch.getAttribute('patchid').split('-')[1:]) == patchid ] - if splitpatch[0][2].issubset(splitpatch[1][2]): + if splitpatch[0][1].issubset(splitpatch[1][1]): log.info('syncing timestamps (%s %s) ' % ( - splitpatch[0][3].timestamp, - splitpatch[1][3].timestamp) + + splitpatch[0][2].timestamp, + splitpatch[1][2].timestamp) + 'across repositories for %s & %s ' % ( - splitpatch[0][1], splitpatch[1][1]) + - 'to superset timestamp %s' % splitpatch[1][3].timestamp) - splitpatch[0][3].timestamp = splitpatch[1][3].timestamp - elif splitpatch[1][2].issubset(splitpatch[0][2]): + splitpatch[0][0], splitpatch[1][0]) + + 'to superset timestamp %s' % splitpatch[1][2].timestamp) + splitpatch[0][2].timestamp = splitpatch[1][2].timestamp + elif splitpatch[1][1].issubset(splitpatch[0][1]): log.info('syncing timestamps (%s %s) ' % ( - splitpatch[0][3].timestamp, - splitpatch[1][3].timestamp) + + splitpatch[0][2].timestamp, + splitpatch[1][2].timestamp) + 'across repositories for %s & %s ' % ( - splitpatch[0][1], splitpatch[1][1]) + - 'to superset timestamp %s' % splitpatch[0][3].timestamp) - splitpatch[1][3].timestamp = splitpatch[0][3].timestamp + splitpatch[0][0], splitpatch[1][0]) + + 'to superset timestamp %s' % splitpatch[0][2].timestamp) + splitpatch[1][2].timestamp = splitpatch[0][2].timestamp # So far this has only been tested in pure-subset cases. else: - raise RuntimeError , 'neither %s nor %s is a subset of the other' % (splitpatch[0][1], splitpatch[1][1]) + raise RuntimeError , 'neither %s nor %s is a subset of the other' % (splitpatch[0][0], splitpatch[1][0]) advPkgMap = {} nevras = {} From juphoff at rpath.com Fri Aug 6 10:09:02 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Fri, 06 Aug 2010 14:09:02 +0000 Subject: mirrorball: move patchid-mapping to map_patchids() function Message-ID: <201008061409.o76E92rq003979@scc.eng.rpath.com> changeset: 01b17936f81a user: Jeff Uphoff date: Fri, 06 Aug 2010 10:08:56 -0400 move patchid-mapping to map_patchids() function diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -117,6 +117,21 @@ set()).add(patch.getAttribute('patchid')) return slices + def map_patchids(slices): + """ + Build a dictionary of patchids and their corresponding timestamps. + Used to determine if a patchid across multiple repositories also + has multiple timestamps. + """ + + patchidMap = {} + + for timestamp, patchids in slices.iteritems(): + for patchid in patchids: + patchidMap.setdefault('-'.join(patchid.split('-')[1:]), + set()).add(timestamp) + return patchidMap + def getChannel(pkg): for label, channel in self._channels.iteritems(): if label in pkg.location: @@ -139,18 +154,14 @@ pkg.location = path + '/' + pkg.location patches.add(patch) - # ...and (time-)slice it up + # ...and (time-)slice it up. slices = slice(patches) for label in self._pkgSource._clients: self._channels[label] = Channel(label) - - patchmap = {} - for timestamp, patchids in slices.iteritems(): - for patchid in patchids: - patchmap.setdefault('-'.join(patchid.split('-')[1:]), - set()).add(timestamp) + # This maps patchid (without regard to repos) to timeslice. + patchidMap = map_patchids(slices) # This requires no more than two timestamps per patchid; # one each for slesp3 and sdkp3 (bails out otherwise): @@ -160,7 +171,7 @@ # # Also needs tightening up! # - for patchid, timestamps in patchmap.iteritems(): + for patchid, timestamps in patchidMap.iteritems(): if len(timestamps) > 1: # Untested beyond 2. assert(len(timestamps) == 2) From juphoff at rpath.com Fri Aug 6 21:57:12 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Sat, 07 Aug 2010 01:57:12 +0000 Subject: mirrorball: encapsulated sles 10sp3 patch ordering (checkpoint) Message-ID: <201008070157.o771vC6C015201@scc.eng.rpath.com> changeset: 0712923f6c6b user: Jeff Uphoff date: Fri, 06 Aug 2010 21:56:06 -0400 encapsulated sles 10sp3 patch ordering (checkpoint) diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -117,18 +117,28 @@ set()).add(patch.getAttribute('patchid')) return slices + def patchidNoRepo(patchid): + """ + Trims the leading (repository) information from a patchid. + Used for folding patches across repositories, among other things. + """ + return '-'.join(patchid.split('-')[1:]) + def map_patchids(slices): """ - Build a dictionary of patchids and their corresponding timestamps. - Used to determine if a patchid across multiple repositories also - has multiple timestamps. + Build a dictionary of (partial) patchids and their + corresponding timestamps. Used to determine if a patchid + across multiple repositories also has multiple timestamps. """ patchidMap = {} for timestamp, patchids in slices.iteritems(): for patchid in patchids: - patchidMap.setdefault('-'.join(patchid.split('-')[1:]), + # Map excludes leading segment of patchid, which + # carries repository information; we're folding + # across repositories. + patchidMap.setdefault(patchidNoRepo(patchid), set()).add(timestamp) return patchidMap @@ -138,6 +148,18 @@ return channel raise RuntimeError , 'unable to find channel for %s' % pkg.location + def getSrcPkg(binPkg): + for srcPkg, binPkgs in self._pkgSource.srcPkgMap.iteritems(): + if binPkg in binPkgs: + return srcPkg + raise RuntimeError , 'unable to find source package for %s' % binPkg.location + + def getPatchById(patches, patchid): + for patch in patches: + if patch.getAttribute('patchid') == patchid: + return patch + raise RuntimeError , 'unable to find patch %s' % patchid + # make sure the pkg source is loaded. self._pkgSource.load() @@ -154,11 +176,19 @@ pkg.location = path + '/' + pkg.location patches.add(patch) + for label in self._pkgSource._clients: + self._channels[label] = Channel(label) + # ...and (time-)slice it up. slices = slice(patches) - for label in self._pkgSource._clients: - self._channels[label] = Channel(label) + for timeslice, patchSet in slices.iteritems(): + for patchId in patchSet: + patchObj = getPatchById(patches, patchId) + if patchObj.timestamp != timeslice: + log.info('syncing %s timestamp (%s) to slice timestamp %s' % ( + patchId, patchObj.timestamp, timeslice)) + patchObj.timestamp = timeslice # This maps patchid (without regard to repos) to timeslice. patchidMap = map_patchids(slices) @@ -178,7 +208,7 @@ # FIXME: refactor this monster. splitpatch = [ (patch.getAttribute('patchid'), set(patch.packages), patch) for patch in - patches if '-'.join(patch.getAttribute('patchid').split('-')[1:]) == patchid ] + patches if patchidNoRepo(patch.getAttribute('patchid')) == patchid ] if splitpatch[0][1].issubset(splitpatch[1][1]): log.info('syncing timestamps (%s %s) ' % ( splitpatch[0][2].timestamp, @@ -202,20 +232,51 @@ advPkgMap = {} nevras = {} packages = {} + srcPkgAdvMap = {} + srcPkgPatchidMap = {} for patch in patches: advisory = patch.getAttribute('patchid') - issue_date = time.strftime('%Y-%m-%d %H:%M:%S', - time.gmtime(int(patch.timestamp))) + patchid = patchidNoRepo(advisory) for binPkg in patch.packages: nevra = binPkg.getNevra() nevraObj = nevras.setdefault(nevra, Nevra(*nevra)) channelObj = getChannel(binPkg) + +# This was a horrible experiment. +# if srcPkgPatchidMap[srcPkgObj] != set([patchid]): +# print "*** Duplicate srcPackage? %s: %s" % (srcPkgObj, srcPkgAdvMap[srcPkgObj]) +# # Untested beyond two, and expected case is two +# # different advisories issued for the same source +# # package, one each for x86 and x86_64. (Lots of +# # these for the kernel, for instance.) +# assert(len(srcPkgPatchidMap[srcPkgObj]) == 2) +# srcPkgAdvs = [ getPatchById(patches, srcPkgAdv) +# for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ] +# maxTimestamp = max(srcPkgAdvs[0].timestamp, +# srcPkgAdvs[1].timestamp) +# log.info('syncing timestamps (%s %s) ' % ( +# srcPkgAdvs[0].timestamp, srcPkgAdvs[1].timestamp) + +# 'across same-source advisories for %s & %s ' % ( +# srcPkgAdvs[0].getAttribute('patchid'), +# srcPkgAdvs[1].getAttribute('patchid')) + +# 'to later timestamp %s' % maxTimestamp) +# srcPkgAdvs[0].timestamp = srcPkgAdvs[1].timestamp = maxTimestamp + package = Package(channelObj, nevraObj) packageObj = packages.setdefault(package, package) advPkgMap.setdefault(advisory, set()).add(packageObj) + srcPkgObj = getSrcPkg(binPkg) + srcPkgAdvMap.setdefault(srcPkgObj, set()).add(advisory) + srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid) + + # There should be no srcPkgs with more than two patchids. + assert(len([ x for x, y in srcPkgPatchidMap.iteritems() + if len(y) > 2 ]) == 0) + issue_date = time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime(int(patch.timestamp))) log.info('creating advisory: %s' % advisory) adv = Advisory(advisory, patch.summary, issue_date, advPkgMap[advisory]) From juphoff at rpath.com Fri Aug 6 21:58:45 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Sat, 07 Aug 2010 01:58:45 +0000 Subject: mirrorball: tidy up: remove horrible experiment in patch-timestamp syncing Message-ID: <201008070158.o771wkah015260@scc.eng.rpath.com> changeset: d92a9667dfb0 user: Jeff Uphoff date: Fri, 06 Aug 2010 21:58:43 -0400 tidy up: remove horrible experiment in patch-timestamp syncing diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -243,27 +243,6 @@ nevra = binPkg.getNevra() nevraObj = nevras.setdefault(nevra, Nevra(*nevra)) channelObj = getChannel(binPkg) - -# This was a horrible experiment. -# if srcPkgPatchidMap[srcPkgObj] != set([patchid]): -# print "*** Duplicate srcPackage? %s: %s" % (srcPkgObj, srcPkgAdvMap[srcPkgObj]) -# # Untested beyond two, and expected case is two -# # different advisories issued for the same source -# # package, one each for x86 and x86_64. (Lots of -# # these for the kernel, for instance.) -# assert(len(srcPkgPatchidMap[srcPkgObj]) == 2) -# srcPkgAdvs = [ getPatchById(patches, srcPkgAdv) -# for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ] -# maxTimestamp = max(srcPkgAdvs[0].timestamp, -# srcPkgAdvs[1].timestamp) -# log.info('syncing timestamps (%s %s) ' % ( -# srcPkgAdvs[0].timestamp, srcPkgAdvs[1].timestamp) + -# 'across same-source advisories for %s & %s ' % ( -# srcPkgAdvs[0].getAttribute('patchid'), -# srcPkgAdvs[1].getAttribute('patchid')) + -# 'to later timestamp %s' % maxTimestamp) -# srcPkgAdvs[0].timestamp = srcPkgAdvs[1].timestamp = maxTimestamp - package = Package(channelObj, nevraObj) packageObj = packages.setdefault(package, package) advPkgMap.setdefault(advisory, set()).add(packageObj) From juphoff at rpath.com Fri Aug 6 22:20:16 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Sat, 07 Aug 2010 02:20:16 +0000 Subject: mirrorball: encapsulated sles 10sp3 patch ordering (checkpoint) Message-ID: <201008070220.o772KG6r015591@scc.eng.rpath.com> changeset: b16da9dbcb10 user: Jeff Uphoff date: Fri, 06 Aug 2010 22:19:58 -0400 encapsulated sles 10sp3 patch ordering (checkpoint) diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -30,7 +30,7 @@ class AdvisoryManager(common.AdvisoryManager): def __init__(self, pkgSource): common.AdvisoryManager.__init__(self) - + self._pkgSource = pkgSource self._channels = {} @@ -99,7 +99,7 @@ # convert to current day return int(time.mktime(time.strptime(time.strftime('%Y%m%d', - time.gmtime(ts)), '%Y%m%d'))) + time.gmtime(ts)), '%Y%m%d'))) def slice(patches): """ @@ -107,7 +107,7 @@ """ slices = {} - + for patch in patches: # Bin by day: updateId = bin_timestamp(int(patch.timestamp)) @@ -159,7 +159,7 @@ if patch.getAttribute('patchid') == patchid: return patch raise RuntimeError , 'unable to find patch %s' % patchid - + # make sure the pkg source is loaded. self._pkgSource.load() @@ -190,7 +190,10 @@ patchId, patchObj.timestamp, timeslice)) patchObj.timestamp = timeslice - # This maps patchid (without regard to repos) to timeslice. + # slices dict is still current since the above only synced the + # patch timestamps to existing slices. + + # This maps patchid (without regard to repos) to timeslices. patchidMap = map_patchids(slices) # This requires no more than two timestamps per patchid; @@ -232,9 +235,8 @@ advPkgMap = {} nevras = {} packages = {} - srcPkgAdvMap = {} srcPkgPatchidMap = {} - + for patch in patches: advisory = patch.getAttribute('patchid') patchid = patchidNoRepo(advisory) @@ -248,12 +250,12 @@ advPkgMap.setdefault(advisory, set()).add(packageObj) srcPkgObj = getSrcPkg(binPkg) - srcPkgAdvMap.setdefault(srcPkgObj, set()).add(advisory) srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid) # There should be no srcPkgs with more than two patchids. assert(len([ x for x, y in srcPkgPatchidMap.iteritems() if len(y) > 2 ]) == 0) + issue_date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(patch.timestamp))) log.info('creating advisory: %s' % advisory) From juphoff at rpath.com Mon Aug 9 18:41:56 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Mon, 09 Aug 2010 22:41:56 +0000 Subject: mirrorball: add firstErrata configuration option Message-ID: <201008092241.o79Mfu8k010303@scc.eng.rpath.com> changeset: 04b3bbbbf417 user: Jeff Uphoff date: Mon, 09 Aug 2010 18:40:12 -0400 add firstErrata configuration option diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -408,6 +408,14 @@ # bucket listed. mergeUpdates = (CfgList(CfgQuotedLineList(CfgInt)), []) + # Timestamp of first erratum. This is used as a baseline for + # determining if any update packages are missing errata. It should + # auto-detect correctly, but in some cases--for instance, when a + # distribution releases the same package as a baseline package on + # one channel and an update on a parallel channel--this will require + # manual specification. + firstErrata = CfgInt + # Errata timestamp pairs for rescheduling when updates are applied. The # first element is the current timestamp of the update. The second element # is the new timestamp. You may need to use this option if it appears that diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -908,9 +908,13 @@ # separate out golden bits other = [] golden = [] - firstErrata = int(time.time()) - if len(buckets): - firstErrata = sorted(buckets.keys())[0] + if self._cfg.firstErrata: + firstErrata = self._cfg.firstErrata + else: + firstErrata = int(time.time()) + if len(buckets): + firstErrata = sorted(buckets.keys())[0] + for nevra, pkg in nevras.iteritems(): buildtime = int(pkg.buildTimestamp) if buildtime < firstErrata: From juphoff at rpath.com Tue Aug 10 11:30:56 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Tue, 10 Aug 2010 15:30:56 +0000 Subject: mirrorball: sync timestamps for different-named advisories that use the same srpm Message-ID: <201008101530.o7AFUuUt029910@scc.eng.rpath.com> changeset: b2be591fad06 user: Jeff Uphoff date: Tue, 10 Aug 2010 11:30:43 -0400 sync timestamps for different-named advisories that use the same srpm diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -202,7 +202,7 @@ # Pondering how this can be consolidated with above code to # reduce iterating... # - # Also needs tightening up! + # Just so it's clear, I hate this code (i.e. FIXME). # for patchid, timestamps in patchidMap.iteritems(): if len(timestamps) > 1: @@ -235,6 +235,7 @@ advPkgMap = {} nevras = {} packages = {} + srcPkgAdvMap = {} srcPkgPatchidMap = {} for patch in patches: @@ -248,9 +249,34 @@ package = Package(channelObj, nevraObj) packageObj = packages.setdefault(package, package) advPkgMap.setdefault(advisory, set()).add(packageObj) + srcPkgObj = getSrcPkg(binPkg) + srcPkgAdvMap.setdefault(srcPkgObj, set()).add(advisory) + srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid) - srcPkgObj = getSrcPkg(binPkg) - srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid) + # FIXME: I hate this code too. + if srcPkgPatchidMap[srcPkgObj] != set([patchid]): + # Untested beyond two, and expected case is two + # different advisories issued for the same source + # package, one each for x86 and x86_64. (Lots of + # these for the kernel, for instance.) + assert(len(srcPkgPatchidMap[srcPkgObj]) == 2) + srcPkgAdvs = [ getPatchById(patches, srcPkgAdv) + for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ] + # Only sync the same source package once. (It may + # appear for multiple binary packages.) + if srcPkgAdvs[0].timestamp != srcPkgAdvs[1].timestamp: + # Using the min here in case the first advisory + # for this source package has already been + # published. + syncTimestamp = min(srcPkgAdvs[0].timestamp, + srcPkgAdvs[1].timestamp) + log.info('syncing timestamps (%s %s) ' % ( + srcPkgAdvs[0].timestamp, srcPkgAdvs[1].timestamp) + + 'across same-SRPM advisories for %s & %s ' % ( + srcPkgAdvs[0].getAttribute('patchid'), + srcPkgAdvs[1].getAttribute('patchid')) + + 'to earlier timestamp %s' % syncTimestamp) + srcPkgAdvs[0].timestamp = srcPkgAdvs[1].timestamp = syncTimestamp # There should be no srcPkgs with more than two patchids. assert(len([ x for x, y in srcPkgPatchidMap.iteritems() @@ -258,7 +284,8 @@ issue_date = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(int(patch.timestamp))) - log.info('creating advisory: %s' % advisory) + log.info('creating advisory: %s (%s)' % (advisory, + patch.timestamp)) adv = Advisory(advisory, patch.summary, issue_date, advPkgMap[advisory]) self._advisories.add(adv) From juphoff at rpath.com Wed Aug 11 15:13:59 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Wed, 11 Aug 2010 19:13:59 +0000 Subject: mirrorball: handle ordered removal of nosrc packages Message-ID: <201008111913.o7BJDxZs006863@scc.eng.rpath.com> changeset: 8ddd73c3d2a7 user: Jeff Uphoff date: Wed, 11 Aug 2010 15:13:54 -0400 handle ordered removal of nosrc packages diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -420,6 +420,12 @@ x for x in self._pkgSource.binNameMap[binPkg.name] if x.arch == binPkg.arch ]) + # Maybe this is a src or nosrc. + if not pkgs: + pkgs = sorted([ + x for x in self._pkgSource.srcNameMap[binPkg.name] + if x.arch == binPkg.arch ]) + # If running in latest mode we really want to compare to the # latest version of this binary, but if we are running in # ordered we really want the "next" version of this binary. From elliot at rpath.com Thu Aug 12 16:03:23 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:23 +0000 Subject: mirrorball: speedups for rebuilding groups Message-ID: <201008122003.o7CK3NgU030740@scc.eng.rpath.com> changeset: 8b86d003f1f4 user: Elliot Peele date: Thu, 12 Aug 2010 15:45:27 -0400 speedups for rebuilding groups diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -124,26 +124,31 @@ if checkUpdates: log.info('%s: looking up version information for rebuild ' 'packages' % version) - for n, v, f in checkUpdates: - upVer = '/'.join([v.branch().label().asString(), - v.trailingRevision().version]) - binSpecs = self._updater._conaryhelper._repos.findTrove( - v.branch().label(), (n, upVer, None), bestFlavor=False) - assert len(binSpecs) > 1 + req = set() + for n, v, f in checkUpdates: + upVer = '/'.join([v.branch().label().asString(), + v.trailingRevision().version]) + req.add((n, upVer, None)) - latest = sorted(binSpecs)[-1] + binSpecMap = self._updater._conaryhelper.findTroves(req) - if v != latest[1]: - #log.info('%s: found updated version of %s %s -> %s' - # % (version, n, v, latest[1])) + for binSpecs in binSpecMap.itervalues(): + assert len(binSpecs) > 1 - toAdd = set([ x for x in binSpecs if x[1] == latest[1] ]) - nvfs.update(toAdd) - elif f is None: - emptyFlavors.add((n, v, f)) - else: - nvfs.add((n, v, f)) + latest = sorted(binSpecs)[-1] + + if v != latest[1]: + #log.info('%s: found updated version of %s %s -> %s' + # % (version, n, v, latest[1])) + + toAdd = set([ x for x in binSpecs + if x[1] == latest[1] ]) + nvfs.update(toAdd) + elif f is None: + emptyFlavors.add((n, v, f)) + else: + nvfs.add((n, v, f)) # Lookup anything that has an empty flavor. log.info('%s: looking up version information for empty flavors' @@ -212,7 +217,8 @@ # Wait for the first results to make sure the group will rebuild # properly. if not trvMap: - log.info('waiting for first build to complete before continuing') + log.info('waiting for first build to complete before ' + 'continuing') while not res.isDone: time.sleep(1) From elliot at rpath.com Thu Aug 12 16:03:23 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:23 +0000 Subject: mirrorball: take into account all labels in the group model Message-ID: <201008122003.o7CK3NV6030771@scc.eng.rpath.com> changeset: 5d19090156ce user: Elliot Peele date: Thu, 12 Aug 2010 15:46:44 -0400 take into account all labels in the group model diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py --- a/updatebot/groupmgr/sanity.py +++ b/updatebot/groupmgr/sanity.py @@ -186,10 +186,14 @@ # Get all old versions so that we can make sure any version conflicts # were introduced by old version handling. oldVersions = set() + if self._cfg.platformSearchPath: + qlabels = set(self._cfg.platformSearchPath) | labels + else: + qlabels = labels for nvfLst in self._cfg.useOldVersion.itervalues(): for nvf in nvfLst: srcMap = self._helper.getSourceVersionMapFromBinaryVersion(nvf, - labels=self._cfg.platformSearchPath, latest=False) + labels=qlabels, latest=False) oldVersions |= set(itertools.chain(*srcMap.itervalues())) errors = {} From elliot at rpath.com Thu Aug 12 16:03:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:24 +0000 Subject: mirrorball: packages for the rhel rebuild Message-ID: <201008122003.o7CK3OK0030798@scc.eng.rpath.com> changeset: 7a4d4d37a302 user: Elliot Peele date: Thu, 12 Aug 2010 15:47:27 -0400 packages for the rhel rebuild diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -269,6 +269,51 @@ errata.fetch() bot = Bot(cfg, errata) - bot.rebuildgroups(restart=False, resolveTargetVersions=False, readdPackages=False) + + rebuiltPackages = ['kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', + 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', + 'cracklib', 'dbus-python', 'dogtail', 'eruby', 'gamin', 'gnome-applets', + 'gnome-bluetooth', 'gnome-menus', 'gnome-python2', + 'gnome-python2-desktop', 'gnome-python2-extras', 'gtk-vnc', 'hplip', + 'iscsi-initiator-utils', 'java-1.4.2-gcj-compat', 'kdebindings', + 'kudzu', 'lcms', 'libbtctl', 'libieee1284', 'libselinux', 'libsemanage', + 'libuser', 'libxml2', 'libxslt', 'm2crypto', 'mkinitrd', 'mod_python', + 'mx', 'MySQL-python', 'newt', 'notify-python', 'oddjob', 'OpenIPMI', + 'orca', 'pexpect', 'pirut', 'policycoreutils', 'postgresql', + 'postgresql84', 'pycairo', 'pygobject2', 'pygtk2', 'pykickstart', + 'pyOpenSSL', 'pyorbit', 'pyparted', 'PyQt', 'Pyrex', 'pyspi', 'python', + 'python-dmidecode', 'python-elementtree', 'python-imaging', + 'python-iniparse', 'python-ldap', 'python-numeric', 'python-pyblock', + 'python-setuptools', 'python-sqlite', 'python-urlgrabber', + 'pyxf86config', 'PyXML', 'rhel-instnum', 'rhnlib', 'rhpl', 'rhpxl', + 'rpm', 'ruby', 'sabayon', 'setroubleshoot', 'sip', 'sos', 'subversion', + 'system-config-printer', 'vte', 'wireshark', 'yum', + 'yum-metadata-parser', 'mvapich2', 'gnome-doc-utils', + 'system-config-lvm', 'booty', 'smartmontools', 'devhelp', + 'system-switch-mail', 'gedit', 'system-config-nfs', 'ant', + 'kexec-tools', 'inn', 'tetex', 'docbook-style-xsl', 'firstboot', + 'system-config-kickstart', 'kdesdk', 'system-config-display', + 'switchdesk', 'gimp', 'system-config-samba', 'kdevelop', 'hal', 'frysk', + 'redhat-release', 'gettext', 'libglade2', 'system-config-bind', + 'system-config-httpd', 'system-config-keyboard', + 'system-config-language', 'system-config-kdump', 'libhugetlbfs', + 'comps-extras', 'system-config-soundcard', 'systemtap', 'vnc', + 'conary-policy', 'system-config-services', 'createrepo', 'blktrace', + 'yum-utils', 'system-config-netboot', 'sblim', 'setroubleshoot-plugins', + 'xinetd', 'rhn-client-tools', 'libevent', 'mailman', + 'system-config-securitylevel', 'selinux-policy', 'hwbrowser', + 'bluez-utils', 'system-config-users', 'cman', 'yum-rhn-plugin', + 'system-config-boot', 'system-config-network', 'system-config-date', + 'ecryptfs-utils', 'tix', 'anaconda', 'system-config-rootpassword', + 'dstat'] + + tags = [ 'chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango', + 'setup', 'shadow-utils', 'shared-mime-info', 'texinfo', + 'xorg-x11-font-utils' ] + + extraPackages = [ 'xulrunner', 'firefox', 'gnome-vfs2', 'numactl', ] + + bot.rebuildgroups(restart=True, + updatedPackages=rebuiltPackages + tags + extraPackages) import epdb; epdb.st() From elliot at rpath.com Thu Aug 12 16:03:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:24 +0000 Subject: mirrorball: rework rebuildpackages to build more than one package name at a time Message-ID: <201008122003.o7CK3OX6030825@scc.eng.rpath.com> changeset: 9feacd6daa3c user: Elliot Peele date: Thu, 12 Aug 2010 15:47:57 -0400 rework rebuildpackages to build more than one package name at a time diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage --- a/scripts/rebuildpackage +++ b/scripts/rebuildpackage @@ -36,12 +36,12 @@ return sorted(nvfs) - def rebuildpackage(self, name, useLatest=None, + def rebuildpackages(self, names, useLatest=None, additionalResolveTroves=None): """ Rebuild all versions of a given package in order. - @param name: name of the package to rebuild. - @type name: str + @param names: names of the packages to rebuild. + @type names: str @param useLatest: A list of package names to use the latest versions of. For instance, you may want to use the latest version of conary to get fixed dependencies. @@ -51,17 +51,21 @@ @type additionalResolveTroves: list(str, ...) """ - trvSpecs = [ (x[0].split(':')[0], x[1], None) - for x in self._getAllVersions(name) ] + nvfs = set() + for name in names: + trvSpecs = [ (x[0].split(':')[0], x[1], None) + for x in self._getAllVersions(name) ] - # Build only the latest versions of each sources. - latestMap = {} - for n, v, f in reversed(sorted(trvSpecs)): - upVer = v.trailingRevision().version - if upVer not in latestMap: - latestMap[upVer] = (n, v, f) + # Build only the latest versions of each sources. + latestMap = {} + for n, v, f in reversed(sorted(trvSpecs)): + upVer = v.trailingRevision().version + if upVer not in latestMap: + latestMap[upVer] = (n, v, f) - trvMap = self._builder.rebuildmany(latestMap.values(), + nvfs.update(set(latestMap.values())) + + trvMap = self._builder.rebuildmany(list(nvfs), useLatest=useLatest, additionalResolveTroves=additionalResolveTroves) @@ -162,16 +166,18 @@ bot = Bot(cfg, errata) - for pkgName in pkgNames: +# for pkgName in pkgNames: # bot.removeSourceFiles(pkgName) - bot.rebuildpackage(pkgName, - useLatest=['conary', 'conary-build', 'conary-policy', 'rpm', ], - additionalResolveTroves=[ - 'libelf-lgpl=rhel.rpath.com at rpath:rhel-5-devel', - 'conary=rhel.rpath.com at rpath:rhel-5-devel', - 'conary-build=rhel.rpath.com at rpath:rhel-5-devel', - 'conary-policy=rhel.rpath.com at rpath:rhel-5-devel', - 'rpm=rhel.rpath.com at rpath:rhel-5-server-devel', - 'group-os=rhel.rpath.com at rpath:rhel-5-server-devel',]) + + bot.rebuildpackages(pkgNames, + useLatest=['conary', 'conary-build', 'conary-policy', 'rpm', ], + additionalResolveTroves=[ + 'libelf-lgpl=rhel.rpath.com at rpath:rhel-5-devel', + 'conary=rhel.rpath.com at rpath:rhel-5-devel', + 'conary-build=rhel.rpath.com at rpath:rhel-5-devel', + 'conary-policy=rhel.rpath.com at rpath:rhel-5-devel', + 'rpm=rhel.rpath.com at rpath:rhel-5-server-devel', + 'group-os=rhel.rpath.com at rpath:rhel-5-server-devel', + 'group-rpath-packages=rhel.rpath.com at rpath:rhel-5-devel', ]) import epdb; epdb.st() From elliot at rpath.com Thu Aug 12 16:03:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:25 +0000 Subject: mirrorball: verify group contents in both directions Message-ID: <201008122003.o7CK3PrB030852@scc.eng.rpath.com> changeset: 9b8d2897f6ab user: Elliot Peele date: Thu, 12 Aug 2010 15:48:38 -0400 verify group contents in both directions diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -1102,12 +1102,20 @@ # in expected will not be promoted because not all packages are # included in the groups. trvDiff = newPkgs.difference(oldPkgs) + trvInv = oldPkgs.difference(newPkgs) + grpTrvs = set([ (x[0], x[2]) for x in trvLst if not x[0].endswith(':source') ]) + grpDiff = set([ x[0] for x in trvDiff.difference(grpTrvs) ]) + grpInv = set([ x[0] for x in trvInv.difference(grpTrvs) ]) + extraTroves = set([ x[0] for x in extraPromoteTroves | extraExpectedPromoteTroves ]) - if checkPackageList and grpDiff.difference(extraTroves): + + + if (checkPackageList and (grpDiff.difference(extraTroves) or + grpInv.difference(extraTroves))): raise PromoteMismatchError(expected=oldPkgs, actual=newPkgs) if not commit: From elliot at rpath.com Thu Aug 12 16:03:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:25 +0000 Subject: mirrorball: handle extended advisories in promote sanity checking Message-ID: <201008122003.o7CK3PHG030880@scc.eng.rpath.com> changeset: 480441528bbd user: Elliot Peele date: Thu, 12 Aug 2010 15:49:24 -0400 handle extended advisories in promote sanity checking diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -473,6 +473,21 @@ exceptions = dict([ (x[0], x[1]) for x in itertools.chain( *self._getOldVersionExceptions(updateId).itervalues()) ]) + + advisories = [ x['name'] for x in + self._errata.getUpdateDetail(updateId) ] + + for advisory in advisories: + # Handle attaching an update that was caused by changes that we + # made outside of the normal update stream to an existing + # advisory. + for nvf in self._cfg.extendAdvisory.get(advisory, ()): + srcMap = self._updater.getSourceVersionMapFromBinaryVersion( + nvf, labels=self._cfg.platformSearchPath, + latest=False, includeBuildLabel=True) + assert len(srcMap) == 1 + srcPkgMap.update(srcMap) + # These are the binary trove specs that we expect to be promoted. expected = self._filterBinPkgSet( itertools.chain(*srcPkgMap.itervalues()), exceptions) From elliot at rpath.com Thu Aug 12 16:03:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:25 +0000 Subject: mirrorball: switch all threaded builders to using a bounded counter object that raises Message-ID: <201008122003.o7CK3P33030908@scc.eng.rpath.com> changeset: bcd48d057523 user: Elliot Peele date: Thu, 12 Aug 2010 15:50:49 -0400 switch all threaded builders to using a bounded counter object that raises errors on out of bounds conditions diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -44,8 +44,7 @@ def __init__(self, builder, maxSlots): self._builder = builder - self._maxSlots = maxSlots - self._slots = self._maxSlots + self._slots = util.BoundedCounter(0, maxSlots, maxSlots) # jobId: (trv, status, commitData) self._jobs = {} @@ -98,11 +97,8 @@ def __init__(self, builder, maxSlots): AbstractDispatcher.__init__(self, builder, maxSlots) - self._maxStartSlots = 10 - self._startSlots = self._maxStartSlots - - self._maxCommitSlots = 2 - self._commitSlots = self._maxCommitSlots + self._startSlots = util.BoundedCounter(0, 10, 10) + self._commitSlots = util.BoundedCounter(0, 2, 2) self._starter = self._starterClass(self._builder) self._monitor = self._monitorClass(self._builder._helper.client) @@ -123,7 +119,7 @@ while troves or not self._jobDone(): # Only create more jobs once the last batch has been started. - if self._startSlots == self._maxStartSlots: + if self._startSlots == self._startSlots.upperlimit: # fill slots with available troves while (troves and self._slots and self._startSlots and self._availableFDs()): @@ -154,7 +150,7 @@ if status in self._slotdone: self._slots += 1 - if self._slots > self._maxSlots: + if self._slots > self._slots.upperlimit: log.critical('slots is greater than maxSlots') # submit any jobs that are ready to commit as long as there are @@ -178,24 +174,25 @@ # check for commit status for jobId, result in self._committer.getStatus(): + self._commitSlots += 1 # unbatch commit jobs if not isinstance(jobId, tuple): jobId = (jobId, ) for jobId in jobId: self._jobs[jobId][2] = result - self._commitSlots += 1 # process committer errors for jobId, error in self._committer.getErrors(): + self._commitSlots += 1 # unbatch commit jobs if not isinstance(jobId, tuple): jobId = (jobId, ) for jobId in jobId: self._jobs[jobId][1] = JobStatus.ERROR_COMMITTER_FAILURE self._failures.append((jobId, error)) - self._commitSlots += 1 - # Flag job as failed so that monitor worker will exit properly. + # Flag job as failed so that monitor worker will exit + # properly. self._builder.setCommitFailed(jobId, reason=str(error)) # Wait for a bit before polling again. @@ -257,8 +254,7 @@ Dispatcher.__init__(self, builder, maxSlots) # Disable commits by removing all commit slots. - self._maxCommitSlots = 0 - self._commitSlots = 0 + self._commitSlots = util.BoundedCounter(0, 0, 0) def buildmany(self, troveSpecs): """ @@ -322,8 +318,7 @@ self._waitForAllVersions = waitForAllVersions - self._maxCommitSlots = 1 - self._commitSlots = self._maxCommitSlots + self._commitSlots = util.BoundedCounter(0, 1, 1) # Mapping of pkgname to ordered list of trove specs self._pkgs = {} diff --git a/updatebot/build/jobs.py b/updatebot/build/jobs.py --- a/updatebot/build/jobs.py +++ b/updatebot/build/jobs.py @@ -24,6 +24,7 @@ from rmake.build import buildjob +from updatebot.lib import util from updatebot.build.constants import JobStatus from updatebot.build.dispatcher import AbstractDispatcher @@ -121,8 +122,7 @@ AbstractDispatcher.__init__(self, builder, maxSlots) Thread.__init__(self) - self._maxCommitSlots = 1 - self._commitSlots = self._maxCommitSlots + self._commitSlots = util.BoundedCounter(0, 1, 1) self._cooker = LocalGroupCooker(self._builder) self._committer = LocalChangeSetCommitter(self._builder) @@ -299,11 +299,8 @@ AbstractDispatcher.__init__(self, builder, maxSlots) Thread.__init__(self) - self._maxStartSlots = 10 - self._startSlots = self._maxStartSlots - - self._maxCommitSlots = 2 - self._commitSlots = self._maxCommitSlots + self._startSlots = util.BoundedCounter(0, 10, 10) + self._commitSlots = util.BoundedCounter(0, 2, 2) self._starter = self._starterClass(self._builder) self._monitor = self._monitorClass(self._builder._helper.client) @@ -330,7 +327,7 @@ while not self._done: # Only create more jobs once the last batch has been started. - if self._startSlots == self._maxStartSlots: + if self._startSlots == self._startSlots.upperlimit: # fill slots with available troves notStarted = self._getNotStarted() while notStarted and self._slots and self._startSlots: @@ -369,7 +366,7 @@ if status in self._slotdone: self._slots += 1 - if self._slots > self._maxSlots: + if self._slots > self._slots.upperlimit: log.critical('slots is greater than maxSlots') res = self._jobs[jobId][2] @@ -403,16 +400,17 @@ # check for commit status for jobId, result in self._committer.getStatus(): + self._commitSlots += 1 # unbatch commit jobs if not isinstance(jobId, tuple): jobId = (jobId, ) for jobId in jobId: self._jobs[jobId][2].setResults(result) self._jobs[jobId][2].setStatus('committed') - self._commitSlots += 1 # process committer errors for jobId, error in self._committer.getErrors(): + self._commitSlots += 1 # unbatch commit jobs if not isinstance(jobId, tuple): jobId = (jobId, ) @@ -421,7 +419,6 @@ self._jobs[jobId][2].setError(error) self._jobs[jobId][2].setStatus('commit failed') self._failures.append((jobId, error)) - self._commitSlots += 1 # Flag job as failed so that monitor worker will exit properly. self._builder.setCommitFailed(jobId, reason=str(error)) diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -215,3 +215,80 @@ osutil.setproctitle('mirrorball %s' % (title,)) except: pass + +class BoundedCounter(object): + """ + Basic counter that can be incremented and decremented while enforcing + bounds. + """ + + def __init__(self, low, high, cur, boundsErrors=True): + self._low = low + self._high = high + self._cur = cur + self._boundsErrors = boundsErrors + + def __str__(self): + return str(self._cur) + + def __repr__(self): + return '' % (self._low, self._high, self._cur) + + def __bool__(self): + if self._cur == self._low: + return False + else: + return True + + def __len__(self): + return self._cur - self._low + + def __add__(self, other): + if isinstance(other, int): + while other: + self.increment() + other -= 1 + else: + raise RuntimeError, 'Counters only support adding integers' + + return self + + def __sub__(self, other): + if isinstance(other, int): + while other: + self.decrement() + other -= 1 + else: + raise RuntimeError, 'Counters only support subtracting integers' + + return self + + def __cmp__(self, other): + if isinstance(other, int): + return cmp(self._cur, other) + elif isinstance(other, self.__class__): + return cmp(self._cur, other._cur) + else: + raise (RuntimeError, 'Counters only support comparision operations ' + 'against integers and other Counter instances') + + @property + def upperlimit(self): + return self._high + + @property + def lowerlimit(self): + return self._low + + def increment(self): + if self._cur + 1 <= self._high: + self._cur += 1 + elif self._boundsErrors: + raise RuntimeError, 'Counter has been incremented past upper bounds' + + def decrement(self): + if self._cur - 1 >= self._low: + self._cur -= 1 + elif self._boundsErrors: + raise RuntimeError, 'Counter has been decremented past lower bounds' + From elliot at rpath.com Thu Aug 12 16:03:26 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:26 +0000 Subject: mirrorball: report failed jobIds by package name when rebuidling packages Message-ID: <201008122003.o7CK3QRS030935@scc.eng.rpath.com> changeset: a09f621476a0 user: Elliot Peele date: Thu, 12 Aug 2010 15:51:34 -0400 report failed jobIds by package name when rebuidling packages diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -322,6 +322,7 @@ # Mapping of pkgname to ordered list of trove specs self._pkgs = {} + self._failedpkgs = {} def buildmany(self, troveSpecs): """ @@ -335,7 +336,14 @@ troveSpecs = sorted(troveSpecs) - return Dispatcher.buildmany(self, troveSpecs) + trvMap, failed = Dispatcher.buildmany(self, troveSpecs) + + if self._failedpkgs: + log.info('The following jobs failed to commit') + for name, jobLst in self._failedpkgs.iteritems(): + log.info('%s: %s' % (name, jobLst)) + + return trvMap, failed def _getCommitJobs(self): """ @@ -354,8 +362,13 @@ if status == buildjob.JOB_STATE_BUILT: built.setdefault(trove[0], dict())[trove] = jobId + # Check if any packages have failed to commit. + elif (status == JobStatus.ERROR_COMMITTER_FAILURE and + trove[0] not in self._failedpkgs): + self._failedpkgs[trove[0]] = [jobId, ] + for name, jobDict in built.iteritems(): - # Wait for all versions of a package to build. + # Wait for all versions of a package to build. if ((self._waitForAllVersions and len(jobDict) == len(self._pkgs[name])) or # Wait for the first version to be built. @@ -366,8 +379,26 @@ spec = self._pkgs[name].pop(0) jobId = jobDict[spec] + + # If a build of one version of a package fails to commit, mark + # all subsiquent versions as failed. + if name in self._failedpkgs: + self._failedpkgs[name].append(jobId) + self._jobs[jobId][1] = JobStatus.ERROR_COMMITTER_FAILURE + continue + toCommit.add(jobId) + for name, jobDict in built.iteritems(): + order = [] + for spec in self._pkgs[name]: + if spec not in jobDict: + break + order.append(jobDict[spec]) + + if order: + log.info('ordered built jobs for %s: %s' % (name, order)) + return toCommit From elliot at rpath.com Thu Aug 12 16:03:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:27 +0000 Subject: mirrorball: remove references to incorrect variables Message-ID: <201008122003.o7CK3R0e030989@scc.eng.rpath.com> changeset: 6edfaea70951 user: Elliot Peele date: Thu, 12 Aug 2010 15:52:51 -0400 remove references to incorrect variables diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -138,17 +138,12 @@ latest = sorted(binSpecs)[-1] - if v != latest[1]: - #log.info('%s: found updated version of %s %s -> %s' - # % (version, n, v, latest[1])) + #log.info('%s: found updated version of %s %s -> %s' + # % (version, n, v, latest[1])) - toAdd = set([ x for x in binSpecs - if x[1] == latest[1] ]) - nvfs.update(toAdd) - elif f is None: - emptyFlavors.add((n, v, f)) - else: - nvfs.add((n, v, f)) + toAdd = set([ x for x in binSpecs + if x[1] == latest[1] ]) + nvfs.update(toAdd) # Lookup anything that has an empty flavor. log.info('%s: looking up version information for empty flavors' @@ -313,7 +308,7 @@ extraPackages = [ 'xulrunner', 'firefox', 'gnome-vfs2', 'numactl', ] - bot.rebuildgroups(restart=True, + bot.rebuildgroups(restart=False, updatedPackages=rebuiltPackages + tags + extraPackages) import epdb; epdb.st() From elliot at rpath.com Thu Aug 12 16:03:26 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:26 +0000 Subject: mirrorball: add support for replace files when rewriting sources Message-ID: <201008122003.o7CK3QBx030962@scc.eng.rpath.com> changeset: 6237e6dffcc9 user: Elliot Peele date: Thu, 12 Aug 2010 15:52:15 -0400 add support for replace files when rewriting sources diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage --- a/scripts/rebuildpackage +++ b/scripts/rebuildpackage @@ -15,6 +15,7 @@ import os import sys +import shutil import logging mirrorballDir = os.path.abspath('../') @@ -71,7 +72,7 @@ return trvMap - def removeSourceFiles(self, name, keepFiles=None): + def removeSourceFiles(self, name, keepFiles=None, replaceFiles=None): """ Remove all of the files from all source versions of a package that are not listed in keepFiles. @@ -85,6 +86,12 @@ if not keepFiles: keepFiles = set() + if not replaceFiles: + replaceFiles = set() + + replaceMap = dict([ (os.path.basename(x), x) for x in replaceFiles ]) + keepFiles.update(set(replaceMap.keys())) + # Always avoid removing the CONARY file and the manifest. keepFiles.add('CONARY') keepFiles.add('manifest') @@ -107,6 +114,7 @@ n, v, f = newPkgs[0] # Commit the changeset now that it has been validated. + log.info('commiting clone changeset') helper._repos.commitChangeSet(cs) else: n, v, f = spec @@ -114,12 +122,19 @@ # Edit the source to remove files. checkoutDir = helper._edit(n, version=v) files = set(os.listdir(checkoutDir)) + removeFiles = files - keepFiles for fn in removeFiles: helper._removeFile(checkoutDir, fn) + replace = files & set(replaceMap.keys()) + for fn in replace: + dest = os.path.join(checkoutDir, fn) + log.info('replacing %s -> %s' % (replaceMap[fn], dest)) + shutil.copyfile(replaceMap[fn], dest) + # Commit changes if anything changed. - if removeFiles: + if removeFiles or replace: helper.commit(n, version=v, commitMessage='automated file removal') @@ -163,11 +178,13 @@ errata.fetch() pkgNames = sys.argv[2:] + #replaceFiles = sys.argv[3:] + replaceFiles = None bot = Bot(cfg, errata) -# for pkgName in pkgNames: -# bot.removeSourceFiles(pkgName) + #for pkgName in pkgNames: + # bot.removeSourceFiles(pkgName, replaceFiles=replaceFiles) bot.rebuildpackages(pkgNames, useLatest=['conary', 'conary-build', 'conary-policy', 'rpm', ], From elliot at rpath.com Thu Aug 12 16:03:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 12 Aug 2010 20:03:27 +0000 Subject: mirrorball: branch merge Message-ID: <201008122003.o7CK3RWd031016@scc.eng.rpath.com> changeset: 860ebcd40756 user: Elliot Peele date: Thu, 12 Aug 2010 16:03:14 -0400 branch merge diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -16,47 +16,26 @@ Generate update information based on the patch detail in SuSE repositories. """ +import time import logging from errata import common +from errata.common import Nevra +from errata.common import Package +from errata.common import Channel +from errata.common import Advisory log = logging.getLogger('errata') -class Package(common.Package): - """ - Class to represent a package. - """ - - def getNevra(self): - """ - Returns a tuple of (name, epoch, version, release, arch) for - this package. - """ - -class Channel(common.Channel): - """ - Class to represent a repository. - """ - - -class Advisory(common.Advisory): - """ - Class to represent an errata or advisory. - """ - - class AdvisoryManager(common.AdvisoryManager): def __init__(self, pkgSource): + common.AdvisoryManager.__init__(self) + self._pkgSource = pkgSource - self._fetched = False - self._patches = set() - - @common.reqfetch - def getRepositories(self): - """ - Returns a list of repository labels that have been fetched. - """ + self._channels = {} + self._advOrder = {} + self._advisories = set() @common.reqfetch def iterByIssueDate(self): @@ -64,7 +43,10 @@ Yields Errata objects by the issue date of the errata. """ - return [] + # FIXME: this work here? (cribbed from centos.py) + for updateId in sorted(self._advOrder): + for adv in self._advOrder[updateId]: + yield adv def fetch(self): """ @@ -75,7 +57,9 @@ excesive load for anyone's servers. """ + self._order() self._fetched = True + return self._advisories @common.reqfetch def getChannels(self): @@ -84,15 +68,16 @@ @return list of indexed channel names """ - return self._pkgSource._clients.keys() - + return self._channels.keys() def cleanup(self): """ Free all cached results. """ - self._patches = set() + self._channels = {} + self._advOrder = {} + self._advisories = set() def getModifiedErrata(self, updateId): """ @@ -102,18 +87,208 @@ return [] - def _fetchPatches(self): + def _order(self): """ Fetch all patch data from the package source. """ + def bin_timestamp(ts): + """ + Convert a time stamp into the desired time slice. + """ + + # convert to current day + return int(time.mktime(time.strptime(time.strftime('%Y%m%d', + time.gmtime(ts)), '%Y%m%d'))) + + def slice(patches): + """ + Build a dictionary of binned (sliced) timestamps and patches. + """ + + slices = {} + + for patch in patches: + # Bin by day: + updateId = bin_timestamp(int(patch.timestamp)) + # ...or uncomment the below line to disable binning: + #updateId = int(patch.timestamp) + slices.setdefault(updateId, + set()).add(patch.getAttribute('patchid')) + return slices + + def patchidNoRepo(patchid): + """ + Trims the leading (repository) information from a patchid. + Used for folding patches across repositories, among other things. + """ + return '-'.join(patchid.split('-')[1:]) + + def map_patchids(slices): + """ + Build a dictionary of (partial) patchids and their + corresponding timestamps. Used to determine if a patchid + across multiple repositories also has multiple timestamps. + """ + + patchidMap = {} + + for timestamp, patchids in slices.iteritems(): + for patchid in patchids: + # Map excludes leading segment of patchid, which + # carries repository information; we're folding + # across repositories. + patchidMap.setdefault(patchidNoRepo(patchid), + set()).add(timestamp) + return patchidMap + + def getChannel(pkg): + for label, channel in self._channels.iteritems(): + if label in pkg.location: + return channel + raise RuntimeError , 'unable to find channel for %s' % pkg.location + + def getSrcPkg(binPkg): + for srcPkg, binPkgs in self._pkgSource.srcPkgMap.iteritems(): + if binPkg in binPkgs: + return srcPkg + raise RuntimeError , 'unable to find source package for %s' % binPkg.location + + def getPatchById(patches, patchid): + for patch in patches: + if patch.getAttribute('patchid') == patchid: + return patch + raise RuntimeError , 'unable to find patch %s' % patchid + # make sure the pkg source is loaded. self._pkgSource.load() - # now get the patch data + # Each client value is a SLES release/update repository object. + # self._pkgSource._clients.values()[...] + + # now get the patch data... patches = set() + for path, client in self._pkgSource.getClients().iteritems(): - log.info('loading patches for %s' % path) - patches.update(set(client.getPatchDetail())) + log.info('loading patches for path %s' % path) + for patch in client.getPatchDetail(): + for pkg in patch.packages: + pkg.location = path + '/' + pkg.location + patches.add(patch) - return patches + for label in self._pkgSource._clients: + self._channels[label] = Channel(label) + + # ...and (time-)slice it up. + slices = slice(patches) + + for timeslice, patchSet in slices.iteritems(): + for patchId in patchSet: + patchObj = getPatchById(patches, patchId) + if patchObj.timestamp != timeslice: + log.info('syncing %s timestamp (%s) to slice timestamp %s' % ( + patchId, patchObj.timestamp, timeslice)) + patchObj.timestamp = timeslice + + # slices dict is still current since the above only synced the + # patch timestamps to existing slices. + + # This maps patchid (without regard to repos) to timeslices. + patchidMap = map_patchids(slices) + + # This requires no more than two timestamps per patchid; + # one each for slesp3 and sdkp3 (bails out otherwise): + # + # Pondering how this can be consolidated with above code to + # reduce iterating... + # + # Just so it's clear, I hate this code (i.e. FIXME). + # + for patchid, timestamps in patchidMap.iteritems(): + if len(timestamps) > 1: + # Untested beyond 2. + assert(len(timestamps) == 2) + # FIXME: refactor this monster. + splitpatch = [ (patch.getAttribute('patchid'), + set(patch.packages), patch) for patch in + patches if patchidNoRepo(patch.getAttribute('patchid')) == patchid ] + if splitpatch[0][1].issubset(splitpatch[1][1]): + log.info('syncing timestamps (%s %s) ' % ( + splitpatch[0][2].timestamp, + splitpatch[1][2].timestamp) + + 'across repositories for %s & %s ' % ( + splitpatch[0][0], splitpatch[1][0]) + + 'to superset timestamp %s' % splitpatch[1][2].timestamp) + splitpatch[0][2].timestamp = splitpatch[1][2].timestamp + elif splitpatch[1][1].issubset(splitpatch[0][1]): + log.info('syncing timestamps (%s %s) ' % ( + splitpatch[0][2].timestamp, + splitpatch[1][2].timestamp) + + 'across repositories for %s & %s ' % ( + splitpatch[0][0], splitpatch[1][0]) + + 'to superset timestamp %s' % splitpatch[0][2].timestamp) + splitpatch[1][2].timestamp = splitpatch[0][2].timestamp + # So far this has only been tested in pure-subset cases. + else: + raise RuntimeError , 'neither %s nor %s is a subset of the other' % (splitpatch[0][0], splitpatch[1][0]) + + advPkgMap = {} + nevras = {} + packages = {} + srcPkgAdvMap = {} + srcPkgPatchidMap = {} + + for patch in patches: + advisory = patch.getAttribute('patchid') + patchid = patchidNoRepo(advisory) + + for binPkg in patch.packages: + nevra = binPkg.getNevra() + nevraObj = nevras.setdefault(nevra, Nevra(*nevra)) + channelObj = getChannel(binPkg) + package = Package(channelObj, nevraObj) + packageObj = packages.setdefault(package, package) + advPkgMap.setdefault(advisory, set()).add(packageObj) + srcPkgObj = getSrcPkg(binPkg) + srcPkgAdvMap.setdefault(srcPkgObj, set()).add(advisory) + srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid) + + # FIXME: I hate this code too. + if srcPkgPatchidMap[srcPkgObj] != set([patchid]): + # Untested beyond two, and expected case is two + # different advisories issued for the same source + # package, one each for x86 and x86_64. (Lots of + # these for the kernel, for instance.) + assert(len(srcPkgPatchidMap[srcPkgObj]) == 2) + srcPkgAdvs = [ getPatchById(patches, srcPkgAdv) + for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ] + # Only sync the same source package once. (It may + # appear for multiple binary packages.) + if srcPkgAdvs[0].timestamp != srcPkgAdvs[1].timestamp: + # Using the min here in case the first advisory + # for this source package has already been + # published. + syncTimestamp = min(srcPkgAdvs[0].timestamp, + srcPkgAdvs[1].timestamp) + log.info('syncing timestamps (%s %s) ' % ( + srcPkgAdvs[0].timestamp, srcPkgAdvs[1].timestamp) + + 'across same-SRPM advisories for %s & %s ' % ( + srcPkgAdvs[0].getAttribute('patchid'), + srcPkgAdvs[1].getAttribute('patchid')) + + 'to earlier timestamp %s' % syncTimestamp) + srcPkgAdvs[0].timestamp = srcPkgAdvs[1].timestamp = syncTimestamp + + # There should be no srcPkgs with more than two patchids. + assert(len([ x for x, y in srcPkgPatchidMap.iteritems() + if len(y) > 2 ]) == 0) + + issue_date = time.strftime('%Y-%m-%d %H:%M:%S', + time.gmtime(int(patch.timestamp))) + log.info('creating advisory: %s (%s)' % (advisory, + patch.timestamp)) + adv = Advisory(advisory, patch.summary, issue_date, + advPkgMap[advisory]) + self._advisories.add(adv) + self._advOrder.setdefault(int(patch.timestamp), set()).add(adv) + + import epdb ; epdb.st() diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -155,6 +155,13 @@ if pkgcmp != 0: return pkgcmp + # Compare arch before checksum to catch cases of multiple + # arch-specific packages that happen to have same content + # (e.g. SLES xorg-x11-fonts packages). + archcmp = cmp(self.arch, other.arch) + if archcmp != 0: + return archcmp + # Compare checksum only for equality, otherwise sorting will result in # checksum ordering. if (self.checksum and other.checksum and diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -1,5 +1,5 @@ # -# Copryright (c) 2008-2009 rPath, Inc. +# Copryright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -16,6 +16,10 @@ Module for parsing patch-*.xml files from the repository metadata. """ +import logging + +log = logging.getLogger('repomd') + __all__ = ('PatchXml', ) from rpath_xmllib import api1 as xmllib @@ -36,7 +40,13 @@ 'release', 'requires', 'recommends', 'rebootNeeded', 'licenseToConfirm', 'packageManager', 'category', 'packages', 'provides', 'supplements', 'conflicts', - 'obsoletes') + 'obsoletes', 'timestamp') + + def __init__(self, *args, **kwargs): + SlotNode.__init__(self, *args, **kwargs) + # Need access to this so it can be modified when syncing a + # patch's timestamp across architectures. + self.timestamp = self.getAttribute('timestamp') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. @@ -105,6 +115,14 @@ if desccmp != 0: return desccmp + if self.timestamp != other.timestamp: + maxtime = max(self.timestamp, other.timestamp) + log.info('syncing timestamps (%s %s) ' % (self.timestamp, + other.timestamp) + + 'for %s-%s to %s' % (self.name, self.version, maxtime)) + self.timestamp = other.timestamp = maxtime + # Don't return here--they're now equal. + for pkg in other.packages: if pkg not in self.packages: self.packages.append(pkg) diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -57,6 +57,7 @@ versionMap = { 'rhel': {'4': '4.0', '5': '5.0', }, + 'centos': {'5': '0', }, } # Make sure to include any packages that were built from the same @@ -141,10 +142,8 @@ #log.info('%s: found updated version of %s %s -> %s' # % (version, n, v, latest[1])) - toAdd = set([ x for x in binSpecs - if x[1] == latest[1] ]) + toAdd = set([ x for x in binSpecs if x[1] == latest[1] ]) nvfs.update(toAdd) - # Lookup anything that has an empty flavor. log.info('%s: looking up version information for empty flavors' % version) @@ -212,8 +211,7 @@ # Wait for the first results to make sure the group will rebuild # properly. if not trvMap: - log.info('waiting for first build to complete before ' - 'continuing') + log.info('waiting for first build to complete before continuing') while not res.isDone: time.sleep(1) @@ -243,8 +241,6 @@ from conary.lib import util sys.excepthook = util.genExcepthook() - import rhnmirror - from updatebot import config from updatebot import log as logSetup @@ -257,13 +253,30 @@ cfg = config.UpdateBotConfig() cfg.read(confDir + '/updatebotrc') - mcfg = rhnmirror.MirrorConfig() - mcfg.read(confDir + '/erratarc') + if cfg.platformName == 'rhel': + import rhnmirror - errata = rhnmirror.Errata(mcfg) - errata.fetch() + mcfg = rhnmirror.MirrorConfig() + mcfg.read(confDir + '/erratarc') - bot = Bot(cfg, errata) + errata = rhnmirror.Errata(mcfg) + + bot = Bot(cfg, errata) + + else: + bot = Bot(cfg, None) + + if cfg.platformName == 'sles': + from errata.sles import AdvisoryManager as Errata + + elif cfg.platformName == 'centos': + from errata.centos import AdvisoryManager as Errata + + else: + raise RuntimeError, 'no errata source found for %s' % cfg.platformName + + errata = Errata(bot._pkgSource) + bot._errata._errata = errata rebuiltPackages = ['kernel', 'xenpv', 'lpfc-kmod', 'be2net-kmod', 'alacarte', 'alchemist', 'audit', 'authconfig', 'avahi', 'beecrypt', @@ -308,7 +321,7 @@ extraPackages = [ 'xulrunner', 'firefox', 'gnome-vfs2', 'numactl', ] - bot.rebuildgroups(restart=False, + bot.rebuildgroups(restart=False, resolveTargetVersions=False, readdPackages=True, updatedPackages=rebuiltPackages + tags + extraPackages) import epdb; epdb.st() diff --git a/scripts/sleorder.py b/scripts/sleorder.py --- a/scripts/sleorder.py +++ b/scripts/sleorder.py @@ -3,12 +3,8 @@ import os import sys import time -import tempfile sys.path.insert(0, os.environ['HOME'] + '/hg/conary') -sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-xmllib') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-capsule-indexer') from conary.lib import util sys.excepthook = util.genExcepthook() @@ -19,10 +15,8 @@ confDir = os.path.join(mbdir, 'config', sys.argv[1]) from updatebot import log -from updatebot import cmdline -from updatebot import pkgsource +from updatebot.ordered import Bot from updatebot import UpdateBotConfig -from updatebot.ordered import Bot from errata.sles import AdvisoryManager as Errata @@ -31,14 +25,12 @@ cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) -ui = cmdline.UserInterface() +bot = Bot(cfg, None) +errata = Errata(bot._pkgSource) +bot._errata._errata = errata -pkgSource = pkgsource.PackageSource(cfg, ui) - -errata = Errata(pkgSource) errata.fetch() -bot = Bot(cfg, errata) bot._pkgSource.load() bot._errata._orderErrata() diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -408,6 +408,14 @@ # bucket listed. mergeUpdates = (CfgList(CfgQuotedLineList(CfgInt)), []) + # Timestamp of first erratum. This is used as a baseline for + # determining if any update packages are missing errata. It should + # auto-detect correctly, but in some cases--for instance, when a + # distribution releases the same package as a baseline package on + # one channel and an update on a parallel channel--this will require + # manual specification. + firstErrata = CfgInt + # Errata timestamp pairs for rescheduling when updates are applied. The # first element is the current timestamp of the update. The second element # is the new timestamp. You may need to use this option if it appears that diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -593,7 +593,7 @@ for src, bins in srcMap.iteritems(): # Pull out any package sets that look like they are incomplete. - if len(bins) != len(self._pkgSource.srcPkgMap[src]) - 1: + if len(bins) != len(set([ (x.name, x.arch) for x in self._pkgSource.srcPkgMap[src] ])) - 1: extras[src] = bins continue @@ -908,9 +908,13 @@ # separate out golden bits other = [] golden = [] - firstErrata = int(time.time()) - if len(buckets): - firstErrata = sorted(buckets.keys())[0] + if self._cfg.firstErrata: + firstErrata = self._cfg.firstErrata + else: + firstErrata = int(time.time()) + if len(buckets): + firstErrata = sorted(buckets.keys())[0] + for nevra, pkg in nevras.iteritems(): buildtime = int(pkg.buildTimestamp) if buildtime < firstErrata: diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -308,6 +308,15 @@ expectedRemovals=expectedRemovals, allowPackageDowngrades=allowDowngrades, **kwargs)) + # Comment out the above pkgMap.update() call and + # uncomment the below call with list of rmake job id's + # in order to commit already-built packages. (Useful in + # cases where a large commit has failed and a retry + # would save time.) Also, set a breakpoint at end of + # enclosing loop so mirrorball won't keep trying this + # over and over! + #pkgMap.update(self._builder.commit([job1,job2,...])) + # When deriving from an upstream platform sometimes we don't want # the latest versions. oldVersions = self._cfg.useOldVersion.get(updateId, None) @@ -402,6 +411,10 @@ log.info('published update %s in %s seconds' % (advTime, totalTime)) count += 1 + # Set this breakpoint when you're committing a list of + # already-built job id's. (See long comment above.) + #import epdb ; epdb.st() + log.info('update completed') log.info('applied %s updates in %s seconds' % (count, time.time() - startime)) diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -420,6 +420,12 @@ x for x in self._pkgSource.binNameMap[binPkg.name] if x.arch == binPkg.arch ]) + # Maybe this is a src or nosrc. + if not pkgs: + pkgs = sorted([ + x for x in self._pkgSource.srcNameMap[binPkg.name] + if x.arch == binPkg.arch ]) + # If running in latest mode we really want to compare to the # latest version of this binary, but if we are running in # ordered we really want the "next" version of this binary. From juphoff at rpath.com Thu Aug 19 12:54:47 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Thu, 19 Aug 2010 16:54:47 +0000 Subject: mirrorball: add ignoreSourceUpdate config option Message-ID: <201008191654.o7JGslK2029507@scc.eng.rpath.com> changeset: c2fec89a15c6 user: Jeff Uphoff date: Thu, 19 Aug 2010 12:21:14 -0400 add ignoreSourceUpdate config option diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -459,6 +459,12 @@ # from the package model removeSource = (CfgIntDict(CfgList(CfgNevra)), {}) + # updateId sourceNevra + # At updateId, ignore the source package update specified by + # sourceNevra and continue to use whatever previous version was in + # the model. + ignoreSourceUpdate = (CfgIntDict(CfgList(CfgNevra)), {}) + # updateId binaryNevra # As of updateId, I expect the code to think this nevra should be removed, # but I want to keep it. diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -274,6 +274,10 @@ expectedKeepRemovals = self._cfg.keepRemoved.get(updateId, []) explicitSourceRemovals = self._cfg.removeSource.get(updateId, set()) explicitBinaryRemovals = self._cfg.removeObsoleted.get(updateId, set()) + explicitIgnoreSources = self._cfg.ignoreSourceUpdate.get(updateId, set()) + if explicitIgnoreSources: + log.info('explicitly ignoring %s in update %s' % + (explicitIgnoreSources, updateId)) explicitPackageDowngrades = downgraded.get(updateId, None) assert len(self._order[updateId]) @@ -303,14 +307,19 @@ if srpm.getNevra() in explicitSourceRemovals: log.error('Removing %s in %s would cause it never to be promoted' % (str(' '.join(srpm.getNevra())), updateId)) - current[srpm.name] = srpm - version = updater.update(nvf, srpm) - assert (not version or - not updater.isPlatformTrove(version)) - if version: - parentPackages.append(((nvf, srpm), version)) + + if srpm.getNevra() in explicitIgnoreSources: + log.warn('Ignoring %s in %s will cause it never to be promoted' % + (str(' '.join(srpm.getNevra())), updateId)) else: - childPackages.append(((nvf, srpm), None)) + current[srpm.name] = srpm + version = updater.update(nvf, srpm) + assert (not version or + not updater.isPlatformTrove(version)) + if version: + parentPackages.append(((nvf, srpm), version)) + else: + childPackages.append(((nvf, srpm), None)) # all package names obsoleted by packages in the current set obsoleteNames = set() @@ -325,6 +334,9 @@ if srpm.getNevra() in explicitSourceRemovals: current.pop(srpm.name, None) continue + if srpm.getNevra() in explicitIgnoreSources: + log.info('explicitly ignoring source package update %s' % [ignoreSource]) + continue for pkg in sorted(self._pkgSource.srcPkgMap[srpm]): if pkg.arch == 'src': continue From juphoff at rpath.com Sat Aug 21 16:08:31 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Sat, 21 Aug 2010 20:08:31 +0000 Subject: mirrorball: munge order for ignoreSourceUpdate config option Message-ID: <201008212008.o7LK8Vgl018784@scc.eng.rpath.com> changeset: 98add62f13b5 user: Jeff Uphoff date: Sat, 21 Aug 2010 16:03:44 -0400 munge order for ignoreSourceUpdate config option diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -669,6 +669,13 @@ for source, dest, nevra in self._cfg.reorderSource: self._reorderSource(source, dest, nevra) + ignoredCount = 0 + # remove any source packages we're deliberately ignoring: + for source, nevras in self._cfg.ignoreSourceUpdate.iteritems(): + for nevra in nevras: + self._reorderSource(source, None, nevra) + ignoredCount += 1 + # add a source to a specific bucket, used to "promote" newer versions # forward. nevras = dict([ (x.getNevra(), x) @@ -685,7 +692,7 @@ for pkgSet in self._order.itervalues(): pkgs.update(pkgSet) assert len(pkgs) == totalPkgs2 - diffCount - assert totalPkgs2 == totalPkgs + diffCount + assert totalPkgs2 == totalPkgs + diffCount - ignoredCount def _mergeUpdates(self, mergeList): """ @@ -792,9 +799,13 @@ def _reorderSource(self, source, dest, nevra): """ Reschedule an individual srpm to another bucket. + If destination bucket is None, simply remove srpm from source bucket. """ - log.info('rescheduling %s %s -> %s' % (nevra, source, dest)) + if dest: + log.info('rescheduling %s %s -> %s' % (nevra, source, dest)) + else: + log.info('removing %s from %s' % (nevra, source)) # Remove specified source nevra from the source bucket bucketNevras = dict([ (x.getNevra(), x) @@ -812,8 +823,9 @@ if not len(self._advPkgMap[advisory]): del self._advPkgMap[advisory] - # Move srpm to destination bucket - self._order.setdefault(dest, set()).add(srpm) + if dest: + # Move srpm to destination bucket if not a removal. + self._order.setdefault(dest, set()).add(srpm) def _getNevra(self, pkg): """ From juphoff at rpath.com Sat Aug 21 16:10:01 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Sat, 21 Aug 2010 20:10:01 +0000 Subject: mirrorball: tweak log message for removing ignored srpms Message-ID: <201008212010.o7LKA1t3018864@scc.eng.rpath.com> changeset: ef9a988d54bd user: Jeff Uphoff date: Sat, 21 Aug 2010 16:09:56 -0400 tweak log message for removing ignored srpms diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -805,7 +805,7 @@ if dest: log.info('rescheduling %s %s -> %s' % (nevra, source, dest)) else: - log.info('removing %s from %s' % (nevra, source)) + log.info('removing ignored %s from %s' % (nevra, source)) # Remove specified source nevra from the source bucket bucketNevras = dict([ (x.getNevra(), x) From juphoff at rpath.com Mon Aug 23 17:05:51 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Mon, 23 Aug 2010 21:05:51 +0000 Subject: mirrorball: add errataPromoteAfter config option Message-ID: <201008232105.o7NL5pww030918@scc.eng.rpath.com> changeset: 27b9dc2a3428 user: Jeff Uphoff date: Mon, 23 Aug 2010 17:05:47 -0400 add errataPromoteAfter config option diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -416,6 +416,11 @@ # manual specification. firstErrata = CfgInt + # Timestamp after which errata promotions begin. This is useful in + # cases where the baseline distribution must be split across + # multiple updateId's in order to de-dupe the package list. + errataPromoteAfter = (CfgInt, 0) + # Errata timestamp pairs for rescheduling when updates are applied. The # first element is the current timestamp of the update. The second element # is the new timestamp. You may need to use this option if it appears that diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -452,6 +452,10 @@ for updateId, bucket in self._errata.iterByIssueDate(current=1): upver = self._errata.getBucketVersion(updateId) + if updateId <= self._cfg.errataPromoteAfter: + log.info('version %s (%s) at or below promotion timestamp threshold (%s), skipping' % (upver, updateId, self._cfg.errataPromoteAfter)) + continue + # Don't try to promote buckets that have already been promoted. if upver in targetLatest: log.info('%s found on target label, skipping' % upver) From elliot at rpath.com Tue Aug 24 17:02:50 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 24 Aug 2010 21:02:50 +0000 Subject: mirrorball: add a cache to conaryhelper.findTroves Message-ID: <201008242102.o7OL2oqK016814@scc.eng.rpath.com> changeset: 6526e64f0756 user: Elliot Peele date: Tue, 24 Aug 2010 16:50:11 -0400 add a cache to conaryhelper.findTroves diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -47,6 +47,7 @@ from updatebot.errors import MirrorFailedError from updatebot.errors import BinariesNotFoundForSourceVersion +from updatebot.lib.findtroves import FindTrovesCache from updatebot.lib.conarycallbacks import UpdateBotCloneCallback log = logging.getLogger('updatebot.conaryhelper') @@ -132,6 +133,8 @@ dir=self._cache.sharedTmpDir, prefix='conaryhelper-%s-' % cfg.platformName) + self._findTrovesCache = FindTrovesCache(self._repos) + def clearCache(self): """ Clear the trove query cache. @@ -169,7 +172,8 @@ labels = self._ccfg.buildLabel try: - return self._repos.findTroves(labels, troveList, *args, **kwargs) + return self._findTrovesCache.findTroves(labels, troveList, + *args, **kwargs) except conaryerrors.TroveNotFound, e: return {} diff --git a/updatebot/lib/findtroves.py b/updatebot/lib/findtroves.py new file mode 100644 --- /dev/null +++ b/updatebot/lib/findtroves.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +This is a caching implentation of the findTroves call from conary netclient. +""" + +import logging + +log = logging.getLogger('updatebot.lib.findtroves') + +class FindTrovesCache(dict): + """ + Caching layer for findTroves + """ + + def __init__(self, repos): + dict.__init__(self) + + self._repos = repos + + def findTroves(self, labelPath, troves, getLeaves=True): + """ + Emulate the behavior of repos.findTroves while caching results. + """ + + found = set() + needed = {} + + if not isinstance(labelPath, (list, tuple, set)): + tlp = (labelPath, ) + else: + tlp = tuple(labelPath) + + # Separate out items that have already been cached from those that have + # not. + for trove in troves: + key = (tlp, tuple(trove), getLeaves) + + if key in self: + found.add(key) + else: + needed[trove] = key + + # Ask the repo for anything that hasn't been cached. + if needed: + if found: + log.info('CACHE hit on %s troves' % len(found)) + log.info('CACHE querying repository for %s troves' % len(needed)) + results = self._repos.findTroves(labelPath, needed, + getLeaves=getLeaves) + else: + log.info('CACHE hit on all troves') + results = {} + + # Cache the results. + for req, res in results.iteritems(): + self[needed[req]] = res + + # Build actual result + results.update(dict((x[1], self[x]) for x in found)) + + return results From elliot at rpath.com Tue Aug 24 17:02:51 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 24 Aug 2010 21:02:51 +0000 Subject: mirrorball: add a config option for mapping updateIds to version strings Message-ID: <201008242102.o7OL2ppg016845@scc.eng.rpath.com> changeset: e025fafda802 user: Elliot Peele date: Tue, 24 Aug 2010 16:50:51 -0400 add a config option for mapping updateIds to version strings diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -402,6 +402,10 @@ # a magic number, but at least it is configurable? maxBuildSize = (CfgInt, 10) + # Map of updateIds to upstream versions to use if you don't want to use the + # normal versioning scheme. + upstreamVersionMap = (CfgIntDict(CfgString), {}) + # List of errata timestamps to merge together. This is used when one errata # leaves the platform in a non dependency closed state and a later update # should solve the dependency problem. All updates are folded into the first diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -138,7 +138,10 @@ @type bucketId: integer (unix time) """ - version = time.strftime('%Y.%m.%d_%H%M.%S', time.gmtime(bucketId)) + version = self._cfg.upstreamVersionMap.get(bucketId, None) + if not version: + version = time.strftime('%Y.%m.%d_%H%M.%S', time.gmtime(bucketId)) + return version @loadErrata From elliot at rpath.com Tue Aug 24 17:02:51 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 24 Aug 2010 21:02:51 +0000 Subject: mirrorball: actually stop at the end of the available stream of groups rather than Message-ID: <201008242102.o7OL2pxU016873@scc.eng.rpath.com> changeset: 2306ce7ef1c8 user: Elliot Peele date: Tue, 24 Aug 2010 16:53:51 -0400 actually stop at the end of the available stream of groups rather than triggering a taceback diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -102,6 +102,10 @@ log.info('%s: retrieving group model information' % version) group = self._groupmgr.getGroup(version=version) + # Stop when we get to a version that isn't in the repository. + if group is None: + break + # Get all of the nvfs from the group model. nvfs = set() checkUpdates = set() From elliot at rpath.com Tue Aug 24 17:02:51 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 24 Aug 2010 21:02:51 +0000 Subject: mirrorball: add a flag to promote for enforcing that all expected packages were promoted Message-ID: <201008242102.o7OL2pSu016901@scc.eng.rpath.com> changeset: aaf6d05ae270 user: Elliot Peele date: Tue, 24 Aug 2010 16:54:30 -0400 add a flag to promote for enforcing that all expected packages were promoted diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -65,6 +65,6 @@ errata = Errata(bot._pkgSource) bot._errata._errata = errata -bot.promote() +bot.promote(enforceAllExpected=True) import epdb; epdb.st() diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -1012,7 +1012,8 @@ def promote(self, trvLst, expected, sourceLabels, targetLabel, checkPackageList=True, extraPromoteTroves=None, - extraExpectedPromoteTroves=None, commit=True): + extraExpectedPromoteTroves=None, commit=True, + enforceAllExpected=True): """ Promote a group and its contents to a target label. @param trvLst: list of troves to publish @@ -1117,9 +1118,13 @@ extraTroves = set([ x[0] for x in extraPromoteTroves | extraExpectedPromoteTroves ]) - + # grpDiff.difference is checking that no packages outside of the + # expected set are promoted. + # + # grpInv.difference is checking that all packages that we expect to be + # promoted are promoted. if (checkPackageList and (grpDiff.difference(extraTroves) or - grpInv.difference(extraTroves))): + (enforceAllExpected and grpInv.difference(extraTroves)))): raise PromoteMismatchError(expected=oldPkgs, actual=newPkgs) if not commit: diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -421,7 +421,7 @@ return updateSet - def promote(self): + def promote(self, enforceAllExpected=True): """ Promote binary groups from the devel label to the production lable in the order that they were built. @@ -442,6 +442,17 @@ targetLatest = self._updater.getUpstreamVersionMap( (self._cfg.topGroup[0], self._cfg.targetLabel, None)) + log.info('querying target label for cloned from information') + # The targetSpec list tells us if the latest version of each group has + # been promoted to the target label. + sourceSpecs = [ x for x in itertools.chain(*sourceLatest.itervalues()) ] + targetSpecs, failed = self._updater.getTargetVersions(sourceSpecs, + logErrors=False) + targetSpecMap = {} + for spec in targetSpecs: + ver = spec[1].trailingRevision().getVersion() + targetSpecMap.setdefault(ver, set()).add(spec) + log.info('starting promote') count = 0 @@ -449,13 +460,16 @@ # Get all updates after the first bucket. missing = False - for updateId, bucket in self._errata.iterByIssueDate(current=1): + for updateId, bucket in self._errata.iterByIssueDate(current=-1): upver = self._errata.getBucketVersion(updateId) # Don't try to promote buckets that have already been promoted. - if upver in targetLatest: + if upver in targetLatest and upver in targetSpecMap: log.info('%s found on target label, skipping' % upver) continue + elif upver not in targetSpecMap: + log.info('%s found on target label, but newer version ' + 'available on source, will repromote' % upver) # Make sure version has been imported. if upver not in sourceLatest: @@ -511,7 +525,8 @@ def promote(): # Create and validate promote changeset packageList = self._updater.publish(toPromote, expected, - self._cfg.targetLabel, extraExpectedPromoteTroves=extra) + self._cfg.targetLabel, extraExpectedPromoteTroves=extra, + enforceAllExpected=enforceAllExpected) return 0 rc = watchdog.waitOnce(promote) diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -960,7 +960,7 @@ return self._conaryhelper.getBuildRequires(pkgName) def publish(self, trvLst, expected, targetLabel, checkPackageList=True, - extraExpectedPromoteTroves=None): + extraExpectedPromoteTroves=None, enforceAllExpected=True): """ Publish a group and its contents to a target label. @param trvLst: list of troves to publish @@ -987,6 +987,7 @@ checkPackageList=checkPackageList, extraPromoteTroves=self._cfg.extraPromoteTroves, extraExpectedPromoteTroves=extraExpectedPromoteTroves, + enforceAllExpected=enforceAllExpected, ) def mirror(self, fullTroveSync=False): From elliot at rpath.com Tue Aug 24 17:02:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 24 Aug 2010 21:02:52 +0000 Subject: mirrorball: allow the mirror config file name to be overridden Message-ID: <201008242102.o7OL2qii016935@scc.eng.rpath.com> changeset: cf38664b66b0 user: Elliot Peele date: Tue, 24 Aug 2010 16:54:54 -0400 allow the mirror config file name to be overridden diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -84,7 +84,7 @@ _cache = ConaryHelperSharedCache() - def __init__(self, cfg): + def __init__(self, cfg, mirrorCfgFn=None): self._groupFlavorCount = len(cfg.groupFlavors) if not self._cache.sharedTmpDir: @@ -114,8 +114,11 @@ self._cache.conaryConfigCache[conaryCfgFile] = self._ccfg + if not mirrorCfgFn: + mirrorCfgFn = 'mirror.conf' + self._mcfg = None - mcfgfn = util.join(cfg.configPath, 'mirror.conf') + mcfgfn = util.join(cfg.configPath, mirrorCfgFn) if mcfgfn in self._cache.conaryConfigCache: self._mcfg = self._cache.conaryConfigCache[mcfgfn] elif os.path.exists(mcfgfn): From elliot at rpath.com Tue Aug 24 17:02:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 24 Aug 2010 21:02:52 +0000 Subject: mirrorball: branch merge Message-ID: <201008242102.o7OL2qwe017011@scc.eng.rpath.com> changeset: b32fd9462843 user: Elliot Peele date: Tue, 24 Aug 2010 17:02:48 -0400 branch merge diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -65,6 +65,6 @@ errata = Errata(bot._pkgSource) bot._errata._errata = errata -bot.promote() +bot.promote(enforceAllExpected=True) import epdb; epdb.st() diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -102,6 +102,10 @@ log.info('%s: retrieving group model information' % version) group = self._groupmgr.getGroup(version=version) + # Stop when we get to a version that isn't in the repository. + if group is None: + break + # Get all of the nvfs from the group model. nvfs = set() checkUpdates = set() diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -47,6 +47,7 @@ from updatebot.errors import MirrorFailedError from updatebot.errors import BinariesNotFoundForSourceVersion +from updatebot.lib.findtroves import FindTrovesCache from updatebot.lib.conarycallbacks import UpdateBotCloneCallback log = logging.getLogger('updatebot.conaryhelper') @@ -83,7 +84,7 @@ _cache = ConaryHelperSharedCache() - def __init__(self, cfg): + def __init__(self, cfg, mirrorCfgFn=None): self._groupFlavorCount = len(cfg.groupFlavors) if not self._cache.sharedTmpDir: @@ -113,8 +114,11 @@ self._cache.conaryConfigCache[conaryCfgFile] = self._ccfg + if not mirrorCfgFn: + mirrorCfgFn = 'mirror.conf' + self._mcfg = None - mcfgfn = util.join(cfg.configPath, 'mirror.conf') + mcfgfn = util.join(cfg.configPath, mirrorCfgFn) if mcfgfn in self._cache.conaryConfigCache: self._mcfg = self._cache.conaryConfigCache[mcfgfn] elif os.path.exists(mcfgfn): @@ -132,6 +136,8 @@ dir=self._cache.sharedTmpDir, prefix='conaryhelper-%s-' % cfg.platformName) + self._findTrovesCache = FindTrovesCache(self._repos) + def clearCache(self): """ Clear the trove query cache. @@ -169,7 +175,8 @@ labels = self._ccfg.buildLabel try: - return self._repos.findTroves(labels, troveList, *args, **kwargs) + return self._findTrovesCache.findTroves(labels, troveList, + *args, **kwargs) except conaryerrors.TroveNotFound, e: return {} @@ -1008,7 +1015,8 @@ def promote(self, trvLst, expected, sourceLabels, targetLabel, checkPackageList=True, extraPromoteTroves=None, - extraExpectedPromoteTroves=None, commit=True): + extraExpectedPromoteTroves=None, commit=True, + enforceAllExpected=True): """ Promote a group and its contents to a target label. @param trvLst: list of troves to publish @@ -1113,9 +1121,13 @@ extraTroves = set([ x[0] for x in extraPromoteTroves | extraExpectedPromoteTroves ]) - + # grpDiff.difference is checking that no packages outside of the + # expected set are promoted. + # + # grpInv.difference is checking that all packages that we expect to be + # promoted are promoted. if (checkPackageList and (grpDiff.difference(extraTroves) or - grpInv.difference(extraTroves))): + (enforceAllExpected and grpInv.difference(extraTroves)))): raise PromoteMismatchError(expected=oldPkgs, actual=newPkgs) if not commit: diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -402,6 +402,10 @@ # a magic number, but at least it is configurable? maxBuildSize = (CfgInt, 10) + # Map of updateIds to upstream versions to use if you don't want to use the + # normal versioning scheme. + upstreamVersionMap = (CfgIntDict(CfgString), {}) + # List of errata timestamps to merge together. This is used when one errata # leaves the platform in a non dependency closed state and a later update # should solve the dependency problem. All updates are folded into the first diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -138,7 +138,10 @@ @type bucketId: integer (unix time) """ - version = time.strftime('%Y.%m.%d_%H%M.%S', time.gmtime(bucketId)) + version = self._cfg.upstreamVersionMap.get(bucketId, None) + if not version: + version = time.strftime('%Y.%m.%d_%H%M.%S', time.gmtime(bucketId)) + return version @loadErrata diff --git a/updatebot/lib/findtroves.py b/updatebot/lib/findtroves.py new file mode 100644 --- /dev/null +++ b/updatebot/lib/findtroves.py @@ -0,0 +1,74 @@ +# +# Copyright (c) 2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +This is a caching implentation of the findTroves call from conary netclient. +""" + +import logging + +log = logging.getLogger('updatebot.lib.findtroves') + +class FindTrovesCache(dict): + """ + Caching layer for findTroves + """ + + def __init__(self, repos): + dict.__init__(self) + + self._repos = repos + + def findTroves(self, labelPath, troves, getLeaves=True): + """ + Emulate the behavior of repos.findTroves while caching results. + """ + + found = set() + needed = {} + + if not isinstance(labelPath, (list, tuple, set)): + tlp = (labelPath, ) + else: + tlp = tuple(labelPath) + + # Separate out items that have already been cached from those that have + # not. + for trove in troves: + key = (tlp, tuple(trove), getLeaves) + + if key in self: + found.add(key) + else: + needed[trove] = key + + # Ask the repo for anything that hasn't been cached. + if needed: + if found: + log.info('CACHE hit on %s troves' % len(found)) + log.info('CACHE querying repository for %s troves' % len(needed)) + results = self._repos.findTroves(labelPath, needed, + getLeaves=getLeaves) + else: + log.info('CACHE hit on all troves') + results = {} + + # Cache the results. + for req, res in results.iteritems(): + self[needed[req]] = res + + # Build actual result + results.update(dict((x[1], self[x]) for x in found)) + + return results diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -421,7 +421,7 @@ return updateSet - def promote(self): + def promote(self, enforceAllExpected=True): """ Promote binary groups from the devel label to the production lable in the order that they were built. @@ -442,6 +442,17 @@ targetLatest = self._updater.getUpstreamVersionMap( (self._cfg.topGroup[0], self._cfg.targetLabel, None)) + log.info('querying target label for cloned from information') + # The targetSpec list tells us if the latest version of each group has + # been promoted to the target label. + sourceSpecs = [ x for x in itertools.chain(*sourceLatest.itervalues()) ] + targetSpecs, failed = self._updater.getTargetVersions(sourceSpecs, + logErrors=False) + targetSpecMap = {} + for spec in targetSpecs: + ver = spec[1].trailingRevision().getVersion() + targetSpecMap.setdefault(ver, set()).add(spec) + log.info('starting promote') count = 0 @@ -449,7 +460,7 @@ # Get all updates after the first bucket. missing = False - for updateId, bucket in self._errata.iterByIssueDate(current=1): + for updateId, bucket in self._errata.iterByIssueDate(current=-1): upver = self._errata.getBucketVersion(updateId) if updateId <= self._cfg.errataPromoteAfter: @@ -457,9 +468,12 @@ continue # Don't try to promote buckets that have already been promoted. - if upver in targetLatest: + if upver in targetLatest and upver in targetSpecMap: log.info('%s found on target label, skipping' % upver) continue + elif upver not in targetSpecMap: + log.info('%s found on target label, but newer version ' + 'available on source, will repromote' % upver) # Make sure version has been imported. if upver not in sourceLatest: @@ -515,7 +529,8 @@ def promote(): # Create and validate promote changeset packageList = self._updater.publish(toPromote, expected, - self._cfg.targetLabel, extraExpectedPromoteTroves=extra) + self._cfg.targetLabel, extraExpectedPromoteTroves=extra, + enforceAllExpected=enforceAllExpected) return 0 rc = watchdog.waitOnce(promote) diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -960,7 +960,7 @@ return self._conaryhelper.getBuildRequires(pkgName) def publish(self, trvLst, expected, targetLabel, checkPackageList=True, - extraExpectedPromoteTroves=None): + extraExpectedPromoteTroves=None, enforceAllExpected=True): """ Publish a group and its contents to a target label. @param trvLst: list of troves to publish @@ -987,6 +987,7 @@ checkPackageList=checkPackageList, extraPromoteTroves=self._cfg.extraPromoteTroves, extraExpectedPromoteTroves=extraExpectedPromoteTroves, + enforceAllExpected=enforceAllExpected, ) def mirror(self, fullTroveSync=False): From juphoff at rpath.com Mon Aug 30 09:21:02 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Mon, 30 Aug 2010 13:21:02 +0000 Subject: mirrorball: add sles 10 sp3 to versionMap Message-ID: <201008301321.o7UDL2ff031896@scc.eng.rpath.com> changeset: 3bfd0955dd54 user: Jeff Uphoff date: Tue, 24 Aug 2010 22:36:05 -0400 add sles 10 sp3 to versionMap diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -58,6 +58,7 @@ 'rhel': {'4': '4.0', '5': '5.0', }, 'centos': {'5': '0', }, + 'sles': {'10': 'sp3', }, } # Make sure to include any packages that were built from the same From juphoff at rpath.com Mon Aug 30 09:21:02 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Mon, 30 Aug 2010 13:21:02 +0000 Subject: mirrorball: branch merge Message-ID: <201008301321.o7UDL2I8031923@scc.eng.rpath.com> changeset: 1fee5b4ae3dd user: Jeff Uphoff date: Mon, 30 Aug 2010 09:20:59 -0400 branch merge diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups --- a/scripts/rebuildgroups +++ b/scripts/rebuildgroups @@ -58,6 +58,7 @@ 'rhel': {'4': '4.0', '5': '5.0', }, 'centos': {'5': '0', }, + 'sles': {'10': 'sp3', }, } # Make sure to include any packages that were built from the same From juphoff at rpath.com Tue Aug 31 13:19:40 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Tue, 31 Aug 2010 17:19:40 +0000 Subject: mirrorball: add packageFlavorsMissing configuration option Message-ID: <201008311719.o7VHJe1o008635@scc.eng.rpath.com> changeset: 87ebe33a1bc4 user: Jeff Uphoff date: Tue, 31 Aug 2010 13:19:31 -0400 add packageFlavorsMissing configuration option diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -518,7 +518,7 @@ # the deps modules in conary. name = name.encode() - # Build groups in all of the defined falvors. We don't need a + # Build groups in all of the defined flavors. We don't need a # context here since groups are all built in a single job. if name.startswith('group-'): for flv in self._cfg.groupFlavors: @@ -556,6 +556,13 @@ [ x for x in binaryNames if fltr[1].match(x) ])): troves.append((name, version, flavor, context)) + # Handle any special-case omissions + # (e.g. due to missing packages) + if name in self._cfg.packageFlavorsMissing: + for context, flavor, fltr in self._cfg.packageFlavorsMissing[name]: + if not [ x for x in binaryNames if fltr[1].match(x) ]: + troves.remove((name, version, flavor, context)) + return sorted(set(troves)) @jobInfoExceptionHandler diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -89,6 +89,41 @@ context, flavor = CfgStringFlavor.parseString(self, val) return context, flavor, use +class CfgFlavorFilter(CfgRegExp, CfgFlavor): + """ + Class for parsing (context, flavor, regex) tuples, where flavor and + regex are optional (during parsing, though not necessarily in + implementation). + """ + + def parseString(self, val): + """ + Parse config input. + """ + + try: + splt = val.split(None, 2) + if len(splt) == 1: + context = val + flavor = None + fltr = None + elif len(splt) == 2: + context = splt[0] + # Note: this split can't handle flavors containing spaces... + flavorStr = splt[1] + flavor = CfgFlavor.parseString(self, flavorStr) + fltr = None + else: + context = splt[0] + flavorStr = splt[1] + flavor = CfgFlavor.parseString(self, flavorStr) + # ...but it *can* handle regexes containing spaces: + fltrStr = ' '.join(splt[2:]) + fltr = CfgRegExp.parseString(self, fltrStr) + return context, flavor, fltr + except versions.ParseError, e: + raise ParseError, e + class CfgStringFilter(CfgRegExp): """ @@ -352,6 +387,12 @@ # flavors to build packages in for packages that need specific flavoring. packageFlavors = (CfgDict(CfgList(CfgStringFlavor)), {}) + # 3-tuple of (context, flavor, regex) of arch-specific package + # flavors to omit unless the regex matches a binary package in the + # manifest. Useful for omitting built of otherwise-expected flavors + # when a package is missing from the repository. + packageFlavorsMissing = (CfgDict(CfgList(CfgFlavorFilter)), {}) + # After committing a rMake job to the repository pull the changeset back out # to make sure all of the contents made it into the repository. sanityCheckCommits = (CfgBool, False) From juphoff at rpath.com Tue Aug 31 13:32:25 2010 From: juphoff at rpath.com (Jeff Uphoff) Date: Tue, 31 Aug 2010 17:32:25 +0000 Subject: mirrorball: remove debug breakpoint Message-ID: <201008311732.o7VHWPSA008943@scc.eng.rpath.com> changeset: 264c7c73370a user: Jeff Uphoff date: Tue, 31 Aug 2010 13:32:20 -0400 remove debug breakpoint diff --git a/errata/sles.py b/errata/sles.py --- a/errata/sles.py +++ b/errata/sles.py @@ -290,5 +290,3 @@ advPkgMap[advisory]) self._advisories.add(adv) self._advOrder.setdefault(int(patch.timestamp), set()).add(adv) - - import epdb ; epdb.st() From elliot at rpath.com Tue Aug 31 16:51:09 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 31 Aug 2010 20:51:09 +0000 Subject: mirrorball: 1. make caching optional Message-ID: <201008312051.o7VKp95o013651@scc.eng.rpath.com> changeset: 49709b286ae3 user: Elliot Peele date: Tue, 31 Aug 2010 16:50:47 -0400 1. make caching optional 2. don't cache findtroves calls for groups since there will be new verisons diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py --- a/updatebot/groupmgr/manager.py +++ b/updatebot/groupmgr/manager.py @@ -147,7 +147,8 @@ # retreiving from the repository is the latest. trvs = self._helper.findTrove((self._sourceName, version, None), labels=self._searchLabels, - getLeaves=not allVersions) + getLeaves=not allVersions, + cache=False) if allVersions: return [ x[1] for x in trvs ] diff --git a/updatebot/lib/findtroves.py b/updatebot/lib/findtroves.py --- a/updatebot/lib/findtroves.py +++ b/updatebot/lib/findtroves.py @@ -30,11 +30,14 @@ self._repos = repos - def findTroves(self, labelPath, troves, getLeaves=True): + def findTroves(self, labelPath, troves, getLeaves=True, cache=True): """ Emulate the behavior of repos.findTroves while caching results. """ + if not cache: + return self._repos.findTroves(labelPath, troves, getLeaves=getLeaves) + found = set() needed = {} From elliot at rpath.com Tue Aug 31 16:51:09 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 31 Aug 2010 20:51:09 +0000 Subject: mirrorball: branch merge Message-ID: <201008312051.o7VKp9mu013682@scc.eng.rpath.com> changeset: 255279055844 user: Elliot Peele date: Tue, 31 Aug 2010 16:51:07 -0400 branch merge diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py --- a/updatebot/groupmgr/manager.py +++ b/updatebot/groupmgr/manager.py @@ -147,7 +147,8 @@ # retreiving from the repository is the latest. trvs = self._helper.findTrove((self._sourceName, version, None), labels=self._searchLabels, - getLeaves=not allVersions) + getLeaves=not allVersions, + cache=False) if allVersions: return [ x[1] for x in trvs ] diff --git a/updatebot/lib/findtroves.py b/updatebot/lib/findtroves.py --- a/updatebot/lib/findtroves.py +++ b/updatebot/lib/findtroves.py @@ -30,11 +30,14 @@ self._repos = repos - def findTroves(self, labelPath, troves, getLeaves=True): + def findTroves(self, labelPath, troves, getLeaves=True, cache=True): """ Emulate the behavior of repos.findTroves while caching results. """ + if not cache: + return self._repos.findTroves(labelPath, troves, getLeaves=getLeaves) + found = set() needed = {}