From juphoff at rpath.com Mon Nov 1 15:43:59 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Mon, 01 Nov 2010 19:43:59 +0000
Subject: mirrorball: fix platformName logic error
Message-ID: <201011011943.oA1JhxRs017791@scc.eng.rpath.com>
changeset: 0ad5d2dff663
user: Jeff Uphoff
date: Mon, 01 Nov 2010 15:43:54 -0400
fix platformName logic error
diff --git a/scripts/order_promote.py b/scripts/order_promote.py
--- a/scripts/order_promote.py
+++ b/scripts/order_promote.py
@@ -56,7 +56,7 @@
if cfg.platformName == 'sles':
from errata.sles import AdvisoryManager as Errata
- if cfg.platformName == 'sles11':
+ elif cfg.platformName == 'sles11':
from errata.sles11 import AdvisoryManager11 as Errata
elif cfg.platformName == 'centos':
From juphoff at rpath.com Fri Nov 5 17:36:34 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Fri, 05 Nov 2010 21:36:34 +0000
Subject: mirrorball: add sles10sp3 debug code
Message-ID: <201011052136.oA5LaYlF022858@scc.eng.rpath.com>
changeset: 4f7586b4825f
user: Jeff Uphoff
date: Fri, 05 Nov 2010 17:36:26 -0400
add sles10sp3 debug code
diff --git a/errata/sles.py b/errata/sles.py
--- a/errata/sles.py
+++ b/errata/sles.py
@@ -290,3 +290,5 @@
advPkgMap[advisory])
self._advisories.add(adv)
self._advOrder.setdefault(int(patch.timestamp), set()).add(adv)
+
+ import epdb ; epdb.st()
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -881,8 +881,8 @@
indexedChannels = set(self._errata.getChannels())
# FIXME: This should not be a hard coded set of arches.
arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch')
- log.info('processing %s errata' %
- len([x for x in self._errata.iterByIssueDate()]))
+ #log.info('processing %s errata' %
+ # len([x for x in self._errata.iterByIssueDate()]))
for e in self._errata.iterByIssueDate():
bucket = []
allocated = []
@@ -905,13 +905,14 @@
# move nevra to errata buckets
if nevra in nevras:
binPkg = nevras.pop(nevra)
- log.info ('dropping %s into bucket' % binPkg)
+ log.info('dropping %s into bucket' % binPkg)
bucket.append(binPkg)
allocated.append(nevra)
# nevra is already part of another bucket
elif nevra in nevraMap:
bucketId = nevraMap[nevra]
+ log.info('...reusing bucket %s' % bucketId)
# raise error if we can't find the required package
else:
@@ -931,6 +932,8 @@
if bucketId is None:
bucketId = int(time.mktime(time.strptime(e.issue_date,
'%Y-%m-%d %H:%M:%S')))
+ log.info('...creating bucket %s for (%s)' % (bucketId,
+ time.strptime(e.issue_date, '%Y-%m-%d %H:%M:%S')))
if bucketId not in buckets:
buckets[bucketId] = set()
From juphoff at rpath.com Mon Nov 8 22:40:50 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 09 Nov 2010 03:40:50 +0000
Subject: mirrorball: fix advisory-ordering bug on SLES10 [SLES11 fix
pending] (PFM-820)
Message-ID: <201011090340.oA93eoDS015705@scc.eng.rpath.com>
changeset: 1db2054f2f13
user: Jeff Uphoff
date: Mon, 08 Nov 2010 22:40:31 -0500
fix advisory-ordering bug on SLES10 [SLES11 fix pending] (PFM-820)
diff --git a/errata/sles.py b/errata/sles.py
--- a/errata/sles.py
+++ b/errata/sles.py
@@ -262,26 +262,26 @@
assert(len(srcPkgPatchidMap[srcPkgObj]) == 2)
srcPkgAdvs = [ getPatchById(patches, srcPkgAdv)
for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ]
- # Only sync the same source package once. (It may
- # appear for multiple binary packages.)
- if srcPkgAdvs[0].timestamp != srcPkgAdvs[1].timestamp:
- # Using the min here in case the first advisory
- # for this source package has already been
- # published.
- syncTimestamp = min(srcPkgAdvs[0].timestamp,
- srcPkgAdvs[1].timestamp)
- log.info('syncing timestamps (%s %s) ' % (
- srcPkgAdvs[0].timestamp, srcPkgAdvs[1].timestamp) +
- 'across same-SRPM advisories for %s & %s ' % (
- srcPkgAdvs[0].getAttribute('patchid'),
- srcPkgAdvs[1].getAttribute('patchid')) +
- 'to earlier timestamp %s' % syncTimestamp)
- srcPkgAdvs[0].timestamp = srcPkgAdvs[1].timestamp = syncTimestamp
+ syncTimestamp = min([ x.timestamp for x in srcPkgAdvs ])
+
+ for srcPkgAdv in srcPkgAdvs:
+ if srcPkgAdv.timestamp != syncTimestamp:
+ log.info('syncing timestamp (%s) of %s to %s ' % (
+ srcPkgAdv.timestamp, srcPkgAdv.getAttribute('patchid'), syncTimestamp) +
+ 'across same-SRPM advisories for %s' % (
+ srcPkgAdvs))
+ srcPkgAdv.timestamp = syncTimestamp
# There should be no srcPkgs with more than two patchids.
assert(len([ x for x, y in srcPkgPatchidMap.iteritems()
if len(y) > 2 ]) == 0)
+ # Now that all timestamps have been munged, make second pass to
+ # establish order & create advisories.
+ for patch in patches:
+ advisory = patch.getAttribute('patchid')
+ patchid = patchidNoRepo(advisory)
+
issue_date = time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(int(patch.timestamp)))
log.info('creating advisory: %s (%s)' % (advisory,
diff --git a/errata/sles11.py b/errata/sles11.py
--- a/errata/sles11.py
+++ b/errata/sles11.py
@@ -205,6 +205,9 @@
srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid)
# FIXME: I hate this code too.
+ #
+ # FIXME HARDER: This is broken and only works because
+ # it's lucky. See sles.py for pending fix.
if srcPkgPatchidMap[srcPkgObj] != set([patchid]):
# Untested beyond two, and expected case is two
# different advisories issued for the same source
diff --git a/repomd/patchxml.py b/repomd/patchxml.py
--- a/repomd/patchxml.py
+++ b/repomd/patchxml.py
@@ -48,6 +48,9 @@
# patch's timestamp across architectures.
self.timestamp = self.getAttribute('timestamp')
+ def __repr__(self):
+ return self.name + '-' + self.version
+
# All attributes are defined in __init__ by iterating over __slots__,
# this confuses pylint.
# W0201 - Attribute $foo defined outside __init__
From juphoff at rpath.com Tue Nov 9 11:31:20 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 09 Nov 2010 16:31:20 +0000
Subject: mirrorball: SLES11 fix (untested in anger) for advisory-ordering
bug found on SLES10 (PFM-820)
Message-ID: <201011091631.oA9GVKod027521@scc.eng.rpath.com>
changeset: ec059b8d3ba0
user: Jeff Uphoff
date: Tue, 09 Nov 2010 11:31:08 -0500
SLES11 fix (untested in anger) for advisory-ordering bug found on SLES10 (PFM-820)
diff --git a/errata/sles.py b/errata/sles.py
--- a/errata/sles.py
+++ b/errata/sles.py
@@ -263,7 +263,6 @@
srcPkgAdvs = [ getPatchById(patches, srcPkgAdv)
for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ]
syncTimestamp = min([ x.timestamp for x in srcPkgAdvs ])
-
for srcPkgAdv in srcPkgAdvs:
if srcPkgAdv.timestamp != syncTimestamp:
log.info('syncing timestamp (%s) of %s to %s ' % (
diff --git a/errata/sles11.py b/errata/sles11.py
--- a/errata/sles11.py
+++ b/errata/sles11.py
@@ -205,9 +205,9 @@
srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid)
# FIXME: I hate this code too.
- #
- # FIXME HARDER: This is broken and only works because
- # it's lucky. See sles.py for pending fix.
+ # Note that to date this has not been tested in anger
+ # on SLES11, as no advisory releases have spanned
+ # multiple buckets.
if srcPkgPatchidMap[srcPkgObj] != set([patchid]):
# Untested beyond two, and expected case is two
# different advisories issued for the same source
@@ -216,26 +216,24 @@
assert(len(srcPkgPatchidMap[srcPkgObj]) == 2)
srcPkgAdvs = [ getPatchById(patches, srcPkgAdv)
for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ]
- # Only sync the same source package once. (It may
- # appear for multiple binary packages.)
- if srcPkgAdvs[0].issued != srcPkgAdvs[1].issued:
- # Using the min here in case the first advisory
- # for this source package has already been
- # published.
- syncTimestamp = min(srcPkgAdvs[0].issued,
- srcPkgAdvs[1].issued)
- log.info('syncing timestamps (%s %s) ' % (
- srcPkgAdvs[0].issued, srcPkgAdvs[1].issued) +
- 'across same-SRPM advisories for %s & %s ' % (
- srcPkgAdvs[0].id,
- srcPkgAdvs[1].id) +
- 'to earlier timestamp %s' % syncTimestamp)
- srcPkgAdvs[0].issued = srcPkgAdvs[1].issued = syncTimestamp
+ syncTimestamp = min([ x.issued for x in srcPkgAdvs ])
+ for srcPkgAdv in srcPkgAdvs:
+ if srcPkgAdv.issued != syncTimestamp:
+ log.info('syncing timestamp (%s) of %s to %s ' % (
+ srcPkgAdv.issued, srcPkgAdv.id, syncTimestamp) +
+ 'across same-SRPM advisories for %s' % (
+ srcPkgAdvs))
+ srcPkgAdv.issued = syncTimestamp
# There should be no srcPkgs with more than two patchids.
assert(len([ x for x, y in srcPkgPatchidMap.iteritems()
if len(y) > 2 ]) == 0)
+ # Now that all timestamps have been munged, make second pass to
+ # establish order & create advisories.
+ for patch in patches:
+ advisory = patch.id
+
issue_date = time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(int(patch.issued)))
log.info('creating advisory: %s (%s)' % (advisory,
@@ -245,6 +243,4 @@
self._advisories.add(adv)
self._advOrder.setdefault(int(patch.issued), set()).add(adv)
-
- #AdvisoryManager._order(self)
import epdb ; epdb.st()
diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py
--- a/repomd/updateinfoxml.py
+++ b/repomd/updateinfoxml.py
@@ -92,8 +92,6 @@
Parse the children of update.
"""
- #import epdb ; epdb.st()
-
n = child.getName()
if n == 'id':
# Make this behave like SLES10 patchid
@@ -151,6 +149,9 @@
def __hash__(self):
return hash((self.id, self.release, self.summary, self.description))
+ def __repr__(self):
+ return self.id
+
class _References(SlotNode):
"""
Represent a list of references in updateinfo.xml.
From juphoff at rpath.com Wed Nov 10 15:34:07 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Wed, 10 Nov 2010 20:34:07 +0000
Subject: mirrorball: fix mental typo
Message-ID: <201011102034.oAAKY7dx000361@scc.eng.rpath.com>
changeset: 3e2e1d6467ff
user: Jeff Uphoff
date: Wed, 10 Nov 2010 15:29:52 -0500
fix mental typo
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -74,7 +74,7 @@
if cfg.platformName == 'sles':
from errata.sles import AdvisoryManager as Errata
- if cfg.platformName == 'sles11':
+ elif cfg.platformName == 'sles11':
from errata.sles11 import AdvisoryManager11 as Errata
elif cfg.platformName == 'centos':
From juphoff at rpath.com Wed Nov 10 17:07:59 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Wed, 10 Nov 2010 22:07:59 +0000
Subject: mirrorball: consolidate sleorder & sle11order scripts
Message-ID: <201011102207.oAAM7xWE003004@scc.eng.rpath.com>
changeset: 534686efdfc9
user: Jeff Uphoff
date: Wed, 10 Nov 2010 17:07:42 -0500
consolidate sleorder & sle11order scripts
diff --git a/scripts/sle11order.py b/scripts/sle11order.py
deleted file mode 100755
--- a/scripts/sle11order.py
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/python
-
-import os
-import sys
-import time
-
-sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
-
-from conary.lib import util
-sys.excepthook = util.genExcepthook()
-
-mbdir = os.path.abspath('../')
-sys.path.insert(0, mbdir)
-
-confDir = os.path.join(mbdir, 'config', sys.argv[1])
-
-from updatebot import log
-from updatebot.ordered import Bot
-from updatebot import UpdateBotConfig
-
-from errata.sles11 import AdvisoryManager11 as Errata
-
-slog = log.addRootLogger()
-
-cfg = UpdateBotConfig()
-cfg.read(os.path.join(confDir, 'updatebotrc'))
-
-bot = Bot(cfg, None)
-errata = Errata(bot._pkgSource)
-bot._errata._errata = errata
-
-errata.fetch()
-
-bot._pkgSource.load()
-bot._errata._orderErrata()
-
-order = bot._errata._order
-advMap = bot._errata._advMap
-sorder = sorted(order)
-
-def tconv(tstamp):
- return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp))
-
-childPackages, parentPackages = bot._errata.sanityCheckOrder()
-
-import epdb; epdb.st()
diff --git a/scripts/sleorder.py b/scripts/sleorder.py
--- a/scripts/sleorder.py
+++ b/scripts/sleorder.py
@@ -18,13 +18,19 @@
from updatebot.ordered import Bot
from updatebot import UpdateBotConfig
-from errata.sles import AdvisoryManager as Errata
slog = log.addRootLogger()
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
+if cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
+elif cfg.platformName == 'sles11':
+ from errata.sles11 import AdvisoryManager11 as Errata
+else:
+ raise RuntimeError, 'unsupported platformName'
+
bot = Bot(cfg, None)
errata = Errata(bot._pkgSource)
bot._errata._errata = errata
From juphoff at rpath.com Sat Nov 13 02:08:34 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Sat, 13 Nov 2010 07:08:34 +0000
Subject: mirrorball: add retrospective repository sequence inspection
Message-ID: <201011130708.oAD78YG0031579@scc.eng.rpath.com>
changeset: 300f5efa9d38
user: Jeff Uphoff
date: Sat, 13 Nov 2010 02:08:11 -0500
add retrospective repository sequence inspection
diff --git a/scripts/order_promote.py b/scripts/order_promote.py
--- a/scripts/order_promote.py
+++ b/scripts/order_promote.py
@@ -68,6 +68,6 @@
errata = Errata(bot._pkgSource)
bot._errata._errata = errata
-bot.promote(enforceAllExpected=True)
+bot.promote(enforceAllExpected=True, checkMissingPackages=False)
import epdb; epdb.st()
diff --git a/scripts/sleorder-check.py b/scripts/sleorder-check.py
new file mode 100755
--- /dev/null
+++ b/scripts/sleorder-check.py
@@ -0,0 +1,49 @@
+#!/usr/bin/python
+
+import os
+import sys
+import time
+
+sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
+
+from conary.lib import util
+sys.excepthook = util.genExcepthook()
+
+mbdir = os.path.abspath('../')
+sys.path.insert(0, mbdir)
+
+confDir = os.path.join(mbdir, 'config', sys.argv[1])
+
+from updatebot import log
+from updatebot.ordered import Bot
+from updatebot import UpdateBotConfig
+
+slog = log.addRootLogger()
+
+cfg = UpdateBotConfig()
+cfg.read(os.path.join(confDir, 'updatebotrc'))
+
+if cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
+elif cfg.platformName == 'sles11':
+ from errata.sles11 import AdvisoryManager11 as Errata
+else:
+ raise RuntimeError, 'unsupported platformName'
+
+bot = Bot(cfg, None)
+errata = Errata(bot._pkgSource)
+bot._errata._errata = errata
+
+errata.fetch()
+
+bot._pkgSource.load()
+bot._errata._orderErrata()
+
+# For easy inspection.
+order = bot._errata._order
+
+childPackages, parentPackages = bot._errata.sanityCheckOrder()
+
+bot._checkMissingPackages()
+
+import epdb; epdb.st()
diff --git a/scripts/sleorder.py b/scripts/sleorder.py
--- a/scripts/sleorder.py
+++ b/scripts/sleorder.py
@@ -18,7 +18,6 @@
from updatebot.ordered import Bot
from updatebot import UpdateBotConfig
-
slog = log.addRootLogger()
cfg = UpdateBotConfig()
@@ -40,13 +39,11 @@
bot._pkgSource.load()
bot._errata._orderErrata()
+# For easy inspection.
order = bot._errata._order
-advMap = bot._errata._advMap
-sorder = sorted(order)
-
-def tconv(tstamp):
- return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp))
childPackages, parentPackages = bot._errata.sanityCheckOrder()
+bot._checkMissingPackages()
+
import epdb; epdb.st()
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -881,8 +881,6 @@
indexedChannels = set(self._errata.getChannels())
# FIXME: This should not be a hard coded set of arches.
arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch')
- #log.info('processing %s errata' %
- # len([x for x in self._errata.iterByIssueDate()]))
for e in self._errata.iterByIssueDate():
bucket = []
allocated = []
@@ -905,14 +903,12 @@
# move nevra to errata buckets
if nevra in nevras:
binPkg = nevras.pop(nevra)
- log.info('dropping %s into bucket' % binPkg)
bucket.append(binPkg)
allocated.append(nevra)
# nevra is already part of another bucket
elif nevra in nevraMap:
bucketId = nevraMap[nevra]
- log.info('...reusing bucket %s' % bucketId)
# raise error if we can't find the required package
else:
@@ -932,8 +928,6 @@
if bucketId is None:
bucketId = int(time.mktime(time.strptime(e.issue_date,
'%Y-%m-%d %H:%M:%S')))
- log.info('...creating bucket %s for (%s)' % (bucketId,
- time.strptime(e.issue_date, '%Y-%m-%d %H:%M:%S')))
if bucketId not in buckets:
buckets[bucketId] = set()
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -183,6 +183,48 @@
return pkgMap, failures
+ def _checkMissingPackages(self):
+ """
+ Check previously ordered buckets for any missing source
+ packages. These are usually caused by delayed release of errata
+ into repositories.
+ """
+
+ log.info('checking for any packages missing from repository')
+
+ group = self._groupmgr.getGroup()
+ log.info('querying buildLabel %s for all committed package versions' %
+ self._updater._conaryhelper._ccfg.buildLabel)
+ allPackageVersions = self._updater._conaryhelper._repos.getTroveVersionsByLabel({None: {self._updater._conaryhelper._ccfg.buildLabel: None}})
+ allSourceVersions = {}
+
+ for source, versions in allPackageVersions.iteritems():
+ if source.endswith(':source'):
+ allSourceVersions[source.replace(':source', '')] = [ x.versions[1].version for x in versions.keys() ]
+
+ missingPackages = {}
+ missingOrder = {}
+
+ for bucket, packages in sorted(self._errata._order.iteritems()):
+ if bucket <= group.errataState:
+ for package in packages:
+ ver = package.version + '_' + package.release
+ try:
+ if ver not in allSourceVersions[package.name]:
+ log.warn('missing %s (%s)' % (package, bucket))
+ missingPackages[package] = bucket
+ missingOrder.setdefault(bucket, set()).add(package)
+ except KeyError:
+ log.warn('missing (KeyError) %s (%s)' % (package, bucket))
+ missingPackages[package] = bucket
+ missingOrder.setdefault(bucket, set()).add(package)
+
+ if len(missingPackages) > 0:
+ log.error('missing %s ordered source packages from repository; inspect missingPackages and missingOrder' % len(missingPackages))
+ import epdb ; epdb.st()
+ else:
+ log.info('all expected source packages found in repository')
+
def update(self, *args, **kwargs):
"""
Handle update case.
@@ -193,6 +235,7 @@
# Get current group
group = self._groupmgr.getGroup()
+ #import epdb ; epdb.st()
# Get current timestamp
current = group.errataState
@@ -211,6 +254,9 @@
# Sanity check errata ordering.
self._errata.sanityCheckOrder()
+ # Ensure no packages are missing from repository.
+ self._checkMissingPackages()
+
# Check for updated errata that may require some manual changes to the
# repository. These are errata that were issued before the current
# errata state, but have been modified in the upstream errata source.
@@ -421,7 +467,7 @@
return updateSet
- def promote(self, enforceAllExpected=True):
+ def promote(self, enforceAllExpected=True, checkMissingPackages=False):
"""
Promote binary groups from the devel label to the production lable in
the order that they were built.
@@ -435,6 +481,10 @@
# laod package source
self._pkgSource.load()
+ if checkMissingPackages:
+ self._errata.sanityCheckOrder()
+ self._checkMissingPackages()
+
log.info('querying repository for all group versions')
sourceLatest = self._updater.getUpstreamVersionMap(self._cfg.topGroup)
From juphoff at rpath.com Sat Nov 13 02:09:12 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Sat, 13 Nov 2010 07:09:12 +0000
Subject: mirrorball: consolidated into sleorder.py
Message-ID: <201011130709.oAD79C27031639@scc.eng.rpath.com>
changeset: 105bdd9f0b52
user: Jeff Uphoff
date: Sat, 13 Nov 2010 02:09:09 -0500
consolidated into sleorder.py
diff --git a/scripts/sleorder-check.py b/scripts/sleorder-check.py
deleted file mode 100755
--- a/scripts/sleorder-check.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/python
-
-import os
-import sys
-import time
-
-sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
-
-from conary.lib import util
-sys.excepthook = util.genExcepthook()
-
-mbdir = os.path.abspath('../')
-sys.path.insert(0, mbdir)
-
-confDir = os.path.join(mbdir, 'config', sys.argv[1])
-
-from updatebot import log
-from updatebot.ordered import Bot
-from updatebot import UpdateBotConfig
-
-slog = log.addRootLogger()
-
-cfg = UpdateBotConfig()
-cfg.read(os.path.join(confDir, 'updatebotrc'))
-
-if cfg.platformName == 'sles':
- from errata.sles import AdvisoryManager as Errata
-elif cfg.platformName == 'sles11':
- from errata.sles11 import AdvisoryManager11 as Errata
-else:
- raise RuntimeError, 'unsupported platformName'
-
-bot = Bot(cfg, None)
-errata = Errata(bot._pkgSource)
-bot._errata._errata = errata
-
-errata.fetch()
-
-bot._pkgSource.load()
-bot._errata._orderErrata()
-
-# For easy inspection.
-order = bot._errata._order
-
-childPackages, parentPackages = bot._errata.sanityCheckOrder()
-
-bot._checkMissingPackages()
-
-import epdb; epdb.st()
From juphoff at rpath.com Sat Nov 13 02:10:50 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Sat, 13 Nov 2010 07:10:50 +0000
Subject: mirrorball: tidy debug leftovers
Message-ID: <201011130710.oAD7AoHP031716@scc.eng.rpath.com>
changeset: 77a09cf4f8d4
user: Jeff Uphoff
date: Sat, 13 Nov 2010 02:10:48 -0500
tidy debug leftovers
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -235,7 +235,6 @@
# Get current group
group = self._groupmgr.getGroup()
- #import epdb ; epdb.st()
# Get current timestamp
current = group.errataState
From juphoff at rpath.com Sat Nov 13 02:19:51 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Sat, 13 Nov 2010 07:19:51 +0000
Subject: mirrorball: add comments to _checkMissingPackages()
Message-ID: <201011130719.oAD7Jq4D031874@scc.eng.rpath.com>
changeset: 2f22823ccfaf
user: Jeff Uphoff
date: Sat, 13 Nov 2010 02:19:50 -0500
add comments to _checkMissingPackages()
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -185,19 +185,25 @@
def _checkMissingPackages(self):
"""
- Check previously ordered buckets for any missing source
- packages. These are usually caused by delayed release of errata
- into repositories.
+ Check previously committed buckets for any missing source
+ packages. These are usually caused by delayed releases of errata
+ into repositories, which play havoc with timestamp ordering.
"""
log.info('checking for any packages missing from repository')
+ # Get current group; needed for latest commit timestamp.
group = self._groupmgr.getGroup()
+
log.info('querying buildLabel %s for all committed package versions' %
self._updater._conaryhelper._ccfg.buildLabel)
+
+ # Get all versions of all on buildLabel committed to repo.
allPackageVersions = self._updater._conaryhelper._repos.getTroveVersionsByLabel({None: {self._updater._conaryhelper._ccfg.buildLabel: None}})
+
allSourceVersions = {}
+ # Build dict of all source versions found in repo.
for source, versions in allPackageVersions.iteritems():
if source.endswith(':source'):
allSourceVersions[source.replace(':source', '')] = [ x.versions[1].version for x in versions.keys() ]
@@ -205,6 +211,8 @@
missingPackages = {}
missingOrder = {}
+ # Check all previously ordered/committed buckets for packages
+ # missing from the repo.
for bucket, packages in sorted(self._errata._order.iteritems()):
if bucket <= group.errataState:
for package in packages:
From juphoff at rpath.com Sat Nov 13 10:26:46 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Sat, 13 Nov 2010 15:26:46 +0000
Subject: mirrorball: improve return handling, output, exceptions for
checkMissingPackages
Message-ID: <201011131526.oADFQkLQ005635@scc.eng.rpath.com>
changeset: 7853acb1fdbc
user: Jeff Uphoff
date: Sat, 13 Nov 2010 10:26:43 -0500
improve return handling, output, exceptions for checkMissingPackages
diff --git a/scripts/order_promote.py b/scripts/order_promote.py
--- a/scripts/order_promote.py
+++ b/scripts/order_promote.py
@@ -68,6 +68,6 @@
errata = Errata(bot._pkgSource)
bot._errata._errata = errata
-bot.promote(enforceAllExpected=True, checkMissingPackages=False)
+bot.promote(enforceAllExpected=True, checkMissingPackages=True)
import epdb; epdb.st()
diff --git a/scripts/sleorder.py b/scripts/sleorder.py
--- a/scripts/sleorder.py
+++ b/scripts/sleorder.py
@@ -44,6 +44,6 @@
childPackages, parentPackages = bot._errata.sanityCheckOrder()
-bot._checkMissingPackages()
+missingPackages, missingOrder = bot._checkMissingPackages()
import epdb; epdb.st()
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -38,6 +38,7 @@
from updatebot.errors import PromoteFlavorMismatchError
from updatebot.errors import PlatformAlreadyImportedError
from updatebot.errors import FoundModifiedNotImportedErrataError
+from updatebot.errors import UnhandledUpdateError
log = logging.getLogger('updatebot.ordered')
@@ -219,11 +220,13 @@
ver = package.version + '_' + package.release
try:
if ver not in allSourceVersions[package.name]:
- log.warn('missing %s (%s)' % (package, bucket))
- missingPackages[package] = bucket
- missingOrder.setdefault(bucket, set()).add(package)
+ # Handle all missing-version cases as exception.
+ raise KeyError
except KeyError:
- log.warn('missing (KeyError) %s (%s)' % (package, bucket))
+ log.warn('%s missing from repository' % package)
+ log.info('? reorderSource %s otherId>%s %s' % (
+ bucket, group.errataState,
+ ' '.join(package.getNevra())))
missingPackages[package] = bucket
missingOrder.setdefault(bucket, set()).add(package)
@@ -233,6 +236,8 @@
else:
log.info('all expected source packages found in repository')
+ return missingPackages, missingOrder
+
def update(self, *args, **kwargs):
"""
Handle update case.
@@ -262,7 +267,9 @@
self._errata.sanityCheckOrder()
# Ensure no packages are missing from repository.
- self._checkMissingPackages()
+ missingPackages, missingOrder = self._checkMissingPackages()
+ if len(missingPackages):
+ raise UnhandledUpdateError(why='missing %s ordered source packages from repository' % len(missingPackages))
# Check for updated errata that may require some manual changes to the
# repository. These are errata that were issued before the current
@@ -490,7 +497,9 @@
if checkMissingPackages:
self._errata.sanityCheckOrder()
- self._checkMissingPackages()
+ missingPackages, missingOrder = self._checkMissingPackages()
+ if len(missingPackages):
+ raise UnhandledUpdateError(why='missing %s ordered source packages from repository' % len(missingPackages))
log.info('querying repository for all group versions')
sourceLatest = self._updater.getUpstreamVersionMap(self._cfg.topGroup)
From juphoff at rpath.com Sat Nov 13 17:52:23 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Sat, 13 Nov 2010 22:52:23 +0000
Subject: mirrorball: disable a couple breakpoints, tweak log messages
Message-ID: <201011132252.oADMqNYd014006@scc.eng.rpath.com>
changeset: 354370869164
user: Jeff Uphoff
date: Sat, 13 Nov 2010 17:52:20 -0500
disable a couple breakpoints, tweak log messages
diff --git a/errata/sles.py b/errata/sles.py
--- a/errata/sles.py
+++ b/errata/sles.py
@@ -290,4 +290,4 @@
self._advisories.add(adv)
self._advOrder.setdefault(int(patch.timestamp), set()).add(adv)
- import epdb ; epdb.st()
+# import epdb ; epdb.st()
diff --git a/errata/sles11.py b/errata/sles11.py
--- a/errata/sles11.py
+++ b/errata/sles11.py
@@ -243,4 +243,4 @@
self._advisories.add(adv)
self._advOrder.setdefault(int(patch.issued), set()).add(adv)
- import epdb ; epdb.st()
+# import epdb ; epdb.st()
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -191,12 +191,12 @@
into repositories, which play havoc with timestamp ordering.
"""
- log.info('checking for any packages missing from repository')
+ log.info('checking for any source packages missing from repository')
# Get current group; needed for latest commit timestamp.
group = self._groupmgr.getGroup()
- log.info('querying buildLabel %s for all committed package versions' %
+ log.info('querying buildLabel %s for all committed packages' %
self._updater._conaryhelper._ccfg.buildLabel)
# Get all versions of all on buildLabel committed to repo.
From juphoff at rpath.com Mon Nov 22 16:48:58 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Mon, 22 Nov 2010 21:48:58 +0000
Subject: mirrorball: add comment re confusing log message
Message-ID: <201011222148.oAMLmwQg031822@scc.eng.rpath.com>
changeset: 2d891ab3fb84
user: Jeff Uphoff
date: Mon, 15 Nov 2010 13:42:36 -0500
add comment re confusing log message
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -538,6 +538,7 @@
log.info('%s found on target label, skipping' % upver)
continue
elif upver not in targetSpecMap:
+ # FIXME: This log message is confusing.
log.info('%s found on target label, but newer version '
'available on source, will repromote' % upver)
From juphoff at rpath.com Mon Nov 22 16:48:59 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Mon, 22 Nov 2010 21:48:59 +0000
Subject: mirrorball: make checkMissingPackages True by default, add
allowMissingPackage option for per-nevra opt-out
Message-ID: <201011222148.oAMLmxNO031853@scc.eng.rpath.com>
changeset: a7235756957c
user: Jeff Uphoff
date: Mon, 22 Nov 2010 16:48:38 -0500
make checkMissingPackages True by default, add allowMissingPackage option for per-nevra opt-out
diff --git a/scripts/centosorder.py b/scripts/centosorder.py
--- a/scripts/centosorder.py
+++ b/scripts/centosorder.py
@@ -18,13 +18,13 @@
from updatebot.ordered import Bot
from updatebot import UpdateBotConfig
-from errata.centos import AdvisoryManager as Errata
-
slog = log.addRootLogger()
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
+from errata.centos import AdvisoryManager as Errata
+
bot = Bot(cfg, None)
errata = Errata(bot._pkgSource)
bot._errata._errata = errata
@@ -34,13 +34,14 @@
bot._pkgSource.load()
bot._errata._orderErrata()
+# For easy inspection.
order = bot._errata._order
-advMap = bot._errata._advMap
-sorder = sorted(order)
def tconv(tstamp):
return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp))
childPackages, parentPackages = bot._errata.sanityCheckOrder()
+missingPackages, missingOrder = bot._checkMissingPackages()
+
import epdb; epdb.st()
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -86,6 +86,7 @@
errata = Errata(bot._pkgSource)
bot._errata._errata = errata
-pkgMap = bot.update(fltr=fltr, restoreFile=restoreFile)
+pkgMap = bot.update(fltr=fltr, restoreFile=restoreFile,
+ checkMissingPackages=True)
import epdb; epdb.st()
diff --git a/scripts/sleorder.py b/scripts/sleorder.py
--- a/scripts/sleorder.py
+++ b/scripts/sleorder.py
@@ -42,6 +42,9 @@
# For easy inspection.
order = bot._errata._order
+def tconv(tstamp):
+ return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp))
+
childPackages, parentPackages = bot._errata.sanityCheckOrder()
missingPackages, missingOrder = bot._checkMissingPackages()
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -519,6 +519,13 @@
# the model.
ignoreSourceUpdate = (CfgIntDict(CfgList(CfgNevra)), {})
+ # updateId sourceNevra
+ # At updateId, allow the source package (and any related binary
+ # packages) specified by sourceNevra to be missing from the
+ # repository. Useful for working around selected gaps uncovered by
+ # OrderedBot._checkMissingPackages() during updates or promotes.
+ allowMissingPackage = (CfgIntDict(CfgList(CfgNevra)), {})
+
# updateId binaryNevra
# As of updateId, I expect the code to think this nevra should be removed,
# but I want to keep it.
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -223,6 +223,13 @@
# Handle all missing-version cases as exception.
raise KeyError
except KeyError:
+ #import epdb ; epdb.st()
+ try:
+ if package.getNevra() in self._cfg.allowMissingPackage[bucket]:
+ log.warn('explicitly allowing missing repository package %s at %s' % (package, bucket))
+ continue
+ except KeyError:
+ pass
log.warn('%s missing from repository' % package)
log.info('? reorderSource %s otherId>%s %s' % (
bucket, group.errataState,
@@ -245,6 +252,7 @@
# Load specific kwargs
restoreFile = kwargs.pop('restoreFile', None)
+ checkMissingPackages = kwargs.pop('checkMissingPackages', True)
# Get current group
group = self._groupmgr.getGroup()
@@ -266,10 +274,11 @@
# Sanity check errata ordering.
self._errata.sanityCheckOrder()
- # Ensure no packages are missing from repository.
- missingPackages, missingOrder = self._checkMissingPackages()
- if len(missingPackages):
- raise UnhandledUpdateError(why='missing %s ordered source packages from repository' % len(missingPackages))
+ if checkMissingPackages:
+ # Ensure no packages are missing from repository.
+ missingPackages, missingOrder = self._checkMissingPackages()
+ if len(missingPackages):
+ raise UnhandledUpdateError(why='missing %s ordered source packages from repository' % len(missingPackages))
# Check for updated errata that may require some manual changes to the
# repository. These are errata that were issued before the current
@@ -481,7 +490,7 @@
return updateSet
- def promote(self, enforceAllExpected=True, checkMissingPackages=False):
+ def promote(self, enforceAllExpected=True, checkMissingPackages=True):
"""
Promote binary groups from the devel label to the production lable in
the order that they were built.
@@ -496,6 +505,7 @@
self._pkgSource.load()
if checkMissingPackages:
+ # Ensure no packages are missing from repository.
self._errata.sanityCheckOrder()
missingPackages, missingOrder = self._checkMissingPackages()
if len(missingPackages):
From juphoff at rpath.com Tue Nov 23 11:30:42 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 23 Nov 2010 16:30:42 +0000
Subject: mirrorball: consolidate centosorder.py & sleorder.py into
platform-order.py (rhelorder.py still pending)
Message-ID: <201011231630.oANGUgN9025397@scc.eng.rpath.com>
changeset: 10b6ff0c6bcc
user: Jeff Uphoff
date: Tue, 23 Nov 2010 11:30:32 -0500
consolidate centosorder.py & sleorder.py into platform-order.py (rhelorder.py still pending)
diff --git a/scripts/centosorder.py b/scripts/centosorder.py
deleted file mode 100755
--- a/scripts/centosorder.py
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/usr/bin/python
-
-import os
-import sys
-import time
-
-sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
-
-from conary.lib import util
-sys.excepthook = util.genExcepthook()
-
-mbdir = os.path.abspath('../')
-sys.path.insert(0, mbdir)
-
-confDir = os.path.join(mbdir, 'config', sys.argv[1])
-
-from updatebot import log
-from updatebot.ordered import Bot
-from updatebot import UpdateBotConfig
-
-slog = log.addRootLogger()
-
-cfg = UpdateBotConfig()
-cfg.read(os.path.join(confDir, 'updatebotrc'))
-
-from errata.centos import AdvisoryManager as Errata
-
-bot = Bot(cfg, None)
-errata = Errata(bot._pkgSource)
-bot._errata._errata = errata
-
-errata.fetch()
-
-bot._pkgSource.load()
-bot._errata._orderErrata()
-
-# For easy inspection.
-order = bot._errata._order
-
-def tconv(tstamp):
- return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp))
-
-childPackages, parentPackages = bot._errata.sanityCheckOrder()
-
-missingPackages, missingOrder = bot._checkMissingPackages()
-
-import epdb; epdb.st()
diff --git a/scripts/sleorder.py b/scripts/platform-order.py
rename from scripts/sleorder.py
rename to scripts/platform-order.py
--- a/scripts/sleorder.py
+++ b/scripts/platform-order.py
@@ -27,6 +27,8 @@
from errata.sles import AdvisoryManager as Errata
elif cfg.platformName == 'sles11':
from errata.sles11 import AdvisoryManager11 as Errata
+elif cfg.platformName == 'centos':
+ from errata.centos import AdvisoryManager as Errata
else:
raise RuntimeError, 'unsupported platformName'