From juphoff at rpath.com Tue Sep 7 22:06:18 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Wed, 08 Sep 2010 02:06:18 +0000
Subject: mirrorball: adding sles11e functionality (checkpoint)
Message-ID: <201009080206.o8826Iwr014250@scc.eng.rpath.com>
changeset: 8509207a081d
user: Jeff Uphoff
date: Tue, 07 Sep 2010 22:04:32 -0400
adding sles11e functionality (checkpoint)
diff --git a/errata/sles11.py b/errata/sles11.py
new file mode 100644
--- /dev/null
+++ b/errata/sles11.py
@@ -0,0 +1,242 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Generate update information based on the patch detail in SuSE repositories
+for SLES11.
+"""
+
+import time
+import logging
+
+from errata.common import Nevra
+from errata.common import Package
+from errata.common import Channel
+from errata.sles import AdvisoryManager
+
+log = logging.getLogger('errata')
+
+class AdvisoryManager11(AdvisoryManager):
+ def _order(self):
+ """
+ Fetch all patch data from the package source.
+ """
+
+ def bin_timestamp(ts):
+ """
+ Convert a time stamp into the desired time slice.
+ """
+
+ # convert to current day
+ return int(time.mktime(time.strptime(time.strftime('%Y%m%d',
+ time.gmtime(ts)), '%Y%m%d')))
+
+ def slice(patches):
+ """
+ Build a dictionary of binned (sliced) timestamps and patches.
+ """
+
+ slices = {}
+
+ for patch in patches:
+ # Bin by day:
+ updateId = bin_timestamp(int(patch.issued))
+ # ...or uncomment the below line to disable binning:
+ #updateId = int(patch.issued)
+ slices.setdefault(updateId,
+ set()).add(patch.id)
+ return slices
+
+ def patchidNoRepo(patchid):
+ """
+ Trims the leading (repository) information from a patchid.
+ Used for folding patches across repositories, among other things.
+ """
+ return '-'.join(patchid.split('-')[1:])
+
+ def map_patchids(slices):
+ """
+ Build a dictionary of (partial) patchids and their
+ corresponding timestamps. Used to determine if a patchid
+ across multiple repositories also has multiple timestamps.
+ """
+
+ patchidMap = {}
+
+ for timestamp, patchids in slices.iteritems():
+ for patchid in patchids:
+ # Map excludes leading segment of patchid, which
+ # carries repository information; we're folding
+ # across repositories.
+ patchidMap.setdefault(patchidNoRepo(patchid),
+ set()).add(timestamp)
+ return patchidMap
+
+ def getChannel(pkg):
+ for label, channel in self._channels.iteritems():
+ if label in pkg.location:
+ return channel
+ raise RuntimeError , 'unable to find channel for %s' % pkg.location
+
+ def getSrcPkg(binPkg):
+ for srcPkg, binPkgs in self._pkgSource.srcPkgMap.iteritems():
+ if binPkg in binPkgs:
+ return srcPkg
+ raise RuntimeError , 'unable to find source package for %s' % binPkg.location
+
+ def getPatchById(patches, patchid):
+ for patch in patches:
+ if patch.id == patchid:
+ return patch
+ raise RuntimeError , 'unable to find patch %s' % patchid
+
+ # make sure the pkg source is loaded.
+ self._pkgSource.load()
+
+ # Each client value is a SLES release/update repository object.
+ # self._pkgSource._clients.values()[...]
+
+ # now get the patch data...
+ patches = set()
+
+ for path, client in self._pkgSource.getClients().iteritems():
+ log.info('loading patches for path %s' % path)
+ #import epdb ; epdb.st()
+ #for patch in client.getPatchDetail():
+ for patch in client.getUpdateInfo():
+ for pkg in patch.pkglist:
+ pkg.location = path + '/' + pkg.filename
+ # Schema change with SLES11:
+ patches.add(patch)
+
+ for label in self._pkgSource._clients:
+ self._channels[label] = Channel(label)
+
+ # ...and (time-)slice it up.
+ slices = slice(patches)
+
+ for timeslice, patchSet in slices.iteritems():
+ for patchId in patchSet:
+ patchObj = getPatchById(patches, patchId)
+ if patchObj.issued != timeslice:
+ log.info('syncing %s timestamp (%s) to slice timestamp %s' % (
+ patchId, patchObj.issued, timeslice))
+ patchObj.issued = timeslice
+
+ # slices dict is still current since the above only synced the
+ # patch timestamps to existing slices.
+
+ # This maps patchid (without regard to repos) to timeslices.
+ patchidMap = map_patchids(slices)
+
+ # This requires no more than two timestamps per patchid;
+ # one each for slesp3 and sdkp3 (bails out otherwise):
+ #
+ # Pondering how this can be consolidated with above code to
+ # reduce iterating...
+ #
+ # Just so it's clear, I hate this code (i.e. FIXME).
+ #
+ for patchid, timestamps in patchidMap.iteritems():
+ if len(timestamps) > 1:
+ import epdb ; epdb.st()
+ # Untested beyond 2.
+ assert(len(timestamps) == 2)
+ # FIXME: refactor this monster.
+ splitpatch = [ (patch.id,
+ set([x.filename for x in patch.pkglist]),
+ patch) for patch in
+ patches if patchidNoRepo(patch.id) == patchid ]
+ if splitpatch[0][1].issubset(splitpatch[1][1]):
+ log.info('syncing timestamps (%s %s) ' % (
+ splitpatch[0][2].issued,
+ splitpatch[1][2].issued) +
+ 'across repositories for %s & %s ' % (
+ splitpatch[0][0], splitpatch[1][0]) +
+ 'to superset timestamp %s' % splitpatch[1][2].issued)
+ splitpatch[0][2].issued = splitpatch[1][2].issued
+ elif splitpatch[1][1].issubset(splitpatch[0][1]):
+ log.info('syncing timestamps (%s %s) ' % (
+ splitpatch[0][2].issued,
+ splitpatch[1][2].issued) +
+ 'across repositories for %s & %s ' % (
+ splitpatch[0][0], splitpatch[1][0]) +
+ 'to superset timestamp %s' % splitpatch[0][2].issued)
+ splitpatch[1][2].issued = splitpatch[0][2].issued
+ # So far this has only been tested in pure-subset cases.
+ else:
+ raise RuntimeError , 'neither %s nor %s is a subset of the other' % (splitpatch[0][0], splitpatch[1][0])
+
+ advPkgMap = {}
+ nevras = {}
+ packages = {}
+ srcPkgAdvMap = {}
+ srcPkgPatchidMap = {}
+
+ for patch in patches:
+ advisory = patch.id
+ patchid = patchidNoRepo(advisory)
+
+ for binPkg in patch.pkglist:
+ nevra = binPkg.getNevra()
+ nevraObj = nevras.setdefault(nevra, Nevra(*nevra))
+ channelObj = getChannel(binPkg)
+ package = Package(channelObj, nevraObj)
+ packageObj = packages.setdefault(package, package)
+ advPkgMap.setdefault(advisory, set()).add(packageObj)
+ srcPkgObj = getSrcPkg(binPkg)
+ srcPkgAdvMap.setdefault(srcPkgObj, set()).add(advisory)
+ srcPkgPatchidMap.setdefault(srcPkgObj, set()).add(patchid)
+
+ # FIXME: I hate this code too.
+ if srcPkgPatchidMap[srcPkgObj] != set([patchid]):
+ # Untested beyond two, and expected case is two
+ # different advisories issued for the same source
+ # package, one each for x86 and x86_64. (Lots of
+ # these for the kernel, for instance.)
+ assert(len(srcPkgPatchidMap[srcPkgObj]) == 2)
+ srcPkgAdvs = [ getPatchById(patches, srcPkgAdv)
+ for srcPkgAdv in srcPkgAdvMap[srcPkgObj] ]
+ # Only sync the same source package once. (It may
+ # appear for multiple binary packages.)
+ if srcPkgAdvs[0].issued != srcPkgAdvs[1].issued:
+ # Using the min here in case the first advisory
+ # for this source package has already been
+ # published.
+ syncTimestamp = min(srcPkgAdvs[0].issued,
+ srcPkgAdvs[1].issued)
+ log.info('syncing timestamps (%s %s) ' % (
+ srcPkgAdvs[0].issued, srcPkgAdvs[1].issued) +
+ 'across same-SRPM advisories for %s & %s ' % (
+ srcPkgAdvs[0].id,
+ srcPkgAdvs[1].id) +
+ 'to earlier timestamp %s' % syncTimestamp)
+ srcPkgAdvs[0].issued = srcPkgAdvs[1].issued = syncTimestamp
+
+ # There should be no srcPkgs with more than two patchids.
+ assert(len([ x for x, y in srcPkgPatchidMap.iteritems()
+ if len(y) > 2 ]) == 0)
+
+ issue_date = time.strftime('%Y-%m-%d %H:%M:%S',
+ time.gmtime(int(patch.issued)))
+ log.info('creating advisory: %s (%s)' % (advisory,
+ patch.issued))
+ adv = Advisory(advisory, patch.summary, issue_date,
+ advPkgMap[advisory])
+ self._advisories.add(adv)
+ self._advOrder.setdefault(int(patch.issued), set()).add(adv)
+
+
+ #AdvisoryManager._order(self)
+ #import epdb ; epdb.st()
diff --git a/repomd/packagexml.py b/repomd/packagexml.py
--- a/repomd/packagexml.py
+++ b/repomd/packagexml.py
@@ -181,7 +181,7 @@
def getNevra(self):
"""
- Return the name, epoch, version, release, and arch the package.
+ Return the name, epoch, version, release, and arch of the package.
"""
return (self.name, self.epoch, self.version, self.release, self.arch)
diff --git a/repomd/repomdxml.py b/repomd/repomdxml.py
--- a/repomd/repomdxml.py
+++ b/repomd/repomdxml.py
@@ -33,7 +33,7 @@
Python representation of repomd.xml from the repository metadata.
"""
- __slots__ = ('revision', )
+ __slots__ = ('revision', 'tags')
def addChild(self, child):
"""
@@ -46,6 +46,9 @@
name = child.getName()
if name == 'revision':
self.revision = child.finalize()
+ elif name == 'tags':
+ # FIXME: Is this complete? -jau
+ self.tags = child.finalize()
elif name == 'data':
child.type = child.getAttribute('type')
if child.type == 'patches':
diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py
--- a/repomd/updateinfoxml.py
+++ b/repomd/updateinfoxml.py
@@ -17,6 +17,10 @@
Refer to patchxml.py for previous versions.
"""
+import logging
+
+log = logging.getLogger('repomd')
+
__all__ = ('UpdateInfoXml', )
from rpath_xmllib import api1 as xmllib
@@ -43,6 +47,7 @@
child.status = None
child.emailfrom = None
child.type = None
+ child.version = None
for attr, value in child.iterAttributes():
if attr == 'status':
@@ -52,7 +57,7 @@
elif attr == 'type':
child.type = value
elif attr == 'version':
- pass
+ child.version = value
else:
raise UnknownAttributeError(child, attr)
@@ -73,7 +78,7 @@
__slots__ = ('status', 'emailfrom', 'type', 'id', 'title', 'release',
'issued', 'references', 'description', 'pkglist', 'packages',
- 'summary', 'packages')
+ 'summary', 'packages', 'version')
# All attributes are defined in __init__ by iterating over __slots__,
# this confuses pylint.
@@ -85,9 +90,12 @@
Parse the children of update.
"""
+ #import epdb ; epdb.st()
+
n = child.getName()
if n == 'id':
- self.id = child.finalize()
+ # Make this behave like SLES10 patchid
+ self.id = child.finalize() + '-' + self.getAttribute('version')
elif n == 'title':
self.title = child.finalize()
elif n == 'release':
@@ -105,6 +113,40 @@
else:
raise UnknownElementError(child)
+ def __cmp__(self, other):
+ vercmp = cmp(self.version, other.version)
+ if vercmp != 0:
+ return vercmp
+
+ relcmp = cmp(self.release, other.release)
+ if relcmp != 0:
+ return relcmp
+
+ # Is there even a summary in the schema?!?
+ sumcmp = cmp(self.summary, other.summary)
+ if sumcmp != 0:
+ return sumcmp
+
+ desccmp = cmp(self.description, other.description)
+ if desccmp != 0:
+ return desccmp
+
+ if self.issued != other.issued:
+ maxtime = max(self.issued, other.issued)
+ log.info('syncing timestamps (%s %s) ' % (self.issued,
+ other.issued) +
+ 'for %s to %s' % (self.id, maxtime))
+ self.issued = other.issued = maxtime
+ # Don't return here--they're now equal.
+
+ for pkg in other.pkglist:
+ if pkg not in self.pkglist:
+ self.pkglist.append(pkg)
+
+ return 0
+
+ def __hash__(self):
+ return hash((self.id, self.release, self.summary, self.description))
class _References(SlotNode):
"""
@@ -164,6 +206,9 @@
child.arch = None
child.version = None
child.release = None
+ # SLES11 updateinfo.xml doesn't provide checksums or archive sizes.
+ child.checksum = None
+ child.archiveSize = None
child.location = ''
@@ -184,7 +229,7 @@
class _UpdateInfoPackage(SlotNode, PackageCompare):
"""
- Represnts a package entry in a pkglist of an update in updateinfo.xml.
+ Represents a package entry in a pkglist of an update in updateinfo.xml.
"""
__slots__ = ('filename', 'name', 'arch', 'version', 'release',
@@ -213,6 +258,13 @@
else:
raise UnknownElementError(child)
+ def getNevra(self):
+ """
+ Return the name, epoch, version, release, and arch of the package.
+ """
+
+ return (self.name, self.epoch, self.version, self.release, self.arch)
+
class UpdateInfoXml(XmlFileParser):
"""
diff --git a/scripts/sle11order.py b/scripts/sle11order.py
new file mode 100755
--- /dev/null
+++ b/scripts/sle11order.py
@@ -0,0 +1,46 @@
+#!/usr/bin/python
+
+import os
+import sys
+import time
+
+sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
+
+from conary.lib import util
+sys.excepthook = util.genExcepthook()
+
+mbdir = os.path.abspath('../')
+sys.path.insert(0, mbdir)
+
+confDir = os.path.join(mbdir, 'config', sys.argv[1])
+
+from updatebot import log
+from updatebot.ordered import Bot
+from updatebot import UpdateBotConfig
+
+from errata.sles11 import AdvisoryManager11 as Errata
+
+slog = log.addRootLogger()
+
+cfg = UpdateBotConfig()
+cfg.read(os.path.join(confDir, 'updatebotrc'))
+
+bot = Bot(cfg, None)
+errata = Errata(bot._pkgSource)
+bot._errata._errata = errata
+
+errata.fetch()
+
+bot._pkgSource.load()
+bot._errata._orderErrata()
+
+order = bot._errata._order
+advMap = bot._errata._advMap
+sorder = sorted(order)
+
+def tconv(tstamp):
+ return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp))
+
+childPackages, parentPackages = bot._errata.sanityCheckOrder()
+
+import epdb; epdb.st()
From juphoff at rpath.com Mon Sep 20 22:09:09 2010
From: juphoff at rpath.com (Jeff Uphoff)
Date: Tue, 21 Sep 2010 02:09:09 +0000
Subject: mirrorball: add sles11 platform to order_import.py
Message-ID: <201009210209.o8L299mj020329@scc.eng.rpath.com>
changeset: 3c56c7d8fd99
user: Jeff Uphoff
date: Mon, 20 Sep 2010 22:08:51 -0400
add sles11 platform to order_import.py
diff --git a/scripts/order_import.py b/scripts/order_import.py
--- a/scripts/order_import.py
+++ b/scripts/order_import.py
@@ -74,6 +74,9 @@
elif cfg.platformName == 'centos':
from errata.centos import AdvisoryManager as Errata
+ elif cfg.platformName == 'sles11':
+ from errata.sles11 import AdvisoryManager11 as Errata
+
else:
raise RuntimeError, 'no errata source found for %s' % cfg.platformName
From agrimm at rpath.com Wed Sep 29 10:52:44 2010
From: agrimm at rpath.com (Andy Grimm)
Date: Wed, 29 Sep 2010 14:52:44 +0000
Subject: mirrorball: minor bi-arch changes, centos point release changes
Message-ID: <201009291452.o8TEqi1P026968@scc.eng.rpath.com>
changeset: d1842fd19a6d
user: Andy Grimm
date: Wed, 29 Sep 2010 10:52:39 -0400
minor bi-arch changes, centos point release changes
diff --git a/errata/centos.py b/errata/centos.py
--- a/errata/centos.py
+++ b/errata/centos.py
@@ -113,7 +113,12 @@
# Sort packages by build timestamp.
slices = {}
for srcPkg in self._pkgSource.srcPkgMap:
- updateId = slice(int(srcPkg.buildTimestamp))
+ updateId = slice(int(srcPkg.fileTimestamp or srcPkg.buildTimestamp))
+ # If package comes from a base path, override updateId
+ for basePath in self._pkgSource._cfg.repositoryBasePaths:
+ if basePath[1].match(srcPkg.location) is not None:
+ updateId = 0
+ break
slices.setdefault(updateId, set()).add(srcPkg)
# find labels
diff --git a/scripts/gengroupmodel b/scripts/gengroupmodel
--- a/scripts/gengroupmodel
+++ b/scripts/gengroupmodel
@@ -49,7 +49,9 @@
# load package source
self._pkgSource.load()
- mgr = groupmgr.GroupManager(self._cfg, useMap=self._pkgSource.useMap)
+ ui = UserInterface()
+
+ mgr = groupmgr.GroupManager(self._cfg, ui, useMap=self._pkgSource.useMap)
mgr.setReadOnly()
lastAvailableUpdate = mgr.latest.errataState
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
--- a/scripts/rebuildgroups
+++ b/scripts/rebuildgroups
@@ -68,7 +68,12 @@
log.info('looking up siblings')
req = [ (x, None, None) for x in updatedPackages ]
- nvfMap = self._updater._conaryhelper.findTroves(req, getLeaves=False)
+
+ labels = [ self._updater._conaryhelper.getConaryConfig().buildLabel, ]
+ if self._cfg.platformSearchPath:
+ labels += self._cfg.platformSearchPath
+
+ nvfMap = self._updater._conaryhelper.findTroves(req, labels=labels, getLeaves=False)
nvfs = [ x for x in itertools.chain(*nvfMap.itervalues()) ]
siblingMap = self._updater._conaryhelper.getSiblingPackages(nvfs,
@@ -113,6 +118,9 @@
emptyFlavors = set()
for pkg in group.iterpackages():
n = str(pkg.name)
+ # if n.find('spice-usb') != -1:
+ # v = versions.ThawVersion(str(pkg.version).replace('workstation', 'workstation-devel'))
+ # else:
v = versions.ThawVersion(str(pkg.version))
f = deps.ThawFlavor(str(pkg.flavor))
@@ -318,7 +326,8 @@
'bluez-utils', 'system-config-users', 'cman', 'yum-rhn-plugin',
'system-config-boot', 'system-config-network', 'system-config-date',
'ecryptfs-utils', 'tix', 'anaconda', 'system-config-rootpassword',
- 'dstat']
+ 'dstat', 'gaim', 'gnome-games', 'kdeaddons', 'pidgin', 'planner',
+ 'redhat-release', 'rhythmbox', 'openoffice.org', 'spice-usb-redirector']
tags = [ 'chkconfig', 'desktop-file-utils', 'fontconfig', 'gtk+', 'pango',
'setup', 'shadow-utils', 'shared-mime-info', 'texinfo',
diff --git a/scripts/rebuildpackage b/scripts/rebuildpackage
--- a/scripts/rebuildpackage
+++ b/scripts/rebuildpackage
@@ -187,7 +187,7 @@
# bot.removeSourceFiles(pkgName, replaceFiles=replaceFiles)
bot.rebuildpackages(pkgNames,
- useLatest=['conary', 'conary-build', 'conary-policy', 'rpm', ],
+ useLatest=['conary', 'conary-build', 'conary-policy', 'rpm', 'capsule-kmod-import'],
additionalResolveTroves=[
'libelf-lgpl=rhel.rpath.com at rpath:rhel-5-devel',
'conary=rhel.rpath.com at rpath:rhel-5-devel',
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -332,6 +332,9 @@
name = req.name()
version = req.version()
if useLatest and name.split(':')[0] in useLatest:
+ # XXX - this broke for me at some point, so I
+ # switched to using "continue" here; I've now
+ # forgotten the reason, and so changed it back. - AG
version = version.branch()
reqs.add((name, version))
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -286,6 +286,10 @@
# Paths based off of the repositoryUrl to get to individual repositories.
repositoryPaths = (CfgList(CfgString), ['/'])
+ # Treat any path matching this spec as "base" (ISO) content.
+ # In other words, these packages are the golden bits, not errata.
+ repositoryBasePaths = (CfgList(CfgRegExp), [])
+
# Arch strings for each repository to signify what base architecture each
# repository is meant for.
# repositoryName archString