From elliot at rpath.com Mon May 3 21:22:54 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 04 May 2010 01:22:54 +0000
Subject: mirrorball: allow contexts to be filtered by binary names
Message-ID: <201005040122.o441MsfH013089@scc.eng.rpath.com>
changeset: de24b1ab88f2
user: Elliot Peele
date: Mon, 03 May 2010 21:12:05 -0400
allow contexts to be filtered by binary names
diff --git a/scripts/buildpackages b/scripts/buildpackages
--- a/scripts/buildpackages
+++ b/scripts/buildpackages
@@ -7,14 +7,28 @@
Script for cooking packages with updatebot config.
"""
+import os
+
from header import *
+from updatebot import conaryhelper
if len(sys.argv) < 3:
usage()
+helper = conaryhelper.ConaryHelper(cfg)
+
+def validateInput(input):
+ for pkgName in input:
+ manifest = helper.getManifest(pkgName)
+ paths = [ os.path.basename(x) for x in manifest ]
+ for context, fltr in cfg.archContexts:
+ if fltr and [ x for x in paths if fltr[1].match(x) ]:
+ raise RuntimeError, 'Found package that may not be built'
+ return input
+
trvs = set()
label = cfg.topSourceGroup[1]
-for pkg in sys.argv[2:]:
+for pkg in validateInput(sys.argv[2:]):
trvs.add((pkg, label, None))
trvMap = builder.build(trvs)
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -57,6 +57,22 @@
return [ x for x in itertools.chain(*setDict.itervalues()) ]
+ def _formatBuildTroves(self, buildSet):
+ """
+ Format a list of trove specs and source package objects into something
+ the build subsystem can deal with.
+ """
+
+ toBuild = set()
+ for nvf, srcPkg in buildSet:
+ binaryNames = None
+ if srcPkg:
+ binaryNames = [ x.name
+ for x in self._pkgSource.srcPkgMap[srcPkg] ]
+ toBuild.add((nvf, binaryNames))
+
+ return sorted(toBuild)
+
def create(self, rebuild=False, recreate=None, toCreate=None):
"""
Do initial imports.
@@ -107,7 +123,7 @@
# Import sources into repository.
- toBuild, parentPkgMap, fail = self._updater.create(
+ buildSet, parentPkgMap, fail = self._updater.create(
toPackage,
buildAll=rebuild,
recreate=bool(recreate),
@@ -118,17 +134,20 @@
trvMap = {}
failed = ()
+
+ toBuild = self._formatBuildTroves(buildSet)
+
if len(toBuild):
if not rebuild or (rebuild and toCreate):
# Build all newly imported packages.
- trvMap, failed = self._builder.buildmany(sorted(toBuild))
+ trvMap, failed = self._builder.buildmany(toBuild)
log.info('failed to import %s packages' % len(failed))
if len(failed):
for pkg in failed:
log.warn('%s' % (pkg, ))
else:
# ReBuild all packages.
- trvMap = self._builder.buildsplitarch(sorted(toBuild))
+ trvMap = self._builder.buildsplitarch(toBuild)
log.info('import completed successfully')
log.info('imported %s source packages' % (len(toBuild), ))
else:
@@ -220,7 +239,7 @@
# Make sure to build everything in the toAdvise list, there may be
# sources that have been updated, but not built.
- buildTroves = set([ x[0] for x in toAdvise ])
+ buildTroves = self._formatBuildTroves(toAdvise)
# If importing specific packages, they might require each other so
# always use buildmany, but wait to commit.
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -43,6 +43,7 @@
from updatebot.errors import JobFailedError
from updatebot.errors import CommitFailedError
from updatebot.errors import UnhandledKernelModule
+from updatebot.errors import InvalidBuildTroveInputError
from updatebot.errors import FailedToRetrieveChangesetError
from updatebot.errors import ChangesetValidationFailedError
@@ -312,7 +313,8 @@
"""
Formats the list of troves provided into a job list for rMake.
@param troveSpecs: set of name, version, flavor tuples
- @type troveSpecs: set([(name, version, flavor), ..])
+ @type troveSpecs: set([(name, version, flavor, optional list of binary
+ names), ..])
@return list((name, version, flavor, context), ...)
"""
@@ -324,7 +326,15 @@
# Build all troves in defined contexts.
troves = []
- for name, version, flavor in troveSpecs:
+ for trv in troveSpecs:
+ if len(trv) == 3:
+ name, version, flavor = trv
+ binaryNames = None
+ elif len(trv) == 4:
+ name, version, flavor, binaryNames = trv
+ else:
+ raise InvalidBuildTroveInputError(input=trv)
+
# Make sure name is not an unicode string, it causes breakage in
# the deps modules in conary.
name = name.encode()
@@ -365,7 +375,12 @@
# All other packages.
else:
# Build all packages as x86 and x86_64.
- for context in self._cfg.archContexts:
+ for context, fltr in self._cfg.archContexts:
+ # If there is a filter and no binary file names or no files
+ # in the binary names match the filter skip this context.
+ if (fltr and not binaryNames or (binaryNames and
+ not [ x for x in binaryNames if fltr[1].match(x) ])):
+ continue
troves.append((name, version, flavor, context))
return troves
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -68,6 +68,29 @@
raise ParseError, e
+class CfgContextFilter(CfgRegExp):
+ """
+ Class for parsing context name/regex tuples.
+ """
+
+ def parseString(self, val):
+ """
+ Parse config input.
+ """
+
+ try:
+ splt = val.split()
+ if len(splt) == 1:
+ context = val
+ fltr = None
+ else:
+ context, fltrStr = splt
+ fltr = CfgRegExp.parseString(self, fltrStr)
+ return context, fltr
+ except versions.ParseError, e:
+ raise ParseError, e
+
+
class CfgAdvisoryOrder(CfgString):
"""
Class for parsing advisor order config.
@@ -256,7 +279,7 @@
listArchiveStartDate = CfgString
# list of contexts that all packages are built in.
- archContexts = CfgList(CfgString)
+ archContexts = CfgList(CfgContextFilter)
# flavors to build the source group.
groupFlavors = (CfgList(CfgFlavor), [])
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -41,7 +41,16 @@
return '%s(%s)' % (self.__class__, params)
-class CommitFailedError(UpdateBotError):
+class BuildError(UpdateBotError):
+ """
+ BuildError, abstract error for all other build related errors.
+ """
+
+ _parms = []
+ _template = 'A build error has occured'
+
+
+class CommitFailedError(BuildError):
"""
CommitFailedError, raised when failing to commit to a repository.
"""
@@ -69,7 +78,7 @@
'validation because:\n%(reason)s')
-class JobFailedError(UpdateBotError):
+class JobFailedError(BuildError):
"""
JobFailedError, raised when an rMake job fails.
"""
@@ -78,7 +87,7 @@
_template = 'rMake job %(jobId)s failed: %(why)s'
-class JobNotCompleteError(UpdateBotError):
+class JobNotCompleteError(BuildError):
"""
JobNotCompleteError, raised when the build dispatcher thinks that the job
should be done, but it isn't.
@@ -88,7 +97,17 @@
_template = 'Build job not complete %(jobId)s'
-class UnhandledKernelModule(UpdateBotError):
+class InvalidBuildTroveInputError(BuildError):
+ """
+ InvalidBuildTroveInputError, raised when validation of a build request
+ fails.
+ """
+
+ _params = ['input', ]
+ _template = 'Invalid input to build system: %(input)s'
+
+
+class UnhandledKernelModule(BuildError):
"""
UnhandledKernelModule, raised when trying to create a build job with a
package that looks as if it might be a kernel module that does not have
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -595,20 +595,19 @@
if not verCache.get(pkg.name) or buildAll or recreate:
if self.isPlatformTrove(version):
- toBuild.add((pkg.name, version, None))
+ toBuild.add(((pkg.name, version, None), pkg))
else:
parentPackages.add((pkg.name, version, None))
else:
log.info('not building %s' % pkg.name)
preBuiltPackages.add((pkg.name, version, None))
except Exception, e:
- raise
log.error('failed to import %s: %s' % (pkg, e))
fail.add((pkg, e))
if buildAll and pkgs and pkgNames:
toBuild.update(
- [ (x, self._conaryhelper.getLatestSourceVersion(x), None)
+ [ ((x, self._conaryhelper.getLatestSourceVersion(x), None), None)
for x in pkgs if not self._fltrPkg(x) ]
)
From elliot at rpath.com Mon May 3 21:22:54 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 04 May 2010 01:22:54 +0000
Subject: mirrorball: 1. add config option for excluding "32bit" packages
on older sles platforms
Message-ID: <201005040122.o441MsGI013121@scc.eng.rpath.com>
changeset: c83817a1de00
user: Elliot Peele
date: Mon, 03 May 2010 21:18:16 -0400
1. add config option for excluding "32bit" packages on older sles platforms
2. If a source package can not be matched up after filtering out all handled
cases, just make one up.
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -210,6 +210,11 @@
# Paths based off of the repositoryUrl to get to individual repositories.
repositoryPaths = (CfgList(CfgString), ['/'])
+ # Control the exclusion of 32bit packages from a package source.
+ # NOTE: This is only here for backwards compatibility with older sles
+ # imports.
+ exclude32bitPackages = (CfgBool, False)
+
# Data source for determining platform version information, only used for
# group versioning.
versionSources = (CfgDict(CfgString), {})
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -103,7 +103,7 @@
for pkg in client.getPackageDetail():
# ignore the 32-bit compatibility libs - we will
# simply use the 32-bit components from the repository
- if '32bit' in pkg.name:
+ if '32bit' in pkg.name and self._cfg.exclude32bitPackages:
continue
# Don't use all arches.
@@ -352,6 +352,18 @@
return srcPkg
+ def synthesizeSource(srcPkg):
+ # add source to structures
+ if srcPkg.getNevra() not in self._srcMap:
+ log.warn('synthesizing source package %s' % srcPkg)
+ self._procSrc(srcPkg)
+
+ # Add location mappings for packages that may have once been
+ # synthesized so that parsing old manifest files still works.
+ elif srcPkg.location not in self.locationMap:
+ pkg = self._srcMap[srcPkg.getNevra()]
+ self.locationMap[srcPkg.location] = pkg
+
# Return if sources should be available in repos.
if not self._cfg.synthesizeSources:
return
@@ -368,16 +380,8 @@
deffer.add(nevra)
continue
- # add source to structures
- if nevra not in self._srcMap:
- log.warn('synthesizing source package %s' % srcPkg)
- self._procSrc(srcPkg)
-
- # Add location mappings for packages that may have once been
- # synthesized so that parsing old manifest files still works.
- elif srcPkg.location not in self.locationMap:
- pkg = self._srcMap[nevra]
- self.locationMap[srcPkg.location] = pkg
+ # Synthesize the source
+ synthesizeSource(srcPkg)
broken = set()
# Make an attempt to sort out the binaries that have different names
@@ -399,7 +403,10 @@
log.warn('found binary without matching source name %s'
% list(bins)[0].name)
- broken.add((nevra, tuple(bins)))
+ # If this isn't a case of a missmatched epoch, just go ahead and
+ # make up a source. What could go wrong?
+ synthesizeSource(srcPkg)
+ #broken.add((nevra, tuple(bins)))
# Raise an exception if this ever happens. We can figure out the right
# thing to do then, purhaps on a case by case basis.
From elliot at rpath.com Mon May 3 21:22:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 04 May 2010 01:22:55 +0000
Subject: mirrorball: begining of script to verify rpm repos
Message-ID: <201005040122.o441MtVQ013149@scc.eng.rpath.com>
changeset: f7b491510b61
user: Elliot Peele
date: Mon, 03 May 2010 21:19:12 -0400
begining of script to verify rpm repos
diff --git a/scripts/pkgsource b/scripts/verifyrepos
copy from scripts/pkgsource
copy to scripts/verifyrepos
--- a/scripts/pkgsource
+++ b/scripts/verifyrepos
@@ -15,6 +15,8 @@
import os
import sys
+import copy
+import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -28,20 +30,31 @@
updatebot.log.addRootLogger()
log = logging.getLogger('test')
+import rpmutils
+
from updatebot import config
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
-pkgSource.load()
+import epdb; epdb.st()
+paths = copy.copy(cfg.repositoryPaths)
+for path in paths:
+ cfg.repositoryPaths = [ path, ]
+ pkgSource = pkgsource.PackageSource(cfg)
+ pkgSource.load()
+
+ for location in pkgSource.locationMap:
+ if not location.startswith(path):
+ log.info('skipping %s' % location)
+ continue
+ url = cfg.repositoryUrl + '/' + location
+ try:
+ rpmutils.readHeader(url)
+ except Exception, e:
+ raise
+ log.info('failed to open %s, %s' % (url, e))
+
import epdb; epdb.st()
-
-updates = []
-for path, client in pkgSource._clients.iteritems():
- if 'Updates' in path:
- updates.extend(client.getUpdateInfo())
-
-import epdb; epdb.st()
From elliot at rpath.com Mon May 3 21:22:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 04 May 2010 01:22:55 +0000
Subject: mirrorball: changes to pkgsource for verifing sles
Message-ID: <201005040122.o441Mt3k013176@scc.eng.rpath.com>
changeset: 79afbe372992
user: Elliot Peele
date: Mon, 03 May 2010 21:19:40 -0400
changes to pkgsource for verifing sles
diff --git a/scripts/pkgsource b/scripts/pkgsource
--- a/scripts/pkgsource
+++ b/scripts/pkgsource
@@ -15,6 +15,7 @@
import os
import sys
+import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -37,6 +38,91 @@
pkgSource = pkgsource.PackageSource(cfg)
pkgSource.load()
+srcs = {}
+srcNevras = {}
+for src, bins in pkgSource.srcPkgMap.iteritems():
+ srcs.setdefault(src.getNevra(), bins)
+ srcNevras.setdefault(src.getNevra(), src)
+
+names = {}
+for nevra, bins in srcs.iteritems():
+ if len(bins) > 1:
+ names.setdefault(nevra[0], set()).add(nevra)
+
+count = {}
+arch = {}
+for name, nevras in names.iteritems():
+ first = None
+ seen = None
+ for nevra in nevras:
+ bins = srcs[nevra]
+ archs = set([ x.arch for x in bins ])
+ if not seen:
+ first = (nevra, bins)
+ seen = archs
+ elif len(seen) != len(archs):
+ count[first[0]] = first[1]
+ count[nevra] = bins
+ elif seen != archs:
+ arch.setdefault(first[0], set()).update(first[1])
+ arch.setdefault(nevra, set()).update(bins)
+
+removed = {}
+#for nevra in itertools.chain(count, arch):
+# removed.setdefault(nevra, srcNevras.pop(nevra))
+for nevra in itertools.chain(*[ x for x in names.values() if len(x) > 1 ]):
+ removed.setdefault(nevra, srcNevras.pop(nevra))
+
+toCreate = set(srcNevras.values())
+
+import epdb; epdb.st()
+
+order = {}
+
+def srtByRPMVerCmp(a, b):
+ from updatebot.lib import util
+ return util.packagevercmp(a, b)
+
+def srtByBuildstamp(a, b):
+ assert hasattr(a, 'buildTimestamp')
+ assert hasattr(b, 'buildTimestamp')
+ assert a.buildTimestamp not in ('0', '', 0)
+ assert b.buildTimestamp not in ('0', '', 0)
+ return cmp(int(a.buildTimestamp), int(b.buildTimestamp))
+
+srcNames = {}
+for srcPkg in pkgSource.srcPkgMap:
+ srcNames.setdefault(srcPkg.name, set()).add(srcPkg)
+
+binOrder = {}
+for srcName, srcPkgs in srcNames.iteritems():
+ uSrcPkgs = dict((x.getNevra(), x) for x in srcPkgs).values()
+
+ ver = sorted(uSrcPkgs, cmp=srtByRPMVerCmp)
+
+ for srcPkg in uSrcPkgs:
+ if srcPkg.buildTimestamp is None:
+ srcPkg.buildTimestamp = sorted([ x for x in pkgSource.srcPkgMap[srcPkg] if x.arch != 'src' ])[0].buildTimestamp
+
+ buildstamp = sorted(uSrcPkgs, cmp=srtByBuildstamp)
+
+ assert ver == buildstamp
+
+ for srcPkg in uSrcPkgs:
+ ts = int(srcPkg.buildTimestamp)
+ bins = pkgSource.srcPkgMap[srcPkg]
+ binOrder.setdefault(ts, set()).update(bins)
+
+def tsToDay(ts):
+ import time
+ return int(time.mktime(time.strptime(time.strftime('%Y%m%d', time.gmtime(ts)), '%Y%m%d')))
+
+# collapse by day
+for ts in sorted(binOrder):
+ day = tsToDay(ts)
+ bins = binOrder[ts]
+ order.setdefault(day, set()).update(bins)
+
import epdb; epdb.st()
updates = []
From elliot at rpath.com Mon May 3 21:22:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 04 May 2010 01:22:55 +0000
Subject: mirrorball: rpmhelper now handles reading into the file for you
Message-ID: <201005040122.o441MthM013204@scc.eng.rpath.com>
changeset: 2a74e22b8ddb
user: Elliot Peele
date: Mon, 03 May 2010 21:21:00 -0400
rpmhelper now handles reading into the file for you
diff --git a/rpmutils/header.py b/rpmutils/header.py
--- a/rpmutils/header.py
+++ b/rpmutils/header.py
@@ -81,7 +81,6 @@
# Have to read into the file a bit to get to the begining of the header
# that we care about.
- fh.read(96)
rpmhelper.RpmHeader(fh)
header = rpmhelper.RpmHeader(fh)
From elliot at rpath.com Mon May 3 21:22:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 04 May 2010 01:22:55 +0000
Subject: mirrorball: branch merge
Message-ID: <201005040122.o441MuoM013231@scc.eng.rpath.com>
changeset: e2cb909c90f1
user: Elliot Peele
date: Mon, 03 May 2010 21:23:01 -0400
branch merge
diff --git a/rpmutils/header.py b/rpmutils/header.py
--- a/rpmutils/header.py
+++ b/rpmutils/header.py
@@ -81,7 +81,6 @@
# Have to read into the file a bit to get to the begining of the header
# that we care about.
- fh.read(96)
rpmhelper.RpmHeader(fh)
header = rpmhelper.RpmHeader(fh)
diff --git a/scripts/buildpackages b/scripts/buildpackages
--- a/scripts/buildpackages
+++ b/scripts/buildpackages
@@ -7,14 +7,28 @@
Script for cooking packages with updatebot config.
"""
+import os
+
from header import *
+from updatebot import conaryhelper
if len(sys.argv) < 3:
usage()
+helper = conaryhelper.ConaryHelper(cfg)
+
+def validateInput(input):
+ for pkgName in input:
+ manifest = helper.getManifest(pkgName)
+ paths = [ os.path.basename(x) for x in manifest ]
+ for context, fltr in cfg.archContexts:
+ if fltr and [ x for x in paths if fltr[1].match(x) ]:
+ raise RuntimeError, 'Found package that may not be built'
+ return input
+
trvs = set()
label = cfg.topSourceGroup[1]
-for pkg in sys.argv[2:]:
+for pkg in validateInput(sys.argv[2:]):
trvs.add((pkg, label, None))
trvMap = builder.build(trvs)
diff --git a/scripts/pkgsource b/scripts/pkgsource
--- a/scripts/pkgsource
+++ b/scripts/pkgsource
@@ -15,6 +15,7 @@
import os
import sys
+import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -37,6 +38,91 @@
pkgSource = pkgsource.PackageSource(cfg)
pkgSource.load()
+srcs = {}
+srcNevras = {}
+for src, bins in pkgSource.srcPkgMap.iteritems():
+ srcs.setdefault(src.getNevra(), bins)
+ srcNevras.setdefault(src.getNevra(), src)
+
+names = {}
+for nevra, bins in srcs.iteritems():
+ if len(bins) > 1:
+ names.setdefault(nevra[0], set()).add(nevra)
+
+count = {}
+arch = {}
+for name, nevras in names.iteritems():
+ first = None
+ seen = None
+ for nevra in nevras:
+ bins = srcs[nevra]
+ archs = set([ x.arch for x in bins ])
+ if not seen:
+ first = (nevra, bins)
+ seen = archs
+ elif len(seen) != len(archs):
+ count[first[0]] = first[1]
+ count[nevra] = bins
+ elif seen != archs:
+ arch.setdefault(first[0], set()).update(first[1])
+ arch.setdefault(nevra, set()).update(bins)
+
+removed = {}
+#for nevra in itertools.chain(count, arch):
+# removed.setdefault(nevra, srcNevras.pop(nevra))
+for nevra in itertools.chain(*[ x for x in names.values() if len(x) > 1 ]):
+ removed.setdefault(nevra, srcNevras.pop(nevra))
+
+toCreate = set(srcNevras.values())
+
+import epdb; epdb.st()
+
+order = {}
+
+def srtByRPMVerCmp(a, b):
+ from updatebot.lib import util
+ return util.packagevercmp(a, b)
+
+def srtByBuildstamp(a, b):
+ assert hasattr(a, 'buildTimestamp')
+ assert hasattr(b, 'buildTimestamp')
+ assert a.buildTimestamp not in ('0', '', 0)
+ assert b.buildTimestamp not in ('0', '', 0)
+ return cmp(int(a.buildTimestamp), int(b.buildTimestamp))
+
+srcNames = {}
+for srcPkg in pkgSource.srcPkgMap:
+ srcNames.setdefault(srcPkg.name, set()).add(srcPkg)
+
+binOrder = {}
+for srcName, srcPkgs in srcNames.iteritems():
+ uSrcPkgs = dict((x.getNevra(), x) for x in srcPkgs).values()
+
+ ver = sorted(uSrcPkgs, cmp=srtByRPMVerCmp)
+
+ for srcPkg in uSrcPkgs:
+ if srcPkg.buildTimestamp is None:
+ srcPkg.buildTimestamp = sorted([ x for x in pkgSource.srcPkgMap[srcPkg] if x.arch != 'src' ])[0].buildTimestamp
+
+ buildstamp = sorted(uSrcPkgs, cmp=srtByBuildstamp)
+
+ assert ver == buildstamp
+
+ for srcPkg in uSrcPkgs:
+ ts = int(srcPkg.buildTimestamp)
+ bins = pkgSource.srcPkgMap[srcPkg]
+ binOrder.setdefault(ts, set()).update(bins)
+
+def tsToDay(ts):
+ import time
+ return int(time.mktime(time.strptime(time.strftime('%Y%m%d', time.gmtime(ts)), '%Y%m%d')))
+
+# collapse by day
+for ts in sorted(binOrder):
+ day = tsToDay(ts)
+ bins = binOrder[ts]
+ order.setdefault(day, set()).update(bins)
+
import epdb; epdb.st()
updates = []
diff --git a/scripts/pkgsource b/scripts/verifyrepos
copy from scripts/pkgsource
copy to scripts/verifyrepos
--- a/scripts/pkgsource
+++ b/scripts/verifyrepos
@@ -15,6 +15,8 @@
import os
import sys
+import copy
+import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -28,20 +30,31 @@
updatebot.log.addRootLogger()
log = logging.getLogger('test')
+import rpmutils
+
from updatebot import config
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
-pkgSource.load()
+import epdb; epdb.st()
+paths = copy.copy(cfg.repositoryPaths)
+for path in paths:
+ cfg.repositoryPaths = [ path, ]
+ pkgSource = pkgsource.PackageSource(cfg)
+ pkgSource.load()
+
+ for location in pkgSource.locationMap:
+ if not location.startswith(path):
+ log.info('skipping %s' % location)
+ continue
+ url = cfg.repositoryUrl + '/' + location
+ try:
+ rpmutils.readHeader(url)
+ except Exception, e:
+ raise
+ log.info('failed to open %s, %s' % (url, e))
+
import epdb; epdb.st()
-
-updates = []
-for path, client in pkgSource._clients.iteritems():
- if 'Updates' in path:
- updates.extend(client.getUpdateInfo())
-
-import epdb; epdb.st()
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -57,6 +57,22 @@
return [ x for x in itertools.chain(*setDict.itervalues()) ]
+ def _formatBuildTroves(self, buildSet):
+ """
+ Format a list of trove specs and source package objects into something
+ the build subsystem can deal with.
+ """
+
+ toBuild = set()
+ for nvf, srcPkg in buildSet:
+ binaryNames = None
+ if srcPkg:
+ binaryNames = [ x.name
+ for x in self._pkgSource.srcPkgMap[srcPkg] ]
+ toBuild.add((nvf, binaryNames))
+
+ return sorted(toBuild)
+
def create(self, rebuild=False, recreate=None, toCreate=None):
"""
Do initial imports.
@@ -107,7 +123,7 @@
# Import sources into repository.
- toBuild, parentPkgMap, fail = self._updater.create(
+ buildSet, parentPkgMap, fail = self._updater.create(
toPackage,
buildAll=rebuild,
recreate=bool(recreate),
@@ -118,17 +134,20 @@
trvMap = {}
failed = ()
+
+ toBuild = self._formatBuildTroves(buildSet)
+
if len(toBuild):
if not rebuild or (rebuild and toCreate):
# Build all newly imported packages.
- trvMap, failed = self._builder.buildmany(sorted(toBuild))
+ trvMap, failed = self._builder.buildmany(toBuild)
log.info('failed to import %s packages' % len(failed))
if len(failed):
for pkg in failed:
log.warn('%s' % (pkg, ))
else:
# ReBuild all packages.
- trvMap = self._builder.buildsplitarch(sorted(toBuild))
+ trvMap = self._builder.buildsplitarch(toBuild)
log.info('import completed successfully')
log.info('imported %s source packages' % (len(toBuild), ))
else:
@@ -220,7 +239,7 @@
# Make sure to build everything in the toAdvise list, there may be
# sources that have been updated, but not built.
- buildTroves = set([ x[0] for x in toAdvise ])
+ buildTroves = self._formatBuildTroves(toAdvise)
# If importing specific packages, they might require each other so
# always use buildmany, but wait to commit.
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -43,6 +43,7 @@
from updatebot.errors import JobFailedError
from updatebot.errors import CommitFailedError
from updatebot.errors import UnhandledKernelModule
+from updatebot.errors import InvalidBuildTroveInputError
from updatebot.errors import FailedToRetrieveChangesetError
from updatebot.errors import ChangesetValidationFailedError
@@ -312,7 +313,8 @@
"""
Formats the list of troves provided into a job list for rMake.
@param troveSpecs: set of name, version, flavor tuples
- @type troveSpecs: set([(name, version, flavor), ..])
+ @type troveSpecs: set([(name, version, flavor, optional list of binary
+ names), ..])
@return list((name, version, flavor, context), ...)
"""
@@ -324,7 +326,15 @@
# Build all troves in defined contexts.
troves = []
- for name, version, flavor in troveSpecs:
+ for trv in troveSpecs:
+ if len(trv) == 3:
+ name, version, flavor = trv
+ binaryNames = None
+ elif len(trv) == 4:
+ name, version, flavor, binaryNames = trv
+ else:
+ raise InvalidBuildTroveInputError(input=trv)
+
# Make sure name is not an unicode string, it causes breakage in
# the deps modules in conary.
name = name.encode()
@@ -365,7 +375,12 @@
# All other packages.
else:
# Build all packages as x86 and x86_64.
- for context in self._cfg.archContexts:
+ for context, fltr in self._cfg.archContexts:
+ # If there is a filter and no binary file names or no files
+ # in the binary names match the filter skip this context.
+ if (fltr and not binaryNames or (binaryNames and
+ not [ x for x in binaryNames if fltr[1].match(x) ])):
+ continue
troves.append((name, version, flavor, context))
return troves
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -68,6 +68,29 @@
raise ParseError, e
+class CfgContextFilter(CfgRegExp):
+ """
+ Class for parsing context name/regex tuples.
+ """
+
+ def parseString(self, val):
+ """
+ Parse config input.
+ """
+
+ try:
+ splt = val.split()
+ if len(splt) == 1:
+ context = val
+ fltr = None
+ else:
+ context, fltrStr = splt
+ fltr = CfgRegExp.parseString(self, fltrStr)
+ return context, fltr
+ except versions.ParseError, e:
+ raise ParseError, e
+
+
class CfgAdvisoryOrder(CfgString):
"""
Class for parsing advisor order config.
@@ -260,7 +283,7 @@
listArchiveStartDate = CfgString
# list of contexts that all packages are built in.
- archContexts = CfgList(CfgString)
+ archContexts = CfgList(CfgContextFilter)
# flavors to build the source group.
groupFlavors = (CfgList(CfgFlavor), [])
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -41,7 +41,16 @@
return '%s(%s)' % (self.__class__, params)
-class CommitFailedError(UpdateBotError):
+class BuildError(UpdateBotError):
+ """
+ BuildError, abstract error for all other build related errors.
+ """
+
+ _parms = []
+ _template = 'A build error has occured'
+
+
+class CommitFailedError(BuildError):
"""
CommitFailedError, raised when failing to commit to a repository.
"""
@@ -69,7 +78,7 @@
'validation because:\n%(reason)s')
-class JobFailedError(UpdateBotError):
+class JobFailedError(BuildError):
"""
JobFailedError, raised when an rMake job fails.
"""
@@ -78,7 +87,7 @@
_template = 'rMake job %(jobId)s failed: %(why)s'
-class JobNotCompleteError(UpdateBotError):
+class JobNotCompleteError(BuildError):
"""
JobNotCompleteError, raised when the build dispatcher thinks that the job
should be done, but it isn't.
@@ -88,7 +97,17 @@
_template = 'Build job not complete %(jobId)s'
-class UnhandledKernelModule(UpdateBotError):
+class InvalidBuildTroveInputError(BuildError):
+ """
+ InvalidBuildTroveInputError, raised when validation of a build request
+ fails.
+ """
+
+ _params = ['input', ]
+ _template = 'Invalid input to build system: %(input)s'
+
+
+class UnhandledKernelModule(BuildError):
"""
UnhandledKernelModule, raised when trying to create a build job with a
package that looks as if it might be a kernel module that does not have
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -352,6 +352,18 @@
return srcPkg
+ def synthesizeSource(srcPkg):
+ # add source to structures
+ if srcPkg.getNevra() not in self._srcMap:
+ log.warn('synthesizing source package %s' % srcPkg)
+ self._procSrc(srcPkg)
+
+ # Add location mappings for packages that may have once been
+ # synthesized so that parsing old manifest files still works.
+ elif srcPkg.location not in self.locationMap:
+ pkg = self._srcMap[srcPkg.getNevra()]
+ self.locationMap[srcPkg.location] = pkg
+
# Return if sources should be available in repos.
if not self._cfg.synthesizeSources:
return
@@ -368,16 +380,8 @@
deffer.add(nevra)
continue
- # add source to structures
- if nevra not in self._srcMap:
- log.warn('synthesizing source package %s' % srcPkg)
- self._procSrc(srcPkg)
-
- # Add location mappings for packages that may have once been
- # synthesized so that parsing old manifest files still works.
- elif srcPkg.location not in self.locationMap:
- pkg = self._srcMap[nevra]
- self.locationMap[srcPkg.location] = pkg
+ # Synthesize the source
+ synthesizeSource(srcPkg)
broken = set()
# Make an attempt to sort out the binaries that have different names
@@ -399,7 +403,10 @@
log.warn('found binary without matching source name %s'
% list(bins)[0].name)
- broken.add((nevra, tuple(bins)))
+ # If this isn't a case of a missmatched epoch, just go ahead and
+ # make up a source. What could go wrong?
+ synthesizeSource(srcPkg)
+ #broken.add((nevra, tuple(bins)))
# Raise an exception if this ever happens. We can figure out the right
# thing to do then, purhaps on a case by case basis.
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -595,20 +595,19 @@
if not verCache.get(pkg.name) or buildAll or recreate:
if self.isPlatformTrove(version):
- toBuild.add((pkg.name, version, None))
+ toBuild.add(((pkg.name, version, None), pkg))
else:
parentPackages.add((pkg.name, version, None))
else:
log.info('not building %s' % pkg.name)
preBuiltPackages.add((pkg.name, version, None))
except Exception, e:
- raise
log.error('failed to import %s: %s' % (pkg, e))
fail.add((pkg, e))
if buildAll and pkgs and pkgNames:
toBuild.update(
- [ (x, self._conaryhelper.getLatestSourceVersion(x), None)
+ [ ((x, self._conaryhelper.getLatestSourceVersion(x), None), None)
for x in pkgs if not self._fltrPkg(x) ]
)
From elliot at rpath.com Fri May 7 11:43:44 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:44 +0000
Subject: mirrorball: create build jobs starting at the begining of the
list
Message-ID: <201005071543.o47Fhi5S009274@scc.eng.rpath.com>
changeset: 853bf302396e
user: Elliot Peele
date: Tue, 04 May 2010 23:45:57 -0400
create build jobs starting at the begining of the list
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -92,7 +92,7 @@
while (troves and self._slots and self._startSlots and
self._availableFDs()):
# get trove to work on
- trove = troves.pop()
+ trove = troves.pop(0)
# start build job
self._starter.startJob(trove)
From elliot at rpath.com Fri May 7 11:43:44 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:44 +0000
Subject: mirrorball: format build troves list properly
Message-ID: <201005071543.o47Fhijn009305@scc.eng.rpath.com>
changeset: f870ab757e0e
user: Elliot Peele
date: Tue, 04 May 2010 23:46:21 -0400
format build troves list properly
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -64,12 +64,12 @@
"""
toBuild = set()
- for nvf, srcPkg in buildSet:
+ for (n, v, f), srcPkg in buildSet:
binaryNames = None
if srcPkg:
- binaryNames = [ x.name
- for x in self._pkgSource.srcPkgMap[srcPkg] ]
- toBuild.add((nvf, binaryNames))
+ binaryNames = tuple([ x.name
+ for x in self._pkgSource.srcPkgMap[srcPkg] ])
+ toBuild.add((n, v, f, binaryNames))
return sorted(toBuild)
@@ -129,14 +129,14 @@
recreate=bool(recreate),
toCreate=toCreate)
+ toBuild = self._formatBuildTroves(buildSet)
+
log.info('failed to create %s packages' % len(fail))
log.info('found %s packages to build' % len(toBuild))
trvMap = {}
failed = ()
- toBuild = self._formatBuildTroves(buildSet)
-
if len(toBuild):
if not rebuild or (rebuild and toCreate):
# Build all newly imported packages.
From elliot at rpath.com Fri May 7 11:43:45 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:45 +0000
Subject: mirrorball: 1. increase number of workers to take advantage of
all build infrastructure
Message-ID: <201005071543.o47Fhj3v009333@scc.eng.rpath.com>
changeset: a8cd244ebe03
user: Elliot Peele
date: Tue, 04 May 2010 23:48:16 -0400
1. increase number of workers to take advantage of all build infrastructure
2. fix bugs related to changes in trove input formatting
3. add reporting of current count of source troves
4. build packages that have a newer source version than the latest binary
5. whitespace fixes
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -177,7 +177,7 @@
@return troveMap: dictionary of troveSpecs to built troves
"""
- workers = 20
+ workers = 30
if not lateCommit:
dispatcher = Dispatcher(self, workers)
else:
@@ -319,7 +319,7 @@
"""
# Make sure troveSpecs is an iterable of three tuples.
- if (len(troveSpecs) == 3 and
+ if (len(troveSpecs) in (3, 4) and
not isinstance(list(troveSpecs)[0], (list, set, tuple))):
# Assume that (n,v,f) was passed in
troveSpecs = [ troveSpecs, ]
@@ -378,8 +378,8 @@
for context, fltr in self._cfg.archContexts:
# If there is a filter and no binary file names or no files
# in the binary names match the filter skip this context.
- if (fltr and not binaryNames or (binaryNames and
- not [ x for x in binaryNames if fltr[1].match(x) ])):
+ if (fltr and (not binaryNames or (binaryNames and
+ not [ x for x in binaryNames if fltr[1].match(x) ]))):
continue
troves.append((name, version, flavor, context))
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -518,12 +518,14 @@
return ret
- def create(self, pkgNames=None, buildAll=False, recreate=False, toCreate=None):
+ def create(self, pkgNames=None, buildAll=False, recreate=False,
+ toCreate=None):
"""
Import a new package into the repository.
@param pkgNames: list of packages to import
@type pkgNames: list
- @param buildAll: return a list of all troves found rather than just the new ones.
+ @param buildAll: return a list of all troves found rather than just the
+ new ones.
@type buildAll: boolean
@param recreate: a package manifest even if it already exists.
@type recreate: boolean
@@ -585,15 +587,25 @@
toBuild = set()
preBuiltPackages = set()
parentPackages = set()
+ total = len(toCreate)
+ current = 1
+ start = False
for pkg in sorted(toCreate):
+ if pkg.name.startswith('n'):
+ start = True
+
try:
# Only import packages that haven't been imported before
version = verCache.get('%s:source' % pkg.name)
if not version or recreate:
- log.info('attempting to import %s' % pkg)
+ log.info('attempting to import %s (%s/%s)'
+ % (pkg, current, total))
version = self.update((pkg.name, None, None), pkg)
- if not verCache.get(pkg.name) or buildAll or recreate:
+ if (not verCache.get(pkg.name) or
+ verCache.get(pkg.name).getSourceVersion() != version or
+ buildAll or recreate):
+
if self.isPlatformTrove(version):
toBuild.add(((pkg.name, version, None), pkg))
else:
@@ -604,10 +616,12 @@
except Exception, e:
log.error('failed to import %s: %s' % (pkg, e))
fail.add((pkg, e))
+ current += 1
if buildAll and pkgs and pkgNames:
toBuild.update(
- [ ((x, self._conaryhelper.getLatestSourceVersion(x), None), None)
+ [ ((x, self._conaryhelper.getLatestSourceVersion(x), None),
+ None)
for x in pkgs if not self._fltrPkg(x) ]
)
@@ -615,7 +629,8 @@
pkgMap = {}
if parentPackages:
# Find all of the binaries that match the upstream platform sources.
- log.info('looking up binary versions of all parent platform packages')
+ log.info('looking up binary versions of all parent platform '
+ 'packages')
parentPkgMap = self.getBinaryVersions(parentPackages,
labels=self._cfg.platformSearchPath)
@@ -857,7 +872,8 @@
src = self._pkgSource.binPkgMap[latest]
srcname = src.name
else:
- log.warn('found virtual requires %s in pkg %s' % (name, srcPkg.name))
+ log.warn('found virtual requires %s in pkg %s'
+ % (name, srcPkg.name))
srcname = 'virtual'
reqs.append((name, srcname))
From elliot at rpath.com Fri May 7 11:43:45 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:45 +0000
Subject: mirrorball: skip troves that don't have the trove info that we
are looking for
Message-ID: <201005071543.o47Fhjts009360@scc.eng.rpath.com>
changeset: 563c27741c57
user: Elliot Peele
date: Fri, 07 May 2010 11:40:31 -0400
skip troves that don't have the trove info that we are looking for
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -318,6 +318,12 @@
tiMap = {}
tiLst = self._repos.getTroveInfo(tiType, req)
for i, nvf in enumerate(uncached):
+ # If this trove doesn't have this piece of trove info, log a warning
+ # and skip over it.
+ if tiLst[i] is None:
+ log.warn('found missing trove info for %s, skipping' % (nvf, ))
+ continue
+
ti = tiLst[i]()
if tiFunc:
ti = tiFunc(ti, nvf)
From elliot at rpath.com Fri May 7 11:43:45 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:45 +0000
Subject: mirrorball: support specifying arch for a given repository
Message-ID: <201005071543.o47FhjaQ009388@scc.eng.rpath.com>
changeset: 1abf6ae032c0
user: Elliot Peele
date: Tue, 06 Apr 2010 11:29:43 -0400
support specifying arch for a given repository
reflect repository contents in group model
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -187,6 +187,11 @@
# Paths based off of the repositoryUrl to get to individual repositories.
repositoryPaths = (CfgList(CfgString), ['/'])
+ # Arch strings for each repository to signify what base architecture each
+ # repository is meant for.
+ # repositoryName archString
+ repositoryArch = (CfgDict(CfgString), {})
+
# Data source for determining platform version information, only used for
# group versioning.
versionSources = (CfgDict(CfgString), {})
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -65,8 +65,9 @@
_helperClass = GroupHelper
_sanityCheckerClass = GroupSanityChecker
- def __init__(self, cfg, parentGroup=False, targetGroup=False):
+ def __init__(self, cfg, parentGroup=False, targetGroup=False, useMap=None):
self._cfg = cfg
+ self._useMap = useMap or dict()
self._helper = self._helperClass(self._cfg)
self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups')
self._sanity = self._sanityCheckerClass(self._cfg, self._helper)
@@ -101,6 +102,7 @@
self._readonly = False
self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName
+ self._stdGroupName = 'group-%s-standard' % self._cfg.platformName
self._checkedout = False
self._groups = {}
@@ -121,7 +123,7 @@
if self._sourceVersion and not copyToLatest:
log.error('refusing to commit out of date source')
- raise NotCommittingOutOfDateSourceError()
+ raise NotCommittingOutOfDateSourceError
# Copy forward data when we are fixing up old group versions so that
# this is the latest source.
@@ -285,127 +287,92 @@
x86_64 = deps.parseFlavor('is: x86_64')
# Count the flavors for later use.
+ flvMap = {}
flvCount = {x86: 0, x86_64: 0, plain: 0}
for flavor in flavors:
if flavor.satisfies(x86):
flvCount[x86] += 1
+ flvMap[flavor] = 'x86'
elif flavor.satisfies(x86_64):
flvCount[x86_64] += 1
+ flvMap[flavor] = 'x86_64'
elif flavor.freeze() == '':
flvCount[plain] += 1
+ flvMap[flavor] = None
else:
raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
- if len(flavors) == 1:
- flavor = flavors[0]
- # noarch package, add unconditionally
- if flavor == plain:
- self.add(name, version=version)
+ def add():
+ upver = version.trailingRevision().version()
+ for flv in flavors:
+ if self._useMap:
+ for useStr in self._useMap[(name, upver, flvMap[flv])]:
+ self.add(name, version=version, flavor=flavor,
+ use=useStr)
+ else:
+ self.add(name, version=version, flavor=flavor,
+ use=flvMap[flv])
- # x86, add with use=x86
- elif flavor.satisfies(x86):
- self.add(name, version=version, flavor=flavor, use='x86')
-
- # x86_64, add with use=x86_64
- elif flavor.satisfies(x86_64):
- self.add(name, version=version, flavor=flavor, use='x86_64')
-
- else:
- raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
-
+ # If this package has one or two flavors and one of those flavors is
+ # x86, x86_64, or plain then handle it like a normal package without
+ # doing any more sanity checking.
+ if sum([ x for x in flvCount.itervalues() if x in (0, 1) ]) in (1, 2):
+ add()
return
- elif (len(flavors) == 2 and
- not [ x for x, y in flvCount.iteritems() if y > 1 ]):
- # This is most likely a normal package with both x86 and x86_64, but
- # lets make sure anyway.
+ # Handle all other odd flavor cases:
+ # 1. kernels
+ # 2. kernel modules
+ # 3. packages with specifically defined flavor sets
- # An exception for python/perl where x86_64 packages are noarch, but
- # x86 packages are flavored.
- assert (flvCount[x86_64] > 0) ^ (flvCount[plain] > 0)
- # make sure there is only one instance of x86 and one instance of
- # x86_64 in the flavor list.
- assert len([ x for x, y in flvCount.iteritems() if y == 0 ]) == 1
+ # Check if this package is configured to have multiple flavors.
+ # Get source trove name.
+ log.info('retrieving trove info for %s' % name)
+ srcTroveMap = self._helper._getSourceTroves((name, version, flavors[0]))
+ srcTroveName = srcTroveMap.keys()[0][0].split(':')[0]
- # In this case just add the package unconditionally
- self.add(name, version=version)
+ # Check if this is package that we have specifically defined a build
+ # flavor for.
+ if srcTroveName in self._cfg.packageFlavors:
+ # separate packages into x86 and x86_64 by context name
+ # TODO: If we were really smart we would load the conary
+ # contexts and see what buildFlavors they contained.
+ flavorCtxCount = {x86: 0, x86_64: 0}
+ for context, bldflv in self._cfg.packageFlavors[srcTroveName]:
+ if context in ('i386', 'i486', 'i586', 'i686', 'x86'):
+ flavorCtxCount[x86] += 1
+ elif context in ('x86_64', ):
+ flavorCtxCount[x86_64] += 1
+ else:
+ raise UnknownBuildContextError(name=name, flavor=context)
+ # Sanity check flavors to make sure we built all the flavors
+ # that we expected.
+ extra = set(flvMap) - set([ x86, x86_64 ])
+ if extra:
+ raise UnsupportedTroveFlavorError(name=name, flavor=extra)
+
+ if (flvCount[x86] != flavorCtxCount[x86] or
+ flvCount[x86_64] != flavorCtxCount[x86_64]):
+ raise FlavorCountMismatchError(name=name)
+
+ # Add packages to the group.
+ add()
return
- # These are special cases.
- else:
- # The way I see it there are a few ways you could end up here.
- # 1. this is a kernel
- # 2. this is a kernel module
- # 3. this is a special package like glibc or openssl where there
- # are i386, i686, and x86_64 varients.
- # 4. this is a package with package flags that I don't know
- # about.
- # Lets see if we know about this package and think it should have
- # more than two flavors.
-
-
- # Get source trove name.
- log.info('retrieving trove info for %s' % name)
- srcTroveMap = self._helper._getSourceTroves(
- (name, version, flavors[0])
- )
- srcTroveName = srcTroveMap.keys()[0][0].split(':')[0]
-
- # handle kernels.
- if srcTroveName == 'kernel' or util.isKernelModulePackage(name):
- # add all x86ish flavors with use=x86 and all x86_64ish flavors
- # with use=x86_64
- for flavor in flavors:
- if flavor.satisfies(x86):
- self.add(name, version=version, flavor=flavor, use='x86')
- elif flavor.satisfies(x86_64):
- self.add(name, version=version, flavor=flavor, use='x86_64')
- else:
- raise UnsupportedTroveFlavorError(name=name,
- flavor=flavor)
-
- # maybe this is one of the special flavors we know about.
- elif srcTroveName in self._cfg.packageFlavors:
- # separate packages into x86 and x86_64 by context name
- # TODO: If we were really smart we would load the conary
- # contexts and see what buildFlavors they contained.
- flavorCount = {'x86': 0, 'x86_64': 0}
- for context, bldflv in self._cfg.packageFlavors[srcTroveName]:
- if context in ('i386', 'i486', 'i586', 'i686', 'x86'):
- flavorCount['x86'] += 1
- elif context in ('x86_64', ):
- flavorCount['x86_64'] += 1
- else:
- raise UnknownBuildContextError(name=name,
- flavor=context)
-
- for flavor in flavors:
- if flavor.satisfies(x86):
- flavorCount['x86'] -= 1
- elif flavor.satisfies(x86_64):
- flavorCount['x86_64'] -= 1
- else:
- raise UnsupportedTroveFlavorError(name=name,
- flavor=flavor)
-
- errors = [ x for x, y in flavorCount.iteritems() if y != 0 ]
- if errors:
- raise FlavorCountMismatchError(name=name)
-
- for flavor in flavors:
- if flavor.satisfies(x86):
- self.add(name, version=version,
- flavor=flavor, use='x86')
- elif flavor.satisfies(x86_64):
- self.add(name, version=version,
- flavor=flavor, use='x86_64')
- else:
- raise UnsupportedTroveFlavorError(name=name,
- flavor=flavor)
+ # handle kernels.
+ if srcTroveName == 'kernel' or srcTroveName in self._cfg.kernelModules:
+ # add all x86ish flavors with use=x86 and all x86_64ish flavors
+ # with use=x86_64
+ for flavor in flavors:
+ if flvMap[flavor] in ('x86', 'x86_64'):
+ self.add(name, version=version, flavor=flavor,
+ use=flvMap[flavor])
+ else:
+ raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
return
- # Unknown state.
+ # don't know how to deal with this package.
raise UnhandledPackageAdditionError(name=name)
@checkout
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -51,7 +51,8 @@
BotSuperClass.__init__(self, cfg)
self._errata = errata.ErrataFilter(self._cfg, self._pkgSource,
errataSource)
- self._groupmgr = groupmgr.GroupManager(self._cfg)
+ self._groupmgr = groupmgr.GroupManager(self._cfg,
+ useMap=self._pkgSource.useMap)
if self._cfg.platformSearchPath:
self._parentGroup = groupmgr.GroupManager(self._cfg,
diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py
--- a/updatebot/pkgsource/common.py
+++ b/updatebot/pkgsource/common.py
@@ -53,6 +53,9 @@
# {binPkg: set(binNames) }
self.obsoletesMap = dict()
+ # {(binName, srcConaryVersion, archStr): [archStr, archStr, ...]}
+ self.useMap = dict()
+
def __copy__(self):
log.info('copying pkgsource')
cls = self.__class__
@@ -62,6 +65,7 @@
obj.binPkgMap = copy.copy(self.binPkgMap)
obj.srcNameMap = copy.copy(self.srcNameMap)
obj.binNameMap = copy.copy(self.binNameMap)
+ obj.useMap = copy.copy(self.useMap)
return obj
def __deepcopy__(self, memo):
@@ -73,6 +77,7 @@
obj.binPkgMap = copy.deepcopy(self.binPkgMap, memo)
obj.srcNameMap = copy.deepcopy(self.srcNameMap, memo)
obj.binNameMap = copy.deepcopy(self.binNameMap, memo)
+ obj.useMap = copy.deepcopy(self.useMap, memo)
return obj
def getClients(self):
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -56,6 +56,10 @@
# set of all src pkg objects
self._srcPkgs = set()
+ # mapping of what arch repository each binary package came from
+ # {binPkg: set([archStr, ..])}
+ self._repoMap = dict()
+
def setLoaded(self):
self._loaded = True
@@ -68,14 +72,15 @@
for repo in self._cfg.repositoryPaths:
log.info('loading repository data %s' % repo)
client = repomd.Client(self._cfg.repositoryUrl + '/' + repo)
- self.loadFromClient(client, repo)
+ archStr = self._cfg.repositoryArch.get(repo, None)
+ self.loadFromClient(client, repo, archStr=archStr)
self._clients[repo] = client
self.finalize()
self._loaded = True
@loaded
- def loadFromUrl(self, url, basePath=''):
+ def loadFromUrl(self, url, basePath='', archStr=None):
"""
Walk the yum repository rooted at url/basePath and collect information
about rpms found.
@@ -87,10 +92,10 @@
log.info('loading repository data %s/%s' % (url, basePath))
client = repomd.Client(url + '/' + basePath)
- self.loadFromClient(client, basePath=basePath)
+ self.loadFromClient(client, basePath=basePath, archStr=archStr)
@loaded
- def loadFromClient(self, client, basePath=''):
+ def loadFromClient(self, client, basePath='', archStr=None):
"""
Walk the yum repository rooted at url/basePath and collect information
about rpms found.
@@ -126,7 +131,7 @@
if pkg.sourcerpm == '' or pkg.sourcerpm is None:
self._procSrc(pkg)
else:
- self._procBin(pkg)
+ self._procBin(pkg, archStr=archStr)
def _procSrc(self, package):
"""
@@ -152,7 +157,7 @@
self._srcMap[(package.name, package.epoch, package.version,
package.release, package.arch)] = package
- def _procBin(self, package):
+ def _procBin(self, package, archStr=None):
"""
Process binary rpms.
@param package: package object
@@ -200,6 +205,9 @@
self.locationMap[package.location] = package
+ if archStr:
+ self._repoMap.setdefault(package, set()).add(archStr)
+
def _excludeLocation(self, location):
"""
Method for filtering packages based on locaiton.
@@ -293,6 +301,28 @@
for loc in sorted(locs):
log.warn('\t%s' % loc)
+ if self._repoMap:
+ for binPkg, archStr in self._repoMap.iteritems():
+ # lookup the source for the binary package
+ srcPkg = self.binPkgMap[binPkg]
+
+ # get the conary version from the source
+ conaryVersion = srcPkg.getConaryVersion()
+
+ if binPkg.arch == 'x86_64':
+ arch = 'x86_64'
+ elif binPkg.arch == 'noarch':
+ arch = ''
+ elif (binPkg.arch.startswith('i') and
+ binPkg.arch.endswith('86') and
+ len(binPkg.arch) == 4):
+ arch = 'x86'
+ else:
+ raise RuntimeError
+
+ trvSpec = (binPkg.name, conaryVersion, arch)
+ self.useMap.setdefault(trvSpec, set()).update(archStr)
+
def loadFileLists(self, client, basePath):
"""
Parse file information.
From elliot at rpath.com Fri May 7 11:43:46 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:46 +0000
Subject: mirrorball: handle package names with or without the :source
Message-ID: <201005071543.o47Fhk2c009415@scc.eng.rpath.com>
changeset: 365d874e8285
user: Elliot Peele
date: Thu, 06 May 2010 17:02:35 -0400
handle package names with or without the :source
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -911,7 +911,10 @@
@type pkgname: string
"""
- versions = self._getVersionsByName('%s:source' % pkgname)
+ if not pkgname.endswith(':source'):
+ pkgname = '%s:source' % pkgname
+
+ versions = self._getVersionsByName(pkgname)
# FIXME: This is a hack to work around the fact that ubuntu has some
# shadows and packages that overlap on the label,
From elliot at rpath.com Fri May 7 11:43:46 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:46 +0000
Subject: mirrorball: take advantage of the new group model
Message-ID: <201005071543.o47Fhkna009470@scc.eng.rpath.com>
changeset: f171a457ea7b
user: Elliot Peele
date: Thu, 06 May 2010 17:03:21 -0400
take advantage of the new group model
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -58,7 +58,7 @@
self._parentGroup = groupmgr.GroupManager(self._cfg,
parentGroup=True)
- def _addPackages(self, pkgMap):
+ def _addPackages(self, pkgMap, group):
"""
Add pkgMap to group.
"""
@@ -82,7 +82,7 @@
log.info('adding %s=%s' % (name, version))
for f in flavors:
log.info('\t%s' % f)
- self._groupmgr.addPackage(name, version, flavors)
+ group.addPackage(name, version, flavors)
def _savePackages(self, pkgMap, fn=None):
"""
@@ -134,8 +134,10 @@
Handle initial import case.
"""
+ group = self._groupmgr.getGroup()
+
# Make sure this platform has not already been imported.
- if self._groupmgr.getErrataState() is not None:
+ if group.errataState is not None:
raise PlatformAlreadyImportedError
self._pkgSource.load()
@@ -144,17 +146,18 @@
pkgMap, failures = self._create(*args, toCreate=toCreate, **kwargs)
# Insert package map into group.
- self._addPackages(pkgMap)
+ self._addPackages(pkgMap, group)
# Save group changes if there are any failures.
if failures:
- self._groupmgr.save()
+ self._groupmgr.setGroup(group)
# Try to build the group if everything imported.
else:
- self._groupmgr.setErrataState('0')
- self._groupmgr.setVersion('0')
- self._groupmgr.build()
+ group.errataState = '0'
+ group.version = '0'
+ group.commit()
+ group.build()
return pkgMap, failures
@@ -166,18 +169,19 @@
# Load specific kwargs
restoreFile = kwargs.pop('restoreFile', None)
+ # Get current group
+ group = self._groupmgr.getGroup()
+
# Get current timestamp
- current = self._groupmgr.getErrataState()
+ current = group.errataState
if current is None:
raise PlatformNotImportedError
# Check to see if there is a binary version if the current group.
# This handles restarts where the group failed to build, but we don't
# want to rebuild all of the packages again.
- if not self._groupmgr.hasBinaryVersion():
- # grpmgr.build will make sure to refresh the group model and sync
- # up the standard group contents before building.
- self._groupmgr.build()
+ if not group.hasBinaryVersion():
+ group.build()
# Load package source.
self._pkgSource.load()
@@ -279,7 +283,7 @@
self._savePackages(pkgMap)
# Store current updateId.
- self._groupmgr.setErrataState(updateId)
+ group.errataState = updateId
# Remove any packages that are scheduled for removal.
# NOTE: This should always be done before adding packages so that
@@ -289,12 +293,12 @@
log.info('removing the following packages from the managed '
'group: %s' % ', '.join(requiredRemovals))
for pkg in requiredRemovals:
- self._groupmgr.remove(pkg)
+ group.removePackage(pkg)
if removeObsoleted:
log.info('removing any of obsoleted packages from the managed '
'group: %s' % ', '.join(removeObsoleted))
for pkg in removeObsoleted:
- self._groupmgr.remove(pkg, missingOk=True)
+ group.removePackage(pkg, missingOk=True)
# Handle the case of entire source being obsoleted, this causes all
# binaries from that source to be removed from the group model.
@@ -316,10 +320,10 @@
# remove all binary names from the group.
binNames = set([ x.name for x in nevraMap[nevra] ])
for name in binNames:
- self._groupmgr.remove(name)
+ group.removePackage(name)
# Make sure built troves are part of the group.
- self._addPackages(pkgMap)
+ self._addPackages(pkgMap, group)
# Get timestamp version.
version = self._errata.getBucketVersion(updateId)
@@ -328,8 +332,9 @@
# Build groups.
log.info('setting version %s' % version)
- self._groupmgr.setVersion(version)
- grpTrvMap = self._groupmgr.build()
+ group.version = version
+ newGroup = group.save()
+ grpTrvMap = group.build()
updateSet.update(pkgMap)
@@ -340,6 +345,8 @@
log.info('published update %s in %s seconds' % (advTime, totalTime))
count += 1
+ group = newGroup
+
log.info('update completed')
log.info('applied %s updates in %s seconds'
% (count, time.time() - startime))
@@ -353,7 +360,7 @@
"""
# Get current timestamp
- current = self._groupmgr.getErrataState()
+ current = self._groupmgr.latest.errataState
if current is None:
raise PlatformNotImportedError
@@ -400,10 +407,10 @@
# Make sure we have the expected number of flavors
if len(set(x[2] for x in toPromote)) != len(self._cfg.groupFlavors):
- slog.error('did not find expected number of flavors')
+ log.error('did not find expected number of flavors')
raise PromoteFlavorMismatchError(
- cfgFlavors=self._cfg.groupFlavors, troves=topPromote,
- version=topPromote[0][1])
+ cfgFlavors=self._cfg.groupFlavors, troves=toPromote,
+ version=toPromote[0][1])
# Find excepted promote packages.
srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages(
@@ -440,7 +447,7 @@
"""
# Get current timestamp
- current = self._groupmgr.getErrataState()
+ current = self._groupmgr.latest.errataState
if current is None:
raise PlatformNotImportedError
@@ -450,7 +457,7 @@
# Get latest errataState from the targetLabel so that we can fence group
# building based on the target label state.
targetGroup = groupmgr.GroupManager(self._cfg, targetGroup=True)
- targetErrataState = targetGroup.getErrataState()
+ targetErrataState = targetGroup.latest.errataState
log.info('starting errata group processing')
@@ -468,7 +475,7 @@
# Make sure the group representing the current updateId has been
# imorted and promoted to the production label.
version = self._errata.getBucketVersion(updateId)
- if not targetGroup.hasBinaryVersion(version=version):
+ if not targetGroup.hasBinaryVersion(sourceVersion=version):
raise TargetVersionNotFoundError(version=version,
updateId=updateId)
@@ -500,9 +507,9 @@
log.info('%s: found existing version, skipping' % advisory)
continue
- grp = mgr.newGroup(groupNames[advisory])
- grp.setVersion(version)
- grp.setErrataState(updateId)
+ grp = mgr.newGroup(groupNames[advisory]).getGroup()
+ grp.version = version
+ grp.errataState = updateId
log.info('%s: finding built packages' % advisory)
binTrvMap = \
From elliot at rpath.com Fri May 7 11:43:46 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:46 +0000
Subject: mirrorball: new biarch group model
Message-ID: <201005071543.o47Fhk5E009443@scc.eng.rpath.com>
changeset: a00a3f67fbb9
user: Elliot Peele
date: Thu, 06 May 2010 17:03:09 -0400
new biarch group model
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -122,6 +122,21 @@
return obsoleter, obsoleted
+class CfgNameFlavor(CfgString):
+ """
+ Class for parsing name/flavor pairs.
+ """
+
+ def parseString(self, val):
+ splt = val.split()
+ name = splt[0]
+ if len(splt) > 1:
+ flv = ' '.join(splt[1:])
+ else:
+ flv = ''
+ return name, flv
+
+
class CfgIntDict(CfgDict):
"""
Config class to represent dictionaries keyed by integers rather than
@@ -397,6 +412,12 @@
# latest that matches the current rpm version.
useOldVersion = (CfgIntDict(CfgList(CfgTroveSpec)), {})
+ # Add a package to a specific group
+ addPackage = (CfgDict(CfgDict(CfgList(CfgNameFlavor))), {})
+
+ # Remove a package from a specific group
+ removePackage = (CfgDict(CfgDict(CfgList(CfgNameFlavor))), {})
+
class UpdateBotConfig(cfg.SectionedConfigFile):
"""
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
new file mode 100644
--- /dev/null
+++ b/updatebot/groupmgr/group.py
@@ -0,0 +1,444 @@
+#
+# Copyright (c) 2009-2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Module for modeling the contents of a top level group.
+"""
+
+import logging
+
+from conary.deps import deps
+
+from updatebot.errors import FlavorCountMismatchError
+from updatebot.errors import UnknownBuildContextError
+from updatebot.errors import UnsupportedTroveFlavorError
+from updatebot.errors import UnhandledPackageAdditionError
+from updatebot.errors import UnknownPackageFoundInManagedGroupError
+
+from updatebot.groupmgr.model import GroupContentsModel
+
+log = logging.getLogger('updatebot.groupmgr')
+
+def require_write(func):
+ def wrapper(self, *args, **kwargs):
+ if not hasattr(self, '_readOnly'):
+ log.warn('instance has no attribute _readOnly, assuming writable')
+ readOnly = True
+ else:
+ readOnly = self._readOnly
+
+ if readOnly:
+ raise RuntimeError, 'This group is marked as readonly.'
+ else:
+ self._dirty = True
+ return func(self, *args, **kwargs)
+ return wrapper
+
+def enforce_readonly(attr):
+ def set(self, value):
+ if self._readOnly:
+ raise RuntimeError, 'This attribute is marked as read only.'
+ else:
+ self._dirty = True
+ setattr(self, attr, value)
+ def get(self):
+ return getattr(self, attr)
+ return property(get, set)
+
+
+class Group(object):
+ """
+ Class for managing group contents.
+ """
+
+ def __init__(self, cfg, useMap, sanityChecker, groupmgr, pkgGroupName,
+ groups, errataState, version, conaryVersion):
+ self._cfg = cfg
+ self._groups = groups
+ self._useMap = useMap
+ self._sanity = sanityChecker
+ self._mgr = groupmgr
+
+ self._pkgGroupName = pkgGroupName
+
+ self._errataState = errataState
+ self._version = version
+ self._conaryVersion = conaryVersion
+
+ self._dirty = False
+ self._committed = False
+ self._readOnly = False
+
+ errataState = enforce_readonly('_errataState')
+ version = enforce_readonly('_version')
+ conaryVersion = enforce_readonly('_conaryVersion')
+
+ def setReadOnly(self):
+ """
+ Make this group read only.
+ """
+
+ self._readOnly = True
+
+ @property
+ def dirty(self):
+ """
+ Check if an instance has been modified in some way.
+ """
+
+ return self._dirty
+
+ @property
+ def committed(self):
+ """
+ Check if an instances has been marked as committed.
+ """
+
+ return self._committed or not self._dirty
+
+ def setCommitted(self):
+ """
+ Mark this group as committed.
+ """
+
+ self._dirty = False
+ self._committed = True
+
+ def __hash__(self):
+ """
+ Make groups hashable.
+ """
+
+ return hash(self._conaryVersion)
+
+ def __cmp__(self, other):
+ """
+ Compare groups to other groups.
+ """
+
+ return cmp(self._conaryVersion, other._conaryVersion)
+
+ def __iter__(self):
+ """
+ Iterate over group model instances.
+ """
+
+ return self._groups.itervalues()
+
+ def iteritems(self):
+ """
+ Iterate over groupName, groupModel pairs.
+ """
+
+ return self._groups.iteritems()
+
+ ###
+ # Start of group manager interface
+ #
+ # Since we sever any relation between the group manager and group instance
+ # at commit time we should avoid the circular reference loop.
+ ###
+
+ def commit(self, copyToLatest=False):
+ """
+ Save this group to the repository.
+ """
+
+ return self._mgr.setGroup(self, copyToLatest=copyToLatest)
+
+ def build(self):
+ """
+ Build this group.
+ """
+
+ return self._mgr.buildGroup(self)
+
+ def hasBinaryVersion(self):
+ """
+ Check if this group has a binary version.
+ """
+
+ return self._mgr.hasBinaryVersion(sourceVersion=self.conaryVersion)
+
+ ###
+ # end group manager interface
+ ###
+
+ @require_write
+ def _add(self, groupName=None, *args, **kwargs):
+ """
+ Add a trove to the package group contents.
+ """
+
+ if not groupName:
+ groupName = self._pkgGroupName
+
+ # create package group model if it does not exist.
+ if groupName not in self._groups:
+ self._groups[groupName] = GroupContentsModel(groupName)
+ for key, value in self._cfg.groupContents.get(groupName, []):
+ value = value == 'True' and True or False
+ setattr(self._groups[groupName], key, value)
+
+ self._groups[groupName].add(*args, **kwargs)
+
+ @require_write
+ def addPackage(self, name, version, flavors, groupName=None):
+ """
+ Add a package to the model.
+ @param name: name of the package
+ @type name: str
+ @param version: conary version from string object
+ @type version: conary.versions.VersionFromString
+ @param flavors: list of flavors
+ @type flavors: [conary.deps.deps.Flavor, ...]
+ """
+
+ # Now that versions are actually used for something make sure they
+ # are always present.
+ assert version
+ assert len(flavors)
+ flavors = list(flavors)
+
+ # Remove all versions and flavors of this name before adding this
+ # package. This avoids flavor change issues by replacing all flavors.
+ if self.hasPackage(name):
+ self.remove(name)
+
+ plain = deps.parseFlavor('')
+ x86 = deps.parseFlavor('is: x86')
+ x86_64 = deps.parseFlavor('is: x86_64')
+ biarch = deps.parseFlavor('is: x86 x86_64')
+
+ # Count the flavors for later use.
+ flvMap = {}
+ flvCount = {x86: 0, x86_64: 0, plain: 0, biarch: 0}
+ for flavor in flavors:
+ # NOTE: Biarch must come first since a biarch flavored binary also
+ # saitisfies both x86 and x86_64.
+ if flavor.satisfies(biarch):
+ flvCount[biarch] += 1
+ flvMap[flavor] = 'x86_64'
+ elif flavor.satisfies(x86):
+ flvCount[x86] += 1
+ flvMap[flavor] = 'x86'
+ elif flavor.satisfies(x86_64):
+ flvCount[x86_64] += 1
+ flvMap[flavor] = 'x86_64'
+ elif flavor.freeze() == '':
+ flvCount[plain] += 1
+ flvMap[flavor] = None
+ else:
+ raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
+
+ def add():
+ upver = version.trailingRevision().version()
+ for flv in flavors:
+ if self._useMap:
+ for useStr in self._useMap[(name, upver, flvMap[flv])]:
+ self._add(name, version=version, flavor=flavor,
+ use=useStr, groupName=groupName)
+ else:
+ self._add(name, version=version, flavor=flavor,
+ use=flvMap[flv], groupName=groupName)
+
+ # If this package has one or two flavors and one of those flavors is
+ # x86, x86_64, biarch, or plain then handle it like a normal package
+ # without doing any more sanity checking.
+ total = 0
+ for flv, count in flvCount.iteritems():
+ if len(count) > 1:
+ break
+ total += count
+ else:
+ if total in (1, 2):
+ add()
+ return
+
+ # Handle all other odd flavor cases:
+ # 1. kernels
+ # 2. kernel modules
+ # 3. packages with specifically defined flavor sets
+
+ # Check if this package is configured to have multiple flavors.
+ # Get source trove name.
+ log.info('retrieving trove info for %s' % name)
+ srcTroveMap = self._helper._getSourceTroves((name, version, flavors[0]))
+ srcTroveName = srcTroveMap.keys()[0][0].split(':')[0]
+
+ # Check if this is package that we have specifically defined a build
+ # flavor for.
+ if srcTroveName in self._cfg.packageFlavors:
+ # separate packages into x86 and x86_64 by context name
+ # TODO: If we were really smart we would load the conary
+ # contexts and see what buildFlavors they contained.
+ flavorCtxCount = {x86: 0, x86_64: 0, biarch: 0}
+ for context, bldflv in self._cfg.packageFlavors[srcTroveName]:
+ if context in ('i386', 'i486', 'i586', 'i686', 'x86'):
+ flavorCtxCount[x86] += 1
+ elif context in ('x86_64', ):
+ flavorCtxCount[x86_64] += 1
+ elif context in ('biarch', ):
+ flavorCtxCount[biarch] += 1
+ else:
+ raise UnknownBuildContextError(name=name, flavor=context)
+
+ # Sanity check flavors to make sure we built all the flavors
+ # that we expected.
+ extra = set(flvMap) - set([ x86, x86_64, biarch ])
+ if extra:
+ raise UnsupportedTroveFlavorError(name=name, flavor=extra)
+
+ if (flvCount[x86] != flavorCtxCount[x86] or
+ flvCount[x86_64] != flavorCtxCount[x86_64] or
+ flvCount[biarch] != flavorCtxCount[biarch]):
+ raise FlavorCountMismatchError(name=name)
+
+ # Add packages to the group.
+ add()
+ return
+
+ # handle kernels.
+ if srcTroveName == 'kernel' or srcTroveName in self._cfg.kernelModules:
+ # add all x86ish flavors with use=x86 and all x86_64ish flavors
+ # with use=x86_64
+ for flavor in flavors:
+ if flvMap[flavor] in ('x86', 'x86_64'):
+ self._add(name, version=version, flavor=flavor,
+ use=flvMap[flavor], groupName=groupName)
+ else:
+ raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
+ return
+
+ # don't know how to deal with this package.
+ raise UnhandledPackageAdditionError(name=name)
+
+ @require_write
+ def removePackage(self, name, missingOk=False):
+ """
+ Remove a given trove from the package group contents.
+ """
+
+ if self._pkgGroupName not in self._groups:
+ return
+
+ return self._groups[self._pkgGroupName].remove(name,
+ missingOk=missingOk)
+
+ def hasPackage(self, name):
+ """
+ Check if a given package name is in the group.
+ """
+
+ return (self._pkgGroupName in self._groups and
+ name in self._groups[self._pkgGroupName])
+
+ __contains__ = hasPackage
+
+ @require_write
+ def modifyContents(self, additions=None, removals=None):
+ """
+ Modify the contents of the group model by adding and/or removing
+ packages.
+ @param additions: dictionary of group names to add packages to.
+ @type additions: dict(groupName=[(pkgName, frzPkgFlavor), ...])
+ @param removals: dictionary of group names to remove packages from.
+ @type additions: dict(groupName=[(pkgName, frzPkgFlavor), ...])
+ """
+
+ assert additions or removals
+
+ if additions is None:
+ additions = {}
+ if removals is None:
+ removals = {}
+
+ # 1. Apply removals before additions in case we are changing flavors
+ # 2. If flavor is specified, only modify that single flavor, otherwise
+ # following normal addition rules as stated in addPackage.
+
+ # Remove requested packages.
+ for groupName, pkgs in removals.iteritems():
+ group = self._groups[groupName]
+ for pkgName, pkgFlv in pkgs:
+ if pkgFlv:
+ group.removePackageFlavor(pkgName, pkgFlv)
+ else:
+ self.remove(pkgName)
+
+ # Add requested packages.
+ for groupName, pkgs in additions.iteritems():
+ flavoredPackages = {}
+ for pkgName, pkgFlv in pkgs:
+ # deffer packages with specifc flavors for later.
+ if pkgFlv:
+ flavoredPackages.setdefault(pkgName, set()).add(pkgFlv)
+
+ # handle packages where flavor is not specified
+ else:
+ # copy packages from the packages group.
+ for pkg in self._groups[self._pkgGroupName]:
+ if pkg.name == pkgName:
+ self._add(pkg.name, version=None,
+ flavor=pkg.flavor, use=pkg.use,
+ groupName=groupName)
+
+ # Add all specifically flavored packages.
+ for pkgName, flavors in flavoredPackages.iteritems():
+ self.addPackage(pkgName, None, flavors, groupName=groupName)
+
+ @require_write
+ def _copyVersions(self):
+ """
+ Copy versions from the packages group to the other managed groups.
+ """
+
+ # Get the versions of all packge names.
+ pkgs = dict([ (x.name, x) for x in self._groups[self._pkgGroupName] ])
+
+ for group in self:
+ # skip over package group since it is the version source.
+ if group.groupName == self._pkgGroupName:
+ continue
+
+ # for all other groups iterate over contents and set versions to
+ # match package group.
+ for pkg in group:
+ if pkg.name in pkgs:
+ pkg.version = pkgs[pkg.name].version
+ else:
+ raise UnknownPackageFoundInManagedGroupError(what=pkg.name)
+
+ def _sanityCheck(self):
+ """
+ Validate the group contents. This will raise an exception if any errors
+ are found.
+ """
+
+ self._sanity.check(self._groups, self.errataState)
+
+ @require_write
+ def finalize(self):
+ """
+ Handle any steps to prepair the group model before saving to disk.
+ """
+
+ # Copy versions from the package group to all other groups.
+ self._copyVersions()
+
+ # Check the sanity of all group models.
+ self._sanityCheck()
+
+ # Make as readonly.
+ self.setReadOnly()
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -17,49 +17,43 @@
"""
import logging
-import itertools
-from conary.deps import deps
+from conary import versions
-from updatebot.lib import util
from updatebot.build import Builder
-from updatebot.errors import FlavorCountMismatchError
-from updatebot.errors import UnknownBuildContextError
-from updatebot.errors import UnsupportedTroveFlavorError
-from updatebot.errors import UnhandledPackageAdditionError
from updatebot.errors import NotCommittingOutOfDateSourceError
-from updatebot.errors import UnknownPackageFoundInManagedGroupError
+from updatebot.groupmgr.group import Group
+from updatebot.groupmgr.group import require_write
from updatebot.groupmgr.helper import GroupHelper
from updatebot.groupmgr.sanity import GroupSanityChecker
-from updatebot.groupmgr.model import GroupContentsModel
log = logging.getLogger('updatebot.groupmgr')
-def checkout(func):
- def wrapper(self, *args, **kwargs):
- if not self._checkedout:
- self._checkout()
-
- return func(self, *args, **kwargs)
- return wrapper
-
-def commit(func):
- def wrapper(self, *args, **kwargs):
- if self._readonly:
- return
-
- if self._checkedout:
- self._commit()
-
- return func(self, *args, **kwargs)
- return wrapper
-
-
class GroupManager(object):
"""
- Manage group of all packages for a platform.
+ Class for managing groups.
+ @param cfg: updatebot configuration object
+ @type cfg: updatebot.config.UpdateBotConfig
+ @param parentGroup: optional argument, if set to True will setup manager to
+ interact with the parent platform group contents. This
+ is automatically set to readonly to avoid writing
+ anything to the parent platform.
+ @type parentGroup: boolean
+ @param targetGroup: optional argument, if set to True will setup manager to
+ interact with the target "production" branch for the
+ configured platform. This manager instance will
+ automatically be set to readonly to avoid modifying the
+ target label.
+ @type targetGroup: boolean
+ @param useMap: optional argument, A dictionary of package names mapped to a
+ list of use flags for the given package name. This is used to
+ determine use flags for packages that are added to group
+ contents models. If not specified all x86 packages will be
+ added to the x86 group and all x86_64 packages will be added
+ to the x86_64 group.
+ @type useMap: dict
"""
_helperClass = GroupHelper
@@ -74,360 +68,269 @@
assert not (parentGroup and targetGroup)
+ srcName = '%s:source' % self._cfg.topSourceGroup[0]
+ srcLabel = self._cfg.topSourceGroup[1]
+ labels = None
+
if targetGroup:
- srcName = '%s:source' % self._cfg.topSourceGroup[0]
- trvs = self._helper.findTrove(
- (srcName, self._cfg.targetLabel, None))
+ srcLabel = self._cfg.tagetLabel
+ elif parentGroup:
+ srcName = self._cfg.topParentSourceGroup[0]
+ srcLabel = self._cfg.topParentSourceGroup[1]
+ labels = self._cfg.platforSearchPath
- assert len(trvs)
+ self._sourceName = srcName
+ self._sourceLabel = srcLabel
+ self._searchLabels = labels
- self._sourceName = self._cfg.topSourceGroup[0]
- self._sourceVersion = trvs[0][1]
- self._readonly = True
- elif parentGroup:
- topGroup = list(self._cfg.topParentSourceGroup)
- topGroup[0] = '%s:source' % topGroup[0]
- trvs = self._helper.findTrove(tuple(topGroup),
- labels=self._cfg.platformSearchPath)
+ # FIXME: Should figure out a better way to handle package group.
+ self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName
- assert len(trvs)
+ assert self._sourceName.endswith(':source')
- self._sourceName = self._cfg.topParentSourceGroup
- self._sourceVersion = trvs[0][1]
+ self._readOnly = False
+ if targetGroup or parentGroup:
+ self._readOnly = True
- self._readonly = True
- else:
- self._sourceName = self._cfg.topSourceGroup[0]
- self._sourceVersion = None
- self._readonly = False
+ self._groupCache = {}
+ self._latest = None
- self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName
- self._stdGroupName = 'group-%s-standard' % self._cfg.platformName
-
- self._checkedout = False
- self._groups = {}
-
- def _checkout(self):
+ def setReadOnly(self):
"""
- Get current group state from the repository.
+ Mark the group manager as read only. You will not be able to modify or
+ build any groups requested through this manager instance.
"""
- self._groups = self._helper.getModel(self._sourceName,
- version=self._sourceVersion)
- self._checkedout = True
+ self._readOnly = True
- def _commit(self, copyToLatest=False):
+ @property
+ def latest(self):
+ if self._latest is None:
+ self._latest = self.getGroup()
+ return self._latest
+
+ def _findVersion(self, version=None, allVersions=False):
"""
- Commit current changes to the group.
+ Find the conary version(s) that match the specified version.
+ @param version: The conary source version to load or a string
+ representation of the version that will be looked up in
+ the repository. The latest source version matching the
+ specified version will be used. If no version is
+ specified, the latest version will be retreived.
+ @type version: conary.versions.VersionFromString, str, or None
+ @param allVersions: By default this method only finds the latest
+ versions. If you would like to find all versions
+ that match the specified version set this option
+ to True.
+ @type allVersions: boolean
+ @return conary version(s) found
+ @rtype conary.versions.VersionFromString
+ @rtype list(conary.versions.VersionFromString)
"""
- if self._sourceVersion and not copyToLatest:
+ # Make sure version is of an acceptable type.
+ assert (isinstance(version, (str, versions.VersionSequence)) or
+ version is None)
+
+ # Find the latest version so that we can check if the version we are
+ # retreiving from the repository is the latest.
+ trvs = self._helper.findTrove((self._sourceName, version, None),
+ labels=self._searchLabels,
+ getLeaves=not allVersions)
+
+ if allVersions:
+ return [ x[1] for x in trvs ]
+ elif len(trvs):
+ return trvs[0][1]
+ else:
+ return None
+
+ def getGroup(self, version=None):
+ """
+ Retrieve a group model from the repository.
+ @param version: The conary source version to load or a string
+ representation of the version that will be looked up in
+ the repository. The latest source version matching the
+ specified version will be used. If no version is
+ specified, the latest version will be retreived.
+ @type version: conary.versions.VersionFromString, str, or None
+ @return group model object
+ @rtype updatebot.groupmgr.group.Group
+ """
+
+ # Make sure version is of an acceptable type.
+ assert (isinstance(version, (str, versions.VersionSequence)) or
+ version is None)
+
+ # Find the latest version so that we can check if the version we are
+ # retreiving from the repository is the latest.
+ latest = self._findVersion()
+
+ # Find the conary version object if we don't have one yet.
+ if version is None:
+ conaryVersion = latest
+ elif isinstance(version, str):
+ conaryVersion = self._findVersion(version=version)
+ # If the user requested a version that doesn't exist return None.
+ if conaryVersion is None:
+ return None
+ else:
+ conaryVersion = version
+
+ # Make sure this is a source version.
+ assert conaryVersion is None or conaryVersion.isSourceVersion()
+
+ # Check the cache for this version first.
+ if conaryVersion in self._groupCache:
+ return self._groupCache[conaryVersion]
+
+ # Get model information from the source.
+ groups = self._helper.getModel(self._sourceName, version=conaryVersion)
+ errataState = self._helper.getErrataState(self._sourceName,
+ version=conaryVersion)
+ upstreamVersion = self._helper.getVersion(self._sourceName,
+ version=conaryVersion)
+
+ # Instantiate a group instance.
+ group = Group(self._cfg, self._useMap, self._sanity, self,
+ self._pkgGroupName, groups, errataState, upstreamVersion,
+ conaryVersion)
+
+ # If this was the latest version, store it as "latest"
+ if conaryVersion == latest:
+ self._latest = group
+
+ # Cache reference to group.
+ self._groupCache[conaryVersion] = group
+
+ return group
+
+ @require_write
+ def setGroup(self, group, copyToLatest=False):
+ """
+ Freeze group model and commit state to the repository. Note that this
+ puts the group model object into read only mode.
+ @param group: group object to commit
+ @type group: updatebot.groupmgr.group.Group
+ @param copyToLatest: Optional parameter to enable committing a model
+ that was not gerenated from the latest version.
+ @type copyToLatest: boolean
+ @return committed group model object
+ @rtype updatebot.groupmgr.group.Group
+ """
+
+ # Make sure model hasn't already been committed.
+ assert not group.committed
+
+ # Don't attempt to commit out of date sources unless requested to do so.
+ if (group.conaryVersion != self.latest.conaryVersion and
+ not copyToLatest):
+
log.error('refusing to commit out of date source')
raise NotCommittingOutOfDateSourceError
# Copy forward data when we are fixing up old group versions so that
# this is the latest source.
if copyToLatest:
- log.info('copying information to latest version')
- # Get data from the old versoin
- version = self._helper.getVersion(self._sourceName,
- version=self._sourceVersion)
- errataState = self._helper.getErrataState(self._sourceName,
- version=self._sourceVersion)
- groups = self._groups
+ log.info('committing %s model as latest' % group.conaryVersion)
+ log.info('version: %s' % group.version)
+ log.info('errataState: %s' % group.errataState)
- log.info('version: %s' % version)
- log.info('errataState: %s' % errataState)
+ conaryVersion = self.latest.conaryVersion
- # Set version to None to get the latest source.
- self._sourceVersion = None
+ # Default to modifying the source verison of the group model.
+ else:
+ conaryVersion = group.conaryVersion
- # Checkout latest source.
- self._checkout()
+ # Finalizing the group performs any sanity checking and marks it as
+ # readonly.
+ group.finalize()
- # Set back to old data
- self.setVersion(version)
- self.setErrataState(errataState)
- self._groups = groups
+ # set version
+ self._helper.setVersion(self._sourceName, group.version,
+ version=conaryVersion)
- # sync versions from the package group to the other managed groups.
- self._copyVersions()
-
- # validate group contents.
- self._sanity.check(self._groups, self.getErrataState())
+ # set errata state
+ self._helper.setErrataState(self._sourceName, group.errataState,
+ version=conaryVersion)
# write out the model data
- self._helper.setModel(self._sourceName, self._groups)
+ self._helper.setModel(self._sourceName, group, version=conaryVersion)
# commit to the repository
- version = self._helper.commit(self._sourceName,
- version=self._sourceVersion,
- commitMessage='automated commit')
- if self._sourceVersion:
- self._sourceVersion = version
- self._checkedout = False
- return version
+ newVersion = self._helper.commit(self._sourceName,
+ version=conaryVersion,
+ commitMessage=self._cfg.commitMessage)
- save = _commit
+ # Remove the cached version of the already committed group.
+ self._groupCache.pop(group.conaryVersion)
- def hasBinaryVersion(self, version=None):
+ # Mark group as committed.
+ group.setCommitted()
+
+ # Get the model for the source version that we just committed.
+ return self.getGroup(sourceVersion=newVersion)
+
+ @require_write
+ def buildGroup(self, group):
"""
- Check if there is a binary version for the current source version.
+ Build the binary version of a given group.
+ @param group: group model to build.
+ @type group: updatebot.groupmgr.group.Group
+ @return mapping of built troves.
+ @rtype dict(sourceTrv=[binTrv, ..])
"""
- if not version:
- verison = self._sourceVersion
+ # Make sure this group has been committed to the repository before
+ # attempting to build it.
+ assert group.committed
- # Search the label from the source version.
- if self._sourceVersion:
- labels = (self._sourceVersion.trailingLabel(), )
- else:
- labels = (self._helper.getConaryConfig().buildLabel, )
+ # Make sure this group is not marked as dirty. This means that things
+ # have been changed about the group since it was committed.
+ assert not group.dirty
- # Get a mapping of all source version to binary versions for all
- # existing binary versions.
- srcVersions = dict([ (x[1].getSourceVersion(), x[1])
- for x in self._helper.findTrove(
- (self._sourceName, None, None),
- getLeaves=False,
- labels=labels,
- )
- ])
+ # Create a build job and build groups using cvc.
+ job = ((self._sourceName, group.conaryVersion, None), )
+ results = self._builder.cvc.cook(job)
+ return results
- # Get the version of the specified source, usually the latest
- # source version.
- srcVersion = self._helper.findTrove(
- ('%s:source' % self._sourceName, version, None),
- labels=labels)
+ def getSourceVersions(self):
+ """
+ Retrieve a list of all available group source versions.
+ @return list of conary versions
+ @rtype list(conary.versions.VersionFromString, ...)
+ """
- if not srcVersion:
+ return self._findVersion(allVersions=True)
+
+ def hasBinaryVersion(self, sourceVersion=None):
+ """
+ Check if there is a binary for a given source version.
+ @param sourceVersion: If specified check for a binary version for the
+ given source verison.
+ @type sourceVersion: conary.versions.VersionFromString
+ @return True if the binary version exists, otherwise False.
+ @rtype boolean
+ """
+
+ # Default to the latest source version if none is specified.
+ if sourceVersion is None:
+ sourceVersion = self.latest.conaryVersion
+
+ # Resolve version to a conary version.
+ sourceVersion = self._findVersion(version=sourceVersion)
+
+ # If the version doesn't exist in the repository return False.
+ if sourceVersion is None:
return False
- # Check to see if the latest source version is in the map of
- # binary versions.
- return srcVersion[0][1] in srcVersions
+ # Make sure it is really a source version.
+ assert sourceVersion.isSourceVersion()
- @commit
- def getBuildJob(self):
- """
- Get the list of trove specs to submit to the build system.
- """
+ # Get the list of binaries for this source from the repository.
+ # FIXME: This should not call into the conary client itself, instead
+ # there should be a call in the conary helper.
+ trvs = self._helper._client.getTrovesBySource(self._sourceName,
+ sourceVersion)
- return ((self._sourceName, self._sourceVersion, None), )
-
- @checkout
- @commit
- def build(self):
- """
- Build all configured flavors of the group.
- """
-
- groupTroves = self.getBuildJob()
- built = self._builder.cvc.cook(groupTroves)
- return built
-
- @checkout
- def add(self, *args, **kwargs):
- """
- Add a trove to the package group contents.
- """
-
- # create package group model if it does not exist.
- if self._pkgGroupName not in self._groups:
- model = GroupContentsModel(self._pkgGroupName)
- self._groups[self._pkgGroupName] = model
-
- self._groups[self._pkgGroupName].add(*args, **kwargs)
-
- @checkout
- def remove(self, name, missingOk=False):
- """
- Remove a given trove from the package group contents.
- """
-
- if self._pkgGroupName not in self._groups:
- return
-
- return self._groups[self._pkgGroupName].remove(name,
- missingOk=missingOk)
-
- @checkout
- def hasPackage(self, name):
- """
- Check if a given package name is in the group.
- """
-
- return (self._pkgGroupName in self._groups and
- name in self._groups[self._pkgGroupName])
-
- def addPackage(self, name, version, flavors):
- """
- Add a package to the model.
- @param name: name of the package
- @type name: str
- @param version: conary version from string object
- @type version: conary.versions.VersionFromString
- @param flavors: list of flavors
- @type flavors: [conary.deps.deps.Flavor, ...]
- """
-
- # Now that versions are actually used for something make sure they
- # are always present.
- assert version
- assert len(flavors)
- flavors = list(flavors)
-
- # Remove all versions and flavors of this name before adding this
- # package. This avoids flavor change issues by replacing all flavors.
- if self.hasPackage(name):
- self.remove(name)
-
- plain = deps.parseFlavor('')
- x86 = deps.parseFlavor('is: x86')
- x86_64 = deps.parseFlavor('is: x86_64')
-
- # Count the flavors for later use.
- flvMap = {}
- flvCount = {x86: 0, x86_64: 0, plain: 0}
- for flavor in flavors:
- if flavor.satisfies(x86):
- flvCount[x86] += 1
- flvMap[flavor] = 'x86'
- elif flavor.satisfies(x86_64):
- flvCount[x86_64] += 1
- flvMap[flavor] = 'x86_64'
- elif flavor.freeze() == '':
- flvCount[plain] += 1
- flvMap[flavor] = None
- else:
- raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
-
- def add():
- upver = version.trailingRevision().version()
- for flv in flavors:
- if self._useMap:
- for useStr in self._useMap[(name, upver, flvMap[flv])]:
- self.add(name, version=version, flavor=flavor,
- use=useStr)
- else:
- self.add(name, version=version, flavor=flavor,
- use=flvMap[flv])
-
- # If this package has one or two flavors and one of those flavors is
- # x86, x86_64, or plain then handle it like a normal package without
- # doing any more sanity checking.
- if sum([ x for x in flvCount.itervalues() if x in (0, 1) ]) in (1, 2):
- add()
- return
-
- # Handle all other odd flavor cases:
- # 1. kernels
- # 2. kernel modules
- # 3. packages with specifically defined flavor sets
-
- # Check if this package is configured to have multiple flavors.
- # Get source trove name.
- log.info('retrieving trove info for %s' % name)
- srcTroveMap = self._helper._getSourceTroves((name, version, flavors[0]))
- srcTroveName = srcTroveMap.keys()[0][0].split(':')[0]
-
- # Check if this is package that we have specifically defined a build
- # flavor for.
- if srcTroveName in self._cfg.packageFlavors:
- # separate packages into x86 and x86_64 by context name
- # TODO: If we were really smart we would load the conary
- # contexts and see what buildFlavors they contained.
- flavorCtxCount = {x86: 0, x86_64: 0}
- for context, bldflv in self._cfg.packageFlavors[srcTroveName]:
- if context in ('i386', 'i486', 'i586', 'i686', 'x86'):
- flavorCtxCount[x86] += 1
- elif context in ('x86_64', ):
- flavorCtxCount[x86_64] += 1
- else:
- raise UnknownBuildContextError(name=name, flavor=context)
-
- # Sanity check flavors to make sure we built all the flavors
- # that we expected.
- extra = set(flvMap) - set([ x86, x86_64 ])
- if extra:
- raise UnsupportedTroveFlavorError(name=name, flavor=extra)
-
- if (flvCount[x86] != flavorCtxCount[x86] or
- flvCount[x86_64] != flavorCtxCount[x86_64]):
- raise FlavorCountMismatchError(name=name)
-
- # Add packages to the group.
- add()
- return
-
- # handle kernels.
- if srcTroveName == 'kernel' or srcTroveName in self._cfg.kernelModules:
- # add all x86ish flavors with use=x86 and all x86_64ish flavors
- # with use=x86_64
- for flavor in flavors:
- if flvMap[flavor] in ('x86', 'x86_64'):
- self.add(name, version=version, flavor=flavor,
- use=flvMap[flavor])
- else:
- raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
- return
-
- # don't know how to deal with this package.
- raise UnhandledPackageAdditionError(name=name)
-
- @checkout
- def setVersion(self, version):
- """
- Set the version of the managed group.
- """
-
- self._helper.setVersion(self._sourceName, version)
-
- @checkout
- def setErrataState(self, state):
- """
- Set errata state info for the managed platform.
- """
-
- self._helper.setErrataState(self._sourceName, state)
-
- @checkout
- def getErrataState(self, version=None):
- """
- Get the errata state info.
- """
-
- if version is None:
- version = self._sourceVersion
-
- return self._helper.getErrataState(self._sourceName,
- version=version)
-
- def getVersions(self, pkgSet):
- """
- Get the set of versions that are represented by the given set of
- packages from the version factory.
- """
-
- return set()
-
- def _copyVersions(self):
- """
- Copy versions from the packages group to the other managed groups.
- """
-
- pkgs = dict([ (x[1].name, x[1]) for x in
- self._groups[self._pkgGroupName].iteritems() ])
-
- for group in self._groups.itervalues():
- # skip over package group since it is the version source.
- if group.groupName == self._pkgGroupName:
- continue
-
- # for all other groups iterate over contents and set versions to
- # match package group.
- for k, pkg in group.iteritems():
- if pkg.name in pkgs:
- pkg.version = pkgs[pkg.name].version
- else:
- raise UnknownPackageFoundInManagedGroupError(what=pkg.name)
+ return bool(len(trvs))
diff --git a/updatebot/groupmgr/model.py b/updatebot/groupmgr/model.py
--- a/updatebot/groupmgr/model.py
+++ b/updatebot/groupmgr/model.py
@@ -93,6 +93,13 @@
return self._data.iteritems()
+ def __iter__(self):
+ """
+ Iterate over the packages of this group.
+ """
+
+ return self._data.itervalues()
+
def add(self, *args, **kwargs):
"""
Add an data element.
@@ -144,3 +151,20 @@
# figure out file name based on group name
name = ''.join([ x.capitalize() for x in self.groupName.split('-') ])
self.fileName = name[0].lower() + name[1:] + '.xml'
+
+ def removePackageFlavor(self, name, frzFlavor):
+ """
+ Remove a specific flavor from the group.
+ """
+
+ removed = []
+ for pkg in self._nameMap[name]:
+ if pkg.flavor == frzFlavor:
+ self._data.pop(pkg.key)
+ removed.append(pkg)
+
+ for pkg in removed:
+ self._nameMap[name].remove(pkg)
+
+ if not self._nameMap[name]:
+ self._nameMap.pop(name)
diff --git a/updatebot/groupmgr/single.py b/updatebot/groupmgr/single.py
--- a/updatebot/groupmgr/single.py
+++ b/updatebot/groupmgr/single.py
@@ -39,8 +39,8 @@
def __init__(self, name, *args, **kwargs):
GroupManager.__init__(self, *args, **kwargs)
- self._sourceName = 'group-%s' % name
- self._pkgGroupName = self._sourceName
+ self._sourceName = 'group-%s:source' % name
+ self._pkgGroupName = 'group-%s' % name
class SingleGroupManagerSet(object):
@@ -74,7 +74,7 @@
pkgMap = {}
for group in self._groups.itervalues():
- pkgMap.update(group.build())
+ pkgMap.update(group.buildGroup(group.latest))
return pkgMap
diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py
--- a/updatebot/lib/xobjects.py
+++ b/updatebot/lib/xobjects.py
@@ -207,7 +207,7 @@
@property
def key(self):
- return (self.name, self.flavor)
+ return (self.name, self.flavor, self.use)
class XPackageData(XItemList):
From elliot at rpath.com Fri May 7 11:43:46 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:46 +0000
Subject: mirrorball: add script for generating group configs based on
repository history
Message-ID: <201005071543.o47Fhk7T009500@scc.eng.rpath.com>
changeset: e56cc8a2e16d
user: Elliot Peele
date: Thu, 06 May 2010 17:04:26 -0400
add script for generating group configs based on repository history
diff --git a/scripts/gengroupmodel b/scripts/gengroupmodel
new file mode 100755
--- /dev/null
+++ b/scripts/gengroupmodel
@@ -0,0 +1,123 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+import os
+import sys
+
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
+
+import rmake
+import conary
+import updatebot
+
+print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
+print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
+print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
+
+from conary.lib import util
+sys.excepthook = util.genExcepthook()
+
+import logging
+
+from updatebot import groupmgr
+from updatebot import OrderedBot
+
+log = logging.getLogger('tmplogger')
+
+class Bot(OrderedBot):
+ def generateGroupModel(self):
+ """
+ Generate config for standard group contents based on repository history.
+ """
+
+ # load package source
+ self._pkgSource.load()
+
+ mgr = groupmgr.GroupManager(self._cfg, useMap=self._pkgSource.useMap)
+ mgr.setReadOnly()
+
+ lastAvailableUpdate = mgr.latest.errataState
+
+ current = set()
+ changes = []
+ for updateId, updates in self._errata.iterByIssueDate(current=-1):
+ if updateId > lastAvailableUpdate:
+ log.info('%s not yet imported' % updateId)
+ continue
+
+ if updateId == 0:
+ version = '5.0'
+ else:
+ version = self._errata.getBucketVersion(updateId)
+
+ grp = mgr.getGroup(version=version)
+ stdGroupName, stdModel = [ (x, y) for x, y in grp.iteritems()
+ if x != grp._pkgGroupName ][0]
+
+ latest = set()
+ for (name, flavor, use), pkg in stdModel.iteritems():
+ latest.add((name, flavor))
+
+ added = latest - current
+ removed = current - latest
+
+ for name, flavor in added:
+ if not flavor:
+ flavor = ''
+ change = 'addPackage %s %s %s %s' % (updateId, stdGroupName, name, flavor)
+ changes.append(change)
+ log.info(change)
+
+ for name, flavor in removed:
+ if not flavor:
+ flavor = ''
+ change = 'removePackage %s %s %s %s' % (updateId, stdGroupName, name, flavor)
+ changes.append(change)
+ log.info(change)
+
+ current = latest
+
+ return changes
+
+
+if __name__ == '__main__':
+ import rhnmirror
+
+ from updatebot import config
+ from updatebot import log as logSetup
+
+ logSetup.addRootLogger()
+
+ log = logging.getLogger('grouprebuild')
+
+ cfg = config.UpdateBotConfig()
+ cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+
+ mcfg = rhnmirror.MirrorConfig()
+ mcfg.read(cfg.configPath + '/erratarc')
+
+ errata = rhnmirror.Errata(mcfg)
+ errata.fetch()
+
+ bot = Bot(cfg, errata)
+ changes = bot.generateGroupModel()
+
+ print '\n'.join(changes)
+
+ import epdb; epdb.st()
From elliot at rpath.com Fri May 7 11:43:47 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:47 +0000
Subject: mirrorball: begining of script to rebuild all groups on a label
Message-ID: <201005071543.o47FhlN1009528@scc.eng.rpath.com>
changeset: 6d4e31e6318f
user: Elliot Peele
date: Thu, 06 May 2010 17:04:43 -0400
begining of script to rebuild all groups on a label
diff --git a/scripts/rebuildgroups b/scripts/rebuildgroups
new file mode 100644
--- /dev/null
+++ b/scripts/rebuildgroups
@@ -0,0 +1,90 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+import logging
+
+from updatebot import groupmgr
+from updatebot import OrderedBot
+
+log = logging.getLogger('tmplogger')
+
+class Bot(OrderedBot):
+ def rebuildgroups(self):
+ """
+ Rebuild all groups on the devel label. This requires rewriting the group
+ model to point at the target label and readding content.
+ """
+
+ # load package source
+ self._pkgSource.load()
+
+ for updateId, updates in self._errata.iterByIssueDate(current=-1):
+ mgr = groupmgr.GroupManager(self._cfg,
+ useMap=self._pkgSource.useMap)
+
+
+
+ # Find any version exceptions for this update.
+ multiVersionExceptions = dict([
+ (x[0], x[1]) for x in itertools.chain(
+ self._updater.getTargetVersions(itertools.chain(
+ *self._getOldVersionExceptions(updateId).itervalues()
+ ))[0]
+ )
+ ])
+
+if __name__ == '__main__':
+ import os
+ import sys
+
+ mirrorballDir = os.path.abspath('../')
+ sys.path.insert(0, mirrorballDir)
+
+ if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
+
+ import rmake
+ import conary
+ import updatebot
+
+ print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
+ print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
+ print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
+
+ from conary.lib import util
+ sys.excepthook = util.genExcepthook()
+
+ import rhnmirror
+
+ from updatebot import config
+ from updatebot import log as logSetup
+
+ logSetup.addRootLogger()
+
+ log = logging.getLogger('grouprebuild')
+
+ cfg = config.UpdateBotConfig()
+ cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+
+ mcfg = rhnmirror.MirrorConfig()
+ mcfg.read(confDir + '/erratarc')
+
+ errata = rhnmirror.Errata(mcfg)
+ errata.fetch()
+
+ bot = Bot(cfg, errata)
+ bot.rebuildgroups()
+
+ import epdb; epdb.st()
From elliot at rpath.com Fri May 7 11:43:47 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:47 +0000
Subject: mirrorball: branch merge
Message-ID: <201005071543.o47Fhlpl009555@scc.eng.rpath.com>
changeset: a9a454bc86c2
user: Elliot Peele
date: Thu, 06 May 2010 17:55:34 -0400
branch merge
diff --git a/errata/__init__.py b/errata/__init__.py
new file mode 100644
--- /dev/null
+++ b/errata/__init__.py
@@ -0,0 +1,13 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
diff --git a/errata/common.py b/errata/common.py
new file mode 100644
--- /dev/null
+++ b/errata/common.py
@@ -0,0 +1,125 @@
+#
+# Copyright (c) 2009 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+This module is here as an example of the data model and interfaces that
+mirrorball is looking for when importing and updating platforms in advisory
+order. These are not meant as superclasses for any sort of implementation. The
+variables and methods that are defined below must be available.
+"""
+
+def reqfetch(func):
+ """
+ Decorator to make sure manager data is loaded before a method is called.
+ """
+
+ def wrap(self, *args, **kwargs):
+ if not self._fetched:
+ self.fetch()
+ return func(self, *args, **kwargs)
+ return wrap
+
+
+class Package(object):
+ """
+ Class to represent a package.
+
+ @param channels: List of channel objects that this package can be found in.
+ @type channels: list(Repository, ...)
+ """
+
+ def __init__(self, channels):
+ for ch in channels:
+ assert isinstance(ch, Channel)
+
+ self.channels = channels
+
+ def getNevra(self):
+ """
+ Returns a tuple of (name, epoch, version, release, arch) for
+ this package.
+ """
+
+ raise NotImplemetedError
+
+
+class Channel(object):
+ """
+ Class to represent a repository.
+
+ @param label: Unique key for the name of a repository.
+ @type label: str
+ """
+
+ def __init__(self, label):
+ self.label = label
+
+
+class Advisory(object):
+ """
+ Class to represent an errata or advisory.
+
+ @param issue_date: Date the advisory was issued in the following format.
+ (format characters are defined in the python time
+ module). '%Y-%m-%d %H:%M:%S'
+ @type issue_date: str
+ @param packages: List of package objects.
+ @type packages: list(Package, ...)
+ @param advisory: Unique key for the advisory
+ @type advisory: str
+ @param synopsis: Brief description of the advisory.
+ @type synopsis: str
+ """
+
+ def __init__(self, advisory, synopsis, issue_date, packages):
+ self.advisory = advisory
+ self.synposis = synopsis
+ self.issue_date = issue_date
+
+ for pkg in packages:
+ assert isinstance(pkg, Package)
+
+ self.packages = packages
+
+
+class AdvisoryManager(object):
+ """
+ Class to provide an interface for accessing advisory information for a
+ platform that can then be matched up to a package source.
+ """
+
+ def getRepositories(self):
+ """
+ Returns a list of repository labels that have been fetched.
+ """
+
+ raise NotImplementedError
+
+ def iterByIssueDate(self):
+ """
+ Yields Errata objects by the issue date of the errata.
+ """
+
+ raise NotImplementedError
+
+ def fetch(self):
+ """
+ Retrieve all required advisory data.
+
+ This is probably going to cache any data, probably in a database, that
+ is being fetched from the internet somewhere so that we don't cause
+ excesive load for anyone's servers.
+ """
+
+ raise NotImplementedError
diff --git a/errata/sles.py b/errata/sles.py
new file mode 100644
--- /dev/null
+++ b/errata/sles.py
@@ -0,0 +1,94 @@
+#
+# Copyright (c) 2010 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Generate update information based on the patch detail in SuSE repositories.
+"""
+
+import logging
+
+from errata import common
+
+log = logging.getLogger('errata')
+
+class Package(common.Package):
+ """
+ Class to represent a package.
+ """
+
+ def getNevra(self):
+ """
+ Returns a tuple of (name, epoch, version, release, arch) for
+ this package.
+ """
+
+class Repository(common.Repository):
+ """
+ Class to represent a repository.
+ """
+
+
+class Advisory(common.Advisory):
+ """
+ Class to represent an errata or advisory.
+ """
+
+
+class AdvisoryManager(common.AdvisoryManager):
+ def __init__(self, pkgSource):
+ self._pkgSource = pkgSource
+
+ slef._fetched = False
+ self._patches = set()
+
+ @common.reqfetch
+ def getRepositories(self):
+ """
+ Returns a list of repository labels that have been fetched.
+ """
+
+ @common.reqfetch
+ def iterByIssueDate(self):
+ """
+ Yields Errata objects by the issue date of the errata.
+ """
+
+ return []
+
+ def fetch(self):
+ """
+ Retrieve all required advisory data.
+
+ This is probably going to cache any data, probably in a database, that
+ is being fetched from the internet somewhere so that we don't cause
+ excesive load for anyone's servers.
+ """
+
+ self._fetched = True
+
+ def _fetchPatches(self):
+ """
+ Fetch all patch data from the package source.
+ """
+
+ # make sure the pkg source is loaded.
+ self._pkgSource.load()
+
+ # now get the patch data
+ patches = set()
+ for path, client in self._pkgSource.getClients().iteritems():
+ log.info('loading patches for %s' % path)
+ patches.update(set(client.getPatchDetail()))
+
+ return patches
diff --git a/repomd/packagexml.py b/repomd/packagexml.py
--- a/repomd/packagexml.py
+++ b/repomd/packagexml.py
@@ -180,6 +180,14 @@
ver = "_".join(nameVerRelease.split("-")[-2:])
return ver
+ def getFileName(self):
+ """
+ Returns the expected package file name.
+ """
+
+ return '%s-%s-%s.%s.rpm' % (self.name, self.version,
+ self.release, self.arch)
+
class _RpmEntry(SlotNode):
"""
diff --git a/rpmutils/header.py b/rpmutils/header.py
--- a/rpmutils/header.py
+++ b/rpmutils/header.py
@@ -81,7 +81,6 @@
# Have to read into the file a bit to get to the begining of the header
# that we care about.
- fh.read(96)
rpmhelper.RpmHeader(fh)
header = rpmhelper.RpmHeader(fh)
diff --git a/scripts/buildpackages b/scripts/buildpackages
--- a/scripts/buildpackages
+++ b/scripts/buildpackages
@@ -7,14 +7,28 @@
Script for cooking packages with updatebot config.
"""
+import os
+
from header import *
+from updatebot import conaryhelper
if len(sys.argv) < 3:
usage()
+helper = conaryhelper.ConaryHelper(cfg)
+
+def validateInput(input):
+ for pkgName in input:
+ manifest = helper.getManifest(pkgName)
+ paths = [ os.path.basename(x) for x in manifest ]
+ for context, fltr in cfg.archContexts:
+ if fltr and [ x for x in paths if fltr[1].match(x) ]:
+ raise RuntimeError, 'Found package that may not be built'
+ return input
+
trvs = set()
label = cfg.topSourceGroup[1]
-for pkg in sys.argv[2:]:
+for pkg in validateInput(sys.argv[2:]):
trvs.add((pkg, label, None))
trvMap = builder.build(trvs)
diff --git a/scripts/grouptoxml.py b/scripts/creategroup.py
copy from scripts/grouptoxml.py
copy to scripts/creategroup.py
--- a/scripts/grouptoxml.py
+++ b/scripts/creategroup.py
@@ -10,395 +10,554 @@
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
+# full details.
#
-"""
-Script to used to generate xml model for standard rhel groups.
-"""
-
-rhel4corePackages = (
- ('kernel', 'kernel.smp,!kernel.largesmp,!kernel.hugemem,!xen,!domU,!dom0'),
- 'acl',
- 'ash',
- 'attr',
- 'audit',
- 'audit-libs',
- 'authconfig',
- 'basesystem',
- 'bash',
- 'beecrypt',
- 'bzip2',
- 'bzip2-libs',
- 'checkpolicy',
- 'chkconfig',
- 'coreutils',
- 'cpio',
- 'cracklib',
- 'cracklib-dicts',
- 'crontabs',
- 'cyrus-sasl',
- 'cyrus-sasl-md5',
- 'db4',
- 'dbus',
- 'dbus-glib',
- 'device-mapper',
- 'dhclient',
- 'diffutils',
- 'dmraid',
- 'e2fsprogs',
- 'ed',
- 'elfutils-libelf',
- 'ethtool',
- 'expat',
- 'file',
- 'filesystem',
- 'findutils',
- 'fontconfig',
- 'freetype',
- 'gawk',
- 'gdbm',
- 'glib2',
- 'glibc',
- 'glibc-common',
- 'gmp',
- 'grep',
- 'grub',
- 'gzip',
- 'hal',
- 'hdparm',
- 'hesiod',
- 'hotplug',
- 'hwdata',
- 'indexhtml',
- 'info',
- 'initscripts',
- 'iproute',
- 'iptables',
- 'iputils',
- 'kbd',
- 'krb5-libs',
- 'kudzu',
- 'less',
- 'libacl',
- 'libattr',
- 'libcap',
- 'libgcc',
- 'libgcrypt',
- 'libgpg-error',
- 'libselinux',
- 'libsepol',
- 'libstdc++',
- 'libtermcap',
- 'libuser',
- 'libxml2',
- 'libxml2-python',
- 'libxslt',
- 'libxslt-python',
- 'logrotate',
- 'lvm2',
- 'MAKEDEV',
- 'mdadm',
- 'mingetty',
- 'mkinitrd',
- 'mktemp',
- 'module-init-tools',
- 'ncurses',
- 'net-tools',
- 'newt',
- 'nscd',
- 'openldap',
- 'openssl',
- 'pam',
- 'parted',
- 'passwd',
- 'pciutils',
- 'pcre',
- 'perl',
- 'perl-Filter',
- 'policycoreutils',
- 'popt',
- 'prelink',
- 'procmail',
- 'procps',
- 'psmisc',
- 'python',
- 'pyxf86config',
- 'readline',
- 'redhat-logos',
- 'redhat-release',
- 'rhpl',
- 'rootfiles',
- 'rpm',
- 'rpmdb-redhat',
- 'rpm-libs',
- 'sed',
- 'selinux-doc',
- 'selinux-policy-targeted',
- 'sendmail',
- 'setools',
- 'setserial',
- 'setup',
- 'shadow-utils',
- 'slang',
- 'sysklogd',
- 'system-config-mouse',
- 'SysVinit',
- 'tar',
- 'tcp_wrappers',
- 'termcap',
- 'tmpwatch',
- 'tzdata',
- 'udev',
- 'usbutils',
- 'usermode',
- 'util-linux',
- 'vim-minimal',
- 'vixie-cron',
- 'wireless-tools',
- 'xorg-x11-libs',
- 'xorg-x11-Mesa-libGL',
- 'zlib',
-)
-
-
-rhel5corePackages = (
- ('kernel', 'kernel.smp,!kernel.debug,!kernel.pae,!xen,!domU,!dom0'),
- 'acl',
- 'alchemist',
- 'atk',
- 'attr',
- 'audit',
- 'audit-libs',
- 'audit-libs-python',
- 'authconfig',
- 'basesystem',
- 'bash',
- 'beecrypt',
- 'bzip2',
- 'bzip2-libs',
- 'cairo',
- 'checkpolicy',
- 'chkconfig',
- 'coreutils',
- 'cpio',
- 'cracklib',
- 'cracklib-dicts',
- 'crontabs',
- 'cryptsetup-luks',
- 'cups-libs',
- 'cyrus-sasl',
- 'cyrus-sasl-lib',
- 'cyrus-sasl-md5',
- 'db4',
- 'dbus',
- 'dbus-glib',
- 'Deployment_Guide-en-US',
- 'device-mapper',
- 'device-mapper-multipath',
- 'dhclient',
- 'diffutils',
- 'dmidecode',
- 'dmraid',
- 'e2fsprogs',
- 'e2fsprogs-libs',
- 'ed',
- 'elfutils-libelf',
- 'ethtool',
- 'expat',
- 'file',
- 'filesystem',
- 'findutils',
- 'fontconfig',
- 'freetype',
- 'gawk',
- 'gdbm',
- 'glib2',
- 'glibc',
- 'glibc-common',
- 'gmp',
- 'gnutls',
- 'grep',
- 'grub',
- 'gtk2',
- 'gzip',
- 'hal',
- 'hdparm',
- 'hesiod',
- 'hicolor-icon-theme',
- 'hwdata',
- 'info',
- 'initscripts',
- 'iproute',
- 'iptables',
- 'iputils',
- 'kbd',
- 'kernel-headers',
- 'keyutils-libs',
- 'kpartx',
- 'krb5-libs',
- 'kudzu',
- 'less',
- 'libacl',
- 'libattr',
- 'libcap',
- 'libgcc',
- 'libgcrypt',
- 'libgpg-error',
- 'libhugetlbfs',
- 'libhugetlbfs-lib',
- 'libjpeg',
- 'libpng',
- 'libselinux',
- 'libselinux-python',
- 'libsemanage',
- 'libsepol',
- 'libstdc++',
- 'libsysfs',
- 'libtermcap',
- 'libtiff',
- 'libusb',
- 'libuser',
- 'libvolume_id',
- 'libX11',
- 'libXau',
- 'libXcursor',
- 'libXdmcp',
- 'libXext',
- 'libXfixes',
- 'libXft',
- 'libXi',
- 'libXinerama',
- 'libxml2',
- 'libxml2-python',
- 'libXrandr',
- 'libXrender',
- 'libxslt',
- 'libxslt-python',
- 'logrotate',
- 'lvm2',
- 'MAKEDEV',
- 'mcstrans',
- 'mdadm',
- 'mingetty',
- 'mkinitrd',
- 'mktemp',
- 'module-init-tools',
- 'nash',
- 'ncurses',
- 'net-tools',
- 'newt',
- 'nscd',
- 'openldap',
- 'openssl',
- 'pam',
- 'pango',
- 'parted',
- 'passwd',
- 'pciutils',
- 'pcre',
- 'pm-utils',
- 'policycoreutils',
- 'popt',
- 'prelink',
- 'procmail',
- 'procps',
- 'psmisc',
- 'pycairo',
- 'pygobject2',
- 'pygtk2',
- 'python',
- 'python-numeric',
- 'pyxf86config',
- 'readline',
- 'redhat-logos',
- 'redhat-release',
- 'redhat-release-notes',
- 'rhpl',
- 'rootfiles',
- 'rpm',
- 'rpm-libs',
- 'sed',
- 'selinux-policy',
- 'selinux-policy-targeted',
- 'sendmail',
- 'setools',
- 'setserial',
- 'setup',
- 'shadow-utils',
- 'slang',
- 'sqlite',
- 'sysfsutils',
- 'sysklogd',
- 'SysVinit',
- 'tar',
- 'tcl',
- 'tcp_wrappers',
- 'termcap',
- 'tmpwatch',
- 'tzdata',
- 'udev',
- 'usbutils',
- 'usermode',
- 'util-linux',
- 'vim-minimal',
- 'vixie-cron',
- 'wireless-tools',
- 'xorg-x11-filesystem',
- 'zlib',
-)
-
import os
import sys
-import time
-import logging
-sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
-sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py')
-sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib')
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+if 'CONARY_PATH' in os.environ:
+ sys.path.insert(0, os.environ['CONARY_PATH'])
+
+import rmake
+import conary
+import updatebot
+
+print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__)
+print >>sys.stderr, 'using rmake from', os.path.dirname(rmake.__file__)
+print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__)
from conary.lib import util
sys.excepthook = util.genExcepthook()
-from conary.deps import deps
+import logging
-mbdir = os.path.abspath('../')
-sys.path.insert(0, mbdir)
+from updatebot import groupmgr
+from updatebot import OrderedBot
+from updatebot.groupmgr.model import GroupContentsModel
-from updatebot import log
-from updatebot import groupmgr
+log = logging.getLogger('tmplogger')
-slog = log.addRootLogger()
+class Bot(OrderedBot):
+ def generateInitialGroup(self):
+ """
+ Generate config for standard group contents based on repository history.
+ """
-def toxml(pkgList, toFile):
- groupName = 'group-rhel-standard'
- byDefault = True
- depCheck = True
+ standard = (
+ 'aaa_base',
+ 'aaa_skel',
+ 'acl',
+ 'acpid',
+ 'ash',
+ 'at',
+# 'atk',
+# 'atk-devel',
+# 'atk-doc',
+ 'attr',
+# 'audiofile',
+# 'audiofile-devel',
+ 'audit',
+ 'audit-devel',
+ 'audit-libs',
+# 'autoconf',
+# 'autofs',
+ 'bash',
+# 'bc',
+# 'bin86',
+# 'bind',
+# 'bind-chrootenv',
+# 'bind-devel',
+# 'bind-doc',
+# 'bind-libs',
+# 'bind-utils',
+ 'binutils',
+# 'bison',
+ 'blt',
+ 'blocxx',
+ 'busybox',
+ 'bzip2',
+# 'cairo',
+# 'cairo-devel',
+# 'cairo-doc',
+# 'cdrecord',
+# 'cdrecord-devel',
+# 'cifs-mount',
+# 'compat-libstdc++',
+# 'compat-openssl097g',
+ 'coreutils',
+ 'cpio',
+# 'cpp',
+ 'cracklib',
+# 'cracklib-devel',
+ 'cron',
+# 'curl',
+# 'curl-devel',
+# 'cvs',
+# 'cvs-doc',
+ 'cyrus-sasl',
+# 'cyrus-sasl-crammd5',
+# 'cyrus-sasl-devel',
+# 'cyrus-sasl-digestmd5',
+# 'cyrus-sasl-gssapi',
+# 'cyrus-sasl-otp',
+# 'cyrus-sasl-plain',
+# 'cyrus-sasl-sqlauxprop',
+ 'db',
+ 'db42',
+# 'db-devel',
+# 'db-utils',
+ 'dbus-1',
+# 'dbus-1-devel',
+ 'dbus-1-glib',
+# 'dbus-1-gtk',
+# 'dbus-1-java',
+# 'dbus-1-mono',
+# 'dbus-1-python',
+# 'dbus-1-qt3',
+# 'dbus-1-qt3-devel',
+# 'dbus-1-x11',
+# 'dev86',
+ 'device-mapper',
+# 'device-mapper-devel',
+ 'dhcpcd',
+ 'diffutils',
+ 'e2fsprogs',
+# 'e2fsprogs-devel',
+# 'eject',
+# 'esound',
+# 'esound-devel',
+ 'ethtool',
+ 'expat',
+# 'expect',
+ 'file',
+# 'file-devel',
+ 'filesystem',
+ 'fillup',
+ 'findutils',
+# 'findutils-locate',
+ 'fontconfig',
+# 'fontconfig-devel',
+# 'freetype',
+# 'freetype-tools',
+ 'freetype2',
+# 'freetype2-devel',
+ 'gawk',
+# 'gawk-doc',
+# 'gcc',
+# 'gcc-c++',
+# 'gcc-fortran',
+# 'gcc-info',
+# 'gcc-java',
+# 'gcc-locale',
+# 'gcc-obj-c++',
+# 'gcc-objc',
+# 'gconf2',
+# 'gconf2-devel',
+# 'gconf2-doc',
+# 'gdb',
+ 'gdbm',
+# 'gdbm-devel',
+ 'gettext',
+# 'gettext-devel',
+ 'glib2',
+# 'glib2-devel',
+# 'glib2-doc',
+ 'glibc',
+# 'glibc-debuginfo',
+# 'glibc-devel',
+# 'glibc-html',
+# 'glibc-i18ndata',
+# 'glibc-info',
+# 'glibc-locale',
+# 'glibc-profile',
+# 'glitz',
+# 'glitz-devel',
+# 'gmp',
+# 'gmp-devel',
+ 'gnome-filesystem',
+# 'gnome-vfs2',
+# 'gnome-vfs2-devel',
+# 'gnome-vfs2-doc',
+# 'gnuplot',
+# 'gnutls',
+# 'gnutls-devel',
+ 'gpg',
+# 'gpg2',
+# 'gpm',
+ 'grep',
+# 'groff',
+ 'grub',
+# 'gtk2',
+# 'gtk2-devel',
+# 'gtk2-doc',
+# 'gvim',
+# 'gxdview',
+ 'gzip',
+ 'hal',
+# 'hal-devel',
+# 'hal-gnome',
+# 'hdparm',
+ 'hwinfo',
+# 'hwinfo-devel',
+ 'info',
+ 'insserv',
+ 'iproute2',
+ 'iptables',
+# 'iptables-devel',
+ 'iputils',
+# 'jpeg',
+# 'kbd',
+ 'klogd',
+ 'krb5',
+# 'krb5-apps-clients',
+# 'krb5-apps-servers',
+# 'krb5-client',
+# 'krb5-devel',
+# 'krb5-server',
+# 'ksh',
+# 'ksh-devel',
+ 'less',
+ 'libacl',
+# 'libacl-devel',
+ 'libaio',
+# 'libaio-devel',
+# 'libapr-util1',
+# 'libapr-util1-devel',
+# 'libapr1',
+# 'libapr1-devel',
+# 'libart_lgpl',
+# 'libart_lgpl-devel',
+ 'libattr',
+# 'libattr-devel',
+# 'libbonobo',
+# 'libbonobo-devel',
+# 'libbonobo-doc',
+# 'libbonoboui',
+# 'libbonoboui-devel',
+# 'libbonoboui-doc',
+ 'libcap',
+# 'libcap-devel',
+ 'libcom_err',
+ 'libelf',
+ 'libevent',
+ 'libgcc',
+# 'libgcj',
+# 'libgcj-devel',
+ 'libgcrypt',
+# 'libgcrypt-devel',
+# 'libgfortran',
+# 'libgnome',
+# 'libgnome-devel',
+# 'libgnome-doc',
+# 'libgnomecanvas',
+# 'libgnomecanvas-devel',
+# 'libgnomecanvas-doc',
+ 'libgpg-error',
+# 'libgpg-error-devel',
+# 'libgssapi',
+# 'libidn',
+# 'libidn-devel',
+# 'libiniparser',
+# 'libiniparser-devel',
+# 'libjpeg',
+# 'libjpeg-devel',
+# 'libksba',
+# 'libksba-devel',
+# 'libmsrpc',
+# 'libmsrpc-devel',
+# 'libmudflap',
+# 'libnlink',
+ 'libnscd',
+# 'libnscd-devel',
+# 'libobjc',
+# 'libopencdk',
+# 'libopencdk-devel',
+# 'libpcap',
+# 'libpng',
+# 'libpng-devel',
+# 'librpcsecgss',
+# 'libsmbclient',
+# 'libsmbclient-devel',
+ 'libstdc++',
+# 'libstdc++-devel',
+# 'libstdc++-doc',
+ 'libtool',
+ 'libusb',
+ 'libxcrypt',
+# 'libxcrypt-devel',
+ 'libxml2',
+# 'libxml2-devel',
+# 'libxml2-python',
+ 'libxslt',
+# 'libxslt-devel',
+ 'libzio',
+ 'limal',
+ 'limal-bootloader',
+ 'limal-perl',
+ 'logrotate',
+# 'lsof',
+ 'lvm2',
+# 'lzo',
+# 'lzo-devel',
+ 'm4',
+# 'mailx',
+# 'make',
+# 'man',
+# 'mdadm',
+# 'microcode_ctl',
+ 'mingetty',
+ 'mkinitrd',
+# 'mkisofs',
+ 'mktemp',
+# 'mm',
+# 'mm-devel',
+ 'module-init-tools',
+# 'mysql',
+# 'mysql-Max',
+# 'mysql-client',
+# 'mysql-devel',
+# 'mysql-shared',
+# 'nc6',
+ 'ncurses',
+# 'ncurses-devel',
+# 'neon',
+# 'net-snmp',
+# 'net-snmp-devel',
+ 'net-tools',
+ 'netcfg',
+# 'nfs-utils',
+# 'nfsidmap',
+# 'nmap',
+# 'nmap-gtk',
+ 'nscd',
+ 'openct',
+# 'openct-devel',
+ 'openldap2',
+# 'openldap2-back-meta',
+# 'openldap2-back-perl',
+ 'openldap2-client',
+# 'openldap2-devel',
+ 'opensc',
+# 'opensc-devel',
+ 'openslp',
+# 'openslp-devel',
+# 'openslp-server',
+ 'openssh',
+# 'openssh-askpass',
+ 'openssl',
+# 'openssl-devel',
+# 'openssl-doc',
+# 'orbit2',
+# 'orbit2-devel',
+ 'pam',
+# 'pam-devel',
+ 'pam-modules',
+# 'pam_krb5',
+# 'pam_smb',
+# 'pango',
+# 'pango-devel',
+# 'pango-doc',
+# 'parted',
+# 'parted-devel',
+# 'patch',
+ 'pciutils',
+# 'pciutils-devel',
+ 'pciutils-ids',
+ 'pcre',
+# 'pcre-devel',
+ 'pcsc-lite',
+# 'pcsc-lite-devel',
+ 'perl',
+# 'perl-Bit-Vector',
+ 'perl-Bootloader',
+# 'perl-Carp-Clan',
+# 'perl-Compress-Zlib',
+# 'perl-DBD-SQLite',
+# 'perl-DBD-mysql',
+# 'perl-DBI',
+# 'perl-Data-ShowTable',
+# 'perl-Date-Calc',
+# 'perl-Digest-SHA1',
+# 'perl-Net-Daemon',
+# 'perl-PlRPC',
+# 'perl-SNMP',
+# 'perl-TermReadKey',
+# 'perl-URI',
+# 'perl-XML-Parser',
+# 'perl-XML-Writer',
+ 'perl-gettext',
+ 'permissions',
+# 'pinentry',
+# 'pkgconfig',
+# 'pmtools',
+ 'popt',
+# 'popt-devel',
+# 'portmap',
+# 'postgresql',
+# 'postgresql-contrib',
+# 'postgresql-devel',
+# 'postgresql-docs',
+# 'postgresql-libs',
+# 'postgresql-server',
+ 'procmail',
+ 'procps',
+ 'psmisc',
+ 'pwdutils',
+# 'pwdutils-plugin-audit',
+ 'python',
+# 'python-cairo',
+ 'python-curses',
+# 'python-demo',
+ 'python-devel',
+ 'python-gdbm',
+# 'python-gnome',
+# 'python-gtk',
+# 'python-idle',
+# 'python-numeric',
+# 'python-orbit',
+# 'python-pam',
+ 'python-tk',
+ 'python-xml',
+ 'readline',
+# 'readline-devel',
+ 'reiserfs',
+# 'resmgr',
+ 'rpm',
+# 'rpm-devel',
+ 'rpm-python',
+# 'rrdtool',
+# 'rsync',
+# 'samba',
+# 'samba-client',
+# 'samba-krb-printing',
+# 'samba-python',
+# 'samba-vscan',
+# 'samba-winbind',
+ 'sed',
+ 'sendmail',
+# 'sendmail-devel',
+# 'sensors',
+# 'slang',
+# 'slang-devel',
+ 'sles-release',
+# 'smartmontools',
+# 'sqlite',
+# 'sqlite-devel',
+# 'strace',
+# 'sudo',
+ 'sysconfig',
+ 'sysfsutils',
+# 'syslinux',
+ 'syslog-ng',
+ 'syslogd',
+# 'sysstat',
+# 'sysstat-isag',
+ 'sysvinit',
+ 'suse-build-key',
+ 'tar',
+ 'tcl',
+# 'tcl-devel',
+ 'tcpd',
+# 'tcpd-devel',
+# 'tcpdump',
+ 'tcsh',
+# 'telnet',
+# 'telnet-server',
+ 'termcap',
+ 'terminfo',
+# 'texinfo',
+ 'timezone',
+ 'tk',
+# 'tk-devel',
+ 'udev',
+# 'unixODBC',
+# 'unixODBC-devel',
+# 'unzip',
+# 'usbutils',
+ 'util-linux',
+# 'uucp',
+ 'vim',
+ 'wget',
+# 'wireless-tools',
+# 'x11-tools',
+ 'xfsprogs',
+# 'xfsprogs-devel',
+# 'xinetd',
+# 'xkeyboard-config',
+# 'xntp',
+# 'xntp-doc',
+# 'xorg-x11',
+# 'xorg-x11-Xnest',
+# 'xorg-x11-Xvfb',
+# 'xorg-x11-Xvnc',
+# 'xorg-x11-devel',
+# 'xorg-x11-doc',
+# 'xorg-x11-fonts-100dpi',
+# 'xorg-x11-fonts-75dpi',
+# 'xorg-x11-fonts-cyrillic',
+# 'xorg-x11-fonts-scalable',
+# 'xorg-x11-fonts-syriac',
+ 'xorg-x11-libs',
+# 'xorg-x11-man',
+# 'xorg-x11-sdk',
+# 'xorg-x11-server',
+# 'xorg-x11-server-glx',
+# 'yp-tools',
+# 'ypbind',
+# 'zip',
+# 'zisofs-tools',
+ 'zlib',
+# 'zlib-devel',
+ )
- contents = groupmgr.GroupContentsModel(groupName,
- byDefault=byDefault,
- depCheck=depCheck)
+ troves = self._updater._conaryhelper._getLatestTroves()
- for pkg in pkgList:
- slog.info('adding %s' % (pkg, ))
- if type(pkg) == tuple:
- pkg, flavor = pkg
- contents.add(pkg, flavor=deps.parseFlavor(flavor))
- else:
- contents.add(pkg)
+ pkgs = set()
+ for name, vMap in troves.iteritems():
+ if name.endswith(':source'):
+ continue
+ name = name.split(':')[0]
+ for version, flavors in vMap.iteritems():
+ pkgs.add((name, version, tuple(flavors)))
- contents.freeze(toFile)
+ for name, version, flavors in pkgs:
+ log.info('adding %s=%s' % (name, version))
+ for flv in flavors:
+ log.info('\t%s' % flv)
+ self._groupmgr.addPackage(name, version, flavors)
+ self._groupmgr.setErrataState(0)
+ self._groupmgr.setVersion('0')
+
+ pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
+# pkgGroup.depCheck = False
+
+ contents = GroupContentsModel('group-standard', depCheck=True)
+ self._groupmgr._groups['group-standard'] = contents
+
+ for pkgName in standard:
+ for key in pkgGroup._nameMap[pkgName]:
+ contents._addItem(pkgGroup._data[key])
+
+ built = self._groupmgr.build()
+ return built
if __name__ == '__main__':
- platforms = {
- 'rhel4': rhel4corePackages,
- 'rhel5': rhel5corePackages,
- }
+ from updatebot import config
+ from updatebot import log as logSetup
- platform = sys.argv[1]
- assert platform in platforms
+ logSetup.addRootLogger()
- toFile = sys.argv[2]
+ log = logging.getLogger('create group')
- pkgs = platforms[platform]
- toxml(pkgs, toFile)
+ cfg = config.UpdateBotConfig()
+ cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
+
+ bot = Bot(cfg, None)
+ changes = bot.generateInitialGroup()
+
+ import epdb; epdb.st()
diff --git a/scripts/depsolver.py b/scripts/depsolver.py
new file mode 100755
--- /dev/null
+++ b/scripts/depsolver.py
@@ -0,0 +1,102 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2008-2009 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+import os
+import sys
+
+mirrorballDir = os.path.abspath('../')
+sys.path.insert(0, mirrorballDir)
+
+from conary.lib import util
+sys.excepthook = util.genExcepthook()
+
+import copy
+import logging
+import updatebot.log
+
+updatebot.log.addRootLogger()
+log = logging.getLogger('test')
+
+from updatebot import config
+from updatebot import pkgsource
+
+cfg = config.UpdateBotConfig()
+cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
+
+pkgSource = pkgsource.PackageSource(cfg)
+pkgSource.load()
+
+reqSrcPkgs = set()
+for pkgName in cfg.package:
+ bins = sorted(pkgSource.binNameMap.get(pkgName, []))
+ if not bins:
+ continue
+ reqSrcPkgs.add(pkgSource.binPkgMap[bins[-1]])
+
+reqBinPkgs = set()
+for srcPkg in reqSrcPkgs:
+ reqBinPkgs.update(set([ x for x in pkgSource.srcPkgMap[srcPkg]
+ if x.arch not in ('nosrc', 'src') ]))
+
+requires = {}
+provides = {}
+log.info('loading package requires/provides')
+for bin, src in pkgSource.binPkgMap.iteritems():
+ if bin.arch in ('nosrc', 'src'):
+ continue
+ for format in bin.format:
+ if format.getName() == 'rpm:provides':
+ bins = set([ x.name for x in pkgSource.srcPkgMap[src]
+ if x.arch not in ('nosrc', 'src') ])
+ for child in format.iterChildren():
+ provides.setdefault(child.name, set()).update(bins)
+ if format.getName() == 'rpm:requires':
+ for child in format.iterChildren():
+ requires.setdefault(bin.name, set()).add(child.name)
+
+
+solved = set()
+working = set([ x.name for x in reqBinPkgs ])
+
+log.info('resolving deps')
+while working:
+ pkg = working.pop()
+ log.info('resolving %s' % pkg)
+ for req in requires[pkg]:
+ if req.startswith('rpmlib'):
+ continue
+ if req not in provides:
+ log.warn('requirement not found: %s' % req)
+ continue
+ for provPkg in provides[req]:
+ if provPkg not in solved and provPkg != pkg:
+ working.add(provPkg)
+
+ solved.add(pkg)
+ log.info('solved: %s, working set: %s' % (len(solved), len(working)))
+
+needed = set()
+for pkgName in solved:
+ bins = sorted(pkgSource.binNameMap[pkgName])
+ src = pkgSource.binPkgMap[bins[-1]]
+ if src in reqSrcPkgs:
+ continue
+ pkgs = sorted([ x for x in pkgSource.srcPkgMap[src]
+ if x.arch not in ('nosrc', 'src') ])
+ needed.add(pkgs[0])
+
+import epdb; epdb.st()
+
+
diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py
--- a/scripts/grpchecker.py
+++ b/scripts/grpchecker.py
@@ -104,7 +104,7 @@
changes = []
try:
- mgr._validateGroups()
+ mgr._sanity.check(mgr._groups, mgr.getErrataState())
except GroupValidationFailedError, e:
for group, error in e.errors:
if isinstance(error, NameVersionConflictsFoundError):
@@ -190,7 +190,7 @@
mgr.addPackage(n, v, flvs)
mgr._copyVersions()
- mgr._validateGroups()
+ mgr._sanity.check(mgr._groups, mgr.getErrataState())
version = mgr.save(copyToLatest=True)
jobId = mgr._builder.start(((mgr._sourceName, version, None), ))
jobIds.append(jobId)
diff --git a/scripts/pkgsource b/scripts/pkgsource
--- a/scripts/pkgsource
+++ b/scripts/pkgsource
@@ -15,6 +15,7 @@
import os
import sys
+import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -37,6 +38,91 @@
pkgSource = pkgsource.PackageSource(cfg)
pkgSource.load()
+srcs = {}
+srcNevras = {}
+for src, bins in pkgSource.srcPkgMap.iteritems():
+ srcs.setdefault(src.getNevra(), bins)
+ srcNevras.setdefault(src.getNevra(), src)
+
+names = {}
+for nevra, bins in srcs.iteritems():
+ if len(bins) > 1:
+ names.setdefault(nevra[0], set()).add(nevra)
+
+count = {}
+arch = {}
+for name, nevras in names.iteritems():
+ first = None
+ seen = None
+ for nevra in nevras:
+ bins = srcs[nevra]
+ archs = set([ x.arch for x in bins ])
+ if not seen:
+ first = (nevra, bins)
+ seen = archs
+ elif len(seen) != len(archs):
+ count[first[0]] = first[1]
+ count[nevra] = bins
+ elif seen != archs:
+ arch.setdefault(first[0], set()).update(first[1])
+ arch.setdefault(nevra, set()).update(bins)
+
+removed = {}
+#for nevra in itertools.chain(count, arch):
+# removed.setdefault(nevra, srcNevras.pop(nevra))
+for nevra in itertools.chain(*[ x for x in names.values() if len(x) > 1 ]):
+ removed.setdefault(nevra, srcNevras.pop(nevra))
+
+toCreate = set(srcNevras.values())
+
+import epdb; epdb.st()
+
+order = {}
+
+def srtByRPMVerCmp(a, b):
+ from updatebot.lib import util
+ return util.packagevercmp(a, b)
+
+def srtByBuildstamp(a, b):
+ assert hasattr(a, 'buildTimestamp')
+ assert hasattr(b, 'buildTimestamp')
+ assert a.buildTimestamp not in ('0', '', 0)
+ assert b.buildTimestamp not in ('0', '', 0)
+ return cmp(int(a.buildTimestamp), int(b.buildTimestamp))
+
+srcNames = {}
+for srcPkg in pkgSource.srcPkgMap:
+ srcNames.setdefault(srcPkg.name, set()).add(srcPkg)
+
+binOrder = {}
+for srcName, srcPkgs in srcNames.iteritems():
+ uSrcPkgs = dict((x.getNevra(), x) for x in srcPkgs).values()
+
+ ver = sorted(uSrcPkgs, cmp=srtByRPMVerCmp)
+
+ for srcPkg in uSrcPkgs:
+ if srcPkg.buildTimestamp is None:
+ srcPkg.buildTimestamp = sorted([ x for x in pkgSource.srcPkgMap[srcPkg] if x.arch != 'src' ])[0].buildTimestamp
+
+ buildstamp = sorted(uSrcPkgs, cmp=srtByBuildstamp)
+
+ assert ver == buildstamp
+
+ for srcPkg in uSrcPkgs:
+ ts = int(srcPkg.buildTimestamp)
+ bins = pkgSource.srcPkgMap[srcPkg]
+ binOrder.setdefault(ts, set()).update(bins)
+
+def tsToDay(ts):
+ import time
+ return int(time.mktime(time.strptime(time.strftime('%Y%m%d', time.gmtime(ts)), '%Y%m%d')))
+
+# collapse by day
+for ts in sorted(binOrder):
+ day = tsToDay(ts)
+ bins = binOrder[ts]
+ order.setdefault(day, set()).update(bins)
+
import epdb; epdb.st()
updates = []
diff --git a/scripts/sync-centos.sh b/scripts/sync-centos.sh
--- a/scripts/sync-centos.sh
+++ b/scripts/sync-centos.sh
@@ -13,7 +13,7 @@
# full details.
#
-SOURCE=rsync://mirrors.us.kernel.org/CentOS-nodvd
+SOURCE=rsync://mirrors.us.kernel.org/CentOS-incdvd
DEST=/l/CentOS/
date
diff --git a/scripts/pkgsource b/scripts/verifyrepos
copy from scripts/pkgsource
copy to scripts/verifyrepos
--- a/scripts/pkgsource
+++ b/scripts/verifyrepos
@@ -15,6 +15,8 @@
import os
import sys
+import copy
+import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -28,20 +30,31 @@
updatebot.log.addRootLogger()
log = logging.getLogger('test')
+import rpmutils
+
from updatebot import config
from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
-pkgSource.load()
+import epdb; epdb.st()
+paths = copy.copy(cfg.repositoryPaths)
+for path in paths:
+ cfg.repositoryPaths = [ path, ]
+ pkgSource = pkgsource.PackageSource(cfg)
+ pkgSource.load()
+
+ for location in pkgSource.locationMap:
+ if not location.startswith(path):
+ log.info('skipping %s' % location)
+ continue
+ url = cfg.repositoryUrl + '/' + location
+ try:
+ rpmutils.readHeader(url)
+ except Exception, e:
+ raise
+ log.info('failed to open %s, %s' % (url, e))
+
import epdb; epdb.st()
-
-updates = []
-for path, client in pkgSource._clients.iteritems():
- if 'Updates' in path:
- updates.extend(client.getUpdateInfo())
-
-import epdb; epdb.st()
diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -57,6 +57,22 @@
return [ x for x in itertools.chain(*setDict.itervalues()) ]
+ def _formatBuildTroves(self, buildSet):
+ """
+ Format a list of trove specs and source package objects into something
+ the build subsystem can deal with.
+ """
+
+ toBuild = set()
+ for (n, v, f), srcPkg in buildSet:
+ binaryNames = None
+ if srcPkg:
+ binaryNames = tuple([ x.name
+ for x in self._pkgSource.srcPkgMap[srcPkg] ])
+ toBuild.add((n, v, f, binaryNames))
+
+ return sorted(toBuild)
+
def create(self, rebuild=False, recreate=None, toCreate=None):
"""
Do initial imports.
@@ -107,28 +123,31 @@
# Import sources into repository.
- toBuild, parentPkgMap, fail = self._updater.create(
+ buildSet, parentPkgMap, fail = self._updater.create(
toPackage,
buildAll=rebuild,
recreate=bool(recreate),
toCreate=toCreate)
+ toBuild = self._formatBuildTroves(buildSet)
+
log.info('failed to create %s packages' % len(fail))
log.info('found %s packages to build' % len(toBuild))
trvMap = {}
failed = ()
+
if len(toBuild):
if not rebuild or (rebuild and toCreate):
# Build all newly imported packages.
- trvMap, failed = self._builder.buildmany(sorted(toBuild))
+ trvMap, failed = self._builder.buildmany(toBuild)
log.info('failed to import %s packages' % len(failed))
if len(failed):
for pkg in failed:
log.warn('%s' % (pkg, ))
else:
# ReBuild all packages.
- trvMap = self._builder.buildsplitarch(sorted(toBuild))
+ trvMap = self._builder.buildsplitarch(toBuild)
log.info('import completed successfully')
log.info('imported %s source packages' % (len(toBuild), ))
else:
@@ -142,7 +161,8 @@
return trvMap, failed
- def update(self, force=None, updatePkgs=None, expectedRemovals=None):
+ def update(self, force=None, updatePkgs=None, expectedRemovals=None,
+ allowPackageDowngrades=None):
"""
Update the conary repository from the yum repositories.
@param force: list of packages to update without exception
@@ -151,6 +171,9 @@
@type updatePkgs: iterable of source package objects
@param expectedRemovals: set of packages that are expected to be
removed.
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
@type expectedRemovals: set of package names
"""
@@ -171,7 +194,8 @@
# Get troves to update and send advisories.
toAdvise, toUpdate = self._updater.getUpdates(
updateTroves=updateTroves,
- expectedRemovals=expectedRemovals)
+ expectedRemovals=expectedRemovals,
+ allowPackageDowngrades=allowPackageDowngrades)
# If forcing an update, make sure that all packages are listed in
# toAdvise and toUpdate as needed.
@@ -215,7 +239,7 @@
# Make sure to build everything in the toAdvise list, there may be
# sources that have been updated, but not built.
- buildTroves = set([ x[0] for x in toAdvise ])
+ buildTroves = self._formatBuildTroves(toAdvise)
# If importing specific packages, they might require each other so
# always use buildmany, but wait to commit.
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -43,6 +43,7 @@
from updatebot.errors import JobFailedError
from updatebot.errors import CommitFailedError
from updatebot.errors import UnhandledKernelModule
+from updatebot.errors import InvalidBuildTroveInputError
from updatebot.errors import FailedToRetrieveChangesetError
from updatebot.errors import ChangesetValidationFailedError
@@ -176,7 +177,7 @@
@return troveMap: dictionary of troveSpecs to built troves
"""
- workers = 20
+ workers = 30
if not lateCommit:
dispatcher = Dispatcher(self, workers)
else:
@@ -312,19 +313,28 @@
"""
Formats the list of troves provided into a job list for rMake.
@param troveSpecs: set of name, version, flavor tuples
- @type troveSpecs: set([(name, version, flavor), ..])
+ @type troveSpecs: set([(name, version, flavor, optional list of binary
+ names), ..])
@return list((name, version, flavor, context), ...)
"""
# Make sure troveSpecs is an iterable of three tuples.
- if (len(troveSpecs) == 3 and
+ if (len(troveSpecs) in (3, 4) and
not isinstance(list(troveSpecs)[0], (list, set, tuple))):
# Assume that (n,v,f) was passed in
troveSpecs = [ troveSpecs, ]
# Build all troves in defined contexts.
troves = []
- for name, version, flavor in troveSpecs:
+ for trv in troveSpecs:
+ if len(trv) == 3:
+ name, version, flavor = trv
+ binaryNames = None
+ elif len(trv) == 4:
+ name, version, flavor, binaryNames = trv
+ else:
+ raise InvalidBuildTroveInputError(input=trv)
+
# Make sure name is not an unicode string, it causes breakage in
# the deps modules in conary.
name = name.encode()
@@ -365,7 +375,12 @@
# All other packages.
else:
# Build all packages as x86 and x86_64.
- for context in self._cfg.archContexts:
+ for context, fltr in self._cfg.archContexts:
+ # If there is a filter and no binary file names or no files
+ # in the binary names match the filter skip this context.
+ if (fltr and (not binaryNames or (binaryNames and
+ not [ x for x in binaryNames if fltr[1].match(x) ]))):
+ continue
troves.append((name, version, flavor, context))
return troves
diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py
--- a/updatebot/build/cvc.py
+++ b/updatebot/build/cvc.py
@@ -23,8 +23,9 @@
from conary import versions
from conary import conarycfg
+from conary.deps import deps
+from conary.build import cook
from conary import conaryclient
-from conary.build import cook
from updatebot.lib import conarycallbacks
from updatebot.errors import LocalCookFailedError
@@ -56,11 +57,16 @@
self._client = conaryclient.ConaryClient(self._ccfg)
- def cook(self, troveSpecs):
+ def cook(self, troveSpecs, flavorFilter=None):
"""
Cook a set of trove specs, currently limited to groups.
@params troveSpecs: list of name, version, and flavor tuples.
@type troveSpecs: [(name, version, flavor), ... ]
+ @params flavorFilter: Allow caller to filter out the contexts that they
+ want to build. This is mostly used for group
+ building where a given group should not be built
+ for a context.
+ @type flavorFilter: iterable of context names.
"""
# TODO: Look at conary.build.cook.cookCommand for how to setup
@@ -68,6 +74,9 @@
troveSpecs = self._formatInput(troveSpecs)
+ if flavorFilter:
+ troveSpecs = self._filterTroveSpecs(troveSpecs, flavorFilter)
+
# make sure all troves are groups
assert not [ x for x in troveSpecs if not x[0].startswith('group-') ]
@@ -113,3 +122,30 @@
res = { (troveSpecs[0][0], troveSpecs[0][1], None): results }
return res
+
+ def _filterTroveSpecs(self, troveSpecs, useFlags):
+ """
+ Filter trove specs based on a list of use flags. This is only applicable
+ to groups.
+ @param troveSpecs: iterable of nvf tuples.
+ @type troveSpecs: list(tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ @param useFlags: iterable of valid use flags (x86 and x86_64)
+ @type useFlags: list(str, ...)
+ @return modified list of trove specs
+ @rtype list(tuple(str, conary.versions.VersionFromString,
+ conary.deps.deps.Flavor), ...)
+ """
+
+ useMap = {
+ 'x86': deps.parseFlavor('is: x86'),
+ 'x86_64': deps.parseFlavor('is: x86_64'),
+ }
+
+ specs = set()
+ for n, v, f in troveSpecs:
+ for flag in useFlags:
+ if f.satisfies(useMap[flag]):
+ specs.add((n, v, f))
+
+ return list(specs)
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -92,7 +92,7 @@
while (troves and self._slots and self._startSlots and
self._availableFDs()):
# get trove to work on
- trove = troves.pop()
+ trove = troves.pop(0)
# start build job
self._starter.startJob(trove)
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -371,8 +371,8 @@
ret = {}
for f, t in cfMap.iteritems():
- assert len(cfMap[f]) == 1
- ret[f] = list(t)[0]
+ assert len(cfMap[f]) >= 1
+ ret[f] = sorted(t)[-1]
self._cache.labelClonedFromCache[label] = ret
return ret
@@ -1052,7 +1052,8 @@
if version == latestVer:
trvLst.append((name, version, flavor))
- callback = UpdateBotCloneCallback(self._ccfg, 'test', log=log)
+ callback = UpdateBotCloneCallback(self._ccfg, 'automated promote',
+ log=log)
success, cs = self._client.createSiblingCloneChangeSet(
labelMap,
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -68,6 +68,29 @@
raise ParseError, e
+class CfgContextFilter(CfgRegExp):
+ """
+ Class for parsing context name/regex tuples.
+ """
+
+ def parseString(self, val):
+ """
+ Parse config input.
+ """
+
+ try:
+ splt = val.split()
+ if len(splt) == 1:
+ context = val
+ fltr = None
+ else:
+ context, fltrStr = splt
+ fltr = CfgRegExp.parseString(self, fltrStr)
+ return context, fltr
+ except versions.ParseError, e:
+ raise ParseError, e
+
+
class CfgAdvisoryOrder(CfgString):
"""
Class for parsing advisor order config.
@@ -108,7 +131,7 @@
return splt
-class CfgObsoletes(CfgString):
+class CfgNevraTuple(CfgString):
"""
Class for parsing obsolete mappings:
@@ -207,6 +230,10 @@
# repositoryName archString
repositoryArch = (CfgDict(CfgString), {})
+ # Ignore packages with "32bit" in the name. This is intened for use with
+ # SLES based platforms.
+ ignore32bitPackages = (CfgBool, False)
+
# Data source for determining platform version information, only used for
# group versioning.
versionSources = (CfgDict(CfgString), {})
@@ -276,7 +303,7 @@
listArchiveStartDate = CfgString
# list of contexts that all packages are built in.
- archContexts = CfgList(CfgString)
+ archContexts = CfgList(CfgContextFilter)
# flavors to build the source group.
groupFlavors = (CfgList(CfgFlavor), [])
@@ -328,6 +355,10 @@
# package format.
writePackageMetadata = (CfgBool, False)
+ # Write version information to the source trove, generated from the source
+ # version and revision.
+ writePackageVersion = (CfgBool, False)
+
# If sources are not available pkgSource will attempt to build artificial
# source information if this is set to True.
synthesizeSources = (CfgBool, False)
@@ -396,7 +427,7 @@
# but CfgSet and CfgTuple do not exist at this point;
# maybe we can add them later.
# keepObsolete
- keepObsolete = (CfgList(CfgObsoletes), [])
+ keepObsolete = (CfgList(CfgNevraTuple), [])
# updateId packageName [packageName ...]
# remove obsoleted packages when other subpackages of the same
@@ -418,6 +449,21 @@
# Remove a package from a specific group
removePackage = (CfgDict(CfgDict(CfgList(CfgNameFlavor))), {})
+ # Allow updates for a given nevra to be published without matching errata.
+ allowMissingErrata = (CfgList(CfgNevra), [])
+
+ # Allow updates to have versions that go backwards.
+ # updateId: [ (from srcTrvSpec, to srcTrvSpec), ... ]
+ allowPackageDowngrades = (CfgIntDict(CfgList(CfgNevraTuple)), {})
+
+ # Allow updates which don't include all binary packages corresponding
+ # to a given source.
+ allowReusedPackages = (CfgBool, False)
+
+ # Add a source to a specific updateId. This is used to move updates forward
+ # after allowing an update to downgrade the version.
+ addSource = (CfgIntDict(CfgList(CfgNevra)), {})
+
class UpdateBotConfig(cfg.SectionedConfigFile):
"""
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -23,6 +23,7 @@
from updatebot import update
from updatebot import conaryhelper
+from updatebot.errors import MissingErrataError
from updatebot.errors import ErrataPackageNotFoundError
from updatebot.errors import ErrataSourceDataMissingError
from updatebot.errors import PackageNotFoundInBucketError
@@ -260,6 +261,7 @@
parentPackages = []
removals = self._cfg.updateRemovesPackages
replaces = self._cfg.updateReplacesPackages
+ downgraded = self._cfg.allowPackageDowngrades
currentlyRemovedBinaryNevras = set()
foundObsoleteEdges = set()
foundObsoleteSrcs = set()
@@ -269,6 +271,7 @@
expectedReplaces = replaces.get(updateId, [])
explicitSourceRemovals = self._cfg.removeSource.get(updateId, set())
explicitBinaryRemovals = self._cfg.removeObsoleted.get(updateId, set())
+ explicitPackageDowngrades = downgraded.get(updateId, None)
assert len(self._order[updateId])
for srpm in self._order[updateId]:
@@ -276,8 +279,17 @@
# validate updates
try:
- assert updater._sanitizeTrove(nvf, srpm,
- expectedRemovals=expectedRemovals + expectedReplaces)
+ toUpdate = updater._sanitizeTrove(nvf, srpm,
+ expectedRemovals=expectedRemovals + expectedReplaces,
+ allowPackageDowngrades=explicitPackageDowngrades)
+
+ # If a source was manually added to this updateId it may
+ # have already been part of another update, which would
+ # cause the manifest not to change.
+ if (srpm.getNevra() not in
+ self._cfg.addSource.get(updateId, [])):
+ assert toUpdate
+
except (UpdateGoesBackwardsError,
UpdateRemovesPackageError,
UpdateReusesPackageError), e:
@@ -521,12 +533,21 @@
# insert packages that did not have errata and were not in the initial
# set of packages (golden bits)
srcMap = {}
+ missing = set()
for pkg in other:
+ if pkg.getNevra() not in self._cfg.allowMissingErrata:
+ missing.add(pkg)
+
src = self._pkgSource.binPkgMap[pkg]
if src not in srcMap:
srcMap[src] = []
srcMap[src].append(pkg)
+ # Raise an error if there are any packages missing an errata that are
+ # now explicitly allowed by the config.
+ if missing:
+ raise MissingErrataError(packages=list(missing))
+
# insert bins by buildstamp
extras = {}
@@ -612,13 +633,23 @@
for source, dest, nevra in self._cfg.reorderSource:
self._reorderSource(source, dest, nevra)
+ # add a source to a specific bucket, used to "promote" newer versions
+ # forward.
+ nevras = dict([ (x.getNevra(), x)
+ for x in self._pkgSource.srcPkgMap ])
+ diffCount = 0
+ for updateId, srcNevras in self._cfg.addSource.iteritems():
+ sources = set(nevras[x] for x in srcNevras)
+ self._order.setdefault(updateId, set()).update(sources)
+ diffCount += len(srcNevras)
+
# Make sure we don't drop any updates
totalPkgs2 = sum([ len(x) for x in self._order.itervalues() ])
pkgs = set()
for pkgSet in self._order.itervalues():
pkgs.update(pkgSet)
- assert len(pkgs) == totalPkgs2
- assert totalPkgs2 == totalPkgs
+ assert len(pkgs) == totalPkgs2 - diffCount
+ assert totalPkgs2 == totalPkgs + diffCount
def _mergeUpdates(self, mergeList):
"""
diff --git a/updatebot/errors.py b/updatebot/errors.py
--- a/updatebot/errors.py
+++ b/updatebot/errors.py
@@ -41,7 +41,16 @@
return '%s(%s)' % (self.__class__, params)
-class CommitFailedError(UpdateBotError):
+class BuildError(UpdateBotError):
+ """
+ BuildError, abstract error for all other build related errors.
+ """
+
+ _parms = []
+ _template = 'A build error has occured'
+
+
+class CommitFailedError(BuildError):
"""
CommitFailedError, raised when failing to commit to a repository.
"""
@@ -69,7 +78,7 @@
'validation because:\n%(reason)s')
-class JobFailedError(UpdateBotError):
+class JobFailedError(BuildError):
"""
JobFailedError, raised when an rMake job fails.
"""
@@ -78,7 +87,7 @@
_template = 'rMake job %(jobId)s failed: %(why)s'
-class JobNotCompleteError(UpdateBotError):
+class JobNotCompleteError(BuildError):
"""
JobNotCompleteError, raised when the build dispatcher thinks that the job
should be done, but it isn't.
@@ -88,7 +97,17 @@
_template = 'Build job not complete %(jobId)s'
-class UnhandledKernelModule(UpdateBotError):
+class InvalidBuildTroveInputError(BuildError):
+ """
+ InvalidBuildTroveInputError, raised when validation of a build request
+ fails.
+ """
+
+ _params = ['input', ]
+ _template = 'Invalid input to build system: %(input)s'
+
+
+class UnhandledKernelModule(BuildError):
"""
UnhandledKernelModule, raised when trying to create a build job with a
package that looks as if it might be a kernel module that does not have
@@ -182,6 +201,29 @@
_params = ['pkgname']
_template = 'No checked out version of %(pkgname)s was found.'
+class SourceNotImportedError(UnhandledUpdateError):
+ """
+ SourceNotImportedError, raised when sanity checking an existing source
+ package that can not be found in the conary repository.
+ """
+
+ _params = ['srpm', ]
+ _template = ('Source package (%(srpm)s) has not been imported. This '
+ 'usually means that something in the order stream is in an unexpected '
+ 'state.')
+
+class FoundModifiedNotImportedErrataError(UnhandledUpdateError):
+ """
+ FoundModifiedNotImportedErrataError, raised when an errata is found that
+ should have already been imported and has changed in the upstream data
+ source.
+ """
+
+ _params = ['advisories', ]
+ _template = ('The following advisories have been modified upstream, but '
+ 'should have already been imported. Check to make sure the '
+ 'modifications are not important: %(advisories)s')
+
class BinariesNotFoundForSourceVersion(UnhandledUpdateError):
"""
BinariesNotFoundForSourceVersion, raised when querying by source name and
@@ -569,7 +611,7 @@
"""
_params = ['count', ]
- _template = ('Could not find %(count) sources for matching binary '
+ _template = ('Could not find %(count)s sources for matching binary '
'packages. This generally means that there is a binary package with a '
'source of a different name and a source can not be found with a '
'matching source name, version, and release.')
@@ -592,6 +634,15 @@
_template = ('Can not merge %(source)s into %(target)s due to conflicting '
'package %(package)s')
+class MissingErrataError(ErrataFilterError):
+ """
+ MissingErrataError, raised when packages are discovered without an
+ associated errata.
+ """
+
+ _params = [ 'packages', ]
+ _template = 'The following packages do not have an errata: %(packages)s'
+
class ConfigurationError(UpdateBotError):
"""
Generic exception class for configuration related errors.
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -289,9 +289,18 @@
# have been changed about the group since it was committed.
assert not group.dirty
+ # Find all of the use flags used in the group model.
+ use = set()
+ for model in group:
+ for pkg in model:
+ if pkg.use:
+ use.add(pkg.use)
+ else:
+ use.update(set(['x86', 'x86_64']))
+
# Create a build job and build groups using cvc.
job = ((self._sourceName, group.conaryVersion, None), )
- results = self._builder.cvc.cook(job)
+ results = self._builder.cvc.cook(job, flavorFilter=use)
return results
def getSourceVersions(self):
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -30,12 +30,14 @@
from updatebot.lib import watchdog
from updatebot.bot import Bot as BotSuperClass
+from updatebot.errors import SourceNotImportedError
from updatebot.errors import UnknownRemoveSourceError
from updatebot.errors import PlatformNotImportedError
from updatebot.errors import TargetVersionNotFoundError
from updatebot.errors import PromoteMissingVersionError
from updatebot.errors import PromoteFlavorMismatchError
from updatebot.errors import PlatformAlreadyImportedError
+from updatebot.errors import FoundModifiedNotImportedErrataError
log = logging.getLogger('updatebot.ordered')
@@ -196,6 +198,11 @@
# Iterate through changed and verify the current conary repository
# contents against any changes.
if changed:
+ notimported = set()
+ expectedDowngrades = [ x for x in
+ itertools.chain(*self._cfg.allowPackageDowngrades.values()) ]
+ sourceExceptions = dict((x[2], x[1])
+ for x in self._cfg.reorderAdvisory)
log.info('found modified updates, validating repository state')
for advisory, advInfo in changed.iteritems():
log.info('validating %s' % advisory)
@@ -203,7 +210,19 @@
log.info('checking %s' % srpm.name)
# This will raise an exception if any inconsistencies are
# detected.
- self._updater.sanityCheckSource(srpm)
+ try:
+ self._updater.sanityCheckSource(srpm,
+ allowPackageDowngrades=expectedDowngrades)
+ except SourceNotImportedError, e:
+ if (advisory in sourceExceptions and
+ sourceExceptions[advisory] > current):
+ log.info('found exception for advisory')
+ continue
+ notimported.add(advisory)
+
+ if notimported:
+ raise FoundModifiedNotImportedErrataError(
+ advisories=notimported)
log.info('starting update run')
@@ -244,6 +263,8 @@
requiredRemovals = (set(removePackages) |
set(removeReplaced))
+ # Get the list of package that are allowed to be downgraded.
+ allowDowngrades = self._cfg.allowPackageDowngrades.get(updateId, [])
# If recovering from a failure, restore the pkgMap from disk.
if restoreFile:
@@ -253,7 +274,8 @@
# Update package set.
else:
pkgMap = self._update(*args, updatePkgs=updates,
- expectedRemovals=expectedRemovals, **kwargs)
+ expectedRemovals=expectedRemovals,
+ allowPackageDowngrades=allowDowngrades, **kwargs)
# When deriving from an upstream platform sometimes we don't want
# the latest versions.
@@ -415,7 +437,8 @@
# Find excepted promote packages.
srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages(
bucket)
- exceptions = self._getOldVersionExceptions(updateId)
+ exceptions = dict([ (x[0], x[1]) for x in itertools.chain(
+ *self._getOldVersionExceptions(updateId).itervalues()) ])
# These are the binary trove specs that we expect to be promoted.
expected = self._filterBinPkgSet(
@@ -590,9 +613,15 @@
# Handle the case where a package has been rebuilt for some
# reason, but we need to use the old version of the package.
pkgName = n.split(':')[0]
- if len(vMap) > 1 and pkgName in exceptions:
- vMap = dict((x, y) for x, y in vMap.iteritems()
- if x == exceptions[pkgName])
+ if len(vMap) > 1:
+ if pkgName in exceptions:
+ log.info('using old version of %s' % n)
+ vMap = dict((x, y) for x, y in vMap.iteritems()
+ if x == exceptions[pkgName])
+ else:
+ log.info('found multiple versions of %s, using latest' % n)
+ v = sorted(vMap)[-1]
+ vMap = { v: vMap[v], }
assert len(vMap) == 1
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -108,7 +108,7 @@
for pkg in client.getPackageDetail():
# ignore the 32-bit compatibility libs - we will
# simply use the 32-bit components from the repository
- if '32bit' in pkg.name:
+ if self._cfg.ignore32bitPackages and '32bit' in pkg.name:
continue
# Don't use all arches.
@@ -140,10 +140,15 @@
@type package: repomd.packagexml._Package
"""
- if package.name not in self.srcNameMap:
- self.srcNameMap[package.name] = set()
- self.srcNameMap[package.name].add(package)
+ other = package
+ if package in self._srcPkgs:
+ other = [ x for x in self._srcPkgs if x == package ][0]
+ self.srcNameMap[package.name].remove(other)
+ self._srcPkgs.remove(other)
+ if package.getFileName() == os.path.basename(package.location):
+ other = package
+ self.srcNameMap.setdefault(package.name, set()).add(other)
self.locationMap[package.location] = package
# In case the a synthesized source ever turns into real source add the
@@ -153,7 +158,7 @@
if baseLoc not in self.locationMap:
self.locationMap[baseLoc] = package
- self._srcPkgs.add(package)
+ self._srcPkgs.add(other)
self._srcMap[(package.name, package.epoch, package.version,
package.release, package.arch)] = package
@@ -173,10 +178,9 @@
srcRelease = srcParts[-1][:-8] # remove '.src.rpm'
elif srcParts[-1].endswith('.nosrc.rpm'):
srcRelease = srcParts[-1][:-10]
+
rpmMapKey = (srcName, package.epoch, srcVersion, srcRelease, 'src')
- if rpmMapKey not in self._rpmMap:
- self._rpmMap[rpmMapKey] = set()
- self._rpmMap[rpmMapKey].add(package)
+ self._rpmMap.setdefault(rpmMapKey, set()).add(package)
# The normal case of "obsoletes foo < version" (or with
# "requires foo", though that normally also follows the
@@ -247,6 +251,24 @@
self.srcPkgMap[pkg] = self._rpmMap[key]
self.srcPkgMap[pkg].add(pkg)
+
+ # Remove any duplicate sources, favoring sources with the source
+ # version in the file name.
+ # FIXME: This doesn't really work for nosrc rpms
+ sources = sorted([ (os.path.basename(x.location), x)
+ for x in self.srcPkgMap[pkg] if x.arch in ('src', 'nosrc') ])
+ if len(sources) > 1:
+ primary = None
+ for fn, src in sources:
+ if fn == '%s-%s-%s.%s.rpm' % (src.name, src.version, src.release, src.arch):
+ primary = src
+ break
+
+ if primary:
+ for fn, src in sources:
+ if src is not primary:
+ self.srcPkgMap[pkg].remove(src)
+
toDelete.add(key)
for binPkg in self.srcPkgMap[pkg]:
@@ -360,6 +382,18 @@
return srcPkg
+ def synthesizeSource(srcPkg):
+ # add source to structures
+ if srcPkg.getNevra() not in self._srcMap:
+ log.warn('synthesizing source package %s' % srcPkg)
+ self._procSrc(srcPkg)
+
+ # Add location mappings for packages that may have once been
+ # synthesized so that parsing old manifest files still works.
+ elif srcPkg.location not in self.locationMap:
+ pkg = self._srcMap[srcPkg.getNevra()]
+ self.locationMap[srcPkg.location] = pkg
+
# Return if sources should be available in repos.
if not self._cfg.synthesizeSources:
return
@@ -376,16 +410,8 @@
deffer.add(nevra)
continue
- # add source to structures
- if nevra not in self._srcMap:
- log.warn('synthesizing source package %s' % srcPkg)
- self._procSrc(srcPkg)
-
- # Add location mappings for packages that may have once been
- # synthesized so that parsing old manifest files still works.
- elif srcPkg.location not in self.locationMap:
- pkg = self._srcMap[nevra]
- self.locationMap[srcPkg.location] = pkg
+ # Synthesize the source
+ synthesizeSource(srcPkg)
broken = set()
# Make an attempt to sort out the binaries that have different names
@@ -407,7 +433,10 @@
log.warn('found binary without matching source name %s'
% list(bins)[0].name)
- broken.add((nevra, tuple(bins)))
+ # If this isn't a case of a missmatched epoch, just go ahead and
+ # make up a source. What could go wrong?
+ synthesizeSource(srcPkg)
+ #broken.add((nevra, tuple(bins)))
# Raise an exception if this ever happens. We can figure out the right
# thing to do then, purhaps on a case by case basis.
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -27,6 +27,7 @@
from updatebot import conaryhelper
from updatebot.errors import GroupNotFound
from updatebot.errors import NoManifestFoundError
+from updatebot.errors import SourceNotImportedError
from updatebot.errors import OldVersionNotFoundError
from updatebot.errors import UpdateGoesBackwardsError
from updatebot.errors import UpdateRemovesPackageError
@@ -45,7 +46,8 @@
self._conaryhelper = conaryhelper.ConaryHelper(self._cfg)
- def getUpdates(self, updateTroves=None, expectedRemovals=None):
+ def getUpdates(self, updateTroves=None, expectedRemovals=None,
+ allowPackageDowngrades=None):
"""
Find all packages that need updates and/or advisories from a top level
binary group.
@@ -54,6 +56,9 @@
@param expectedRemovals: set of package names that are expected to be
removed.
@type expectedRemovals: set of package names
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
@return list of packages to send advisories for and list of packages
to update
"""
@@ -73,7 +78,8 @@
for nvf, srpm in updateTroves:
# Will raise exception if any errors are found, halting execution.
if self._sanitizeTrove(nvf, srpm,
- expectedRemovals=expectedRemovals):
+ expectedRemovals=expectedRemovals,
+ allowPackageDowngrades=allowPackageDowngrades):
toUpdate.append((nvf, srpm))
toAdvise.append((nvf, srpm))
@@ -313,7 +319,8 @@
srpms.sort(util.packagevercmp)
return srpms[-1]
- def _sanitizeTrove(self, nvf, srpm, expectedRemovals=None):
+ def _sanitizeTrove(self, nvf, srpm, expectedRemovals=None,
+ allowPackageDowngrades=None):
"""
Verifies the package update to make sure it looks correct and is a
case that the bot knows how to handle.
@@ -330,6 +337,9 @@
@param expectedRemovals: set of package names that are expected to be
removed.
@type expectedRemovals: set of package names
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
@return needsUpdate boolean
@raises UpdateGoesBackwardsError
@raises UpdateRemovesPackageError
@@ -341,6 +351,9 @@
removedPackages = set()
reusedPackages = set()
+ if allowPackageDowngrades is None:
+ allowPackageDowngrades = ()
+
try:
manifest = self._conaryhelper.getManifest(nvf[0], version=nvf[1])
except NoManifestFoundError, e:
@@ -374,9 +387,14 @@
# make sure new package is actually newer
if util.packagevercmp(srpm, srcPkg) == -1:
- log.warn('version goes backwards %s -> %s' %
- (srcPkg.getNevra(), srpm.getNevra()))
- raise UpdateGoesBackwardsError(why=(srcPkg, srpm))
+ srcTuple = (srcPkg.getNevra(), srpm.getNevra())
+ log.warn('version goes backwards %s -> %s' % srcTuple)
+ if srcTuple in allowPackageDowngrades:
+ log.info('found version downgrade exception in '
+ 'configuration')
+ needsUpdate = True
+ else:
+ raise UpdateGoesBackwardsError(why=(srcPkg, srpm))
# make sure we aren't trying to remove a package
if ((binPkg.name, binPkg.arch) not in newNames and
@@ -400,7 +418,7 @@
# binary does not come from the same source as it used to
self._pkgSource.binPkgMap[pkg].name != srpm.name):
log.warn('update removes package (%s) %s -> %s'
- % (pkg.name, srpm.getNevra(), srcPkg.getNevra()))
+ % (pkg.name, srcPkg.getNevra(), srpm.getNevra()))
# allow some packages to be removed.
if expectedRemovals and pkg.name in expectedRemovals:
@@ -412,8 +430,8 @@
if not removedPackages:
reusedPackages.add(pkg)
- #log.warn('using old version of package %s' % (pkg, ))
- #self._pkgSource.srcPkgMap[srpm].add(pkg)
+ log.warn('using old version of package %s' % (pkg, ))
+ self._pkgSource.srcPkgMap[srpm].add(pkg)
if removedPackages:
pkgList=sorted(removedPackages)
@@ -423,7 +441,7 @@
oldNevra=str(' '.join(srcPkg.getNevra())),
newNevra=str(' '.join(srpm.getNevra())))
- if reusedPackages:
+ if reusedPackages and not self._cfg.allowReusedPackages:
pkgList=sorted(reusedPackages)
raise UpdateReusesPackageError(pkgList=pkgList,
pkgNames=' '.join([str(x) for x in pkgList]),
@@ -433,12 +451,15 @@
return needsUpdate
- def sanityCheckSource(self, srpm):
+ def sanityCheckSource(self, srpm, allowPackageDowngrades=None):
"""
Look up the matching source version in the conary repository and verify
that the manifest matches the package list in the package source.
@param srpm: src pacakge object
@type srpm: repomd.packagexml._Package
+ @param allowPackageDowngrades: list of source nevra tuples to downgrade
+ from/to.
+ @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
"""
srcQuery = ('%s:source' % srpm.name, srpm.getConaryVersion(), None)
@@ -454,11 +475,17 @@
if nvflst:
return None
+ # Source hasn't been imported.
+ if not nvflst:
+ log.error('source has not been imported: %s' % srpm)
+ raise SourceNotImportedError(srpm=srpm)
+
assert len(nvflst) == 1
n, v, f = nvflst[0]
nvf = (n.split(':')[0], v, None)
- needsUpdate = self._sanitizeTrove(nvf, srpm)
+ needsUpdate = self._sanitizeTrove(nvf, srpm,
+ allowPackageDowngrades=allowPackageDowngrades)
# If anything has chnaged raise an error.
if needsUpdate:
@@ -491,12 +518,14 @@
return ret
- def create(self, pkgNames=None, buildAll=False, recreate=False, toCreate=None):
+ def create(self, pkgNames=None, buildAll=False, recreate=False,
+ toCreate=None):
"""
Import a new package into the repository.
@param pkgNames: list of packages to import
@type pkgNames: list
- @param buildAll: return a list of all troves found rather than just the new ones.
+ @param buildAll: return a list of all troves found rather than just the
+ new ones.
@type buildAll: boolean
@param recreate: a package manifest even if it already exists.
@type recreate: boolean
@@ -558,30 +587,41 @@
toBuild = set()
preBuiltPackages = set()
parentPackages = set()
+ total = len(toCreate)
+ current = 1
+ start = False
for pkg in sorted(toCreate):
+ if pkg.name.startswith('n'):
+ start = True
+
try:
# Only import packages that haven't been imported before
version = verCache.get('%s:source' % pkg.name)
if not version or recreate:
- log.info('attempting to import %s' % pkg)
+ log.info('attempting to import %s (%s/%s)'
+ % (pkg, current, total))
version = self.update((pkg.name, None, None), pkg)
- if not verCache.get(pkg.name) or buildAll or recreate:
+ if (not verCache.get(pkg.name) or
+ verCache.get(pkg.name).getSourceVersion() != version or
+ buildAll or recreate):
+
if self.isPlatformTrove(version):
- toBuild.add((pkg.name, version, None))
+ toBuild.add(((pkg.name, version, None), pkg))
else:
parentPackages.add((pkg.name, version, None))
else:
log.info('not building %s' % pkg.name)
preBuiltPackages.add((pkg.name, version, None))
except Exception, e:
- raise
log.error('failed to import %s: %s' % (pkg, e))
fail.add((pkg, e))
+ current += 1
if buildAll and pkgs and pkgNames:
toBuild.update(
- [ (x, self._conaryhelper.getLatestSourceVersion(x), None)
+ [ ((x, self._conaryhelper.getLatestSourceVersion(x), None),
+ None)
for x in pkgs if not self._fltrPkg(x) ]
)
@@ -589,7 +629,8 @@
pkgMap = {}
if parentPackages:
# Find all of the binaries that match the upstream platform sources.
- log.info('looking up binary versions of all parent platform packages')
+ log.info('looking up binary versions of all parent platform '
+ 'packages')
parentPkgMap = self.getBinaryVersions(parentPackages,
labels=self._cfg.platformSearchPath)
@@ -693,6 +734,10 @@
manifest = self._getManifestFromPkgSource(srcPkg)
self._conaryhelper.setManifest(nvf[0], manifest)
+ if self._cfg.writePackageVersion:
+ self._conaryhelper.setVersion(nvf[0], '%s_%s'
+ % (srcPkg.version, srcPkg.release))
+
# FIXME: This is apt specific for now. Once repomd has been rewritten
# to use something other than rpath-xmllib we should be able to
# convert this to xobj.
@@ -827,7 +872,8 @@
src = self._pkgSource.binPkgMap[latest]
srcname = src.name
else:
- log.warn('found virtual requires %s in pkg %s' % (name, srcPkg.name))
+ log.warn('found virtual requires %s in pkg %s'
+ % (name, srcPkg.name))
srcname = 'virtual'
reqs.append((name, srcname))
From elliot at rpath.com Fri May 7 11:43:47 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:47 +0000
Subject: mirrorball: sort packages
Message-ID: <201005071543.o47Fhlv9009577@scc.eng.rpath.com>
changeset: 4e3cb8292b7b
user: Elliot Peele
date: Fri, 07 May 2010 11:28:54 -0400
sort packages
diff --git a/scripts/gengroupmodel b/scripts/gengroupmodel
--- a/scripts/gengroupmodel
+++ b/scripts/gengroupmodel
@@ -93,6 +93,13 @@
current = latest
+ def srtChangesByPkgName(a, b):
+ cmpa = ' '.join(a.split()[1:])
+ cmpb = ' '.join(b.split()[1:])
+ return cmp(cmpa, cmpb)
+
+ changes.sort(cmp=srtChangesByPkgName)
+
return changes
From elliot at rpath.com Fri May 7 11:43:48 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:48 +0000
Subject: mirrorball: fix typo and make sure srcName always includes
:source
Message-ID: <201005071543.o47Fhmw5009610@scc.eng.rpath.com>
changeset: a84261661bc2
user: Elliot Peele
date: Fri, 07 May 2010 11:29:25 -0400
fix typo and make sure srcName always includes :source
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -77,7 +77,7 @@
elif parentGroup:
srcName = self._cfg.topParentSourceGroup[0]
srcLabel = self._cfg.topParentSourceGroup[1]
- labels = self._cfg.platforSearchPath
+ labels = self._cfg.platformSearchPath
self._sourceName = srcName
self._sourceLabel = srcLabel
@@ -86,7 +86,8 @@
# FIXME: Should figure out a better way to handle package group.
self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName
- assert self._sourceName.endswith(':source')
+ if not srcName.endswith(':source'):
+ srcName = '%s:source' % srcName
self._readOnly = False
if targetGroup or parentGroup:
From elliot at rpath.com Fri May 7 11:43:48 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 07 May 2010 15:43:48 +0000
Subject: mirrorball: branch merge
Message-ID: <201005071543.o47FhmDX009638@scc.eng.rpath.com>
changeset: dc8f6d485cc4
user: Elliot Peele
date: Fri, 07 May 2010 11:41:36 -0400
branch merge
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -318,6 +318,12 @@
tiMap = {}
tiLst = self._repos.getTroveInfo(tiType, req)
for i, nvf in enumerate(uncached):
+ # If this trove doesn't have this piece of trove info, log a warning
+ # and skip over it.
+ if tiLst[i] is None:
+ log.warn('found missing trove info for %s, skipping' % (nvf, ))
+ continue
+
ti = tiLst[i]()
if tiFunc:
ti = tiFunc(ti, nvf)
From agrimm at rpath.com Sat May 8 14:33:50 2010
From: agrimm at rpath.com (agrimm at rpath.com)
Date: Sat, 08 May 2010 18:33:50 +0000
Subject: mirrorball: fix some typos, and things related to SL5 update
Message-ID: <201005081833.o48IXotK031991@scc.eng.rpath.com>
changeset: 1bcec7def5bc
user: Andy Grimm
date: Sat, 08 May 2010 14:31:59 -0400
fix some typos, and things related to SL5 update
diff --git a/scripts/gengroup.py b/scripts/gengroup.py
--- a/scripts/gengroup.py
+++ b/scripts/gengroup.py
@@ -84,7 +84,7 @@
mgr.setVersion('0')
mgr.setErrataState('0')
mgr._copyVersions()
-mgr._validateGroups()
+mgr._sanity.check(mgr._groups, mgr.getErrataState())
mgr._helper.setModel(mgr._sourceName, mgr._groups)
#mgr._commit()
#mgr.build()
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -401,7 +401,7 @@
# Allow updates which don't include all binary packages corresponding
# to a given source.
- allowReusedPackages = (CfgBool, False)
+ allowRemovedPackages = (CfgBool, False)
# Add a source to a specific updateId. This is used to move updates forward
# after allowing an update to downgrade the version.
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -123,6 +123,7 @@
'x86_64' in pkg.location):
continue
+ # Source RPM is one without a "sourcerpm" element
if pkg.sourcerpm == '' or pkg.sourcerpm is None:
self._procSrc(pkg)
else:
@@ -202,7 +203,7 @@
def _excludeLocation(self, location):
"""
- Method for filtering packages based on locaiton.
+ Method for filtering packages based on location.
"""
return False
@@ -222,19 +223,21 @@
# structures.
count = 0
toDelete = set()
+ srcToDelete = set()
for pkg in self._srcPkgs:
key = (pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch)
if pkg in self.srcPkgMap:
continue
if key not in self._rpmMap:
- #log.warn('found source without binary rpms: %s' % pkg)
+ log.warn('found source without binary rpms: %s' % pkg)
#log.debug(key)
#log.debug([ x for x in self._rpmMap if x[0] == key[0] ])
count += 1
if pkg in self.srcNameMap[pkg.name]:
self.srcNameMap[pkg.name].remove(pkg)
+ srcToDelete.add(pkg)
continue
self.srcPkgMap[pkg] = self._rpmMap[key]
@@ -248,6 +251,10 @@
log.warn('found %s source rpms without matching binary '
'rpms' % count)
+ # Remove references to sources that don't match binaries
+ for pkg in srcToDelete:
+ self._srcPkgs.remove(pkg)
+
# Defer deletes, contents of rpmMap are used more than once.
for key in toDelete:
del self._rpmMap[key]
@@ -334,7 +341,7 @@
if not self._cfg.synthesizeSources:
return
- deffer = set()
+ defer = set()
# Create a fake source rpm object for each key in the rpmMap.
for nevra, bins in self._rpmMap.iteritems():
srcPkg = getSourcePackage(nevra, bins)
@@ -343,7 +350,7 @@
# care of with the epoch fuzzing that happens in finalize. This
# should only happen with differently named packages.
if nevra[0] not in [ x.name for x in bins ]:
- deffer.add(nevra)
+ defer.add(nevra)
continue
# add source to structures
@@ -360,7 +367,7 @@
broken = set()
# Make an attempt to sort out the binaries that have different names
# than the related sources.
- for nevra in deffer:
+ for nevra in defer:
bins = self._rpmMap[nevra]
srcPkg = getSourcePackage(nevra, bins)
@@ -377,6 +384,18 @@
log.warn('found binary without matching source name %s'
% list(bins)[0].name)
+ # add source to structures
+ if nevra not in self._srcMap:
+ log.warn('synthesizing source package %s' % srcPkg)
+ self._procSrc(srcPkg)
+
+ # Add location mappings for packages that may have once been
+ # synthesized so that parsing old manifest files still works.
+ elif srcPkg.location not in self.locationMap:
+ pkg = self._srcMap[nevra]
+ self.locationMap[srcPkg.location] = pkg
+
+
broken.add((nevra, tuple(bins)))
# Raise an exception if this ever happens. We can figure out the right
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -369,17 +369,21 @@
if line in self._pkgSource.locationMap:
binPkg = self._pkgSource.locationMap[line]
srcPkg = self._pkgSource.binPkgMap[binPkg]
+ elif line.strip().endswith('.src.rpm') and self._cfg.synthesizeSources:
+ # this is a fake source. Move on.
+ continue
else:
if metadata is None:
pkgs = self._getMetadataFromConaryRepository(nvf[0],
version=nvf[1])
- metadata = util.Metadata(pkgs)
- if metadata:
- binPkg = metadata.locationMap[line]
- srcPkg = metadata.binPkgMap[binPkg]
- else:
- raise OldVersionNotFoundError(
- why="can't find metadata for %s" % line)
+ if pkgs:
+ metadata = util.Metadata(pkgs)
+ if metadata:
+ binPkg = metadata.locationMap[line]
+ srcPkg = metadata.binPkgMap[binPkg]
+ else:
+ raise OldVersionNotFoundError(
+ why="can't find metadata for %s" % line)
# set needsUpdate if version changes
if util.packagevercmp(srpm, srcPkg) == 1:
@@ -413,7 +417,7 @@
if (rpmvercmp(pkg.epoch, srpm.epoch) != 0 or
rpmvercmp(pkg.version, srpm.version) != 0 or
# in the suse case we have to ignore release
- (self._cfg.reuseOldRevisions or
+ (not self._cfg.reuseOldRevisions and
rpmvercmp(pkg.release, srpm.release) != 0) or
# binary does not come from the same source as it used to
self._pkgSource.binPkgMap[pkg].name != srpm.name):
@@ -433,7 +437,7 @@
log.warn('using old version of package %s' % (pkg, ))
self._pkgSource.srcPkgMap[srpm].add(pkg)
- if removedPackages:
+ if removedPackages and not self._cfg.allowRemovedPackages:
pkgList=sorted(removedPackages)
raise UpdateRemovesPackageError(pkgList=pkgList,
pkgNames=' '.join([str(x) for x in pkgList]),
@@ -441,7 +445,7 @@
oldNevra=str(' '.join(srcPkg.getNevra())),
newNevra=str(' '.join(srpm.getNevra())))
- if reusedPackages and not self._cfg.allowReusedPackages:
+ if reusedPackages and not self._cfg.reuseOldRevisions:
pkgList=sorted(reusedPackages)
raise UpdateReusesPackageError(pkgList=pkgList,
pkgNames=' '.join([str(x) for x in pkgList]),
From agrimm at rpath.com Sat May 8 14:33:51 2010
From: agrimm at rpath.com (agrimm at rpath.com)
Date: Sat, 08 May 2010 18:33:51 +0000
Subject: mirrorball: branch merge
Message-ID: <201005081833.o48IXpWj032018@scc.eng.rpath.com>
changeset: 7db3144a66aa
user: Andy Grimm
date: Sat, 08 May 2010 14:33:55 -0400
branch merge
diff --git a/scripts/gengroup.py b/scripts/gengroup.py
--- a/scripts/gengroup.py
+++ b/scripts/gengroup.py
@@ -84,7 +84,7 @@
mgr.setVersion('0')
mgr.setErrataState('0')
mgr._copyVersions()
-mgr._validateGroups()
+mgr._sanity.check(mgr._groups, mgr.getErrataState())
mgr._helper.setModel(mgr._sourceName, mgr._groups)
#mgr._commit()
#mgr.build()
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -458,7 +458,7 @@
# Allow updates which don't include all binary packages corresponding
# to a given source.
- allowReusedPackages = (CfgBool, False)
+ allowRemovedPackages = (CfgBool, False)
# Add a source to a specific updateId. This is used to move updates forward
# after allowing an update to downgrade the version.
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -128,6 +128,7 @@
'x86_64' in pkg.location):
continue
+ # Source RPM is one without a "sourcerpm" element
if pkg.sourcerpm == '' or pkg.sourcerpm is None:
self._procSrc(pkg)
else:
@@ -214,7 +215,7 @@
def _excludeLocation(self, location):
"""
- Method for filtering packages based on locaiton.
+ Method for filtering packages based on location.
"""
return False
@@ -234,19 +235,21 @@
# structures.
count = 0
toDelete = set()
+ srcToDelete = set()
for pkg in self._srcPkgs:
key = (pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch)
if pkg in self.srcPkgMap:
continue
if key not in self._rpmMap:
- #log.warn('found source without binary rpms: %s' % pkg)
+ log.warn('found source without binary rpms: %s' % pkg)
#log.debug(key)
#log.debug([ x for x in self._rpmMap if x[0] == key[0] ])
count += 1
if pkg in self.srcNameMap[pkg.name]:
self.srcNameMap[pkg.name].remove(pkg)
+ srcToDelete.add(pkg)
continue
self.srcPkgMap[pkg] = self._rpmMap[key]
@@ -278,6 +281,10 @@
log.warn('found %s source rpms without matching binary '
'rpms' % count)
+ # Remove references to sources that don't match binaries
+ for pkg in srcToDelete:
+ self._srcPkgs.remove(pkg)
+
# Defer deletes, contents of rpmMap are used more than once.
for key in toDelete:
del self._rpmMap[key]
@@ -398,7 +405,7 @@
if not self._cfg.synthesizeSources:
return
- deffer = set()
+ defer = set()
# Create a fake source rpm object for each key in the rpmMap.
for nevra, bins in self._rpmMap.iteritems():
srcPkg = getSourcePackage(nevra, bins)
@@ -407,7 +414,7 @@
# care of with the epoch fuzzing that happens in finalize. This
# should only happen with differently named packages.
if nevra[0] not in [ x.name for x in bins ]:
- deffer.add(nevra)
+ defer.add(nevra)
continue
# Synthesize the source
@@ -416,7 +423,7 @@
broken = set()
# Make an attempt to sort out the binaries that have different names
# than the related sources.
- for nevra in deffer:
+ for nevra in defer:
bins = self._rpmMap[nevra]
srcPkg = getSourcePackage(nevra, bins)
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -369,17 +369,21 @@
if line in self._pkgSource.locationMap:
binPkg = self._pkgSource.locationMap[line]
srcPkg = self._pkgSource.binPkgMap[binPkg]
+ elif line.strip().endswith('.src.rpm') and self._cfg.synthesizeSources:
+ # this is a fake source. Move on.
+ continue
else:
if metadata is None:
pkgs = self._getMetadataFromConaryRepository(nvf[0],
version=nvf[1])
- metadata = util.Metadata(pkgs)
- if metadata:
- binPkg = metadata.locationMap[line]
- srcPkg = metadata.binPkgMap[binPkg]
- else:
- raise OldVersionNotFoundError(
- why="can't find metadata for %s" % line)
+ if pkgs:
+ metadata = util.Metadata(pkgs)
+ if metadata:
+ binPkg = metadata.locationMap[line]
+ srcPkg = metadata.binPkgMap[binPkg]
+ else:
+ raise OldVersionNotFoundError(
+ why="can't find metadata for %s" % line)
# set needsUpdate if version changes
if util.packagevercmp(srpm, srcPkg) == 1:
@@ -413,7 +417,7 @@
if (rpmvercmp(pkg.epoch, srpm.epoch) != 0 or
rpmvercmp(pkg.version, srpm.version) != 0 or
# in the suse case we have to ignore release
- (self._cfg.reuseOldRevisions or
+ (not self._cfg.reuseOldRevisions and
rpmvercmp(pkg.release, srpm.release) != 0) or
# binary does not come from the same source as it used to
self._pkgSource.binPkgMap[pkg].name != srpm.name):
@@ -433,7 +437,7 @@
log.warn('using old version of package %s' % (pkg, ))
self._pkgSource.srcPkgMap[srpm].add(pkg)
- if removedPackages:
+ if removedPackages and not self._cfg.allowRemovedPackages:
pkgList=sorted(removedPackages)
raise UpdateRemovesPackageError(pkgList=pkgList,
pkgNames=' '.join([str(x) for x in pkgList]),
@@ -441,7 +445,7 @@
oldNevra=str(' '.join(srcPkg.getNevra())),
newNevra=str(' '.join(srpm.getNevra())))
- if reusedPackages and not self._cfg.allowReusedPackages:
+ if reusedPackages and not self._cfg.reuseOldRevisions:
pkgList=sorted(reusedPackages)
raise UpdateReusesPackageError(pkgList=pkgList,
pkgNames=' '.join([str(x) for x in pkgList]),
From elliot at rpath.com Mon May 10 17:24:53 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:53 +0000
Subject: mirrorball: add support for passing in versions for more methods
Message-ID: <201005102124.o4ALOr0I007697@scc.eng.rpath.com>
changeset: 43b7c0e2b4a8
user: Elliot Peele
date: Mon, 10 May 2010 17:12:29 -0400
add support for passing in versions for more methods
try to always transform package names from foo:source -> foo
fix typos
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -694,25 +694,27 @@
version = open(versionFileName).read().strip()
return version
- def setVersion(self, pkgname, version):
+ def setVersion(self, pkgname, upver, version=None):
"""
Set the version of the specified package, for this to be meaningful
there must be a factory that consumes this data.
@param pkgname: name of hte package to edit
@type pkgname: string
- @param version: upstream version of the package, required to be a valid
+ @param upver: upstream version of the package, required to be a valid
conary version.
- @type version: string
+ @type upver: string
+ @param version: optional source version to checkout.
+ @type version: conary.versions.Version
"""
log.info('setting version info for %s' % pkgname)
- recipeDir = self._edit(pkgname)
+ recipeDir = self._edit(pkgname, version=version)
versionFileName = util.join(recipeDir, 'version')
# write version info
versionfh = open(versionFileName, 'w')
- versionfh.write(version)
+ versionfh.write(upver)
# source files must end in a trailing newline
versionfh.write('\n')
@@ -735,6 +737,8 @@
@return version of the source commit.
"""
+ pkgname = self._convSrcName(pkgname)
+
pkgkey = (pkgname, version)
if pkgkey not in self._checkoutCache:
raise NoCheckoutFoundError(pkgname=pkgname)
@@ -752,6 +756,16 @@
assert version is not None
return version
+ def _convSrcName(self, name):
+ """
+ Strip the :source off the end of a name if it is there.
+ """
+
+ # make sure package name does not include :source.
+ if name.endswith(':source'):
+ name = name.split(':')[0]
+ return name
+
def _edit(self, pkgname, version=None):
"""
Checkout/Create source checkout.
@@ -762,6 +776,8 @@
@return path to checkout
"""
+ pkgname = self._convSrcName(pkgname)
+
pkgkey = (pkgname, version)
if pkgkey in self._checkoutCache:
return self._checkoutCache[pkgkey]
@@ -1140,7 +1156,7 @@
Set metadata on a given trove spec.
"""
- if not license and not dec and not shortDesc:
+ if not license and not desc and not shortDesc:
log.warn('no metadata found for %s' % trvSpecs[-1][0])
return
From elliot at rpath.com Mon May 10 17:24:53 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:53 +0000
Subject: mirrorball: add flag for select serve
Message-ID: <201005102124.o4ALOrVw007728@scc.eng.rpath.com>
changeset: d12f41babf23
user: Elliot Peele
date: Mon, 10 May 2010 17:13:05 -0400
add flag for select serve
diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py
--- a/updatebot/lib/util.py
+++ b/updatebot/lib/util.py
@@ -172,13 +172,16 @@
limit = getRLimit()
return limit - openfds
-def setupDebugHandler():
+def setupDebugHandler(serve=False):
"""
Sets up a USR1 signal handler to trigger epdb.serv().
"""
def handler(signum, sigtb):
- epdb.serve()
+ if serve:
+ epdb.serve()
+ else:
+ epdb.st()
signal.signal(signal.SIGUSR1, handler)
From elliot at rpath.com Mon May 10 17:24:54 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:54 +0000
Subject: mirrorball: extend the usemap to be a bit more flexible about
what rpms could possibly
Message-ID: <201005102124.o4ALOslG007755@scc.eng.rpath.com>
changeset: 48caa66f8259
user: Elliot Peele
date: Mon, 10 May 2010 17:16:37 -0400
extend the usemap to be a bit more flexible about what rpms could possibly
generate what conary flavors and make sure to always include the source name
for every arch.
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -331,19 +331,30 @@
# get the conary version from the source
conaryVersion = srcPkg.getConaryVersion()
+ # If the package arch is x86_64, I expect that it should only
+ # ever be built as x86_64.
if binPkg.arch == 'x86_64':
- arch = 'x86_64'
+ arch = ['x86_64', ]
+ # If the package arch is noarch, it could produce a conary
+ # package of any flavor.
elif binPkg.arch == 'noarch':
- arch = ''
+ arch = ['x86', 'x86_64', '']
+ # If the package arch is x86, it could produce either x86 or
+ # x86_64 flavors.
elif (binPkg.arch.startswith('i') and
binPkg.arch.endswith('86') and
len(binPkg.arch) == 4):
- arch = 'x86'
+ arch = ['x86', 'x86_64']
else:
raise RuntimeError
- trvSpec = (binPkg.name, conaryVersion, arch)
- self.useMap.setdefault(trvSpec, set()).update(archStr)
+ for a in arch:
+ trvSpecs = set([
+ (binPkg.name, conaryVersion, a),
+ (srcPkg.name, conaryVersion, a),
+ ])
+ for trvSpec in trvSpecs:
+ self.useMap.setdefault(trvSpec, set()).update(archStr)
def loadFileLists(self, client, basePath):
"""
From elliot at rpath.com Mon May 10 17:24:54 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:54 +0000
Subject: mirrorball: all consumers to specify which version to change for
errataState and version
Message-ID: <201005102124.o4ALOs9p007782@scc.eng.rpath.com>
changeset: daa17504ad14
user: Elliot Peele
date: Mon, 10 May 2010 17:20:33 -0400
all consumers to specify which version to change for errataState and version
diff --git a/updatebot/groupmgr/helper.py b/updatebot/groupmgr/helper.py
--- a/updatebot/groupmgr/helper.py
+++ b/updatebot/groupmgr/helper.py
@@ -122,13 +122,13 @@
return groups
- def setModel(self, pkgName, groups):
+ def setModel(self, pkgName, groups, version=None):
"""
Freeze group model and save to the repository.
"""
log.info('saving model for %s' % pkgName)
- recipeDir = self._edit(pkgName)
+ recipeDir = self._edit(pkgName, version=version)
groupFileName = util.join(recipeDir, 'groups.xml')
groupModel = GroupModel()
@@ -164,14 +164,14 @@
state = int(state)
return state
- def setErrataState(self, pkgname, state):
+ def setErrataState(self, pkgname, state, version=None):
"""
Set the current errata state for the given package.
"""
log.info('storing errata state information in %s' % pkgname)
- recipeDir = self._edit(pkgname)
+ recipeDir = self._edit(pkgname, version=version)
stateFileName = util.join(recipeDir, 'erratastate')
# write state info
From elliot at rpath.com Mon May 10 17:24:54 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:54 +0000
Subject: mirrorball: warn if a flavor of a package is in the repository,
but not in the model iff the
Message-ID: <201005102124.o4ALOsr4007809@scc.eng.rpath.com>
changeset: d94ad7152342
user: Elliot Peele
date: Mon, 10 May 2010 17:21:32 -0400
warn if a flavor of a package is in the repository, but not in the model iff the
alternate flavor is an older version.
diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py
--- a/updatebot/groupmgr/sanity.py
+++ b/updatebot/groupmgr/sanity.py
@@ -214,6 +214,15 @@
log.info('found %s=%s[%s] in oldVersions exceptions'
% (n, v, f))
continue
+
+ # This is probably a flavor that we don't care about
+ # anymore.
+ if cv > v and cv in [ x[1] for x in found ]:
+ log.warn('missing flavors found of %s that are not all '
+ 'included in the group, assuming this '
+ 'intentional.' % cn)
+ continue
+
foundError = True
if foundError:
From elliot at rpath.com Mon May 10 17:24:54 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:54 +0000
Subject: mirrorball: 1. don't overwrite usemap if it is empty, we still
need the reference
Message-ID: <201005102124.o4ALOtjq007836@scc.eng.rpath.com>
changeset: a003462093d1
user: Elliot Peele
date: Mon, 10 May 2010 17:22:30 -0400
1. don't overwrite usemap if it is empty, we still need the reference
2. use the correct keyword arg for getGroup
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -61,7 +61,12 @@
def __init__(self, cfg, parentGroup=False, targetGroup=False, useMap=None):
self._cfg = cfg
- self._useMap = useMap or dict()
+
+ if useMap is None:
+ self._useMap = {}
+ else:
+ self._useMap = useMap
+
self._helper = self._helperClass(self._cfg)
self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups')
self._sanity = self._sanityCheckerClass(self._cfg, self._helper)
@@ -270,7 +275,7 @@
group.setCommitted()
# Get the model for the source version that we just committed.
- return self.getGroup(sourceVersion=newVersion)
+ return self.getGroup(version=newVersion)
@require_write
def buildGroup(self, group):
From elliot at rpath.com Mon May 10 17:24:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:55 +0000
Subject: mirrorball: several fixups to make new group model function
Message-ID: <201005102124.o4ALOtlm007863@scc.eng.rpath.com>
changeset: 7446a1f8a4c3
user: Elliot Peele
date: Mon, 10 May 2010 17:23:01 -0400
several fixups to make new group model function
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -176,13 +176,12 @@
###
@require_write
- def _add(self, groupName=None, *args, **kwargs):
+ def _add(self, *args, **kwargs):
"""
Add a trove to the package group contents.
"""
- if not groupName:
- groupName = self._pkgGroupName
+ groupName = kwargs.pop('groupName', self._pkgGroupName)
# create package group model if it does not exist.
if groupName not in self._groups:
@@ -205,6 +204,9 @@
@type flavors: [conary.deps.deps.Flavor, ...]
"""
+ if not groupName:
+ groupName = self._pkgGroupName
+
# Now that versions are actually used for something make sure they
# are always present.
assert version
@@ -214,7 +216,7 @@
# Remove all versions and flavors of this name before adding this
# package. This avoids flavor change issues by replacing all flavors.
if self.hasPackage(name):
- self.remove(name)
+ self.removePackage(name)
plain = deps.parseFlavor('')
x86 = deps.parseFlavor('is: x86')
@@ -243,26 +245,30 @@
raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
def add():
- upver = version.trailingRevision().version()
+ upver = version.trailingRevision().version
for flv in flavors:
- if self._useMap:
- for useStr in self._useMap[(name, upver, flvMap[flv])]:
- self._add(name, version=version, flavor=flavor,
+ key = (name, upver, flvMap[flv])
+ if key in self._useMap:
+ for useStr in self._useMap[key]:
+ self._add(name, version=version, flavor=flv,
use=useStr, groupName=groupName)
else:
- self._add(name, version=version, flavor=flavor,
- use=flvMap[flv], groupName=groupName)
+ log.warn('%s=%s[%s] not found in useMap, falling back to '
+ 'old method of adding troves to groups'
+ % (name, version, flvMap[flv]))
+ self._add(name, version=version, flavor=flv,
+ use=flvMap[flv], groupName=groupName)
# If this package has one or two flavors and one of those flavors is
# x86, x86_64, biarch, or plain then handle it like a normal package
# without doing any more sanity checking.
total = 0
for flv, count in flvCount.iteritems():
- if len(count) > 1:
+ if count > 1:
break
total += count
else:
- if total in (1, 2):
+ if total in (1, 2, 3):
add()
return
@@ -274,7 +280,7 @@
# Check if this package is configured to have multiple flavors.
# Get source trove name.
log.info('retrieving trove info for %s' % name)
- srcTroveMap = self._helper._getSourceTroves((name, version, flavors[0]))
+ srcTroveMap = self._mgr._helper._getSourceTroves((name, version, flavors[0]))
srcTroveName = srcTroveMap.keys()[0][0].split(':')[0]
# Check if this is package that we have specifically defined a build
@@ -284,25 +290,30 @@
# TODO: If we were really smart we would load the conary
# contexts and see what buildFlavors they contained.
flavorCtxCount = {x86: 0, x86_64: 0, biarch: 0}
+ ctxMap = dict([ (x, y[1]) for x, y in self._cfg.archContexts if y ])
for context, bldflv in self._cfg.packageFlavors[srcTroveName]:
+ fltr = ctxMap.get(context, None)
if context in ('i386', 'i486', 'i586', 'i686', 'x86'):
flavorCtxCount[x86] += 1
elif context in ('x86_64', ):
flavorCtxCount[x86_64] += 1
elif context in ('biarch', ):
- flavorCtxCount[biarch] += 1
+ if fltr and fltr.match(name):
+ flavorCtxCount[biarch] += 1
else:
raise UnknownBuildContextError(name=name, flavor=context)
# Sanity check flavors to make sure we built all the flavors
# that we expected.
- extra = set(flvMap) - set([ x86, x86_64, biarch ])
- if extra:
- raise UnsupportedTroveFlavorError(name=name, flavor=extra)
-
if (flvCount[x86] != flavorCtxCount[x86] or
flvCount[x86_64] != flavorCtxCount[x86_64] or
- flvCount[biarch] != flavorCtxCount[biarch]):
+
+ # Only enforce biarch for packages that we expect to be biarch.
+ # This is a kluge to deal with the fact that biarch builds
+ # produce a byDefault=False package for the source that only
+ # contains the build log.
+ (flavorCtxCount[biarch] > 0 and
+ flvCount[biarch] != flavorCtxCount[biarch])):
raise FlavorCountMismatchError(name=name)
# Add packages to the group.
@@ -375,7 +386,7 @@
if pkgFlv:
group.removePackageFlavor(pkgName, pkgFlv)
else:
- self.remove(pkgName)
+ self.removePackage(pkgName)
# Add requested packages.
for groupName, pkgs in additions.iteritems():
From elliot at rpath.com Mon May 10 17:24:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:55 +0000
Subject: mirrorball: use the new model
Message-ID: <201005102124.o4ALOt1s007890@scc.eng.rpath.com>
changeset: 78736855fe46
user: Elliot Peele
date: Mon, 10 May 2010 17:23:21 -0400
use the new model
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -35,7 +35,6 @@
import logging
-from updatebot import groupmgr
from updatebot import OrderedBot
from updatebot.groupmgr.model import GroupContentsModel
@@ -516,34 +515,48 @@
troves = self._updater._conaryhelper._getLatestTroves()
- pkgs = set()
+ # combine packages of the same name.
+ trvs = {}
for name, vMap in troves.iteritems():
if name.endswith(':source'):
continue
name = name.split(':')[0]
for version, flavors in vMap.iteritems():
- pkgs.add((name, version, tuple(flavors)))
+ for flv in flavors:
+ trvs.setdefault(name, dict()).setdefault(version, set()).add(flv)
+
+ pkgs = set()
+ for name, vMap in trvs.iteritems():
+ if name.endswith(':source'):
+ continue
+ name = name.split(':')[0]
+ for version, flavors in vMap.iteritems():
+ data = (name, version, tuple(flavors))
+ pkgs.add(data)
+
+ group = self._groupmgr.getGroup()
for name, version, flavors in pkgs:
log.info('adding %s=%s' % (name, version))
for flv in flavors:
log.info('\t%s' % flv)
- self._groupmgr.addPackage(name, version, flavors)
+ group.addPackage(name, version, flavors)
- self._groupmgr.setErrataState(0)
- self._groupmgr.setVersion('0')
+ group.errataState = '0'
+ group.version = '0'
- pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
+# pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
# pkgGroup.depCheck = False
- contents = GroupContentsModel('group-standard', depCheck=True)
- self._groupmgr._groups['group-standard'] = contents
+ addReq = dict([ ('group-standard', [ (x, None) for x in standard ]), ])
- for pkgName in standard:
- for key in pkgGroup._nameMap[pkgName]:
- contents._addItem(pkgGroup._data[key])
+ group.modifyContents(additions=addReq)
- built = self._groupmgr.build()
+ group.commit()
+ built = group.build()
+
+ import epdb; epdb.st()
+
return built
if __name__ == '__main__':
@@ -558,6 +571,7 @@
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
bot = Bot(cfg, None)
+ bot._pkgSource.load()
changes = bot.generateInitialGroup()
import epdb; epdb.st()
From elliot at rpath.com Mon May 10 17:24:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:55 +0000
Subject: mirrorball: use the package list in the manifest as a good enough
list of names to filter contexts
Message-ID: <201005102124.o4ALOt8m007918@scc.eng.rpath.com>
changeset: e7cceaa0c05c
user: Elliot Peele
date: Mon, 10 May 2010 17:24:19 -0400
use the package list in the manifest as a good enough list of names to filter contexts
diff --git a/scripts/buildpackages b/scripts/buildpackages
--- a/scripts/buildpackages
+++ b/scripts/buildpackages
@@ -10,6 +10,7 @@
import os
from header import *
+from updatebot import errors
from updatebot import conaryhelper
if len(sys.argv) < 3:
@@ -19,17 +20,22 @@
def validateInput(input):
for pkgName in input:
- manifest = helper.getManifest(pkgName)
+ try:
+ manifest = helper.getManifest(pkgName)
+ except errors.NoManifestFoundError, e:
+ yield pkgName, None
+ continue
paths = [ os.path.basename(x) for x in manifest ]
- for context, fltr in cfg.archContexts:
- if fltr and [ x for x in paths if fltr[1].match(x) ]:
- raise RuntimeError, 'Found package that may not be built'
- return input
+# for context, fltr in cfg.archContexts:
+# if fltr and [ x for x in paths if fltr[1].match(x) ]:
+# raise RuntimeError, 'Found package that may not be built'
+ yield (pkgName, tuple(paths))
+ raise StopIteration
trvs = set()
label = cfg.topSourceGroup[1]
-for pkg in validateInput(sys.argv[2:]):
- trvs.add((pkg, label, None))
+for pkg, manifest in validateInput(sys.argv[2:]):
+ trvs.add((pkg, label, None, manifest))
trvMap = builder.build(trvs)
print "built:\n"
From elliot at rpath.com Mon May 10 17:24:56 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 10 May 2010 21:24:56 +0000
Subject: mirrorball: branch merge
Message-ID: <201005102124.o4ALOuwd007947@scc.eng.rpath.com>
changeset: f4c60c5d6b9f
user: Elliot Peele
date: Mon, 10 May 2010 17:24:42 -0400
branch merge
diff --git a/scripts/buildpackages b/scripts/buildpackages
--- a/scripts/buildpackages
+++ b/scripts/buildpackages
@@ -10,6 +10,7 @@
import os
from header import *
+from updatebot import errors
from updatebot import conaryhelper
if len(sys.argv) < 3:
@@ -19,17 +20,22 @@
def validateInput(input):
for pkgName in input:
- manifest = helper.getManifest(pkgName)
+ try:
+ manifest = helper.getManifest(pkgName)
+ except errors.NoManifestFoundError, e:
+ yield pkgName, None
+ continue
paths = [ os.path.basename(x) for x in manifest ]
- for context, fltr in cfg.archContexts:
- if fltr and [ x for x in paths if fltr[1].match(x) ]:
- raise RuntimeError, 'Found package that may not be built'
- return input
+# for context, fltr in cfg.archContexts:
+# if fltr and [ x for x in paths if fltr[1].match(x) ]:
+# raise RuntimeError, 'Found package that may not be built'
+ yield (pkgName, tuple(paths))
+ raise StopIteration
trvs = set()
label = cfg.topSourceGroup[1]
-for pkg in validateInput(sys.argv[2:]):
- trvs.add((pkg, label, None))
+for pkg, manifest in validateInput(sys.argv[2:]):
+ trvs.add((pkg, label, None, manifest))
trvMap = builder.build(trvs)
print "built:\n"
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -35,7 +35,6 @@
import logging
-from updatebot import groupmgr
from updatebot import OrderedBot
from updatebot.groupmgr.model import GroupContentsModel
@@ -516,34 +515,48 @@
troves = self._updater._conaryhelper._getLatestTroves()
- pkgs = set()
+ # combine packages of the same name.
+ trvs = {}
for name, vMap in troves.iteritems():
if name.endswith(':source'):
continue
name = name.split(':')[0]
for version, flavors in vMap.iteritems():
- pkgs.add((name, version, tuple(flavors)))
+ for flv in flavors:
+ trvs.setdefault(name, dict()).setdefault(version, set()).add(flv)
+
+ pkgs = set()
+ for name, vMap in trvs.iteritems():
+ if name.endswith(':source'):
+ continue
+ name = name.split(':')[0]
+ for version, flavors in vMap.iteritems():
+ data = (name, version, tuple(flavors))
+ pkgs.add(data)
+
+ group = self._groupmgr.getGroup()
for name, version, flavors in pkgs:
log.info('adding %s=%s' % (name, version))
for flv in flavors:
log.info('\t%s' % flv)
- self._groupmgr.addPackage(name, version, flavors)
+ group.addPackage(name, version, flavors)
- self._groupmgr.setErrataState(0)
- self._groupmgr.setVersion('0')
+ group.errataState = '0'
+ group.version = '0'
- pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
+# pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
# pkgGroup.depCheck = False
- contents = GroupContentsModel('group-standard', depCheck=True)
- self._groupmgr._groups['group-standard'] = contents
+ addReq = dict([ ('group-standard', [ (x, None) for x in standard ]), ])
- for pkgName in standard:
- for key in pkgGroup._nameMap[pkgName]:
- contents._addItem(pkgGroup._data[key])
+ group.modifyContents(additions=addReq)
- built = self._groupmgr.build()
+ group.commit()
+ built = group.build()
+
+ import epdb; epdb.st()
+
return built
if __name__ == '__main__':
@@ -558,6 +571,7 @@
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1])
bot = Bot(cfg, None)
+ bot._pkgSource.load()
changes = bot.generateInitialGroup()
import epdb; epdb.st()
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -694,25 +694,27 @@
version = open(versionFileName).read().strip()
return version
- def setVersion(self, pkgname, version):
+ def setVersion(self, pkgname, upver, version=None):
"""
Set the version of the specified package, for this to be meaningful
there must be a factory that consumes this data.
@param pkgname: name of hte package to edit
@type pkgname: string
- @param version: upstream version of the package, required to be a valid
+ @param upver: upstream version of the package, required to be a valid
conary version.
- @type version: string
+ @type upver: string
+ @param version: optional source version to checkout.
+ @type version: conary.versions.Version
"""
log.info('setting version info for %s' % pkgname)
- recipeDir = self._edit(pkgname)
+ recipeDir = self._edit(pkgname, version=version)
versionFileName = util.join(recipeDir, 'version')
# write version info
versionfh = open(versionFileName, 'w')
- versionfh.write(version)
+ versionfh.write(upver)
# source files must end in a trailing newline
versionfh.write('\n')
@@ -735,6 +737,8 @@
@return version of the source commit.
"""
+ pkgname = self._convSrcName(pkgname)
+
pkgkey = (pkgname, version)
if pkgkey not in self._checkoutCache:
raise NoCheckoutFoundError(pkgname=pkgname)
@@ -752,6 +756,16 @@
assert version is not None
return version
+ def _convSrcName(self, name):
+ """
+ Strip the :source off the end of a name if it is there.
+ """
+
+ # make sure package name does not include :source.
+ if name.endswith(':source'):
+ name = name.split(':')[0]
+ return name
+
def _edit(self, pkgname, version=None):
"""
Checkout/Create source checkout.
@@ -762,6 +776,8 @@
@return path to checkout
"""
+ pkgname = self._convSrcName(pkgname)
+
pkgkey = (pkgname, version)
if pkgkey in self._checkoutCache:
return self._checkoutCache[pkgkey]
@@ -1140,7 +1156,7 @@
Set metadata on a given trove spec.
"""
- if not license and not dec and not shortDesc:
+ if not license and not desc and not shortDesc:
log.warn('no metadata found for %s' % trvSpecs[-1][0])
return
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -176,13 +176,12 @@
###
@require_write
- def _add(self, groupName=None, *args, **kwargs):
+ def _add(self, *args, **kwargs):
"""
Add a trove to the package group contents.
"""
- if not groupName:
- groupName = self._pkgGroupName
+ groupName = kwargs.pop('groupName', self._pkgGroupName)
# create package group model if it does not exist.
if groupName not in self._groups:
@@ -205,6 +204,9 @@
@type flavors: [conary.deps.deps.Flavor, ...]
"""
+ if not groupName:
+ groupName = self._pkgGroupName
+
# Now that versions are actually used for something make sure they
# are always present.
assert version
@@ -214,7 +216,7 @@
# Remove all versions and flavors of this name before adding this
# package. This avoids flavor change issues by replacing all flavors.
if self.hasPackage(name):
- self.remove(name)
+ self.removePackage(name)
plain = deps.parseFlavor('')
x86 = deps.parseFlavor('is: x86')
@@ -243,26 +245,30 @@
raise UnsupportedTroveFlavorError(name=name, flavor=flavor)
def add():
- upver = version.trailingRevision().version()
+ upver = version.trailingRevision().version
for flv in flavors:
- if self._useMap:
- for useStr in self._useMap[(name, upver, flvMap[flv])]:
- self._add(name, version=version, flavor=flavor,
+ key = (name, upver, flvMap[flv])
+ if key in self._useMap:
+ for useStr in self._useMap[key]:
+ self._add(name, version=version, flavor=flv,
use=useStr, groupName=groupName)
else:
- self._add(name, version=version, flavor=flavor,
- use=flvMap[flv], groupName=groupName)
+ log.warn('%s=%s[%s] not found in useMap, falling back to '
+ 'old method of adding troves to groups'
+ % (name, version, flvMap[flv]))
+ self._add(name, version=version, flavor=flv,
+ use=flvMap[flv], groupName=groupName)
# If this package has one or two flavors and one of those flavors is
# x86, x86_64, biarch, or plain then handle it like a normal package
# without doing any more sanity checking.
total = 0
for flv, count in flvCount.iteritems():
- if len(count) > 1:
+ if count > 1:
break
total += count
else:
- if total in (1, 2):
+ if total in (1, 2, 3):
add()
return
@@ -274,7 +280,7 @@
# Check if this package is configured to have multiple flavors.
# Get source trove name.
log.info('retrieving trove info for %s' % name)
- srcTroveMap = self._helper._getSourceTroves((name, version, flavors[0]))
+ srcTroveMap = self._mgr._helper._getSourceTroves((name, version, flavors[0]))
srcTroveName = srcTroveMap.keys()[0][0].split(':')[0]
# Check if this is package that we have specifically defined a build
@@ -284,25 +290,30 @@
# TODO: If we were really smart we would load the conary
# contexts and see what buildFlavors they contained.
flavorCtxCount = {x86: 0, x86_64: 0, biarch: 0}
+ ctxMap = dict([ (x, y[1]) for x, y in self._cfg.archContexts if y ])
for context, bldflv in self._cfg.packageFlavors[srcTroveName]:
+ fltr = ctxMap.get(context, None)
if context in ('i386', 'i486', 'i586', 'i686', 'x86'):
flavorCtxCount[x86] += 1
elif context in ('x86_64', ):
flavorCtxCount[x86_64] += 1
elif context in ('biarch', ):
- flavorCtxCount[biarch] += 1
+ if fltr and fltr.match(name):
+ flavorCtxCount[biarch] += 1
else:
raise UnknownBuildContextError(name=name, flavor=context)
# Sanity check flavors to make sure we built all the flavors
# that we expected.
- extra = set(flvMap) - set([ x86, x86_64, biarch ])
- if extra:
- raise UnsupportedTroveFlavorError(name=name, flavor=extra)
-
if (flvCount[x86] != flavorCtxCount[x86] or
flvCount[x86_64] != flavorCtxCount[x86_64] or
- flvCount[biarch] != flavorCtxCount[biarch]):
+
+ # Only enforce biarch for packages that we expect to be biarch.
+ # This is a kluge to deal with the fact that biarch builds
+ # produce a byDefault=False package for the source that only
+ # contains the build log.
+ (flavorCtxCount[biarch] > 0 and
+ flvCount[biarch] != flavorCtxCount[biarch])):
raise FlavorCountMismatchError(name=name)
# Add packages to the group.
@@ -375,7 +386,7 @@
if pkgFlv:
group.removePackageFlavor(pkgName, pkgFlv)
else:
- self.remove(pkgName)
+ self.removePackage(pkgName)
# Add requested packages.
for groupName, pkgs in additions.iteritems():
diff --git a/updatebot/groupmgr/helper.py b/updatebot/groupmgr/helper.py
--- a/updatebot/groupmgr/helper.py
+++ b/updatebot/groupmgr/helper.py
@@ -122,13 +122,13 @@
return groups
- def setModel(self, pkgName, groups):
+ def setModel(self, pkgName, groups, version=None):
"""
Freeze group model and save to the repository.
"""
log.info('saving model for %s' % pkgName)
- recipeDir = self._edit(pkgName)
+ recipeDir = self._edit(pkgName, version=version)
groupFileName = util.join(recipeDir, 'groups.xml')
groupModel = GroupModel()
@@ -164,14 +164,14 @@
state = int(state)
return state
- def setErrataState(self, pkgname, state):
+ def setErrataState(self, pkgname, state, version=None):
"""
Set the current errata state for the given package.
"""
log.info('storing errata state information in %s' % pkgname)
- recipeDir = self._edit(pkgname)
+ recipeDir = self._edit(pkgname, version=version)
stateFileName = util.join(recipeDir, 'erratastate')
# write state info
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -61,7 +61,12 @@
def __init__(self, cfg, parentGroup=False, targetGroup=False, useMap=None):
self._cfg = cfg
- self._useMap = useMap or dict()
+
+ if useMap is None:
+ self._useMap = {}
+ else:
+ self._useMap = useMap
+
self._helper = self._helperClass(self._cfg)
self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups')
self._sanity = self._sanityCheckerClass(self._cfg, self._helper)
@@ -270,7 +275,7 @@
group.setCommitted()
# Get the model for the source version that we just committed.
- return self.getGroup(sourceVersion=newVersion)
+ return self.getGroup(version=newVersion)
@require_write
def buildGroup(self, group):
diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py
--- a/updatebot/groupmgr/sanity.py
+++ b/updatebot/groupmgr/sanity.py
@@ -214,6 +214,15 @@
log.info('found %s=%s[%s] in oldVersions exceptions'
% (n, v, f))
continue
+
+ # This is probably a flavor that we don't care about
+ # anymore.
+ if cv > v and cv in [ x[1] for x in found ]:
+ log.warn('missing flavors found of %s that are not all '
+ 'included in the group, assuming this '
+ 'intentional.' % cn)
+ continue
+
foundError = True
if foundError:
diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py
--- a/updatebot/lib/util.py
+++ b/updatebot/lib/util.py
@@ -172,13 +172,16 @@
limit = getRLimit()
return limit - openfds
-def setupDebugHandler():
+def setupDebugHandler(serve=False):
"""
Sets up a USR1 signal handler to trigger epdb.serv().
"""
def handler(signum, sigtb):
- epdb.serve()
+ if serve:
+ epdb.serve()
+ else:
+ epdb.st()
signal.signal(signal.SIGUSR1, handler)
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -338,19 +338,30 @@
# get the conary version from the source
conaryVersion = srcPkg.getConaryVersion()
+ # If the package arch is x86_64, I expect that it should only
+ # ever be built as x86_64.
if binPkg.arch == 'x86_64':
- arch = 'x86_64'
+ arch = ['x86_64', ]
+ # If the package arch is noarch, it could produce a conary
+ # package of any flavor.
elif binPkg.arch == 'noarch':
- arch = ''
+ arch = ['x86', 'x86_64', '']
+ # If the package arch is x86, it could produce either x86 or
+ # x86_64 flavors.
elif (binPkg.arch.startswith('i') and
binPkg.arch.endswith('86') and
len(binPkg.arch) == 4):
- arch = 'x86'
+ arch = ['x86', 'x86_64']
else:
raise RuntimeError
- trvSpec = (binPkg.name, conaryVersion, arch)
- self.useMap.setdefault(trvSpec, set()).update(archStr)
+ for a in arch:
+ trvSpecs = set([
+ (binPkg.name, conaryVersion, a),
+ (srcPkg.name, conaryVersion, a),
+ ])
+ for trvSpec in trvSpecs:
+ self.useMap.setdefault(trvSpec, set()).update(archStr)
def loadFileLists(self, client, basePath):
"""
From elliot at rpath.com Fri May 14 11:49:36 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 14 May 2010 15:49:36 +0000
Subject: mirrorball: if available, write out the repositoryArch for each
package in the manifest
Message-ID: <201005141549.o4EFnanu008181@scc.eng.rpath.com>
changeset: 4737fee4b6ec
user: Elliot Peele
date: Wed, 12 May 2010 21:12:02 -0400
if available, write out the repositoryArch for each package in the manifest
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -32,6 +32,7 @@
from updatebot.errors import UpdateGoesBackwardsError
from updatebot.errors import UpdateRemovesPackageError
from updatebot.errors import ParentPlatformManifestInconsistencyError
+from updatebot.errors import RepositoryPackageSourceInconsistencyError
log = logging.getLogger('updatebot.update')
@@ -365,7 +366,7 @@
for line in manifest:
# Some manifests were created with double slashes, need to
# normalize the path to work around this problem.
- line = os.path.normpath(line)
+ line = os.path.normpath(line).split('?')[0]
if line in self._pkgSource.locationMap:
binPkg = self._pkgSource.locationMap[line]
srcPkg = self._pkgSource.binPkgMap[binPkg]
@@ -495,8 +496,7 @@
if needsUpdate:
raise RepositoryPackageSourceInconsistencyError(nvf=nvf, srpm=srpm)
- @staticmethod
- def _getLatestOfAvailableArches(pkgLst):
+ def _getLatestOfAvailableArches(self, pkgLst):
"""
Given a list of package objects, find the latest versions of each
package for each name/arch.
@@ -508,7 +508,8 @@
pkgMap = {}
for pkg in pkgLst:
- key = pkg.name + pkg.arch
+ arch = self._getRepositoryArch(pkg.location)
+ key = pkg.name + pkg.arch + arch
if key not in pkgMap:
pkgMap[key] = pkg
continue
@@ -813,6 +814,18 @@
return srcVersion
+ def _getRepositoryArch(self, loc):
+ """
+ Get the architecture of the repository for a given location if
+ repository arch is defined.
+ """
+
+ for repo, arch in self._cfg.repositoryArch.iteritems():
+ if loc.startswith(repo):
+ return arch
+
+ return ''
+
def _getManifestFromPkgSource(self, srcPkg):
"""
Get the contents of the a manifest file from the pkgSource object.
@@ -828,7 +841,11 @@
manifestPkgs = list(self._pkgSource.srcPkgMap[srcPkg])
for pkg in self._getLatestOfAvailableArches(manifestPkgs):
if hasattr(pkg, 'location'):
- manifest.append(pkg.location)
+ arch = self._getRepositoryArch(pkg.location)
+ location = pkg.location
+ if arch:
+ location += '?arch=%s' % arch
+ manifest.append(location)
elif hasattr(pkg, 'files'):
manifest.extend(pkg.files)
return manifest
From elliot at rpath.com Fri May 14 11:49:36 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 14 May 2010 15:49:36 +0000
Subject: mirrorball: add support for keeping removed packages and keeping
obsoleted sources
Message-ID: <201005141549.o4EFnaen008208@scc.eng.rpath.com>
changeset: c907479ef9b8
user: Elliot Peele
date: Fri, 14 May 2010 11:49:33 -0400
add support for keeping removed packages and keeping obsoleted sources
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -416,10 +416,15 @@
# from the package model
removeSource = (CfgIntDict(CfgList(CfgNevra)), {})
+ # updateId binaryNevra
+ # As of updateId, I expect the code to think this nevra should be removed,
+ # but I want to keep it.
+ keepRemoved = (CfgIntDict(CfgList(CfgNevra)), {})
+
# updateId sourceNevra
# As of updateId, the specified src is fully obsoleted, but
# should be retained in groups
- keepObsoleteSource = (CfgIntDict(CfgList(CfgNevra)), {})
+ keepObsoleteSource = (CfgList(CfgNevraTuple), [])
# Some obsoletes are merely conceptual preferences, and should not
# turn into removals.
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -237,6 +237,7 @@
# convert keepObsolete config into set of edges
keepObsolete = set(self._cfg.keepObsolete)
+ keepObsoleteSource = set(self._cfg.keepObsoleteSource)
errors = {}
# Make sure there no buckets that contain the same srpm name twice.
@@ -269,6 +270,7 @@
log.info('validating %s' % updateId)
expectedRemovals = removals.get(updateId, [])
expectedReplaces = replaces.get(updateId, [])
+ expectedKeepRemovals = self._cfg.keepRemoved.get(updateId, [])
explicitSourceRemovals = self._cfg.removeSource.get(updateId, set())
explicitBinaryRemovals = self._cfg.removeObsoleted.get(updateId, set())
explicitPackageDowngrades = downgraded.get(updateId, None)
@@ -281,7 +283,8 @@
try:
toUpdate = updater._sanitizeTrove(nvf, srpm,
expectedRemovals=expectedRemovals + expectedReplaces,
- allowPackageDowngrades=explicitPackageDowngrades)
+ allowPackageDowngrades=explicitPackageDowngrades,
+ keepRemovedPackages=expectedKeepRemovals)
# If a source was manually added to this updateId it may
# have already been part of another update, which would
@@ -389,14 +392,28 @@
obsoletingSrcPkgs = tuple(sorted(set(
self._pkgSource.binPkgMap[x]
for x, y in obsoleteEdgeSet)))
- # we exclude any source only once, not per bucket
- obsoleteSources.add(
- (obsoletedPkg.name,
- obsoletedPkg,
- obsoletingSrcPkgs,
- srcPkg,
- binPkgs))
- foundObsoleteSrcs.add(srcPkg)
+
+ newEdgeSet = set()
+ obsoletingSrcPkgs = set()
+
+ for obsoletingPkg, obsoletedPkg in obsoleteEdgeSet:
+ obsoletingSrcPkg = self._pkgSource.binPkgMap[obsoletingPkg]
+ if (obsoletingSrcPkg.getNevra(), srcPkg.getNevra()) in keepObsoleteSource:
+ continue
+ newEdgeSet.add((obsoletingPkg, obsoletedPkg))
+ obsoletingSrcPkgs.add(obsoletingSrcPkg)
+
+ if newEdgeSet:
+ # we exclude any source only once, not per bucket
+ for obsoletingPkg, obsoletedPkg in newEdgeSet:
+ obsoleteSources.add(
+ (obsoletedPkg.name,
+ obsoletedPkg,
+ tuple(obsoletingSrcPkgs),
+ srcPkg,
+ binPkgs))
+
+ foundObsoleteSrcs.add(srcPkg)
if obsoleteBinaries:
@@ -421,15 +438,17 @@
twoNevra = str(' '.join(two.getNevra()))
log.warn('%s %s -revertsTo-> %s' % (updateId,
oneNevra, twoNevra))
- log.info('%s -revertsTo-> %s' % (
- rhnUrls(srpmToAdvisory[one]),
- rhnUrls(srpmToAdvisory[two])))
- log.info('%s %s (%d) -> %s (%d)' % (updateId,
- tconv(srpmToBucketId[one]), srpmToBucketId[one],
- tconv(srpmToBucketId[two]), srpmToBucketId[two]))
- log.info('? reorderSource %s otherId<%s %s' % (
- updateId, srpmToBucketId[one], twoNevra))
- for errataId in srpmToAdvisory[two]:
+ if one in srpmToAdvisory and two in srpmToAdvisory:
+ log.info('%s -revertsTo-> %s' % (
+ rhnUrls(srpmToAdvisory[one]),
+ rhnUrls(srpmToAdvisory[two])))
+ if one in srpmToBucketId and two in srpmToBucketId:
+ log.info('%s %s (%d) -> %s (%d)' % (updateId,
+ tconv(srpmToBucketId[one]), srpmToBucketId[one],
+ tconv(srpmToBucketId[two]), srpmToBucketId[two]))
+ log.info('? reorderSource %s otherId<%s %s' % (
+ updateId, srpmToBucketId[one], twoNevra))
+ for errataId in srpmToAdvisory.get(two, []):
log.info('? reorderAdvisory %s otherId<%s %s' % (
updateId, srpmToBucketId[one], errataId))
elif isinstance(e, UpdateReusesPackageError):
@@ -442,9 +461,10 @@
elif isinstance(e, UpdateRemovesPackageError):
log.warn('%s %s removed in %s' % (
updateId, e.pkgNames, e.newspkg))
- for name in sorted(set(p.name for p in e.pkgList)):
+ for name, p in sorted(dict((p.name, p) for p in e.pkgList).items()):
log.info('? updateRemovesPackages %s %s' % (
updateId, name))
+ log.info('? keepRemoved %s %s' % (updateId, '%s %s %s %s %s' % p.getNevra()))
elif isinstance(e, tuple):
if e[0] == 'duplicates':
sortedOrder = sorted(self._order)
@@ -453,13 +473,14 @@
dupList = sorted(dupSet)
log.warn('%s contains duplicate %s %s' %(updateId,
dupName, dupList))
- for srcPkg in dupList[:-1]:
+ for srcPkg in sorted(dupList[1:]):
srcNevra = str(' '.join(srcPkg.getNevra()))
- log.info('%s : %s' % (
- srcNevra, rhnUrls(srpmToAdvisory[srcPkg])))
+ if srcPkg in srpmToAdvisory:
+ log.info('%s : %s' % (
+ srcNevra, rhnUrls(srpmToAdvisory[srcPkg])))
log.info('? reorderSource %s earlierId>%s %s' %
(updateId, previousId, srcNevra))
- for errataId in srpmToAdvisory[srcPkg]:
+ for errataId in srpmToAdvisory.get(srcPkg, []):
log.info(
'? reorderAdvisory %s earlierId>%s %r'
% (updateId, previousId, errataId))
@@ -504,6 +525,10 @@
log.info('? removeSource %s %s # %s' % (
updateId, srcNevraStr,
obsoletingSrcPkgNames))
+ for obsoletingSrcPkg in obsoletingSrcPkgs:
+ log.info('? keepObsoleteSource %s %s'
+ % (str(' '.join(obsoletingSrcPkg.getNevra())),
+ srcNevraStr))
log.info(' will remove the following: %s' % pkgList)
elif e[0] == 'removedShouldBeReplaced':
for pkgName in e[1]:
@@ -893,7 +918,9 @@
# separate out golden bits
other = []
golden = []
- firstErrata = sorted(buckets.keys())[0]
+ firstErrata = int(time.time())
+ if len(buckets):
+ firstErrata = sorted(buckets.keys())[0]
for nevra, pkg in nevras.iteritems():
buildtime = int(pkg.buildTimestamp)
if buildtime < firstErrata:
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -321,7 +321,7 @@
return srpms[-1]
def _sanitizeTrove(self, nvf, srpm, expectedRemovals=None,
- allowPackageDowngrades=None):
+ allowPackageDowngrades=None, keepRemovedPackages=None):
"""
Verifies the package update to make sure it looks correct and is a
case that the bot knows how to handle.
@@ -341,6 +341,10 @@
@param allowPackageDowngrades: list of source nevra tuples to downgrade
from/to.
@type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
+ @param keepRemovedPackages: list of package nevras to keep even though
+ they have been removed in the latest version
+ of the source.
+ @type keepRemovedPackages: list(nevra, nevra, ...)
@return needsUpdate boolean
@raises UpdateGoesBackwardsError
@raises UpdateRemovesPackageError
@@ -415,13 +419,18 @@
# Raise an exception if the versions of the packages aren't
# equal or the discovered package comes from a different source.
- if (rpmvercmp(pkg.epoch, srpm.epoch) != 0 or
- rpmvercmp(pkg.version, srpm.version) != 0 or
+ #
+ # Get the source that the package was built from for version
+ # comparison since the source and binary can have different
+ # versions.
+ src = self._pkgSource.binPkgMap[pkg]
+ if (rpmvercmp(src.epoch, srpm.epoch) != 0 or
+ rpmvercmp(src.version, srpm.version) != 0 or
# in the suse case we have to ignore release
(not self._cfg.reuseOldRevisions and
- rpmvercmp(pkg.release, srpm.release) != 0) or
+ rpmvercmp(src.release, srpm.release) != 0) or
# binary does not come from the same source as it used to
- self._pkgSource.binPkgMap[pkg].name != srpm.name):
+ src.name != srpm.name):
log.warn('update removes package (%s) %s -> %s'
% (pkg.name, srcPkg.getNevra(), srpm.getNevra()))
@@ -431,7 +440,8 @@
% pkg.name)
continue
- removedPackages.add(pkg)
+ if pkg.getNevra() not in keepRemovedPackages:
+ removedPackages.add(pkg)
if not removedPackages:
reusedPackages.add(pkg)
From elliot at rpath.com Mon May 17 17:31:04 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 17 May 2010 21:31:04 +0000
Subject: mirrorball: add more method stubs
Message-ID: <201005172131.o4HLV4rj013832@scc.eng.rpath.com>
changeset: b9327658c9ff
user: Elliot Peele
date: Mon, 17 May 2010 17:30:47 -0400
add more method stubs
diff --git a/errata/common.py b/errata/common.py
--- a/errata/common.py
+++ b/errata/common.py
@@ -51,7 +51,7 @@
this package.
"""
- raise NotImplemetedError
+ raise NotImplementedError
class Channel(object):
@@ -123,3 +123,25 @@
"""
raise NotImplementedError
+
+ def getChannels(self):
+ """
+ Return a list of all indexed channels, will trigger a fetch if needed.
+ """
+
+ raise NotImplementedError
+
+ def cleanup(self):
+ """
+ Frees any cached results.
+ """
+
+ raise NotImplementedError
+
+ def getModifiedErrata(self, updateId):
+ """
+ Get a list of any errata that were modified after updateId and were
+ issued before updateId.
+ """
+
+ raise NotImplementedError
From elliot at rpath.com Mon May 17 17:31:04 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 17 May 2010 21:31:04 +0000
Subject: mirrorball: fill in more api for sles
Message-ID: <201005172131.o4HLV4Rt013861@scc.eng.rpath.com>
changeset: 64fb9b76d3f6
user: Elliot Peele
date: Mon, 17 May 2010 17:31:02 -0400
fill in more api for sles
diff --git a/errata/sles.py b/errata/sles.py
--- a/errata/sles.py
+++ b/errata/sles.py
@@ -33,7 +33,7 @@
this package.
"""
-class Repository(common.Repository):
+class Channel(common.Channel):
"""
Class to represent a repository.
"""
@@ -49,7 +49,7 @@
def __init__(self, pkgSource):
self._pkgSource = pkgSource
- slef._fetched = False
+ self._fetched = False
self._patches = set()
@common.reqfetch
@@ -77,6 +77,31 @@
self._fetched = True
+ @common.reqfetch
+ def getChannels(self):
+ """
+ Get a list of indexed channel names.
+ @return list of indexed channel names
+ """
+
+ return self._pkgSource._clients.keys()
+
+
+ def cleanup(self):
+ """
+ Free all cached results.
+ """
+
+ self._patches = set()
+
+ def getModifiedErrata(self, updateId):
+ """
+ Get a list of any errata that were modified after updateId and were
+ issued before updateId.
+ """
+
+ return []
+
def _fetchPatches(self):
"""
Fetch all patch data from the package source.
From elliot at rpath.com Tue May 18 16:01:16 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:16 +0000
Subject: mirrorball: whitespace cleanup
Message-ID: <201005182001.o4IK1GQ2000658@scc.eng.rpath.com>
changeset: 5d58feba684a
user: Elliot Peele
date: Tue, 18 May 2010 15:55:46 -0400
whitespace cleanup
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -782,7 +782,6 @@
self._advMap.setdefault(dest, set()).add(advInfo)
break
-
def _reorderSource(self, source, dest, nevra):
"""
Reschedule an individual srpm to another bucket.
From elliot at rpath.com Tue May 18 16:01:16 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:16 +0000
Subject: mirrorball: call into the correct layer for this method
Message-ID: <201005182001.o4IK1G2I000690@scc.eng.rpath.com>
changeset: f4a48b3c4bc1
user: Elliot Peele
date: Tue, 18 May 2010 15:56:12 -0400
call into the correct layer for this method
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -345,7 +345,7 @@
# Get the list of binaries for this source from the repository.
# FIXME: This should not call into the conary client itself, instead
# there should be a call in the conary helper.
- trvs = self._helper._client.getTrovesBySource(self._sourceName,
+ trvs = self._helper._repos.getTrovesBySource(self._sourceName,
sourceVersion)
return bool(len(trvs))
From elliot at rpath.com Tue May 18 16:01:17 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:17 +0000
Subject: mirrorball: handle empty repository contents
Message-ID: <201005182001.o4IK1HTU000717@scc.eng.rpath.com>
changeset: 673ff9f2c4eb
user: Elliot Peele
date: Tue, 18 May 2010 15:57:02 -0400
handle empty repository contents
diff --git a/repomd/__init__.py b/repomd/__init__.py
--- a/repomd/__init__.py
+++ b/repomd/__init__.py
@@ -78,6 +78,8 @@
"""
node = self._repomd.getRepoData('primary')
+ if node is None:
+ return []
return node.parseChildren().getPackages()
def getFileLists(self):
From elliot at rpath.com Tue May 18 16:01:17 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:17 +0000
Subject: mirrorball: safely create temp files
Message-ID: <201005182001.o4IK1HMI000737@scc.eng.rpath.com>
changeset: a927bcf6c147
user: Elliot Peele
date: Tue, 18 May 2010 15:57:29 -0400
safely create temp files
diff --git a/repomd/repository.py b/repomd/repository.py
--- a/repomd/repository.py
+++ b/repomd/repository.py
@@ -61,7 +61,8 @@
@return name of tempory file
"""
- return tempfile.mktemp(prefix='mdparse')
+ fd, name = tempfile.mkstemp(prefix='mdparse')
+ return name
def _getRealUrl(self, path):
"""
From elliot at rpath.com Tue May 18 16:01:17 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:17 +0000
Subject: mirrorball: Use archiveSize to compare for equality if checksum
is different
Message-ID: <201005182001.o4IK1Hxd000772@scc.eng.rpath.com>
changeset: 3ba32f84ca36
user: Elliot Peele
date: Tue, 18 May 2010 15:58:10 -0400
Use archiveSize to compare for equality if checksum is different
diff --git a/repomd/packagexml.py b/repomd/packagexml.py
--- a/repomd/packagexml.py
+++ b/repomd/packagexml.py
@@ -155,11 +155,21 @@
if pkgcmp != 0:
return pkgcmp
+ # Compare checksum only for equality, otherwise sorting will result in
+ # checksum ordering.
if (self.checksum and other.checksum and
self.checksumType == other.checksumType and
self.checksum == other.checksum):
return 0
+ # Compare on archiveSize for equality only. This is needed for rpms
+ # that have identical contents, but may have been rebuilt. Idealy we
+ # would use file checksums for this, but we don't have the payload
+ # contents available at this time.
+ if (self.archiveSize and other.archiveSize and
+ self.archiveSize == other.archiveSize):
+ return 0
+
return cmp(self.location, other.location)
def getNevra(self):
From elliot at rpath.com Tue May 18 16:01:18 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:18 +0000
Subject: mirrorball: simple script for just loading a repomd
Message-ID: <201005182001.o4IK1IqL000827@scc.eng.rpath.com>
changeset: fb2d3beec934
user: Elliot Peele
date: Tue, 18 May 2010 16:00:47 -0400
simple script for just loading a repomd
diff --git a/scripts/pkgsource b/scripts/repomd
copy from scripts/pkgsource
copy to scripts/repomd
--- a/scripts/pkgsource
+++ b/scripts/repomd
@@ -15,7 +15,6 @@
import os
import sys
-import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -30,104 +29,15 @@
log = logging.getLogger('test')
from updatebot import config
-from updatebot import pkgsource
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-pkgSource = pkgsource.PackageSource(cfg)
-pkgSource.load()
+import repomd
-srcs = {}
-srcNevras = {}
-for src, bins in pkgSource.srcPkgMap.iteritems():
- srcs.setdefault(src.getNevra(), bins)
- srcNevras.setdefault(src.getNevra(), src)
-
-names = {}
-for nevra, bins in srcs.iteritems():
- if len(bins) > 1:
- names.setdefault(nevra[0], set()).add(nevra)
-
-count = {}
-arch = {}
-for name, nevras in names.iteritems():
- first = None
- seen = None
- for nevra in nevras:
- bins = srcs[nevra]
- archs = set([ x.arch for x in bins ])
- if not seen:
- first = (nevra, bins)
- seen = archs
- elif len(seen) != len(archs):
- count[first[0]] = first[1]
- count[nevra] = bins
- elif seen != archs:
- arch.setdefault(first[0], set()).update(first[1])
- arch.setdefault(nevra, set()).update(bins)
-
-removed = {}
-#for nevra in itertools.chain(count, arch):
-# removed.setdefault(nevra, srcNevras.pop(nevra))
-for nevra in itertools.chain(*[ x for x in names.values() if len(x) > 1 ]):
- removed.setdefault(nevra, srcNevras.pop(nevra))
-
-toCreate = set(srcNevras.values())
+contents = []
+for path in cfg.repositoryPaths:
+ client = repomd.Client(cfg.repositoryUrl + '/' + path)
+ contents.extend(client.getPackageDetail())
import epdb; epdb.st()
-
-order = {}
-
-def srtByRPMVerCmp(a, b):
- from updatebot.lib import util
- return util.packagevercmp(a, b)
-
-def srtByBuildstamp(a, b):
- assert hasattr(a, 'buildTimestamp')
- assert hasattr(b, 'buildTimestamp')
- assert a.buildTimestamp not in ('0', '', 0)
- assert b.buildTimestamp not in ('0', '', 0)
- return cmp(int(a.buildTimestamp), int(b.buildTimestamp))
-
-srcNames = {}
-for srcPkg in pkgSource.srcPkgMap:
- srcNames.setdefault(srcPkg.name, set()).add(srcPkg)
-
-binOrder = {}
-for srcName, srcPkgs in srcNames.iteritems():
- uSrcPkgs = dict((x.getNevra(), x) for x in srcPkgs).values()
-
- ver = sorted(uSrcPkgs, cmp=srtByRPMVerCmp)
-
- for srcPkg in uSrcPkgs:
- if srcPkg.buildTimestamp is None:
- srcPkg.buildTimestamp = sorted([ x for x in pkgSource.srcPkgMap[srcPkg] if x.arch != 'src' ])[0].buildTimestamp
-
- buildstamp = sorted(uSrcPkgs, cmp=srtByBuildstamp)
-
- assert ver == buildstamp
-
- for srcPkg in uSrcPkgs:
- ts = int(srcPkg.buildTimestamp)
- bins = pkgSource.srcPkgMap[srcPkg]
- binOrder.setdefault(ts, set()).update(bins)
-
-def tsToDay(ts):
- import time
- return int(time.mktime(time.strptime(time.strftime('%Y%m%d', time.gmtime(ts)), '%Y%m%d')))
-
-# collapse by day
-for ts in sorted(binOrder):
- day = tsToDay(ts)
- bins = binOrder[ts]
- order.setdefault(day, set()).update(bins)
-
-import epdb; epdb.st()
-
-updates = []
-for path, client in pkgSource._clients.iteritems():
- if 'Updates' in path:
- updates.extend(client.getUpdateInfo())
-
-import epdb; epdb.st()
From elliot at rpath.com Tue May 18 16:01:17 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:17 +0000
Subject: mirrorball: use the basename of the manifest packages as a name list
Message-ID: <201005182001.o4IK1HPm000799@scc.eng.rpath.com>
changeset: 818453b4f97a
user: Elliot Peele
date: Tue, 18 May 2010 15:59:17 -0400
use the basename of the manifest packages as a name list
diff --git a/scripts/buildpackages b/scripts/buildpackages
--- a/scripts/buildpackages
+++ b/scripts/buildpackages
@@ -26,10 +26,7 @@
yield pkgName, None
continue
paths = [ os.path.basename(x) for x in manifest ]
-# for context, fltr in cfg.archContexts:
-# if fltr and [ x for x in paths if fltr[1].match(x) ]:
-# raise RuntimeError, 'Found package that may not be built'
- yield (pkgName, tuple(paths))
+ yield pkgName, tuple(paths)
raise StopIteration
trvs = set()
From elliot at rpath.com Tue May 18 16:01:18 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 18 May 2010 20:01:18 +0000
Subject: mirrorball: script for testing sles package ordering
Message-ID: <201005182001.o4IK1I6o000855@scc.eng.rpath.com>
changeset: 704ce65b8a2d
user: Elliot Peele
date: Tue, 18 May 2010 16:01:06 -0400
script for testing sles package ordering
diff --git a/scripts/rhelorder.py b/scripts/sleorder.py
copy from scripts/rhelorder.py
copy to scripts/sleorder.py
--- a/scripts/rhelorder.py
+++ b/scripts/sleorder.py
@@ -21,34 +21,22 @@
from updatebot import log
from updatebot.ordered import Bot
from updatebot import UpdateBotConfig
+from updatebot import pkgsource
-import rhnmirror
+from errata.sles import AdvisoryManager as Errata
slog = log.addRootLogger()
-mcfg = rhnmirror.MirrorConfig()
-mcfg.read(confDir + '/erratarc')
-#mcfg.indexerDb += '5'
-#mcfg.indexerDb = 'sqlite:///%s' % tempfile.mktemp(suffix='.db', prefix='order-')
-
-slog.info('db = %s' % mcfg.indexerDb)
-
-#mcfg.channels = [
-# 'rhel-x86_64-server-5',
-# 'rhel-i386-server-5',
-# 'rhel-x86_64-as-4',
-# 'rhel-i386-as-4',
-#]
-
-errata = rhnmirror.Errata(mcfg)
-errata.fetch()
-
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
+pkgSource = pkgsource.PackageSource(cfg)
+
+errata = Errata(pkgSource)
+errata.fetch()
+
bot = Bot(cfg, errata)
bot._pkgSource.load()
-
bot._errata._orderErrata()
order = bot._errata._order
From elliot at rpath.com Wed May 19 14:59:49 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 19 May 2010 18:59:49 +0000
Subject: mirrorball: remove old debugging code
Message-ID: <201005191859.o4JIxnfj024632@scc.eng.rpath.com>
changeset: edf4d8c5beee
user: Elliot Peele
date: Tue, 18 May 2010 20:55:35 -0400
remove old debugging code
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -604,11 +604,7 @@
parentPackages = set()
total = len(toCreate)
current = 1
- start = False
for pkg in sorted(toCreate):
- if pkg.name.startswith('n'):
- start = True
-
try:
# Only import packages that haven't been imported before
version = verCache.get('%s:source' % pkg.name)
From elliot at rpath.com Wed May 19 14:59:49 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 19 May 2010 18:59:49 +0000
Subject: mirrorball: rework useMap generation a bit to fix problems with
biarch groups
Message-ID: <201005191859.o4JIxnQD024659@scc.eng.rpath.com>
changeset: 3e163c718777
user: Elliot Peele
date: Wed, 19 May 2010 14:41:05 -0400
rework useMap generation a bit to fix problems with biarch groups
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -123,11 +123,6 @@
if self._excludeLocation(pkg.location):
continue
- # ignore 32bit rpms in a 64bit repo.
- if (pkg.arch in ('i386', 'i586', 'i686') and
- 'x86_64' in pkg.location):
- continue
-
# Source RPM is one without a "sourcerpm" element
if pkg.sourcerpm == '' or pkg.sourcerpm is None:
self._procSrc(pkg)
@@ -211,7 +206,11 @@
self.locationMap[package.location] = package
if archStr:
- self._repoMap.setdefault(package, set()).add(archStr)
+ if package.arch == 'x86_64' and archStr == 'x86':
+ log.warn('not adding %s to repoMap since we do not allow 64bit '
+ 'packages in a 32bit repository.' % package)
+ else:
+ self._repoMap.setdefault(package, set()).add(archStr)
def _excludeLocation(self, location):
"""
@@ -331,7 +330,7 @@
log.warn('\t%s' % loc)
if self._repoMap:
- for binPkg, archStr in self._repoMap.iteritems():
+ for binPkg, archSet in self._repoMap.iteritems():
# lookup the source for the binary package
srcPkg = self.binPkgMap[binPkg]
@@ -341,27 +340,38 @@
# If the package arch is x86_64, I expect that it should only
# ever be built as x86_64.
if binPkg.arch == 'x86_64':
- arch = ['x86_64', ]
+ possibleFlavors = ['x86_64', ]
# If the package arch is noarch, it could produce a conary
# package of any flavor.
elif binPkg.arch == 'noarch':
- arch = ['x86', 'x86_64', '']
+ possibleFlavors = ['x86', 'x86_64', ]
# If the package arch is x86, it could produce either x86 or
# x86_64 flavors.
elif (binPkg.arch.startswith('i') and
binPkg.arch.endswith('86') and
len(binPkg.arch) == 4):
- arch = ['x86', 'x86_64']
+ possibleFlavors = ['x86', ]
else:
raise RuntimeError
- for a in arch:
+ for flv in possibleFlavors:
trvSpecs = set([
- (binPkg.name, conaryVersion, a),
- (srcPkg.name, conaryVersion, a),
+ (binPkg.name, conaryVersion, flv),
+ # Always include a package with the source name since
+ # every build will generate a conary package with the
+ # given build flavor, even if the package only contains
+ # a buildlog.
+ (srcPkg.name, conaryVersion, flv),
])
for trvSpec in trvSpecs:
- self.useMap.setdefault(trvSpec, set()).update(archStr)
+ useSet = self.useMap.setdefault(trvSpec, set())
+ if binPkg.arch == 'noarch':
+ if flv == 'x86' and 'x86' in archSet:
+ useSet.add('x86')
+ elif flv == 'x86_64' and 'x86_64' in archSet:
+ useSet.add('x86_64')
+ else:
+ useSet.update(archSet)
def loadFileLists(self, client, basePath):
"""
From elliot at rpath.com Wed May 19 14:59:50 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 19 May 2010 18:59:50 +0000
Subject: mirrorball: split out writing group contents into a separate
method
Message-ID: <201005191859.o4JIxoc2024690@scc.eng.rpath.com>
changeset: 942569599dd8
user: Elliot Peele
date: Wed, 19 May 2010 14:41:59 -0400
split out writing group contents into a separate method
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -151,6 +151,27 @@
else:
return None
+ def _persistGroup(self, group, conaryVersion=None):
+ """
+ Serialize the contents of a group model to disk.
+ @param conaryVersion: The version of the group source to update.
+ @type conaryVersion: conary.versions.VersionFromString
+ """
+
+ if not conaryVersion:
+ conaryVersion = group.conaryVersion
+
+ # set version
+ self._helper.setVersion(self._sourceName, group.version,
+ version=conaryVersion)
+
+ # set errata state
+ self._helper.setErrataState(self._sourceName, group.errataState,
+ version=conaryVersion)
+
+ # write out the model data
+ self._helper.setModel(self._sourceName, group, version=conaryVersion)
+
def getGroup(self, version=None):
"""
Retrieve a group model from the repository.
@@ -252,16 +273,8 @@
# readonly.
group.finalize()
- # set version
- self._helper.setVersion(self._sourceName, group.version,
- version=conaryVersion)
-
- # set errata state
- self._helper.setErrataState(self._sourceName, group.errataState,
- version=conaryVersion)
-
- # write out the model data
- self._helper.setModel(self._sourceName, group, version=conaryVersion)
+ # Serialize the group model.
+ self._persistGroup(group, conaryVersion=conaryVersion)
# commit to the repository
newVersion = self._helper.commit(self._sourceName,
From elliot at rpath.com Wed May 19 14:59:50 2010
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 19 May 2010 18:59:50 +0000
Subject: mirrorball: don't try to open the same rpm twice
Message-ID: <201005191859.o4JIxosm024717@scc.eng.rpath.com>
changeset: de9cd0f6d821
user: Elliot Peele
date: Wed, 19 May 2010 14:42:21 -0400
don't try to open the same rpm twice
diff --git a/rpmutils/header.py b/rpmutils/header.py
--- a/rpmutils/header.py
+++ b/rpmutils/header.py
@@ -78,10 +78,5 @@
"""
fh = SeekableStream(url)
-
- # Have to read into the file a bit to get to the begining of the header
- # that we care about.
- rpmhelper.RpmHeader(fh)
-
header = rpmhelper.RpmHeader(fh)
return header
From elliot at rpath.com Thu May 20 14:04:03 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 18:04:03 +0000
Subject: mirrorball: parallelize repository verification
Message-ID: <201005201804.o4KI43Us018945@scc.eng.rpath.com>
changeset: 16e0653891fb
user: Elliot Peele
date: Thu, 20 May 2010 14:03:41 -0400
parallelize repository verification
diff --git a/scripts/verifyrepos b/scripts/verifyrepos
--- a/scripts/verifyrepos
+++ b/scripts/verifyrepos
@@ -16,7 +16,6 @@
import os
import sys
import copy
-import itertools
mirrorballDir = os.path.abspath('../')
sys.path.insert(0, mirrorballDir)
@@ -38,23 +37,80 @@
cfg = config.UpdateBotConfig()
cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] )
-import epdb; epdb.st()
+headers = []
paths = copy.copy(cfg.repositoryPaths)
+
+from Queue import Empty
+from multiprocessing import Process, Queue
+
+class Worker(Process):
+ def __init__(self, inq, outq, errq, *args, **kwargs):
+ Process.__init__(self, *args, **kwargs)
+
+ self.inq = inq
+ self.outq = outq
+ self.errq = errq
+
+ self.daemon = True
+
+ def run(self):
+ while True:
+ try:
+ platform, repoPath = self.inq.get()
+ cfg = config.UpdateBotConfig()
+ cfg.read(mirrorballDir + '/config/%s/updatebotrc' % platform)
+
+ cfg.repositoryPaths = [ repoPath, ]
+ pkgSource = pkgsource.PackageSource(cfg)
+ pkgSource.load()
+
+ for location in pkgSource.locationMap:
+ if not location.startswith(repoPath):
+ continue
+ url = cfg.repositoryUrl + '/' + location
+ try:
+ log.info('testing %s' % url)
+ header = rpmutils.readHeader(url)
+ except Exception, e:
+ log.info('failed to open %s, %s' % (url, e))
+ self.outq.put((url, str(e)))
+ except Exception, e:
+ self.errq.put(str(e))
+
+
+class Mgr(object):
+ def __init__(self, max=6):
+ self.max = max
+ self.inq = Queue()
+ self.outq = Queue()
+ self.errq = Queue()
+
+ self._procs = []
+
+ self._results = []
+
+ def start(self):
+ for i in range(self.max):
+ proc = Worker(self.inq, self.outq, self.errq)
+ proc.start()
+ self._procs.append(proc)
+
+ def doWork(self, platform, repoPath):
+ self.inq.put((platform, repoPath))
+
+ def getResults(self):
+ while not self.outq.empty():
+ try:
+ res = self.outq.get_nowait()
+ self._results.append(res)
+ except Empty, e:
+ pass
+ return self._results
+
+mgr = Mgr(max=len(paths))
for path in paths:
- cfg.repositoryPaths = [ path, ]
- pkgSource = pkgsource.PackageSource(cfg)
- pkgSource.load()
+ mgr.doWork(sys.argv[1], path)
- for location in pkgSource.locationMap:
- if not location.startswith(path):
- log.info('skipping %s' % location)
- continue
- url = cfg.repositoryUrl + '/' + location
- try:
- rpmutils.readHeader(url)
- except Exception, e:
- raise
- log.info('failed to open %s, %s' % (url, e))
-
+mgr.start()
import epdb; epdb.st()
From elliot at rpath.com Thu May 20 14:04:03 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 18:04:03 +0000
Subject: mirrorball: allow for multiple advisory sources
Message-ID: <201005201804.o4KI435x018976@scc.eng.rpath.com>
changeset: 93214c853a3d
user: Elliot Peele
date: Thu, 20 May 2010 14:04:01 -0400
allow for multiple advisory sources
diff --git a/scripts/order_import.py b/scripts/order_import.py
--- a/scripts/order_import.py
+++ b/scripts/order_import.py
@@ -1,6 +1,6 @@
#!/usr/bin/python
#
-# Copyright (c) 2009 rPath, Inc.
+# Copyright (c) 2009-2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
@@ -32,10 +32,9 @@
from conary.lib import util
sys.excepthook = util.genExcepthook()
-import rhnmirror
-
from updatebot import config
from updatebot import ordered
+from updatebot import pkgsource
from updatebot import log as logSetup
logSetup.addRootLogger()
@@ -54,13 +53,45 @@
cfg = config.UpdateBotConfig()
cfg.read(confDir + '/updatebotrc')
-mcfg = rhnmirror.MirrorConfig()
-mcfg.read(confDir + '/erratarc')
+fltr = None
-errata = rhnmirror.Errata(mcfg)
+if cfg.platformName == 'rhel':
+ import rhnmirror
+
+ mcfg = rhnmirror.MirrorConfig()
+ mcfg.read(confDir + '/erratarc')
+
+ errata = rhnmirror.Errata(mcfg)
+elif cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
+
+ pkgSource = pkgsource.PackageSource(cfg)
+ errata = Errata(pkgSource)
+
+ def fltr(sourceSet):
+ removed = set()
+ removedNames = set()
+ pkgSource.load()
+ for src, bins in pkgSource.srcPkgMap.iteritems():
+ # filter out packages that we don't handle right now.
+ if ([ x for x in bins if 'kmp' in x.name ] or
+ # the kernel needs a recipe
+ 'kernel' in src.name):
+
+ removedNames.add(src.name)
+
+ for src, bins in pkgSource.srcPkgMap.iteritems():
+ if src.name in removedNames:
+ removed.add(src)
+
+ return sourceSet - removed
+
+else:
+ raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+
errata.fetch()
bot = ordered.Bot(cfg, errata)
-pkgMap, failures = bot.create()
+pkgMap, failures = bot.create(fltr=fltr)
import epdb; epdb.st()
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -1,6 +1,6 @@
#!/usr/bin/python
#
-# Copyright (c) 2009 rPath, Inc.
+# Copyright (c) 2009-2010 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
@@ -32,10 +32,9 @@
from conary.lib import util
sys.excepthook = util.genExcepthook()
-import rhnmirror
-
from updatebot import config
from updatebot import ordered
+from updatebot import pkgsource
from updatebot import log as logSetup
logSetup.addRootLogger()
@@ -58,13 +57,45 @@
cfg = config.UpdateBotConfig()
cfg.read(confDir + '/updatebotrc')
-mcfg = rhnmirror.MirrorConfig()
-mcfg.read(confDir + '/erratarc')
+fltr = None
-errata = rhnmirror.Errata(mcfg)
+if cfg.platformName == 'rhel':
+ import rhnmirror
+
+ mcfg = rhnmirror.MirrorConfig()
+ mcfg.read(confDir + '/erratarc')
+
+ errata = rhnmirror.Errata(mcfg)
+elif cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
+
+ pkgSource = pkgsource.PackageSource(cfg)
+ errata = Errata(pkgSource)
+
+ def fltr(sourceSet):
+ removed = set()
+ removedNames = set()
+ pkgSource.load()
+ for src, bins in pkgSource.srcPkgMap.iteritems():
+ # filter out packages that we don't handle right now.
+ if ([ x for x in bins if 'kmp' in x.name ] or
+ # the kernel needs a recipe
+ 'kernel' in src.name):
+
+ removedNames.add(src.name)
+
+ for src, bins in pkgSource.srcPkgMap.iteritems():
+ if src.name in removedNames:
+ removed.add(src)
+
+ return sourceSet - removed
+
+else:
+ raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+
errata.fetch()
bot = ordered.Bot(cfg, errata)
-pkgMap = bot.update(restoreFile=restoreFile)
+pkgMap, failures = bot.update(fltr=fltr)
import epdb; epdb.st()
From agrimm at rpath.com Thu May 20 14:09:41 2010
From: agrimm at rpath.com (Andy Grimm)
Date: Thu, 20 May 2010 18:09:41 +0000
Subject: mirrorball: initialize an empty list
Message-ID: <201005201809.o4KI9fvJ019118@scc.eng.rpath.com>
changeset: 3347789957c2
user: Andy Grimm
date: Thu, 20 May 2010 14:09:34 -0400
initialize an empty list
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -350,6 +350,7 @@
@raises UpdateRemovesPackageError
"""
+ keepRemovedPackages = keepRemovedPackages or []
needsUpdate = False
newNames = [ (x.name, x.arch) for x in self._pkgSource.srcPkgMap[srpm] ]
metadata = None
From elliot at rpath.com Thu May 20 16:54:10 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 20:54:10 +0000
Subject: mirrorball: add missing import
Message-ID: <201005202054.o4KKsAku028269@scc.eng.rpath.com>
changeset: fd8516ab8259
user: Elliot Peele
date: Thu, 20 May 2010 14:20:55 -0400
add missing import
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -30,6 +30,7 @@
from updatebot.errors import SourceNotImportedError
from updatebot.errors import OldVersionNotFoundError
from updatebot.errors import UpdateGoesBackwardsError
+from updatebot.errors import UpdateReusesPackageError
from updatebot.errors import UpdateRemovesPackageError
from updatebot.errors import ParentPlatformManifestInconsistencyError
from updatebot.errors import RepositoryPackageSourceInconsistencyError
@@ -604,6 +605,7 @@
parentPackages = set()
total = len(toCreate)
current = 1
+
for pkg in sorted(toCreate):
try:
# Only import packages that haven't been imported before
From elliot at rpath.com Thu May 20 16:54:10 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 20:54:10 +0000
Subject: mirrorball: 1. avoid adding empty sets to the usemap
Message-ID: <201005202054.o4KKsAV6028300@scc.eng.rpath.com>
changeset: 77bfc1d4246a
user: Elliot Peele
date: Thu, 20 May 2010 15:35:51 -0400
1. avoid adding empty sets to the usemap
2. add versionless specs to the usemap for package versions that may have been
combined with older versions through an update.
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -362,17 +362,24 @@
# given build flavor, even if the package only contains
# a buildlog.
(srcPkg.name, conaryVersion, flv),
+
+ (binPkg.name, flv),
+ (srcPkg.name, flv),
])
for trvSpec in trvSpecs:
- useSet = self.useMap.setdefault(trvSpec, set())
+ useSet = set()
if binPkg.arch == 'noarch':
if flv == 'x86' and 'x86' in archSet:
useSet.add('x86')
elif flv == 'x86_64' and 'x86_64' in archSet:
useSet.add('x86_64')
else:
+ assert archSet
useSet.update(archSet)
+ if useSet:
+ self.useMap.setdefault(trvSpec, set()).update(useSet)
+
def loadFileLists(self, client, basePath):
"""
Parse file information.
From elliot at rpath.com Thu May 20 16:54:10 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 20:54:10 +0000
Subject: mirrorball: search the usemap for a couple of different options
Message-ID: <201005202054.o4KKsAIl028327@scc.eng.rpath.com>
changeset: e764e75b4895
user: Elliot Peele
date: Thu, 20 May 2010 15:36:29 -0400
search the usemap for a couple of different options
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -247,9 +247,11 @@
def add():
upver = version.trailingRevision().version
for flv in flavors:
- key = (name, upver, flvMap[flv])
- if key in self._useMap:
- for useStr in self._useMap[key]:
+ primary = (name, upver, flvMap[flv])
+ secondary = (name, flvMap[flv])
+ use = self._useMap.get(primary, self._useMap.get(secondary, []))
+ if use:
+ for useStr in use:
self._add(name, version=version, flavor=flv,
use=useStr, groupName=groupName)
else:
From elliot at rpath.com Thu May 20 16:54:11 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 20:54:11 +0000
Subject: mirrorball: add capability to move nosrc content into a single
source package
Message-ID: <201005202054.o4KKsBKw028354@scc.eng.rpath.com>
changeset: 83d6de191895
user: Elliot Peele
date: Thu, 20 May 2010 16:49:41 -0400
add capability to move nosrc content into a single source package
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -68,9 +68,9 @@
raise ParseError, e
-class CfgContextFilter(CfgRegExp):
+class CfgStringFilter(CfgRegExp):
"""
- Class for parsing context name/regex tuples.
+ Class for parsing (string, regex) tuples.
"""
def parseString(self, val):
@@ -230,6 +230,10 @@
# repositoryName archString
repositoryArch = (CfgDict(CfgString), {})
+ # Associate binaries generated from a nosrc package with a source package
+ # name if the nosrc package matches a given regular expression.
+ nosrcFilter = (CfgList(CfgStringFilter), [])
+
# Ignore packages with "32bit" in the name. This is intened for use with
# SLES based platforms.
ignore32bitPackages = (CfgBool, False)
@@ -303,7 +307,7 @@
listArchiveStartDate = CfgString
# list of contexts that all packages are built in.
- archContexts = CfgList(CfgContextFilter)
+ archContexts = CfgList(CfgStringFilter)
# flavors to build the source group.
groupFlavors = (CfgList(CfgFlavor), [])
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -380,6 +380,36 @@
if useSet:
self.useMap.setdefault(trvSpec, set()).update(useSet)
+ # In the case of SLES 10 we need to combine several source entries in
+ # the srcPkgMap to create a single unified kernel source package.
+ if self._cfg.nosrcFilter:
+ # Find all nosrc rpms in the srcPKgMap, have to use basename of the
+ # location since nosrc packages have an arch of 'src'.
+ nosrcMap = dict([ (x, y) for x, y in self.srcPkgMap.iteritems()
+ if 'nosrc' in os.path.basename(x.location) ])
+
+ # Build a mapping of version to nosrc package.
+ verMap = {}
+ for src in nosrcMap:
+ verMap.setdefault((src.version, src.release), set()).add(src)
+
+ for srcName, (fltrStr, fltr) in self._cfg.nosrcFilter:
+ for src in self.srcNameMap[srcName]:
+ # Match source version and release to nosrc version and
+ # release. This may be too strong a requirement.
+ if (src.version, src.release) not in verMap:
+ continue
+
+ # Move all binaries associated with the nosrc package into
+ # the source package.
+ for nosrc in verMap[(src.version, src.release)]:
+ if fltr.match(nosrc.name):
+ log.info('relocating package content %s -> %s'
+ % (nosrc, src))
+ nosrcSet = self.srcPkgMap.get(nosrc)
+ self.srcPkgMap[src].update(nosrcSet)
+ self.srcPkgMap[nosrc] = self.srcPkgMap[src]
+
def loadFileLists(self, client, basePath):
"""
Parse file information.
From elliot at rpath.com Thu May 20 16:54:11 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 20:54:11 +0000
Subject: mirrorball: add ability for callers to filter packages
Message-ID: <201005202054.o4KKsBGe028381@scc.eng.rpath.com>
changeset: 0429f41625d0
user: Elliot Peele
date: Thu, 20 May 2010 16:53:03 -0400
add ability for callers to filter packages
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -145,6 +145,10 @@
self._pkgSource.load()
toCreate = self._errata.getInitialPackages()
+ fltr = kwargs.pop('fltr', None)
+ if fltr:
+ toCreate = fltr(toCreate)
+
pkgMap, failures = self._create(*args, toCreate=toCreate, **kwargs)
# Insert package map into group.
@@ -273,6 +277,10 @@
# Update package set.
else:
+ fltr = kwargs.pop('fltr', None)
+ if fltr:
+ updates = fltr(updates)
+
pkgMap = self._update(*args, updatePkgs=updates,
expectedRemovals=expectedRemovals,
allowPackageDowngrades=allowDowngrades, **kwargs)
From elliot at rpath.com Thu May 20 16:54:11 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 20:54:11 +0000
Subject: mirrorball: disable dep check for standard group and write out
generated model
Message-ID: <201005202054.o4KKsBVH028408@scc.eng.rpath.com>
changeset: 8157355f6086
user: Elliot Peele
date: Thu, 20 May 2010 16:53:57 -0400
disable dep check for standard group and write out generated model
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -36,7 +36,6 @@
import logging
from updatebot import OrderedBot
-from updatebot.groupmgr.model import GroupContentsModel
log = logging.getLogger('tmplogger')
@@ -513,6 +512,7 @@
# 'zlib-devel',
)
+ log.info('getting latest troves')
troves = self._updater._conaryhelper._getLatestTroves()
# combine packages of the same name.
@@ -545,12 +545,19 @@
group.errataState = '0'
group.version = '0'
-# pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
-# pkgGroup.depCheck = False
+ addReq = dict([ ('group-standard', [ (x, None) for x in standard ]), ])
+ group.modifyContents(additions=addReq)
- addReq = dict([ ('group-standard', [ (x, None) for x in standard ]), ])
+ group._groups['group-standard'].depCheck = False
- group.modifyContents(additions=addReq)
+ group.removePackage('samba-pdb')
+ group.removePackage('kiwi-desc-xennetboot')
+
+ group._copyVersions()
+ group._sanityCheck()
+ group._mgr._persistGroup(group)
+
+ import epdb; epdb.st()
group.commit()
built = group.build()
From elliot at rpath.com Thu May 20 16:54:11 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 20 May 2010 20:54:11 +0000
Subject: mirrorball: branch merge
Message-ID: <201005202054.o4KKsBqd028435@scc.eng.rpath.com>
changeset: 79d4f2c04ebf
user: Elliot Peele
date: Thu, 20 May 2010 16:54:08 -0400
branch merge
diff --git a/scripts/creategroup.py b/scripts/creategroup.py
--- a/scripts/creategroup.py
+++ b/scripts/creategroup.py
@@ -36,7 +36,6 @@
import logging
from updatebot import OrderedBot
-from updatebot.groupmgr.model import GroupContentsModel
log = logging.getLogger('tmplogger')
@@ -513,6 +512,7 @@
# 'zlib-devel',
)
+ log.info('getting latest troves')
troves = self._updater._conaryhelper._getLatestTroves()
# combine packages of the same name.
@@ -545,12 +545,19 @@
group.errataState = '0'
group.version = '0'
-# pkgGroup = self._groupmgr._groups[self._groupmgr._pkgGroupName]
-# pkgGroup.depCheck = False
+ addReq = dict([ ('group-standard', [ (x, None) for x in standard ]), ])
+ group.modifyContents(additions=addReq)
- addReq = dict([ ('group-standard', [ (x, None) for x in standard ]), ])
+ group._groups['group-standard'].depCheck = False
- group.modifyContents(additions=addReq)
+ group.removePackage('samba-pdb')
+ group.removePackage('kiwi-desc-xennetboot')
+
+ group._copyVersions()
+ group._sanityCheck()
+ group._mgr._persistGroup(group)
+
+ import epdb; epdb.st()
group.commit()
built = group.build()
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -68,9 +68,9 @@
raise ParseError, e
-class CfgContextFilter(CfgRegExp):
+class CfgStringFilter(CfgRegExp):
"""
- Class for parsing context name/regex tuples.
+ Class for parsing (string, regex) tuples.
"""
def parseString(self, val):
@@ -230,6 +230,10 @@
# repositoryName archString
repositoryArch = (CfgDict(CfgString), {})
+ # Associate binaries generated from a nosrc package with a source package
+ # name if the nosrc package matches a given regular expression.
+ nosrcFilter = (CfgList(CfgStringFilter), [])
+
# Ignore packages with "32bit" in the name. This is intened for use with
# SLES based platforms.
ignore32bitPackages = (CfgBool, False)
@@ -303,7 +307,7 @@
listArchiveStartDate = CfgString
# list of contexts that all packages are built in.
- archContexts = CfgList(CfgContextFilter)
+ archContexts = CfgList(CfgStringFilter)
# flavors to build the source group.
groupFlavors = (CfgList(CfgFlavor), [])
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -247,9 +247,11 @@
def add():
upver = version.trailingRevision().version
for flv in flavors:
- key = (name, upver, flvMap[flv])
- if key in self._useMap:
- for useStr in self._useMap[key]:
+ primary = (name, upver, flvMap[flv])
+ secondary = (name, flvMap[flv])
+ use = self._useMap.get(primary, self._useMap.get(secondary, []))
+ if use:
+ for useStr in use:
self._add(name, version=version, flavor=flv,
use=useStr, groupName=groupName)
else:
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -145,6 +145,10 @@
self._pkgSource.load()
toCreate = self._errata.getInitialPackages()
+ fltr = kwargs.pop('fltr', None)
+ if fltr:
+ toCreate = fltr(toCreate)
+
pkgMap, failures = self._create(*args, toCreate=toCreate, **kwargs)
# Insert package map into group.
@@ -273,6 +277,10 @@
# Update package set.
else:
+ fltr = kwargs.pop('fltr', None)
+ if fltr:
+ updates = fltr(updates)
+
pkgMap = self._update(*args, updatePkgs=updates,
expectedRemovals=expectedRemovals,
allowPackageDowngrades=allowDowngrades, **kwargs)
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -362,17 +362,54 @@
# given build flavor, even if the package only contains
# a buildlog.
(srcPkg.name, conaryVersion, flv),
+
+ (binPkg.name, flv),
+ (srcPkg.name, flv),
])
for trvSpec in trvSpecs:
- useSet = self.useMap.setdefault(trvSpec, set())
+ useSet = set()
if binPkg.arch == 'noarch':
if flv == 'x86' and 'x86' in archSet:
useSet.add('x86')
elif flv == 'x86_64' and 'x86_64' in archSet:
useSet.add('x86_64')
else:
+ assert archSet
useSet.update(archSet)
+ if useSet:
+ self.useMap.setdefault(trvSpec, set()).update(useSet)
+
+ # In the case of SLES 10 we need to combine several source entries in
+ # the srcPkgMap to create a single unified kernel source package.
+ if self._cfg.nosrcFilter:
+ # Find all nosrc rpms in the srcPKgMap, have to use basename of the
+ # location since nosrc packages have an arch of 'src'.
+ nosrcMap = dict([ (x, y) for x, y in self.srcPkgMap.iteritems()
+ if 'nosrc' in os.path.basename(x.location) ])
+
+ # Build a mapping of version to nosrc package.
+ verMap = {}
+ for src in nosrcMap:
+ verMap.setdefault((src.version, src.release), set()).add(src)
+
+ for srcName, (fltrStr, fltr) in self._cfg.nosrcFilter:
+ for src in self.srcNameMap[srcName]:
+ # Match source version and release to nosrc version and
+ # release. This may be too strong a requirement.
+ if (src.version, src.release) not in verMap:
+ continue
+
+ # Move all binaries associated with the nosrc package into
+ # the source package.
+ for nosrc in verMap[(src.version, src.release)]:
+ if fltr.match(nosrc.name):
+ log.info('relocating package content %s -> %s'
+ % (nosrc, src))
+ nosrcSet = self.srcPkgMap.get(nosrc)
+ self.srcPkgMap[src].update(nosrcSet)
+ self.srcPkgMap[nosrc] = self.srcPkgMap[src]
+
def loadFileLists(self, client, basePath):
"""
Parse file information.
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -30,6 +30,7 @@
from updatebot.errors import SourceNotImportedError
from updatebot.errors import OldVersionNotFoundError
from updatebot.errors import UpdateGoesBackwardsError
+from updatebot.errors import UpdateReusesPackageError
from updatebot.errors import UpdateRemovesPackageError
from updatebot.errors import ParentPlatformManifestInconsistencyError
from updatebot.errors import RepositoryPackageSourceInconsistencyError
@@ -605,6 +606,7 @@
parentPackages = set()
total = len(toCreate)
current = 1
+
for pkg in sorted(toCreate):
try:
# Only import packages that haven't been imported before
From elliot at rpath.com Mon May 24 13:36:22 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 24 May 2010 17:36:22 +0000
Subject: mirrorball: ignore x86_64 packages found in an x86 repository
Message-ID: <201005241736.o4OHaMOD023833@scc.eng.rpath.com>
changeset: a5fd56f5d4c4
user: Elliot Peele
date: Mon, 24 May 2010 13:36:19 -0400
ignore x86_64 packages found in an x86 repository
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -165,6 +165,11 @@
@type package: repomd.packagexml._Package
"""
+ # Exclude all x86_64 packages that are in an x86 repository.
+ if archStr and archStr == 'x86' and package.arch == 'x86_64':
+ log.warn('ignoring %s because it is an x86_64 package an x86 '
+ 'repository' % package)
+
# FIXME: There should be a better way to figure out the tuple that
# represents the hash of the srcPkg.
srcParts = package.sourcerpm.split('-')
From elliot at rpath.com Mon May 24 13:56:56 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 24 May 2010 17:56:56 +0000
Subject: mirrorball: actually skip the package
Message-ID: <201005241756.o4OHuumX024880@scc.eng.rpath.com>
changeset: 57795f1f0c84
user: Elliot Peele
date: Mon, 24 May 2010 13:56:54 -0400
actually skip the package
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -169,6 +169,7 @@
if archStr and archStr == 'x86' and package.arch == 'x86_64':
log.warn('ignoring %s because it is an x86_64 package an x86 '
'repository' % package)
+ continue
# FIXME: There should be a better way to figure out the tuple that
# represents the hash of the srcPkg.
From elliot at rpath.com Mon May 24 13:58:56 2010
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 24 May 2010 17:58:56 +0000
Subject: mirrorball: really don't add x86_64 packages in an x86 repository
Message-ID: <201005241758.o4OHwuLV024994@scc.eng.rpath.com>
changeset: 22b8ff8cd4c6
user: Elliot Peele
date: Mon, 24 May 2010 13:58:54 -0400
really don't add x86_64 packages in an x86 repository
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -169,7 +169,7 @@
if archStr and archStr == 'x86' and package.arch == 'x86_64':
log.warn('ignoring %s because it is an x86_64 package an x86 '
'repository' % package)
- continue
+ return
# FIXME: There should be a better way to figure out the tuple that
# represents the hash of the srcPkg.
From elliot at rpath.com Mon May 24 22:50:54 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 02:50:54 +0000
Subject: mirrorball: more consistent interface
Message-ID: <201005250250.o4P2os3E007027@scc.eng.rpath.com>
changeset: 5ee3f50cc0b5
user: Elliot Peele
date: Mon, 24 May 2010 22:47:57 -0400
more consistent interface
diff --git a/errata/common.py b/errata/common.py
--- a/errata/common.py
+++ b/errata/common.py
@@ -15,8 +15,7 @@
"""
This module is here as an example of the data model and interfaces that
mirrorball is looking for when importing and updating platforms in advisory
-order. These are not meant as superclasses for any sort of implementation. The
-variables and methods that are defined below must be available.
+order. The variables and methods that are defined below must be available.
"""
def reqfetch(func):
@@ -31,6 +30,28 @@
return wrap
+class Nevra(object):
+ """
+ Class to represent a package nevra.
+ """
+
+ __slots__ = ('name', 'epoch', 'version', 'release', 'arch', )
+
+ def __init__(self, name, epoch, version, release, arch):
+ self.name = name
+ self.epoch = epoch
+ self.version = version
+ self.release = release
+ self.arch = arch
+
+ def getNevra(self):
+ """
+ Return a tuple representation of the nevra.
+ """
+
+ return (self.name, self.epoch, self.version, self.release, self.arch)
+
+
class Package(object):
"""
Class to represent a package.
@@ -39,11 +60,14 @@
@type channels: list(Repository, ...)
"""
- def __init__(self, channels):
- for ch in channels:
- assert isinstance(ch, Channel)
+ __slots__ = ('channel', 'nevra', )
- self.channels = channels
+ def __init__(self, channel, nevra):
+ assert isinstance(channel, Channel)
+ assert isinstance(nevra, Nevra)
+
+ self.channel = channel
+ self.nevra = nevra
def getNevra(self):
"""
@@ -51,7 +75,7 @@
this package.
"""
- raise NotImplementedError
+ return self.nevra.getNevra()
class Channel(object):
@@ -62,6 +86,8 @@
@type label: str
"""
+ __slots__ = ('label', )
+
def __init__(self, label):
self.label = label
@@ -82,15 +108,24 @@
@type synopsis: str
"""
+ __slots__ = ('advisory', 'synopsis', 'issue_date', 'nevraChannels', )
+
def __init__(self, advisory, synopsis, issue_date, packages):
self.advisory = advisory
- self.synposis = synopsis
+ self.synopsis = synopsis
self.issue_date = issue_date
for pkg in packages:
assert isinstance(pkg, Package)
- self.packages = packages
+ self.nevraChannels = packages
+
+ def __hash__(self):
+ return hash((self.advisory, self.synopsis, self.issue_date))
+
+ def __cmp__(self, other):
+ return cmp((self.issue_date, self.advisory, self.synopsis),
+ (other.issue_date, other.advisory, other.synopsis))
class AdvisoryManager(object):
@@ -99,6 +134,9 @@
platform that can then be matched up to a package source.
"""
+ def __init__(self):
+ self._fetched = False
+
def getRepositories(self):
"""
Returns a list of repository labels that have been fetched.
From elliot at rpath.com Mon May 24 22:50:55 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 02:50:55 +0000
Subject: mirrorball: centos errata sources
Message-ID: <201005250250.o4P2otd3007058@scc.eng.rpath.com>
changeset: 463eeb701cdd
user: Elliot Peele
date: Mon, 24 May 2010 22:48:06 -0400
centos errata sources
diff --git a/errata/sles.py b/errata/centos.py
copy from errata/sles.py
copy to errata/centos.py
--- a/errata/sles.py
+++ b/errata/centos.py
@@ -13,50 +13,31 @@
#
"""
-Generate update information based on the patch detail in SuSE repositories.
+Generate update information based on the ordering of a pkgSource.
"""
+import time
import logging
+from updatebot.lib import util
+
from errata import common
+from errata.common import Nevra
+from errata.common import Package
+from errata.common import Channel
+from errata.common import Advisory
log = logging.getLogger('errata')
-class Package(common.Package):
- """
- Class to represent a package.
- """
-
- def getNevra(self):
- """
- Returns a tuple of (name, epoch, version, release, arch) for
- this package.
- """
-
-class Channel(common.Channel):
- """
- Class to represent a repository.
- """
-
-
-class Advisory(common.Advisory):
- """
- Class to represent an errata or advisory.
- """
-
-
class AdvisoryManager(common.AdvisoryManager):
def __init__(self, pkgSource):
+ common.AdvisoryManager.__init__(self)
+
self._pkgSource = pkgSource
- self._fetched = False
- self._patches = set()
-
- @common.reqfetch
- def getRepositories(self):
- """
- Returns a list of repository labels that have been fetched.
- """
+ self._channels = {}
+ self._advOrder = {}
+ self._advisories = set()
@common.reqfetch
def iterByIssueDate(self):
@@ -64,7 +45,9 @@
Yields Errata objects by the issue date of the errata.
"""
- return []
+ for updateId in sorted(self._advOrder):
+ for adv in self._advOrder[updateId]:
+ yield adv
def fetch(self):
"""
@@ -75,7 +58,9 @@
excesive load for anyone's servers.
"""
+ self._order()
self._fetched = True
+ return self._advisories
@common.reqfetch
def getChannels(self):
@@ -84,15 +69,16 @@
@return list of indexed channel names
"""
- return self._pkgSource._clients.keys()
-
+ return self._channels.keys()
def cleanup(self):
"""
Free all cached results.
"""
- self._patches = set()
+ self._channels = {}
+ self._advOrder = {}
+ self._advisories = set()
def getModifiedErrata(self, updateId):
"""
@@ -102,18 +88,87 @@
return []
- def _fetchPatches(self):
+ def _order(self):
"""
Fetch all patch data from the package source.
"""
+ def srtByNevra(a, b):
+ return util.packagevercmp(a, b)
+
+ def srtByBuildTime(a, b):
+ assert hasattr(a, 'buildTimestamp')
+ assert hasattr(b, 'buildTimestamp')
+ assert a.buildTimestamp not in ('0', '', 0)
+ assert b.buildTimestamp not in ('0', '', 0)
+ return cmp(int(a.buildTimestamp), int(b.buildTimestamp))
+
+ def slice(ts):
+ """
+ Convert a time stamp into the desired time slice.
+ """
+
+ # convert to current day
+ return int(time.mktime(time.strptime(time.strftime('%Y%m%d',
+ time.gmtime(ts)), '%Y%m%d')))
+
+ def getChannel(pkg):
+ for label, channel in self._channels.iteritems():
+ if label in pkg.location:
+ return channel
+
# make sure the pkg source is loaded.
self._pkgSource.load()
- # now get the patch data
- patches = set()
- for path, client in self._pkgSource.getClients().iteritems():
- log.info('loading patches for %s' % path)
- patches.update(set(client.getPatchDetail()))
+ # Make sure that nevra order and build time order are the same for each
+ # source name.
+ order = {}
+ for srcName, srcPkgs in self._pkgSource.srcNameMap.iteritems():
+ log.info('checking %s' % srcName)
+ sortedNevras = sorted(srcPkgs, cmp=srtByNevra)
+ sortedBuildTime = sorted(srcPkgs, cmp=srtByBuildTime)
+ assert sortedNevras == sortedBuildTime
- return patches
+ for srcPkg in srcPkgs:
+ assert srcPkg.buildTimestamp is not None
+ order.setdefault(int(srcPkg.buildTimestamp), set()).add(srcPkg)
+
+ slices = {}
+ for updateId, srcPkgs in order.iteritems():
+ slices.setdefault(slice(updateId), set()).update(srcPkgs)
+
+ # find labels
+ for label in self._pkgSource._clients:
+ self._channels[label] = Channel(label)
+
+ # make package objects from binaries
+ nevras = {}
+ packages = {}
+ srcPkgMap = {}
+ for sliceId, srcPkgs in slices.iteritems():
+ for srcPkg in srcPkgs:
+ pkgSet = srcPkgMap.setdefault(srcPkg, set())
+ for binPkg in self._pkgSource.srcPkgMap[srcPkg]:
+ if binPkg.arch == 'src':
+ continue
+ nevra = binPkg.getNevra()
+ nevraObj = nevras.setdefault(nevra, Nevra(*nevra))
+ channelObj = getChannel(binPkg)
+ package = Package(channelObj, nevraObj)
+ packageObj = packages.setdefault(package, package)
+ srcPkgMap.setdefault(srcPkg, set()).add(packageObj)
+
+ # create advisories
+ for sliceId, srcPkgs in slices.iteritems():
+ for srcPkg in srcPkgs:
+ advisory = 'cu-%s' % srcPkg
+ synopsis = 'update of %s' % srcPkg
+ issue_date = time.strftime('%Y-%m-%d %H:%M:%S',
+ time.gmtime(sliceId))
+ packages = srcPkgMap[srcPkg]
+
+ adv = Advisory(advisory, synopsis, issue_date, packages)
+ self._advisories.add(adv)
+ self._advOrder.setdefault(sliceId, set()).add(adv)
+
+ log.info('creating advisory: %s' % advisory)
From elliot at rpath.com Tue May 25 17:00:46 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 21:00:46 +0000
Subject: mirrorball: add script for testing centos ordering
Message-ID: <201005252100.o4PL0keq002560@scc.eng.rpath.com>
changeset: 53267020f313
user: Elliot Peele
date: Tue, 25 May 2010 10:37:34 -0400
add script for testing centos ordering
diff --git a/scripts/sleorder.py b/scripts/centosorder.py
copy from scripts/sleorder.py
copy to scripts/centosorder.py
--- a/scripts/sleorder.py
+++ b/scripts/centosorder.py
@@ -3,12 +3,8 @@
import os
import sys
import time
-import tempfile
sys.path.insert(0, os.environ['HOME'] + '/hg/conary')
-sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror')
-sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-xmllib')
-sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-capsule-indexer')
from conary.lib import util
sys.excepthook = util.genExcepthook()
@@ -23,7 +19,7 @@
from updatebot import UpdateBotConfig
from updatebot import pkgsource
-from errata.sles import AdvisoryManager as Errata
+from errata.centos import AdvisoryManager as Errata
slog = log.addRootLogger()
From elliot at rpath.com Tue May 25 17:00:46 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 21:00:46 +0000
Subject: mirrorball: build kernel modules for sles
Message-ID: <201005252100.o4PL0krJ002591@scc.eng.rpath.com>
changeset: c69b2ae9b807
user: Elliot Peele
date: Tue, 25 May 2010 10:37:55 -0400
build kernel modules for sles
diff --git a/scripts/order_import.py b/scripts/order_import.py
--- a/scripts/order_import.py
+++ b/scripts/order_import.py
@@ -74,7 +74,7 @@
pkgSource.load()
for src, bins in pkgSource.srcPkgMap.iteritems():
# filter out packages that we don't handle right now.
- if ([ x for x in bins if 'kmp' in x.name ] or
+ if (#[ x for x in bins if 'kmp' in x.name ] or
# the kernel needs a recipe
'kernel' in src.name):
From elliot at rpath.com Tue May 25 17:00:46 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 21:00:46 +0000
Subject: mirrorball: code cleanup
Message-ID: <201005252100.o4PL0kIR002618@scc.eng.rpath.com>
changeset: abedb18321ff
user: Elliot Peele
date: Tue, 25 May 2010 13:45:07 -0400
code cleanup
diff --git a/updatebot/errata.py b/updatebot/errata.py
--- a/updatebot/errata.py
+++ b/updatebot/errata.py
@@ -564,9 +564,7 @@
missing.add(pkg)
src = self._pkgSource.binPkgMap[pkg]
- if src not in srcMap:
- srcMap[src] = []
- srcMap[src].append(pkg)
+ srcMap.setdefault(src, list()).append(pkg)
# Raise an error if there are any packages missing an errata that are
# now explicitly allowed by the config.
From elliot at rpath.com Tue May 25 17:00:47 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 21:00:47 +0000
Subject: mirrorball: copy binary timestamps into synthesized sources
Message-ID: <201005252100.o4PL0lFi002645@scc.eng.rpath.com>
changeset: c2e81e8938cb
user: Elliot Peele
date: Tue, 25 May 2010 13:45:41 -0400
copy binary timestamps into synthesized sources
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -451,6 +451,13 @@
# of the file. The factory will take care of the rest.
srcPkg.location = pkg.sourcerpm
+ # Copy the greatest build time from the list of binaries to
+ # determine the source build time.
+ def srtByBuildTime(a, b):
+ return cmp(int(a.buildTimestamp), int(b.buildTimestamp))
+ pkgs = sorted(bins, cmp=srtByBuildTime)
+ srcPkg.buildTimestamp = pkgs[-1].buildTimestamp
+
return srcPkg
def synthesizeSource(srcPkg):
From elliot at rpath.com Tue May 25 17:00:47 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 21:00:47 +0000
Subject: mirrorball: avoid loading a package source twice
Message-ID: <201005252100.o4PL0lOa002672@scc.eng.rpath.com>
changeset: 99b51b673317
user: Elliot Peele
date: Tue, 25 May 2010 13:46:03 -0400
avoid loading a package source twice
diff --git a/scripts/centosorder.py b/scripts/centosorder.py
--- a/scripts/centosorder.py
+++ b/scripts/centosorder.py
@@ -17,7 +17,6 @@
from updatebot import log
from updatebot.ordered import Bot
from updatebot import UpdateBotConfig
-from updatebot import pkgsource
from errata.centos import AdvisoryManager as Errata
@@ -26,12 +25,12 @@
cfg = UpdateBotConfig()
cfg.read(os.path.join(confDir, 'updatebotrc'))
-pkgSource = pkgsource.PackageSource(cfg)
+bot = Bot(cfg, None)
+errata = Errata(bot._pkgSource)
+bot._errata._errata = errata
-errata = Errata(pkgSource)
errata.fetch()
-bot = Bot(cfg, errata)
bot._pkgSource.load()
bot._errata._orderErrata()
From elliot at rpath.com Tue May 25 17:00:48 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 21:00:48 +0000
Subject: mirrorball: 1. just sort by build timestamp, not nevra
Message-ID: <201005252100.o4PL0mnn002699@scc.eng.rpath.com>
changeset: 49a575b51a8b
user: Elliot Peele
date: Tue, 25 May 2010 15:07:10 -0400
1. just sort by build timestamp, not nevra
2. determine base set by looking for the first time a package name is updated
diff --git a/errata/centos.py b/errata/centos.py
--- a/errata/centos.py
+++ b/errata/centos.py
@@ -93,16 +93,6 @@
Fetch all patch data from the package source.
"""
- def srtByNevra(a, b):
- return util.packagevercmp(a, b)
-
- def srtByBuildTime(a, b):
- assert hasattr(a, 'buildTimestamp')
- assert hasattr(b, 'buildTimestamp')
- assert a.buildTimestamp not in ('0', '', 0)
- assert b.buildTimestamp not in ('0', '', 0)
- return cmp(int(a.buildTimestamp), int(b.buildTimestamp))
-
def slice(ts):
"""
Convert a time stamp into the desired time slice.
@@ -120,22 +110,11 @@
# make sure the pkg source is loaded.
self._pkgSource.load()
- # Make sure that nevra order and build time order are the same for each
- # source name.
- order = {}
- for srcName, srcPkgs in self._pkgSource.srcNameMap.iteritems():
- log.info('checking %s' % srcName)
- sortedNevras = sorted(srcPkgs, cmp=srtByNevra)
- sortedBuildTime = sorted(srcPkgs, cmp=srtByBuildTime)
- assert sortedNevras == sortedBuildTime
-
- for srcPkg in srcPkgs:
- assert srcPkg.buildTimestamp is not None
- order.setdefault(int(srcPkg.buildTimestamp), set()).add(srcPkg)
-
+ # Sort packages by build timestamp.
slices = {}
- for updateId, srcPkgs in order.iteritems():
- slices.setdefault(slice(updateId), set()).update(srcPkgs)
+ for srcPkg in self._pkgSource.srcPkgMap:
+ updateId = slice(int(srcPkg.buildTimestamp))
+ slices.setdefault(updateId, set()).set(srcPkg)
# find labels
for label in self._pkgSource._clients:
@@ -145,30 +124,54 @@
nevras = {}
packages = {}
srcPkgMap = {}
- for sliceId, srcPkgs in slices.iteritems():
+
+ seen = set()
+ startedUpdates = False
+ for sliceId, srcPkgs in sorted(slices.iteritems()):
for srcPkg in srcPkgs:
+ # Assume everything before we see a dupllicate source name is in
+ # the base set of packages.
+ if srcPkg.name not in seen and not startedUpdates:
+ seen.add(srcPkg.name)
+ continue
+ startedUpdates = True
+
pkgSet = srcPkgMap.setdefault(srcPkg, set())
for binPkg in self._pkgSource.srcPkgMap[srcPkg]:
if binPkg.arch == 'src':
continue
+
+ # Get a nevra object
nevra = binPkg.getNevra()
nevraObj = nevras.setdefault(nevra, Nevra(*nevra))
+
+ # Get a channel object
channelObj = getChannel(binPkg)
+
+ # Create a package object
package = Package(channelObj, nevraObj)
packageObj = packages.setdefault(package, package)
+
+ # Add packages to the package map
srcPkgMap.setdefault(srcPkg, set()).add(packageObj)
# create advisories
for sliceId, srcPkgs in slices.iteritems():
for srcPkg in srcPkgs:
+ # If this source is not in the srcPkgMap it is probably
+ # considered to be in the base set of packages.
+ if srcPkg not in srcPkgMap:
+ continue
+
+ # Collect everything needed to make an advisory.
advisory = 'cu-%s' % srcPkg
synopsis = 'update of %s' % srcPkg
issue_date = time.strftime('%Y-%m-%d %H:%M:%S',
time.gmtime(sliceId))
packages = srcPkgMap[srcPkg]
+ # Create a fake advisory.
+ log.info('creating advisory: %s' % advisory)
adv = Advisory(advisory, synopsis, issue_date, packages)
self._advisories.add(adv)
self._advOrder.setdefault(sliceId, set()).add(adv)
-
- log.info('creating advisory: %s' % advisory)
From elliot at rpath.com Tue May 25 17:00:48 2010
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 25 May 2010 21:00:48 +0000
Subject: mirrorball: add support for centos errata source and don't load
pkgsource more than once
Message-ID: <201005252100.o4PL0m1h002728@scc.eng.rpath.com>
changeset: c3d2d815856f
user: Elliot Peele
date: Tue, 25 May 2010 15:10:10 -0400
add support for centos errata source and don't load pkgsource more than once
diff --git a/scripts/order_import.py b/scripts/order_import.py
--- a/scripts/order_import.py
+++ b/scripts/order_import.py
@@ -62,36 +62,44 @@
mcfg.read(confDir + '/erratarc')
errata = rhnmirror.Errata(mcfg)
-elif cfg.platformName == 'sles':
- from errata.sles import AdvisoryManager as Errata
+ errata.fetch()
- pkgSource = pkgsource.PackageSource(cfg)
- errata = Errata(pkgSource)
-
- def fltr(sourceSet):
- removed = set()
- removedNames = set()
- pkgSource.load()
- for src, bins in pkgSource.srcPkgMap.iteritems():
- # filter out packages that we don't handle right now.
- if (#[ x for x in bins if 'kmp' in x.name ] or
- # the kernel needs a recipe
- 'kernel' in src.name):
-
- removedNames.add(src.name)
-
- for src, bins in pkgSource.srcPkgMap.iteritems():
- if src.name in removedNames:
- removed.add(src)
-
- return sourceSet - removed
+ bot = ordered.Bot(cfg, errata)
else:
- raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+ bot = ordered.Bot(cfg, None)
-errata.fetch()
+ if cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
-bot = ordered.Bot(cfg, errata)
+ def fltr(sourceSet):
+ removed = set()
+ removedNames = set()
+ bot._pkgSource.load()
+ for src, bins in bot._pkgSource.srcPkgMap.iteritems():
+ # filter out packages that we don't handle right now.
+ if (#[ x for x in bins if 'kmp' in x.name ] or
+ # the kernel needs a recipe
+ 'kernel' in src.name):
+
+ removedNames.add(src.name)
+
+ for src, bins in bot._pkgSource.srcPkgMap.iteritems():
+ if src.name in removedNames:
+ removed.add(src)
+
+ return sourceSet - removed
+
+ elif cfg.platformName == 'centos':
+ from errata.centos import AdvisoryManager as Errata
+
+ else:
+ raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+
+ errata = Errata(bot._pkgSource)
+ bot._errata._errata = errata
+
+
pkgMap, failures = bot.create(fltr=fltr)
import epdb; epdb.st()
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -66,36 +66,43 @@
mcfg.read(confDir + '/erratarc')
errata = rhnmirror.Errata(mcfg)
-elif cfg.platformName == 'sles':
- from errata.sles import AdvisoryManager as Errata
- pkgSource = pkgsource.PackageSource(cfg)
- errata = Errata(pkgSource)
-
- def fltr(sourceSet):
- removed = set()
- removedNames = set()
- pkgSource.load()
- for src, bins in pkgSource.srcPkgMap.iteritems():
- # filter out packages that we don't handle right now.
- if ([ x for x in bins if 'kmp' in x.name ] or
- # the kernel needs a recipe
- 'kernel' in src.name):
-
- removedNames.add(src.name)
-
- for src, bins in pkgSource.srcPkgMap.iteritems():
- if src.name in removedNames:
- removed.add(src)
-
- return sourceSet - removed
+ bot = ordered.Bot(cfg, errata)
else:
- raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+ bot = ordered.Bot(cfg, None)
-errata.fetch()
+ if cfg.platformName == 'sles':
+ from errata.sles import AdvisoryManager as Errata
-bot = ordered.Bot(cfg, errata)
+ def fltr(sourceSet):
+ removed = set()
+ removedNames = set()
+ bot._pkgSource.load()
+ for src, bins in bot._pkgSource.srcPkgMap.iteritems():
+ # filter out packages that we don't handle right now.
+ if (#[ x for x in bins if 'kmp' in x.name ] or
+ # the kernel needs a recipe
+ 'kernel' in src.name):
+
+ removedNames.add(src.name)
+
+ for src, bins in bot._pkgSource.srcPkgMap.iteritems():
+ if src.name in removedNames:
+ removed.add(src)
+
+ return sourceSet - removed
+
+ elif cfg.platformName == 'centos':
+ from errata.centos import AdvisoryManager as Errata
+
+ else:
+ raise RuntimeError, 'no errata source found for %s' % cfg.platformName
+
+ errata = Errata(bot._pkgSource)
+ bot._errata._errata = errata
+
+
pkgMap, failures = bot.update(fltr=fltr)
import epdb; epdb.st()
From agrimm at rpath.com Tue May 25 21:29:52 2010
From: agrimm at rpath.com (Andy Grimm)
Date: Wed, 26 May 2010 01:29:52 +0000
Subject: mirrorball: fix builduaotloadrecipes script to set mirrorballpath
correctly
Message-ID: <201005260129.o4Q1Tqpv006827@scc.eng.rpath.com>
changeset: 6641a10944e8
user: Andy Grimm
date: Tue, 25 May 2010 21:29:49 -0400
fix builduaotloadrecipes script to set mirrorballpath correctly
diff --git a/scripts/buildautoloadrecipes b/scripts/buildautoloadrecipes
--- a/scripts/buildautoloadrecipes
+++ b/scripts/buildautoloadrecipes
@@ -22,7 +22,8 @@
context="x86"
fi
-mirrorballpath="$HOME/hg/mirrorball/mirrorball"
+# magic to determine where the script lives
+mirrorballpath=$(cd ${0%/*} && cd .. && pwd -P)
platformConfig="$mirrorballpath/config/$platform/conaryrc"
if [ ! -f $platformConfig ] ; then
From juphoff at rpath.com Wed May 26 09:29:18 2010
From: juphoff at rpath.com (juphoff at rpath.com)
Date: Wed, 26 May 2010 13:29:18 +0000
Subject: mirrorball: don't suppress passing Flags.debug to SUSE -kmp
modules packages
Message-ID: <201005261329.o4QDTIEm016738@scc.eng.rpath.com>
changeset: b673cb6236b3
user: Jeff Uphoff
date: Wed, 26 May 2010 09:29:13 -0400
don't suppress passing Flags.debug to SUSE -kmp modules packages
diff --git a/scripts/import b/scripts/import
--- a/scripts/import
+++ b/scripts/import
@@ -15,6 +15,7 @@
import os
import sys
+import itertools
sys.path.insert(0, os.path.abspath('../'))
@@ -27,7 +28,39 @@
cfg = config.UpdateBotConfig()
cfg.read(os.path.abspath('../') + '/config/%s/updatebotrc' % sys.argv[1])
obj = bot.Bot(cfg)
-trvMap = obj.create()
+
+# load the package source
+pkgSource = obj._pkgSource
+pkgSource.load()
+
+# filter out sources we don't know how to deal with yet.
+names = {}
+srcNevras = {}
+removed = {}
+removedNames = set()
+for src, bins in pkgSource.srcPkgMap.iteritems():
+ # a source without any binaries
+ if len(bins) <= 1:
+ continue
+
+ # filter out packages that we don't handle right now.
+ if (src.name in removedNames): #or
+ ## the kernel needs a recipe
+ #'kernel' in src.name):
+
+ removed.setdefault(src.getNevra(), src)
+ removedNames.add(src.name)
+ continue
+
+ names.setdefault(src.name, set()).add(src.getNevra())
+ srcNevras.setdefault(src.getNevra(), src)
+
+for nevra in itertools.chain(*[ x for x in names.values() if len(x) > 1 ]):
+ removed.setdefault(nevra, srcNevras.pop(nevra))
+
+toCreate = set(srcNevras.values())
+
+trvMap, failures = obj.create(toCreate=toCreate, recreate=True)
import epdb ; epdb.st()
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -357,9 +357,12 @@
and self._cfg.kernelFlavors):
for context, flavor in self._cfg.kernelFlavors:
# Replace flag name to match package
- if name != 'kernel':
- # Don't build kernel modules with a .debug flag, that
- # is only for kernels.
+ #
+ # Don't build kernel modules with a .debug flag, that
+ # is only for kernels...
+ # ...except in the case of SUSE -kmp module packages.
+ if (name != 'kernel' and
+ not ([bN for bN in binaryNames if '-kmp' in bN])):
if flavor.stronglySatisfies(
deps.parseFlavor('kernel.debug')):
continue
From juphoff at rpath.com Wed May 26 09:40:42 2010
From: juphoff at rpath.com (juphoff at rpath.com)
Date: Wed, 26 May 2010 13:40:42 +0000
Subject: mirrorball: back out accidental commit of throw-away changes to
import script
Message-ID: <201005261340.o4QDegPT016905@scc.eng.rpath.com>
changeset: ff8d58646c9d
user: Jeff Uphoff
date: Wed, 26 May 2010 09:40:37 -0400
back out accidental commit of throw-away changes to import script
diff --git a/scripts/import b/scripts/import
--- a/scripts/import
+++ b/scripts/import
@@ -15,7 +15,6 @@
import os
import sys
-import itertools
sys.path.insert(0, os.path.abspath('../'))
@@ -28,39 +27,7 @@
cfg = config.UpdateBotConfig()
cfg.read(os.path.abspath('../') + '/config/%s/updatebotrc' % sys.argv[1])
obj = bot.Bot(cfg)
-
-# load the package source
-pkgSource = obj._pkgSource
-pkgSource.load()
-
-# filter out sources we don't know how to deal with yet.
-names = {}
-srcNevras = {}
-removed = {}
-removedNames = set()
-for src, bins in pkgSource.srcPkgMap.iteritems():
- # a source without any binaries
- if len(bins) <= 1:
- continue
-
- # filter out packages that we don't handle right now.
- if (src.name in removedNames): #or
- ## the kernel needs a recipe
- #'kernel' in src.name):
-
- removed.setdefault(src.getNevra(), src)
- removedNames.add(src.name)
- continue
-
- names.setdefault(src.name, set()).add(src.getNevra())
- srcNevras.setdefault(src.getNevra(), src)
-
-for nevra in itertools.chain(*[ x for x in names.values() if len(x) > 1 ]):
- removed.setdefault(nevra, srcNevras.pop(nevra))
-
-toCreate = set(srcNevras.values())
-
-trvMap, failures = obj.create(toCreate=toCreate, recreate=True)
+trvMap = obj.create()
import epdb ; epdb.st()
From elliot at rpath.com Wed May 26 22:42:33 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 27 May 2010 02:42:33 +0000
Subject: mirrorball: build debug version anyway
Message-ID: <201005270242.o4R2gXIu008516@scc.eng.rpath.com>
changeset: c2d307b1829e
user: Elliot Peele
date: Wed, 26 May 2010 22:38:01 -0400
build debug version anyway
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -360,9 +360,12 @@
if name != 'kernel':
# Don't build kernel modules with a .debug flag, that
# is only for kernels.
- if flavor.stronglySatisfies(
- deps.parseFlavor('kernel.debug')):
- continue
+ # FIXME: determine what the right thing is here. In SLES
+ # many of the kernel modules include debug
+ # versions.
+ #if flavor.stronglySatisfies(
+ # deps.parseFlavor('kernel.debug')):
+ # continue
flavor = deps.parseFlavor(
str(flavor).replace('kernel', name))
troves.append((name, version, flavor, context))
From elliot at rpath.com Wed May 26 22:42:33 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 27 May 2010 02:42:33 +0000
Subject: mirrorball: remap binPkgMap
Message-ID: <201005270242.o4R2gXdx008547@scc.eng.rpath.com>
changeset: d53a7fdea8d4
user: Elliot Peele
date: Wed, 26 May 2010 22:39:13 -0400
remap binPkgMap
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -412,9 +412,10 @@
if fltr.match(nosrc.name):
log.info('relocating package content %s -> %s'
% (nosrc, src))
- nosrcSet = self.srcPkgMap.get(nosrc)
+ nosrcSet = self.srcPkgMap.pop(nosrc)
self.srcPkgMap[src].update(nosrcSet)
- self.srcPkgMap[nosrc] = self.srcPkgMap[src]
+ for binPkg in nosrcSet:
+ self.binPkgMap[binPkg] = src
def loadFileLists(self, client, basePath):
"""
From elliot at rpath.com Wed May 26 22:42:34 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 27 May 2010 02:42:34 +0000
Subject: mirrorball: remove special filter for sles now that we can build
all packages
Message-ID: <201005270242.o4R2gYDA008574@scc.eng.rpath.com>
changeset: afd7a676cd57
user: Elliot Peele
date: Wed, 26 May 2010 22:40:14 -0400
remove special filter for sles now that we can build all packages
diff --git a/scripts/order_import.py b/scripts/order_import.py
--- a/scripts/order_import.py
+++ b/scripts/order_import.py
@@ -34,7 +34,6 @@
from updatebot import config
from updatebot import ordered
-from updatebot import pkgsource
from updatebot import log as logSetup
logSetup.addRootLogger()
@@ -72,24 +71,6 @@
if cfg.platformName == 'sles':
from errata.sles import AdvisoryManager as Errata
- def fltr(sourceSet):
- removed = set()
- removedNames = set()
- bot._pkgSource.load()
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- # filter out packages that we don't handle right now.
- if (#[ x for x in bins if 'kmp' in x.name ] or
- # the kernel needs a recipe
- 'kernel' in src.name):
-
- removedNames.add(src.name)
-
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- if src.name in removedNames:
- removed.add(src)
-
- return sourceSet - removed
-
elif cfg.platformName == 'centos':
from errata.centos import AdvisoryManager as Errata
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -34,7 +34,6 @@
from updatebot import config
from updatebot import ordered
-from updatebot import pkgsource
from updatebot import log as logSetup
logSetup.addRootLogger()
@@ -75,24 +74,6 @@
if cfg.platformName == 'sles':
from errata.sles import AdvisoryManager as Errata
- def fltr(sourceSet):
- removed = set()
- removedNames = set()
- bot._pkgSource.load()
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- # filter out packages that we don't handle right now.
- if (#[ x for x in bins if 'kmp' in x.name ] or
- # the kernel needs a recipe
- 'kernel' in src.name):
-
- removedNames.add(src.name)
-
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- if src.name in removedNames:
- removed.add(src)
-
- return sourceSet - removed
-
elif cfg.platformName == 'centos':
from errata.centos import AdvisoryManager as Errata
From elliot at rpath.com Wed May 26 22:42:34 2010
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 27 May 2010 02:42:34 +0000
Subject: mirrorball: branch merge
Message-ID: <201005270242.o4R2gYZv008601@scc.eng.rpath.com>
changeset: 33895602e27b
user: Elliot Peele
date: Wed, 26 May 2010 22:42:29 -0400
branch merge
diff --git a/scripts/order_import.py b/scripts/order_import.py
--- a/scripts/order_import.py
+++ b/scripts/order_import.py
@@ -34,7 +34,6 @@
from updatebot import config
from updatebot import ordered
-from updatebot import pkgsource
from updatebot import log as logSetup
logSetup.addRootLogger()
@@ -72,24 +71,6 @@
if cfg.platformName == 'sles':
from errata.sles import AdvisoryManager as Errata
- def fltr(sourceSet):
- removed = set()
- removedNames = set()
- bot._pkgSource.load()
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- # filter out packages that we don't handle right now.
- if (#[ x for x in bins if 'kmp' in x.name ] or
- # the kernel needs a recipe
- 'kernel' in src.name):
-
- removedNames.add(src.name)
-
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- if src.name in removedNames:
- removed.add(src)
-
- return sourceSet - removed
-
elif cfg.platformName == 'centos':
from errata.centos import AdvisoryManager as Errata
diff --git a/scripts/order_update.py b/scripts/order_update.py
--- a/scripts/order_update.py
+++ b/scripts/order_update.py
@@ -34,7 +34,6 @@
from updatebot import config
from updatebot import ordered
-from updatebot import pkgsource
from updatebot import log as logSetup
logSetup.addRootLogger()
@@ -75,24 +74,6 @@
if cfg.platformName == 'sles':
from errata.sles import AdvisoryManager as Errata
- def fltr(sourceSet):
- removed = set()
- removedNames = set()
- bot._pkgSource.load()
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- # filter out packages that we don't handle right now.
- if (#[ x for x in bins if 'kmp' in x.name ] or
- # the kernel needs a recipe
- 'kernel' in src.name):
-
- removedNames.add(src.name)
-
- for src, bins in bot._pkgSource.srcPkgMap.iteritems():
- if src.name in removedNames:
- removed.add(src)
-
- return sourceSet - removed
-
elif cfg.platformName == 'centos':
from errata.centos import AdvisoryManager as Errata
diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -357,15 +357,15 @@
and self._cfg.kernelFlavors):
for context, flavor in self._cfg.kernelFlavors:
# Replace flag name to match package
- #
- # Don't build kernel modules with a .debug flag, that
- # is only for kernels...
- # ...except in the case of SUSE -kmp module packages.
- if (name != 'kernel' and
- not ([bN for bN in binaryNames if '-kmp' in bN])):
- if flavor.stronglySatisfies(
- deps.parseFlavor('kernel.debug')):
- continue
+ if name != 'kernel':
+ # Don't build kernel modules with a .debug flag, that
+ # is only for kernels.
+ # FIXME: determine what the right thing is here. In SLES
+ # many of the kernel modules include debug
+ # versions.
+ #if flavor.stronglySatisfies(
+ # deps.parseFlavor('kernel.debug')):
+ # continue
flavor = deps.parseFlavor(
str(flavor).replace('kernel', name))
troves.append((name, version, flavor, context))
diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py
--- a/updatebot/pkgsource/yumsource.py
+++ b/updatebot/pkgsource/yumsource.py
@@ -412,9 +412,10 @@
if fltr.match(nosrc.name):
log.info('relocating package content %s -> %s'
% (nosrc, src))
- nosrcSet = self.srcPkgMap.get(nosrc)
+ nosrcSet = self.srcPkgMap.pop(nosrc)
self.srcPkgMap[src].update(nosrcSet)
- self.srcPkgMap[nosrc] = self.srcPkgMap[src]
+ for binPkg in nosrcSet:
+ self.binPkgMap[binPkg] = src
def loadFileLists(self, client, basePath):
"""
From elliot at rpath.com Fri May 28 13:10:29 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 28 May 2010 17:10:29 +0000
Subject: mirrorball: hook up group model config handling
Message-ID: <201005281710.o4SHATQL000334@scc.eng.rpath.com>
changeset: b5f7a74b5d6d
user: Elliot Peele
date: Fri, 28 May 2010 13:04:42 -0400
hook up group model config handling
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -45,9 +45,9 @@
raise ParseError, e
-class CfgContextFlavor(CfgFlavor):
+class CfgStringFlavor(CfgFlavor):
"""
- Class for representing both a flavor context name and a build flavor.
+ Class for representing a two tuple of a string and an optional flavor.
"""
def parseString(self, val):
@@ -145,21 +145,6 @@
return obsoleter, obsoleted
-class CfgNameFlavor(CfgString):
- """
- Class for parsing name/flavor pairs.
- """
-
- def parseString(self, val):
- splt = val.split()
- name = splt[0]
- if len(splt) > 1:
- flv = ' '.join(splt[1:])
- else:
- flv = ''
- return name, flv
-
-
class CfgIntDict(CfgDict):
"""
Config class to represent dictionaries keyed by integers rather than
@@ -313,13 +298,13 @@
groupFlavors = (CfgList(CfgFlavor), [])
# flavors to build kernels.
- kernelFlavors = (CfgList(CfgContextFlavor), [])
+ kernelFlavors = (CfgList(CfgStringFlavor), [])
# packages other than "kernel" to be built in kernelFlavers
kernelModules = (CfgList(CfgString), [])
# flavors to build packages in for packages that need specific flavoring.
- packageFlavors = (CfgDict(CfgList(CfgContextFlavor)), {})
+ packageFlavors = (CfgDict(CfgList(CfgStringFlavor)), {})
# After committing a rMake job to the repository pull the changeset back out
# to make sure all of the contents made it into the repository.
@@ -453,10 +438,10 @@
useOldVersion = (CfgIntDict(CfgList(CfgTroveSpec)), {})
# Add a package to a specific group
- addPackage = (CfgDict(CfgDict(CfgList(CfgNameFlavor))), {})
+ addPackage = (CfgIntDict(CfgDict(CfgList(CfgStringFlavor))), {})
# Remove a package from a specific group
- removePackage = (CfgDict(CfgDict(CfgList(CfgNameFlavor))), {})
+ removePackage = (CfgIntDict(CfgDict(CfgList(CfgStringFlavor))), {})
# Allow updates for a given nevra to be published without matching errata.
allowMissingErrata = (CfgList(CfgNevra), [])
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -209,7 +209,8 @@
# Now that versions are actually used for something make sure they
# are always present.
- assert version
+ if groupName == self._pkgGroupName:
+ assert version
assert len(flavors)
flavors = list(flavors)
@@ -370,8 +371,6 @@
@type additions: dict(groupName=[(pkgName, frzPkgFlavor), ...])
"""
- assert additions or removals
-
if additions is None:
additions = {}
if removals is None:
@@ -386,16 +385,21 @@
group = self._groups[groupName]
for pkgName, pkgFlv in pkgs:
if pkgFlv:
- group.removePackageFlavor(pkgName, pkgFlv)
+ group.removePackageFlavor(pkgName, pkgFlv.freeze())
else:
- self.removePackage(pkgName)
+ group.remove(pkgName)
# Add requested packages.
for groupName, pkgs in additions.iteritems():
+ if groupName == self._pkgGroupName:
+ log.warn('modifyContents does not support modifying the package'
+ 'group, please update your config file')
+ continue
+
flavoredPackages = {}
for pkgName, pkgFlv in pkgs:
# deffer packages with specifc flavors for later.
- if pkgFlv:
+ if pkgFlv is not None:
flavoredPackages.setdefault(pkgName, set()).add(pkgFlv)
# handle packages where flavor is not specified
@@ -409,7 +413,9 @@
# Add all specifically flavored packages.
for pkgName, flavors in flavoredPackages.iteritems():
- self.addPackage(pkgName, None, flavors, groupName=groupName)
+ for flv in flavors:
+ self._add(pkgName, version=None, flavor=flv, use=None,
+ groupName=groupName)
@require_write
def _copyVersions(self):
diff --git a/updatebot/ordered.py b/updatebot/ordered.py
--- a/updatebot/ordered.py
+++ b/updatebot/ordered.py
@@ -86,6 +86,19 @@
log.info('\t%s' % f)
group.addPackage(name, version, flavors)
+ def _modifyGroups(self, updateId, group):
+ """
+ Apply the list of modifications, if available, from the config to the
+ group model.
+ """
+
+ addPackages = self._cfg.addPackage.get(updateId, None)
+ removePackages = self._cfg.removePackage.get(updateId, None)
+
+ # Don't taint group model unless something has actually changed.
+ if addPackages or removePackages:
+ group.modifyContents(additions=addPackages, removals=removePackages)
+
def _savePackages(self, pkgMap, fn=None):
"""
Save the package map to a file.
@@ -160,6 +173,7 @@
# Try to build the group if everything imported.
else:
+ self._modifyContents(0, group)
group.errataState = '0'
group.version = '0'
group.commit()
@@ -355,6 +369,9 @@
# Make sure built troves are part of the group.
self._addPackages(pkgMap, group)
+ # Modify any extra groups to match config.
+ self._modifyGroups(updateId, group)
+
# Get timestamp version.
version = self._errata.getBucketVersion(updateId)
if not version:
From elliot at rpath.com Fri May 28 13:10:29 2010
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 28 May 2010 17:10:29 +0000
Subject: mirrorball: make package group name configurable, defaulting to
group-packages
Message-ID: <201005281710.o4SHATTx000365@scc.eng.rpath.com>
changeset: 19204818caed
user: Elliot Peele
date: Fri, 28 May 2010 13:10:26 -0400
make package group name configurable, defaulting to group-packages
diff --git a/updatebot/config.py b/updatebot/config.py
--- a/updatebot/config.py
+++ b/updatebot/config.py
@@ -443,6 +443,9 @@
# Remove a package from a specific group
removePackage = (CfgIntDict(CfgDict(CfgList(CfgStringFlavor))), {})
+ # Group name for group that contains all packages in a platform.
+ packageGroupName = (CfgString, 'group-packages')
+
# Allow updates for a given nevra to be published without matching errata.
allowMissingErrata = (CfgList(CfgNevra), [])
diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py
--- a/updatebot/groupmgr/manager.py
+++ b/updatebot/groupmgr/manager.py
@@ -89,7 +89,7 @@
self._searchLabels = labels
# FIXME: Should figure out a better way to handle package group.
- self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName
+ self._pkgGroupName = self._cfg.packageGroupName
if not srcName.endswith(':source'):
srcName = '%s:source' % srcName