From elliot at rpath.com Mon Mar 15 17:02:06 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:06 +0000 Subject: mirrorball: rpath_common.xmllib has been renamed to rpath_xmllib Message-ID: <201003152102.o2FL26Mo005039@scc.eng.rpath.com> changeset: e34d15dcc263 user: Elliot Peele date: Wed, 07 Oct 2009 10:40:20 -0400 rpath_common.xmllib has been renamed to rpath_xmllib diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -20,7 +20,7 @@ import os -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from updatebot.lib import util diff --git a/repomd/patchesxml.py b/repomd/patchesxml.py --- a/repomd/patchesxml.py +++ b/repomd/patchesxml.py @@ -19,7 +19,7 @@ __all__ = ('PatchesXml', ) # import stable api -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.patchxml import PatchXml from repomd.xmlcommon import XmlFileParser, SlotNode diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -8,7 +8,7 @@ __all__ = ('PatchXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.xmlcommon import XmlFileParser, SlotNode from repomd.packagexml import PackageXmlMixIn diff --git a/repomd/primaryxml.py b/repomd/primaryxml.py --- a/repomd/primaryxml.py +++ b/repomd/primaryxml.py @@ -8,7 +8,7 @@ __all__ = ('PrimaryXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.xmlcommon import XmlFileParser from repomd.packagexml import PackageXmlMixIn diff --git a/repomd/repomdxml.py b/repomd/repomdxml.py --- a/repomd/repomdxml.py +++ b/repomd/repomdxml.py @@ -19,7 +19,7 @@ __all__ = ('RepoMdXml', ) # use stable api -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.primaryxml import PrimaryXml from repomd.patchesxml import PatchesXml diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py --- a/repomd/updateinfoxml.py +++ b/repomd/updateinfoxml.py @@ -19,7 +19,7 @@ __all__ = ('UpdateInfoXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.packagexml import PackageCompare from repomd.xmlcommon import XmlFileParser, SlotNode diff --git a/repomd/xmlcommon.py b/repomd/xmlcommon.py --- a/repomd/xmlcommon.py +++ b/repomd/xmlcommon.py @@ -18,7 +18,7 @@ __all__ = ('XmlFileParser', 'SlotNode') -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib class XmlFileParser(object): """ From elliot at rpath.com Mon Mar 15 17:02:07 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:07 +0000 Subject: mirrorball: enable sanity checks Message-ID: <201003152102.o2FL27ZD005070@scc.eng.rpath.com> changeset: 6fbdfc027574 user: Elliot Peele date: Wed, 07 Oct 2009 10:41:51 -0400 enable sanity checks diff --git a/scripts/header.py b/scripts/header.py --- a/scripts/header.py +++ b/scripts/header.py @@ -14,6 +14,7 @@ import os import sys +import tempfile mirrorballDir = os.path.abspath('../') sys.path.insert(0, mirrorballDir) @@ -45,7 +46,7 @@ cfg = config.UpdateBotConfig() cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) -builder = build.Builder(cfg) +builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp(), sanityCheckCommit=True) def displayTrove(nvf): flavor = '' From elliot at rpath.com Mon Mar 15 17:02:09 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:09 +0000 Subject: mirrorball: use better search for mirrorball dir Message-ID: <201003152102.o2FL29Cu005097@scc.eng.rpath.com> changeset: 3e57f40052b4 user: Elliot Peele date: Wed, 07 Oct 2009 10:42:19 -0400 use better search for mirrorball dir diff --git a/scripts/genmanifest b/scripts/genmanifest --- a/scripts/genmanifest +++ b/scripts/genmanifest @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -25,7 +26,7 @@ log.addRootLogger() cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) obj = bot.Bot(cfg) obj._pkgSource.load() From elliot at rpath.com Mon Mar 15 17:02:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:10 +0000 Subject: mirrorball: update method for finding mirrorball Message-ID: <201003152102.o2FL2BA4005125@scc.eng.rpath.com> changeset: 2241e5ee0b9c user: Elliot Peele date: Wed, 07 Oct 2009 10:43:51 -0400 update method for finding mirrorball diff --git a/scripts/pkgsource b/scripts/pkgsource --- a/scripts/pkgsource +++ b/scripts/pkgsource @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -31,7 +32,7 @@ from updatebot import pkgsource cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1] ) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] ) pkgSource = pkgsource.PackageSource(cfg) pkgSource.load() diff --git a/scripts/recreate b/scripts/recreate --- a/scripts/recreate +++ b/scripts/recreate @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from updatebot import log from updatebot import bot @@ -25,7 +26,7 @@ log.addRootLogger() cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) obj = bot.Bot(cfg) From elliot at rpath.com Mon Mar 15 17:02:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:13 +0000 Subject: mirrorball: handle multiple fedora branches Message-ID: <201003152102.o2FL2DOp005155@scc.eng.rpath.com> changeset: 53d0411fd561 user: Elliot Peele date: Wed, 07 Oct 2009 10:44:24 -0400 handle multiple fedora branches diff --git a/scripts/sync-fedora.sh b/scripts/sync-fedora.sh --- a/scripts/sync-fedora.sh +++ b/scripts/sync-fedora.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,12 +13,24 @@ # full details. # -TARGET=$1 +if [ "$1" = "" ] ; then + # see if $0 has the target in the name + name=$(basename $0) + tmp=${name#sync-fedora-} + target=${tmp%.sh} +else + target=$1 +fi + +if [ "$target" = "" ] ; then + echo "Could not determine target" + exit 1 +fi SOURCE=rsync://mirror.linux.ncsu.edu/fedora-linux- DEST=/l/fedora/linux/ -CMD="rsync -arv --progress --bwlimit=800 --exclude ppc --exclude ppc64" +CMD="rsync -arv --progress --bwlimit=800 --exclude ppc --exclude ppc64 --exclude 9 --exclude 10 --exclude test --exclude testing" date @@ -30,5 +42,3 @@ # cleanup mirror $CMD --delete $SOURCE$target $DEST$target - -./hardlink.py $DEST From elliot at rpath.com Mon Mar 15 17:02:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:15 +0000 Subject: mirrorball: add support for saving changesets and checking for commit sanity Message-ID: <201003152102.o2FL2FZZ005182@scc.eng.rpath.com> changeset: c8d5972ee3f5 user: Elliot Peele date: Wed, 07 Oct 2009 11:45:17 -0400 add support for saving changesets and checking for commit sanity diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -18,6 +18,7 @@ import time import logging +import tempfile import xml from Queue import Queue, Empty @@ -63,15 +64,40 @@ return deco +class _ErrorEvent(object): + def __init__(self): + self.exception = None + self.error = None + + def setError(self, exception, error): + self.exception = exception + self.error = error + + def raiseError(self): + raise self.exception, self.error + + def isSet(self): + return self.exception is not None + + class Builder(object): """ Class for wrapping the rMake api until we can switch to using rBuild. @param cfg: updateBot configuration object @type cfg: config.UpdateBotConfig + @param saveChangeSets: directory to save changesets to + @type saveChangeSets: str + @param errorEvent: object for storing error states. + @type errorEvent: _ErrorEvent + @param sanityCheckCommits: get the changeset that was just committed from the + repository to verify everything made it into the + repository. + @type sanityCheckCommits: boolean """ - def __init__(self, cfg): + def __init__(self, cfg, saveChangeSets=None, errorEvent=None, + sanityCheckCommits=False): self._cfg = cfg self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) @@ -81,6 +107,21 @@ self._client = conaryclient.ConaryClient(self._ccfg) + if saveChangeSets is None and self._cfg.saveChangeSets: + self._saveChangeSets=tempfile.mkdtemp(prefix=self._cfg.platformName, + suffix='-import-changesets') + else: + self._saveChangeSets = saveChangeSets + + self._sanityCheckCommits = self._cfg.sanityCheckCommits + + if errorEvent: + self._errorEvent = errorEvent + self._topLevel = False + else: + self._errorEvent = _ErrorEvent() + self._topLevel = True + # Get default pluginDirs from the rmake cfg object, setup the plugin # manager, then create a new rmake config object so that rmakeUser # will be parsed correctly. @@ -192,15 +233,24 @@ return results, failed def buildmany2(self, troveSpecs): - dispatcher = Dispatcher(self._cfg, 30) + dispatcher = Dispatcher(self._cfg, 10, + saveChangeSets=self._saveChangeSets, + errorEvent=self._errorEvent, + sanityCheckCommits=self._sanityCheckCommits) return dispatcher.buildmany(troveSpecs) def buildmany3(self, troveSpecs): - dispatcher = Dispatcher2(self._cfg, 50) + dispatcher = Dispatcher2(self._cfg, 50, + saveChangeSets=self._saveChangeSets, + errorEvent=self._errorEvent, + sanityCheckCommits=self._sanityCheckCommits) return dispatcher.buildmany(troveSpecs) def buildmany4(self, troveSpecs): - dispatcher = Dispatcher3(self._cfg, 50) + dispatcher = Dispatcher3(self._cfg, 50, + saveChangeSets=self._saveChangeSets, + errorEvent=self._errorEvent, + sanityCheckCommits=self._sanityCheckCommits) return dispatcher.buildmany(troveSpecs) def buildsplitarch(self, troveSpecs): @@ -423,16 +473,44 @@ jobs = [ self._getJob(x) for x in jobIds ] log.info('Starting commit of job %s', jobIdsStr) + if self._saveChangeSets: + csfn = tempfile.mktemp(dir=self._saveChangeSets, suffix='.ccs') + + writeToFile = self._saveChangeSets and csfn or None + self._helper.client.startCommit(jobIds) succeeded, data = commit.commitJobs(self._helper.getConaryClient(), jobs, self._rmakeCfg.reposName, - self._cfg.commitMessage) + self._cfg.commitMessage, + writeToFile=writeToFile) if not succeeded: self._helper.client.commitFailed(jobIds, data) raise CommitFailedError(jobId=jobIdsStr, why=data) + if writeToFile: + log.info('changeset saved to %s' % writeToFile) + log.info('committing changeset to repository') + self._client.repos.commitChangeSetFile(writeToFile) + + if self._sanityCheckCommits: + # sanity check repository + log.info('checking repository for sanity') + jobList = [] + for job in data.itervalues(): + for arch in job.itervalues(): + for n, v, f in arch: + if n.startswith('group-'): continue + jobList.append((n, (None, None), (v, f), True)) + try: + cs = self._client.repos.createChangeSet(jobList, withFiles=True, + withFileContents=False) + except Exception, e: + self._errorEvent.setError(Exception, e) + if self._topLevel: + self._errorEvent.raiseError() + log.info('Commit of job %s completed in %.02f seconds', jobIdsStr, time.time() - startTime) @@ -534,7 +612,9 @@ class BuildWorker(Thread): BuilderClass = Builder - def __init__(self, cfg, toBuild, status, name=None, offset=0): + def __init__(self, cfg, toBuild, status, name=None, offset=0, + saveChangeSets=None, errorEvent=None, + sanityCheckCommits=False): Thread.__init__(self, name=name) self.setDaemon(True) @@ -543,7 +623,9 @@ self.offset = offset self.toBuild = toBuild self.status = status - self.builder = self.BuilderClass(cfg) + self.builder = self.BuilderClass(cfg, saveChangeSets=saveChangeSets, + errorEvent=errorEvent, + sanityCheckCommits=sanityCheckCommits) self.trv = None self.jobId = None @@ -603,10 +685,15 @@ class Dispatcher(object): workerClass = BuildWorker - def __init__(self, cfg, workerCount): + def __init__(self, cfg, workerCount, saveChangeSets=None, errorEvent=None, + sanityCheckCommits=False): self._cfg = cfg self._workerCount = workerCount + self._saveChangeSets = saveChangeSets + self._errorEvent = errorEvent + self._sanityCheckCommits = sanityCheckCommits + self._workers = [] self._started = False @@ -618,7 +705,10 @@ def provisionWorkers(self): for i in range(self._workerCount): worker = self.workerClass(self._cfg, self._toBuild, self._status, - name='Build Worker %s' % i, offset=i) + name='Build Worker %s' % i, offset=i, + saveChangeSets=self._saveChangeSets, + errorEvent=self._errorEvent, + sanityCheckCommits=self._sanityCheckCommits) self._workers.append(worker) def start(self): @@ -649,6 +739,9 @@ except Empty: continue + if self._errorEvent.isSet(): + self._errorEvent.raiseError() + self._processMessage(msg) done = self._buildDone() @@ -682,11 +775,15 @@ class Dispatcher2(object): builderClass = Builder - def __init__(self, cfg, workerCount): + def __init__(self, cfg, workerCount, saveChangeSets=None, errorEvent=None, + sanityCheckCommits=False): self._cfg = cfg self._workerCount = workerCount - self._builder = self.builderClass(self._cfg) + self._builder = self.builderClass(self._cfg, + saveChangeSets=saveChangeSets, + errorEvent=errorEvent, + sanityCheckCommits=sanityCheckCommits) self._activeJobs = [] self._commitJobs = [] @@ -740,6 +837,8 @@ for troveSpec, jobId in self._commitJobs: try: res = self._builder.commit(jobId) + if self._errorEvent.isSet(): + self._errorEvent.raiseError() self._results.update(res) self._completedJobs.append(jobId) except JobFailedError: @@ -750,13 +849,16 @@ class CommitWorker(Thread): BuilderClass = Builder - def __init__(self, cfg, toCommit, results, name=None): + def __init__(self, cfg, toCommit, results, name=None, saveChangeSets=None, + errorEvent=None, sanityCheckCommits=False): Thread.__init__(self, name=name) self.name = name self.toCommit = toCommit self.results = results - self.builder = self.BuilderClass(cfg) + self.builder = self.BuilderClass(cfg, saveChangeSets=saveChangeSets, + errorEvent=errorEvent, + sanityCheckCommits=sanityCheckCommits) self.setDaemon(True) @@ -778,8 +880,12 @@ class Dispatcher3(Dispatcher2): workerClass = CommitWorker - def __init__(self, cfg, workerCount): - Dispatcher2.__init__(self, cfg, workerCount) + def __init__(self, cfg, workerCount, saveChangeSets=None, errorEvent=None, + sanityCheckCommits=False): + Dispatcher2.__init__(self, cfg, workerCount, + saveChangeSets=saveChangeSets, + errorEvent=errorEvent, + sanityCheckCommits=sanityCheckCommits) self._commitQueue = Queue() self._resultQueue = Queue() @@ -787,7 +893,10 @@ self._workers = [] for i in range(10): worker = self.workerClass(self._cfg, self._commitQueue, - self._resultQueue, name='Commit Worker %s' % i) + self._resultQueue, name='Commit Worker %s' % i, + saveChangeSets=saveChangeSets, + errorEvent=errorEvent, + sanityCheckCommits=sanityCheckCommits) worker.start() self._workers.append(worker) diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -166,6 +166,13 @@ # flavors to build packages in for packages that need specific flavoring. packageFlavors = (CfgDict(CfgList(CfgContextFlavor)), {}) + # After committing a rMake job to the repository pull the changeset back out + # to make sure all of the contents made it into the repository. + sanityCheckCommits = (CfgBool, False) + + # Save all binary changesets to disk before committing them. + saveChangeSets = (CfgBool, False) + # email information for sending advisories emailFromName = CfgString emailFrom = CfgString From elliot at rpath.com Mon Mar 15 17:02:17 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:17 +0000 Subject: mirrorball: fix typo Message-ID: <201003152102.o2FL2HN0005212@scc.eng.rpath.com> changeset: 056492064878 user: Elliot Peele date: Thu, 08 Oct 2009 01:16:24 -0400 fix typo diff --git a/scripts/header.py b/scripts/header.py --- a/scripts/header.py +++ b/scripts/header.py @@ -46,7 +46,7 @@ cfg = config.UpdateBotConfig() cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) -builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp(), sanityCheckCommit=True) +builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp(), sanityCheckCommits=True) def displayTrove(nvf): flavor = '' From elliot at rpath.com Mon Mar 15 17:02:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:18 +0000 Subject: mirrorball: use buildmany Message-ID: <201003152102.o2FL2J63005267@scc.eng.rpath.com> changeset: 5f5c16bd3dd4 user: Elliot Peele date: Thu, 08 Oct 2009 01:16:39 -0400 use buildmany diff --git a/scripts/buildmany b/scripts/buildmany --- a/scripts/buildmany +++ b/scripts/buildmany @@ -16,7 +16,7 @@ label = cfg.topSourceGroup[1] for pkg in sys.argv[2:]: trvs.add((pkg, label, None)) -trvMap = builder.buildmany2(trvs) +trvMap = builder.buildmany(trvs) print "built:\n" diff --git a/scripts/compare b/scripts/compare --- a/scripts/compare +++ b/scripts/compare @@ -143,7 +143,7 @@ for source in sorted(toBuild): jobSet.append((source, None, None)) - return self._builder.buildmany2(jobSet) + return self._builder.buildmany(jobSet) class CompareAndCopy(object): From elliot at rpath.com Mon Mar 15 17:02:21 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:21 +0000 Subject: mirrorball: switch to new subscriber model for buildmany implementaiton Message-ID: <201003152102.o2FL2LO4005320@scc.eng.rpath.com> changeset: c5c0acd2f9a3 user: Elliot Peele date: Thu, 08 Oct 2009 01:18:21 -0400 switch to new subscriber model for buildmany implementaiton diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -32,6 +32,7 @@ from rmake.cmdline import helper, monitor, commit from updatebot.lib import util +from updatebot import subscriber from updatebot.errors import JobFailedError, CommitFailedError log = logging.getLogger('updateBot.build') @@ -64,22 +65,6 @@ return deco -class _ErrorEvent(object): - def __init__(self): - self.exception = None - self.error = None - - def setError(self, exception, error): - self.exception = exception - self.error = error - - def raiseError(self): - raise self.exception, self.error - - def isSet(self): - return self.exception is not None - - class Builder(object): """ Class for wrapping the rMake api until we can switch to using rBuild. @@ -88,16 +73,13 @@ @type cfg: config.UpdateBotConfig @param saveChangeSets: directory to save changesets to @type saveChangeSets: str - @param errorEvent: object for storing error states. - @type errorEvent: _ErrorEvent @param sanityCheckCommits: get the changeset that was just committed from the repository to verify everything made it into the repository. @type sanityCheckCommits: boolean """ - def __init__(self, cfg, saveChangeSets=None, errorEvent=None, - sanityCheckCommits=False): + def __init__(self, cfg, saveChangeSets=None, sanityCheckCommits=False): self._cfg = cfg self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) @@ -115,13 +97,6 @@ self._sanityCheckCommits = self._cfg.sanityCheckCommits - if errorEvent: - self._errorEvent = errorEvent - self._topLevel = False - else: - self._errorEvent = _ErrorEvent() - self._topLevel = True - # Get default pluginDirs from the rmake cfg object, setup the plugin # manager, then create a new rmake config object so that rmakeUser # will be parsed correctly. @@ -163,94 +138,13 @@ def buildmany(self, troveSpecs): """ - Build all packages in troveSpecs, 10 at a time, one per job. + Build many troves in separate jobs. @param troveSpecs: list of trove specs @type troveSpecs: [(name, versionObj, flavorObj), ...] @return troveMap: dictionary of troveSpecs to built troves """ - troveSpecs = list(troveSpecs) - def trvSort(a, b): - """ - Sort troves tuples based on the first element. - """ - - return cmp(a[0], b[0]) - troveSpecs.sort(trvSort) - - index = 0 - jobs = {} - for i, trv in enumerate(troveSpecs): - if index not in jobs: - jobs[index] = [] - - jobs[index].append(trv) - - if i % 20 == 0: - index += 1 - - failed = set() - results = {} - for job in jobs.itervalues(): - res, fail = self._buildmany(job) - failed.update(fail) - results.update(res) - - return results, failed - - def _buildmany(self, troveSpecs): - """ - Build a list of packages, one per job. - @param troveSpecs: list of trove specs - @type troveSpecs: [(name, versionObj, flavorObj), ...] - @return troveMap: dictionary of troveSpecs to built troves - """ - - jobs = {} - jobkeys = [] - for trv in troveSpecs: - jobkeys.append(trv) - jobs[trv] = self.start([trv, ]) - - for trv in jobkeys: - jobId = jobs[trv] - job = self._getJob(jobId) - self._wait(jobId) - - failed = set() - results = {} - for trv, jobId in jobs.iteritems(): - job = self._getJob(jobId) - if job.isFailed(): - failed.add((trv, jobId)) - elif job.isFinished(): - try: - res = self.commit(jobId) - results.update(res) - except JobFailedError: - failed.add((trv, jobId)) - - return results, failed - - def buildmany2(self, troveSpecs): - dispatcher = Dispatcher(self._cfg, 10, - saveChangeSets=self._saveChangeSets, - errorEvent=self._errorEvent, - sanityCheckCommits=self._sanityCheckCommits) - return dispatcher.buildmany(troveSpecs) - - def buildmany3(self, troveSpecs): - dispatcher = Dispatcher2(self._cfg, 50, - saveChangeSets=self._saveChangeSets, - errorEvent=self._errorEvent, - sanityCheckCommits=self._sanityCheckCommits) - return dispatcher.buildmany(troveSpecs) - - def buildmany4(self, troveSpecs): - dispatcher = Dispatcher3(self._cfg, 50, - saveChangeSets=self._saveChangeSets, - errorEvent=self._errorEvent, - sanityCheckCommits=self._sanityCheckCommits) + dispatcher = subscriber.Dispatcher(self, 30) return dispatcher.buildmany(troveSpecs) def buildsplitarch(self, troveSpecs): @@ -284,7 +178,7 @@ # Wait for the jobs to finish. log.info('Waiting for jobs to complete') for jobId in jobIds.itervalues(): - self._wait(jobId) + self._monitorJob(jobId) # Sanity check all jobs. for jobId in jobIds.itervalues(): @@ -409,19 +303,6 @@ return jobId - def _wait(self, jobId): - """ - Wait for a job to complete. - @param jobId: rMake job ID - @type jobId: integer - """ - - log.info('waiting for job [%s] to complete' % jobId) - job = self._getJob(jobId) - while not job.isFinished() and not job.isFailed(): - time.sleep(5) - job = self._getJob(jobId) - @jobInfoExceptionHandler def _monitorJob(self, jobId): """ @@ -578,341 +459,3 @@ """ Don't care about the build log """ - -## -# Experimental threaded builder, beware of dragons -## - -MESSAGE_TYPES = { - 0: 'log', - 'log': 0, - 1: 'results', - 'results': 1, - 2: 'error', - 'error': 2, -} - -class StatusMessage(object): - def __init__(self, name, trv, jobId, message, type=0): - assert type in MESSAGE_TYPES - self.name = name - self.trv = trv - self.jobId = jobId - self.message = message - self.type = type - - def __str__(self): - msg = '%(name)s: %(trv)s [%(jobId)s] - ' - if self.type == MESSAGE_TYPES['results']: - msg += 'done' - else: - msg += '%(message)s' - return msg % self.__dict__ - -class BuildWorker(Thread): - BuilderClass = Builder - - def __init__(self, cfg, toBuild, status, name=None, offset=0, - saveChangeSets=None, errorEvent=None, - sanityCheckCommits=False): - Thread.__init__(self, name=name) - - self.setDaemon(True) - - self.name = name - self.offset = offset - self.toBuild = toBuild - self.status = status - self.builder = self.BuilderClass(cfg, saveChangeSets=saveChangeSets, - errorEvent=errorEvent, - sanityCheckCommits=sanityCheckCommits) - - self.trv = None - self.jobId = None - - def run(self): - time.sleep(self.offset * 5) - while True: - self.trv = self.toBuild.get() - self.log('received trv') - - retries = 10 - built = False - while not built and retries: - retries -= 1 - try: - self._doBuild() - except Exception, e: - built = False - self.log('traceback while building %s, retrying' % e) - continue - built = True - - if not built: - self.error('job failed') - - self.toBuild.task_done() - - def _doBuild(self): - self.jobId = self.builder.start([self.trv, ]) - - self.builder._wait(self.jobId) - - job = self.builder._getJob(self.jobId) - if job.isFailed(): - self.error('job failed') - - else: - try: - res = self.builder.commit(self.jobId) - self.results(res) - except JobFailedError: - self.error('job failed') - - def _status(self, msg, type=0): - msg = StatusMessage(self.name, self.trv, self.jobId, msg, type) - self.status.put(msg) - - def error(self, msg): - self._status(msg, type=MESSAGE_TYPES['error']) - - def log(self, msg): - self._status(msg, type=MESSAGE_TYPES['log']) - - def results(self, res): - self._status(res, type=MESSAGE_TYPES['results']) - -class Dispatcher(object): - workerClass = BuildWorker - - def __init__(self, cfg, workerCount, saveChangeSets=None, errorEvent=None, - sanityCheckCommits=False): - self._cfg = cfg - self._workerCount = workerCount - - self._saveChangeSets = saveChangeSets - self._errorEvent = errorEvent - self._sanityCheckCommits = sanityCheckCommits - - self._workers = [] - self._started = False - - self._toBuild = Queue() - self._status = Queue() - - self._trvs = {} - - def provisionWorkers(self): - for i in range(self._workerCount): - worker = self.workerClass(self._cfg, self._toBuild, self._status, - name='Build Worker %s' % i, offset=i, - saveChangeSets=self._saveChangeSets, - errorEvent=self._errorEvent, - sanityCheckCommits=self._sanityCheckCommits) - self._workers.append(worker) - - def start(self): - if self._started: - return - - for wkr in self._workers: - wkr.start() - self._started = True - - def buildmany(self, trvSpecs): - self.provisionWorkers() - self.start() - - for trv in trvSpecs: - self._trvs[trv] = [] - self._toBuild.put(trv) - - results, failed = self.monitorStatus() - return results, failed - - def monitorStatus(self): - done = False - while not done: - try: - log.debug('checking for status messages') - msg = self._status.get(timeout=5) - except Empty: - continue - - if self._errorEvent.isSet(): - self._errorEvent.raiseError() - - self._processMessage(msg) - done = self._buildDone() - - return self._getResultsAndErrors() - - def _processMessage(self, msg): - assert msg.trv in self._trvs - self._trvs[msg.trv].append(msg) - log.info(msg) - - def _buildDone(self): - for trv, msgs in self._trvs.iteritems(): - if len(msgs) == 0: - return False - elif msgs[-1].type not in (MESSAGE_TYPES['results'], MESSAGE_TYPES['error']): - return False - return True - - def _getResultsAndErrors(self): - errors = set() - results = [] - for trv, msgs in self._trvs.iteritems(): - msg = msgs[-1] - if msg.type == MESSAGE_TYPES['error']: - errors.add((trv, msg.jobId)) - elif msg.type == MESSAGE_TYPES['results']: - results.append(msg.message) - return results, errors - - -class Dispatcher2(object): - builderClass = Builder - - def __init__(self, cfg, workerCount, saveChangeSets=None, errorEvent=None, - sanityCheckCommits=False): - self._cfg = cfg - self._workerCount = workerCount - - self._builder = self.builderClass(self._cfg, - saveChangeSets=saveChangeSets, - errorEvent=errorEvent, - sanityCheckCommits=sanityCheckCommits) - - self._activeJobs = [] - self._commitJobs = [] - self._failedJobs = set() - self._completedJobs = [] - - self._results = {} - - def buildmany(self, troveSpecs): - troveSpecs = list(troveSpecs) - - while len(troveSpecs): - # Wait for some amount of jobs to complete. - while len(self._activeJobs) >= self._workerCount: - time.sleep(5) - self._checkStatus() - - # Populate build queue. - while len(self._activeJobs) < self._workerCount: - trvSpec = troveSpecs.pop() - self._start(trvSpec) - - # Commit completed jobs. - self._commit() - - return self._results, self._failedJobs - - @jobInfoExceptionHandler - def _start(self, troveSpec): - jobId = self._builder.start([troveSpec, ]) - self._activeJobs.append((troveSpec, jobId)) - - def _checkStatus(self): - for troveSpec, jobId in self._activeJobs: - log.info('Checking status of %s' % jobId) - job = self._builder._getJob(jobId, retry=10) - if job is None: - log.warn('Failed to retrieve job information for %s' % jobId) - import epdb; epdb.st() - elif job.isFailed(): - self._failedJobs.add((troveSpec, jobId)) - self._activeJobs.remove((troveSpec, jobId)) - elif job.isFinished(): - self._commitJobs.append((troveSpec, jobId)) - self._activeJobs.remove((troveSpec, jobId)) - - # Wait between each status check. - #time.sleep(1) - - def _commit(self): - for troveSpec, jobId in self._commitJobs: - try: - res = self._builder.commit(jobId) - if self._errorEvent.isSet(): - self._errorEvent.raiseError() - self._results.update(res) - self._completedJobs.append(jobId) - except JobFailedError: - self._failedJobs.add((troveSpec, jobId)) - self._commitJobs.remove((troveSpec, jobId)) - - -class CommitWorker(Thread): - BuilderClass = Builder - - def __init__(self, cfg, toCommit, results, name=None, saveChangeSets=None, - errorEvent=None, sanityCheckCommits=False): - Thread.__init__(self, name=name) - - self.name = name - self.toCommit = toCommit - self.results = results - self.builder = self.BuilderClass(cfg, saveChangeSets=saveChangeSets, - errorEvent=errorEvent, - sanityCheckCommits=sanityCheckCommits) - - self.setDaemon(True) - - def run(self): - while True: - trvSpec, jobId = self.toCommit.get() - try: - res = self.builder.commit(jobId) - self.msg(0, trvSpec, jobId, res) - except JobFailedError: - self.msg(1, trvSpec, jobId) - except Exception, e: - log.critical('%s received exception: %s' % (self.name, e)) - - def msg(self, rc, trvSpec, jobId, data=None): - self.results.put((rc, ((trvSpec, jobId), data))) - - -class Dispatcher3(Dispatcher2): - workerClass = CommitWorker - - def __init__(self, cfg, workerCount, saveChangeSets=None, errorEvent=None, - sanityCheckCommits=False): - Dispatcher2.__init__(self, cfg, workerCount, - saveChangeSets=saveChangeSets, - errorEvent=errorEvent, - sanityCheckCommits=sanityCheckCommits) - - self._commitQueue = Queue() - self._resultQueue = Queue() - - self._workers = [] - for i in range(10): - worker = self.workerClass(self._cfg, self._commitQueue, - self._resultQueue, name='Commit Worker %s' % i, - saveChangeSets=saveChangeSets, - errorEvent=errorEvent, - sanityCheckCommits=sanityCheckCommits) - worker.start() - self._workers.append(worker) - - def _commit(self): - for trvSpec, jobId in self._commitJobs: - self._commitQueue.put((trvSpec, jobId)) - - try: - msg = self._resultQueue.get(False) - while msg: - rc, ((trvSpec, jobId), data) = msg - if rc == 0: - self._results.update(data) - self._completedJobs.append(jobId) - elif rc == 1: - self._failedJobs.add((trvSpec, jobId)) - msg = self._resultQueue.get(False) - except Empty: - pass diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py new file mode 100644 --- /dev/null +++ b/updatebot/subscriber.py @@ -0,0 +1,352 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +New implementation of builder module that uses rMake's message bus for job +monitoring. +""" + +import copy +import time +import logging +from threading import Thread +from Queue import Queue, Empty + +from rmake.build import buildjob +from rmake.build import buildtrove +from rmake.cmdline import monitor + +log = logging.getLogger('updatebot.subscriber') + +class MessageTypes(object): + """ + Class for storing message type constants. + """ + + LOG = 0 + DATA = 1 + THREAD_DONE = 2 + THREAD_ERROR = 3 + + +class ThreadTypes(object): + """ + Class for storing thread types. + """ + + MONITOR = 0 + COMMIT = 1 + + names = { + MONITOR: 'Monitor', + COMMIT: 'Commit', + } + + +class JobMonitorCallback(monitor.JobLogDisplay): + """ + Monitor job status changes. + """ + + monitorStates = ( + buildjob.JOB_STATE_STARTED, + buildjob.JOB_STATE_BUILT, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_COMMITTING, + buildjob.JOB_STATE_COMMITTED, + ) + + def __init__(self, status, *args, **kwargs): + # override showBuildLogs since we don't handle writing to the out pipe + kwargs['showBuildLogs'] = False + + monitor.JobLogDisplay.__init__(self, *args, **kwargs) + self._status = status + + def _msg(self, msg, *args): + self._status.put((MessageTypes.LOG, msg)) + + def _data(self, data): + self._status.put((MessageTypes.DATA, data)) + + def _jobStateUpdated(self, jobId, state, status): + monitor.JobLogDisplay(self, jobId, state, None) + if state in self.monitorStates: + self._data((jobId, state)) + + def _troveLogUpdated(self, (jobId, troveTuple), state, status): + pass + + def _jobTrovesSet(self, jobId, troveData): + pass + + def _primeOutput(self, jobId): + # Override parents _primeOutput to avoid sending output to stdout via + # print. + logMark = 0 + while True: + newLogs = self.client.getJobLogs(jobId, logMark) + if not newLogs: + break + logMark += len(newLogs) + for (timeStamp, message, args) in newLogs: + self._msg('[%s] - %s' % (jobId, message)) + + BUILDING = buildtrove.TROVE_STATE_BUILDING + troveTups = self.client.listTrovesByState(jobId, BUILDING).get(BUILDING, []) + for troveTuple in troveTups: + self._tailBuildLog(jobId, troveTuple) + + monitor._AbstractDisplay._primeOutput(self, jobId) + + +class MonitorWorker(Thread): + """ + Worker thread for monitoring jobs and reporting status. + """ + + threadType = ThreadTypes.MONITOR + displayClass = JobMonitorCallback + + def __init__(self, jobId, status, rmakeClient, name=None): + Thread.__init__(self, name=name) + + self.setDaemon(False) + + self.client = rmakeClient + self.jobId = jobId + self.status = status + self.name = name + + def run(self): + """ + Watch the monitor queue and monitor any available jobs. + """ + + # monitor job, job display class supplies status information back to + # the main thread via the status queue. + try: + self.monitorJob(self.jobId) + except Exception, e: + self.status.put((MessageTypes.THREAD_ERROR, + (ThreadType.MONITOR, self.jobId, e))) + + self.status.put((MessageTypes.THREAD_DONE, self.jobId)) + + def monitorJob(self, jobId): + """ + Monitor the specified job. + """ + + # FIXME: This is copied from rmake.cmdlin.monitor for the most part + # because I need to pass extra args to the display class. + + uri, tmpPath = monitor._getUri(self.client) + + try: + display = self.displayClass(self.status, self.client, + showBuildLogs=False, exitOnFinish=True) + client = self.client.listenToEvents(uri, jobId, display, + showTroveDetails=False, + serve=True) + return client + finally: + if tmpPath: + os.remove(tmpPath) + + +class CommitWorker(Thread): + """ + Worker thread for committing jobs. + """ + + threadType = ThreadTypes.COMMIT + + def __init__(self, jobId, status, builder, name=None): + Thread.__init__(self, name=name) + + self.builder = builder + self.jobId = jobId + self.status = status + self.name = name + + def run(self): + """ + Commit the specified job. + """ + + try: + result = self.builder.commit(self.jobId) + self.status.put((MessageTypes.DATA, (self.jobId, result))) + except Exception, e: + self.status.put((MessageTypes.THREAD_ERROR, + (ThreadType.COMMIT, self.jobId, e))) + + self.status.put((MessageTypes.THREAD_DONE, self.jobId)) + + +class AbstractStatusMonitor(object): + """ + Abstract class for implementing monitoring classes. + """ + + workerClass = None + + def __init__(self, threadArgs): + self._threadArgs = threadArgs + + self._status = Queue() + self._workers = {} + + def addJobId(self, jobId): + """ + Add a jobId to the worker pool. + """ + + threadName = ('%s Worker' + % ThreadTypes.names[self.workerClass.threadType]) + worker = self.workerClass(jobId, self._status, *self._threadArgs, + name=threadName) + self._workers[jobId] = worker + worker.daemon = True + worker.start() + + def getStatus(self): + """ + Process all messages in the status queue, returning any data messages. + """ + + data = [] + while True: + try: + msg = self._status.get_nowait() + except Empty: + break + + data.extend(self._processMessage(msg)) + + return data + + def _processMessage(self, msg): + """ + Handle messages. + """ + + data = [] + mtype, payload = msg + + if mtype == MessageTypes.LOG: + log.info(payload) + elif mtype == MessageTypes.DATA: + data.append(payload) + elif mtype == MessageTypes.THREAD_DONE: + jobId = payload + assert not self._workers[jobId].isAlive() + del self._workers[jobId] + elif mtype == MessageTypes.THREAD_ERROR: + threadType, jobId, error = payload + assert not self._workers[jobId].isAlive() + raise error + + return data + + +class JobMonitor(AbstractStatusMonitor): + """ + Abstraction around threaded monitoring model. + """ + + workerClass = MonitorWorker + monitorJob = AbstractStatusMonitor.addJobId + + +class JobCommitter(AbstractStatusMonitor): + """ + Abstraction around threaded commit model. + """ + + workerClass = CommitWorker + commitJob = AbstractStatusMonitor.addJobId + + +class Dispatcher(object): + """ + Manage building a list of troves in a way that doesn't bring rMake to its + knees. + """ + + _completed = (buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_COMMITTED) + + def __init__(self, builder, maxSlots): + self._builder = builder + self._maxSlots = maxSlots + self._slots = maxSlots + + self._monitor = JobMonitor((self._builder._helper.client, )) + self._committer = JobCommitter((self._builder, )) + + # jobId: (trv, status, commitData) + self._jobs = {} + + def buildmany(self, troveSpecs): + """ + Build as many packages as possible until we run out of slots. + """ + + troves = list(troveSpecs) + troves.reverse() + + while troves or not self._jobDone(): + # fill slots with available troves + while troves and self._slots: + # get trove to work on + trove = troves.pop() + + # start build job + jobId = self._builder.start((trove, )) + self._jobs[jobId] = [trove, -1, None] + self._slots -= 1 + self._monitor.monitorJob(jobId) + + # update job status changes + for jobId, status in self._monitor.getStatus(): + self._jobs[jobId][1] = status + if status in self._completed: + self._slots += 1 + assert self._slots <= self._maxSlots + + # commit any jobs that are complete + if status == buildjob.JOB_STATE_BUILT: + self._committer.commitJob(jobId) + + # check for commit status + for jobId, result in self._committer.getStatus(): + self._jobs[jobId][2] = result + + results = {} + for jobId, (trove, status, result) in self._jobs.iteritems(): + # log failed jobs + if status == buildjob.JOB_STATE_FAILED: + log.info('[%s] failed job: %s' % (jobId, trove)) + else: + results[trove] = result + + return results + + def _jobDone(self): + for jobId, (trove, status, result) in self._jobs.iteritems(): + if status not in self._completed: + return False + return True From elliot at rpath.com Mon Mar 15 17:02:22 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:22 +0000 Subject: mirrorball: call buildmany rather than buildmany2 Message-ID: <201003152102.o2FL2NA6005348@scc.eng.rpath.com> changeset: 21a9d714e71d user: Elliot Peele date: Thu, 08 Oct 2009 01:18:42 -0400 call buildmany rather than buildmany2 diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -108,7 +108,7 @@ if len(toBuild): if not rebuild: # Build all newly imported packages. - trvMap, failed = self._builder.buildmany2(sorted(toBuild)) + trvMap, failed = self._builder.buildmany(sorted(toBuild)) log.info('failed to import %s packages' % len(failed)) if len(failed): for pkg in failed: From elliot at rpath.com Mon Mar 15 17:02:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:24 +0000 Subject: mirrorball: remove unused import Message-ID: <201003152102.o2FL2O2P005379@scc.eng.rpath.com> changeset: 311bb3714c8a user: Elliot Peele date: Thu, 08 Oct 2009 01:27:22 -0400 remove unused import diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py --- a/updatebot/subscriber.py +++ b/updatebot/subscriber.py @@ -18,7 +18,6 @@ """ import copy -import time import logging from threading import Thread from Queue import Queue, Empty From elliot at rpath.com Mon Mar 15 17:02:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:27 +0000 Subject: mirrorball: add log files Message-ID: <201003152102.o2FL2RHQ005411@scc.eng.rpath.com> changeset: 59135dcfb158 user: Elliot Peele date: Thu, 08 Oct 2009 14:49:56 -0400 add log files diff --git a/updatebot/log.py b/updatebot/log.py --- a/updatebot/log.py +++ b/updatebot/log.py @@ -17,19 +17,35 @@ """ import sys +import time import logging +import tempfile -def addRootLogger(): +def addRootLogger(logFile=None): """ Setup the root logger that should be inherited by all other loggers. """ + logSize = 1024 * 1024 * 50 + logFile = logFile and logFile or tempfile.mktemp(prefix='updatebot-log-%s-' + % int(time.time())) + rootLog = logging.getLogger('') - handler = logging.StreamHandler(sys.stdout) + + streamHandler = logging.StreamHandler(sys.stdout) + logFileHandler = logging.handlers.RotatingFileHandler(logFile, + maxBytes=logSize, + backupCount=5) + formatter = logging.Formatter('%(asctime)s %(levelname)s ' '%(name)s %(message)s') - handler.setFormatter(formatter) - rootLog.addHandler(handler) + + streamHandler.setFormatter(formatter) + logFileHandler.setFormatter(formatter) + + rootLog.addHandler(streamHandler) + rootLog.addHandler(logFileHandler) + rootLog.setLevel(logging.INFO) # Delete conary's log handler since it puts things on stderr and without From elliot at rpath.com Mon Mar 15 17:02:29 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:29 +0000 Subject: mirrorball: add new elements Message-ID: <201003152102.o2FL2T53005452@scc.eng.rpath.com> changeset: 9552a82b6c5e user: Elliot Peele date: Wed, 21 Oct 2009 16:10:57 -0400 add new elements diff --git a/repomd/repomdxml.py b/repomd/repomdxml.py --- a/repomd/repomdxml.py +++ b/repomd/repomdxml.py @@ -89,7 +89,8 @@ Parser for repomd.xml data elements. """ __slots__ = ('location', 'checksum', 'checksumType', 'timestamp', - 'openChecksum', 'openChecksumType', 'databaseVersion', ) + 'openChecksum', 'openChecksumType', 'databaseVersion', + 'size', 'openSize') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. @@ -114,6 +115,10 @@ self.openChecksumType = child.getAttribute('type') elif name == 'database_version': self.databaseVersion = child.finalize() + elif name == 'size': + self.size = child.finalize() + elif name == 'open-size': + self.openSize = child.finalize() else: raise UnknownElementError(child) From elliot at rpath.com Mon Mar 15 17:02:32 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:32 +0000 Subject: mirrorball: fixup import Message-ID: <201003152102.o2FL2XIi005496@scc.eng.rpath.com> changeset: e2a727457a55 user: Elliot Peele date: Wed, 21 Oct 2009 16:11:22 -0400 fixup import diff --git a/updatebot/log.py b/updatebot/log.py --- a/updatebot/log.py +++ b/updatebot/log.py @@ -20,6 +20,7 @@ import time import logging import tempfile +from logging import handlers def addRootLogger(logFile=None): """ @@ -33,9 +34,9 @@ rootLog = logging.getLogger('') streamHandler = logging.StreamHandler(sys.stdout) - logFileHandler = logging.handlers.RotatingFileHandler(logFile, - maxBytes=logSize, - backupCount=5) + logFileHandler = handlers.RotatingFileHandler(logFile, + maxBytes=logSize, + backupCount=5) formatter = logging.Formatter('%(asctime)s %(levelname)s ' '%(name)s %(message)s') From elliot at rpath.com Mon Mar 15 17:02:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:35 +0000 Subject: mirrorball: bug fixes Message-ID: <201003152102.o2FL2ael005536@scc.eng.rpath.com> changeset: 49abc493c208 user: Elliot Peele date: Wed, 21 Oct 2009 16:11:43 -0400 bug fixes diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py --- a/updatebot/subscriber.py +++ b/updatebot/subscriber.py @@ -90,6 +90,9 @@ def _jobTrovesSet(self, jobId, troveData): pass + def _tailBuildLog(self, jobId, troveTuple): + pass + def _primeOutput(self, jobId): # Override parents _primeOutput to avoid sending output to stdout via # print. @@ -139,7 +142,7 @@ self.monitorJob(self.jobId) except Exception, e: self.status.put((MessageTypes.THREAD_ERROR, - (ThreadType.MONITOR, self.jobId, e))) + (ThreadTypes.MONITOR, self.jobId, e))) self.status.put((MessageTypes.THREAD_DONE, self.jobId)) @@ -190,7 +193,7 @@ self.status.put((MessageTypes.DATA, (self.jobId, result))) except Exception, e: self.status.put((MessageTypes.THREAD_ERROR, - (ThreadType.COMMIT, self.jobId, e))) + (ThreadTypes.COMMIT, self.jobId, e))) self.status.put((MessageTypes.THREAD_DONE, self.jobId)) @@ -251,11 +254,11 @@ data.append(payload) elif mtype == MessageTypes.THREAD_DONE: jobId = payload - assert not self._workers[jobId].isAlive() + #assert not self._workers[jobId].isAlive() del self._workers[jobId] elif mtype == MessageTypes.THREAD_ERROR: threadType, jobId, error = payload - assert not self._workers[jobId].isAlive() + #assert not self._workers[jobId].isAlive() raise error return data @@ -342,6 +345,8 @@ else: results[trove] = result + import epdb; epdb.st() + return results def _jobDone(self): From elliot at rpath.com Mon Mar 15 17:02:38 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:38 +0000 Subject: mirrorball: initial errata ordering poc Message-ID: <201003152102.o2FL2cif005595@scc.eng.rpath.com> changeset: 9e352a8ead7a user: Elliot Peele date: Wed, 21 Oct 2009 16:12:13 -0400 initial errata ordering poc diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py new file mode 100755 --- /dev/null +++ b/scripts/rhelorder.py @@ -0,0 +1,151 @@ +#!/usr/bin/python + +import os +import sys + +sys.path.insert(0, os.environ['HOME'] + '/hg/conary') +sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-capsule-indexer') + +from conary.lib import util +sys.excepthook = util.genExcepthook() + +mbdir = os.path.abspath('../') +sys.path.insert(0, mbdir) + +confDir = os.path.join(mbdir, 'config', 'rhel4') + +from updatebot import log +from updatebot import bot +from updatebot import config + +import time +import logging + +slog = logging.getLogger('script') + +import rhnmirror + +log.addRootLogger() +cfg = config.UpdateBotConfig() +cfg.read(os.path.join(confDir, 'updatebotrc')) + +obj = bot.Bot(cfg) + +mcfg = rhnmirror.MirrorConfig() +mcfg.read(os.path.join(confDir, 'erratarc')) + +errata = rhnmirror.Errata(mcfg) +errata.fetch() + +pkgSource = obj._pkgSource +pkgSource.load() + +# get mapping of advisory to errata obj +advisories = dict((x.advisory, x) + for x in errata.iterByIssueDate(mcfg.channels)) + +# get mapping of nevra to pkg obj +nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) + for x in pkgSource.binPkgMap.keys() if x.arch != 'src') + +# pull nevras into errata sized buckets +buckets = {} +advMap = {} +nevraMap = {} + +arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') +for e in errata.iterByIssueDate(mcfg.channels): + bnevras = [] + bucket = [] + bucketId = None + slog.info('processing %s' % e.advisory) + for pkg in e.packages: + nevra = pkg.getNevra() + + # filter out channels we don't have indexed + channels = set([ x.label for x in pkg.channels ]) + if not set(mcfg.channels) & channels: + continue + + # ignore arches we don't know about. + if nevra[4] not in arches: + continue + + # convert rhn nevra to yum nevra + nevra = list(nevra) + if nevra[1] is None: + nevra[1] = '0' + if type(nevra[1]) == int: + nevra[1] = str(nevra[1]) + nevra = tuple(nevra) + + # move nevra to errata buckets + if nevra in nevras: + binPkg = nevras.pop(nevra) + bucket.append(binPkg) + bnevras.append(nevra) + + # nevra is already part of another bucket + elif nevra in nevraMap: + bucketId = nevraMap[nevra] + + # raise error if we can't find the required package + else: + raise KeyError + + if bucketId is None: + bucketId = int(time.mktime(time.strptime(e.issue_date, + '%Y-%m-%d %H:%M:%S'))) + buckets[bucketId] = bucket + else: + buckets[bucketId].extend(bucket) + + for nevra in bnevras: + nevraMap[nevra] = bucketId + + advMap[e.advisory] = bucketId + +# separate out golden bits +other = [] +golden = [] +firstErrata = sorted(buckets.keys())[0] +for nevra, pkg in nevras.iteritems(): + buildtime = int(pkg.buildTimestamp) + if buildtime < firstErrata: + golden.append(pkg) + else: + other.append(pkg) + +# sort by source package +srcMap = {} +for pkg in other: + src = pkgSource.binPkgMap[pkg] + if src not in srcMap: + srcMap[src] = [] + srcMap[src].append(pkg) + +# insert bins by buildstamp +for src, bins in srcMap.iteritems(): + buildstamp = int(sorted(bins)[0].buildTimestamp) + if buildstamp in buckets: + buckets[buildstamp].extend(bins) + else: + buckets[buildstamp] = bins + +# get sources to build +buildOrder = {0: set()} +for pkg in golden: + # lookup source package + src = pkgSource.binPkgMap[pkg] + buildOrder[0].add(src) + +for bucketId in sorted(buckets.keys()): + bucket = buckets[bucketId] + buildOrder[bucketId] = set() + for pkg in bucket: + src = pkgSource.binPkgMap[pkg] + buildOrder[bucketId].add(src) + +import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:02:41 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:41 +0000 Subject: mirrorball: branch merge Message-ID: <201003152102.o2FL2fGV005656@scc.eng.rpath.com> changeset: 54a0d47f184e user: Elliot Peele date: Wed, 21 Oct 2009 16:12:34 -0400 branch merge diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -20,7 +20,7 @@ import os -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from updatebot.lib import util diff --git a/repomd/patchesxml.py b/repomd/patchesxml.py --- a/repomd/patchesxml.py +++ b/repomd/patchesxml.py @@ -19,7 +19,7 @@ __all__ = ('PatchesXml', ) # import stable api -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.patchxml import PatchXml from repomd.xmlcommon import XmlFileParser, SlotNode diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -8,7 +8,7 @@ __all__ = ('PatchXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.xmlcommon import XmlFileParser, SlotNode from repomd.packagexml import PackageXmlMixIn diff --git a/repomd/primaryxml.py b/repomd/primaryxml.py --- a/repomd/primaryxml.py +++ b/repomd/primaryxml.py @@ -8,7 +8,7 @@ __all__ = ('PrimaryXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.xmlcommon import XmlFileParser from repomd.packagexml import PackageXmlMixIn diff --git a/repomd/repomdxml.py b/repomd/repomdxml.py --- a/repomd/repomdxml.py +++ b/repomd/repomdxml.py @@ -19,7 +19,7 @@ __all__ = ('RepoMdXml', ) # use stable api -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.primaryxml import PrimaryXml from repomd.patchesxml import PatchesXml @@ -89,7 +89,8 @@ Parser for repomd.xml data elements. """ __slots__ = ('location', 'checksum', 'checksumType', 'timestamp', - 'openChecksum', 'openChecksumType', 'databaseVersion', ) + 'openChecksum', 'openChecksumType', 'databaseVersion', + 'size', 'openSize') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. @@ -114,6 +115,10 @@ self.openChecksumType = child.getAttribute('type') elif name == 'database_version': self.databaseVersion = child.finalize() + elif name == 'size': + self.size = child.finalize() + elif name == 'open-size': + self.openSize = child.finalize() else: raise UnknownElementError(child) diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py --- a/repomd/updateinfoxml.py +++ b/repomd/updateinfoxml.py @@ -19,7 +19,7 @@ __all__ = ('UpdateInfoXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.packagexml import PackageCompare from repomd.xmlcommon import XmlFileParser, SlotNode diff --git a/repomd/xmlcommon.py b/repomd/xmlcommon.py --- a/repomd/xmlcommon.py +++ b/repomd/xmlcommon.py @@ -18,7 +18,7 @@ __all__ = ('XmlFileParser', 'SlotNode') -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib class XmlFileParser(object): """ diff --git a/scripts/buildmany b/scripts/buildmany --- a/scripts/buildmany +++ b/scripts/buildmany @@ -16,7 +16,7 @@ label = cfg.topSourceGroup[1] for pkg in sys.argv[2:]: trvs.add((pkg, label, None)) -trvMap = builder.buildmany2(trvs) +trvMap = builder.buildmany(trvs) print "built:\n" diff --git a/scripts/compare b/scripts/compare --- a/scripts/compare +++ b/scripts/compare @@ -143,7 +143,7 @@ for source in sorted(toBuild): jobSet.append((source, None, None)) - return self._builder.buildmany2(jobSet) + return self._builder.buildmany(jobSet) class CompareAndCopy(object): diff --git a/scripts/genmanifest b/scripts/genmanifest --- a/scripts/genmanifest +++ b/scripts/genmanifest @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -25,7 +26,7 @@ log.addRootLogger() cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) obj = bot.Bot(cfg) obj._pkgSource.load() diff --git a/scripts/header.py b/scripts/header.py --- a/scripts/header.py +++ b/scripts/header.py @@ -14,6 +14,7 @@ import os import sys +import tempfile mirrorballDir = os.path.abspath('../') sys.path.insert(0, mirrorballDir) @@ -45,7 +46,7 @@ cfg = config.UpdateBotConfig() cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) -builder = build.Builder(cfg) +builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp(), sanityCheckCommits=True) def displayTrove(nvf): flavor = '' diff --git a/scripts/pkgsource b/scripts/pkgsource --- a/scripts/pkgsource +++ b/scripts/pkgsource @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -31,7 +32,7 @@ from updatebot import pkgsource cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1] ) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] ) pkgSource = pkgsource.PackageSource(cfg) pkgSource.load() diff --git a/scripts/recreate b/scripts/recreate --- a/scripts/recreate +++ b/scripts/recreate @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from updatebot import log from updatebot import bot @@ -25,7 +26,7 @@ log.addRootLogger() cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) obj = bot.Bot(cfg) diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py new file mode 100755 --- /dev/null +++ b/scripts/rhelorder.py @@ -0,0 +1,151 @@ +#!/usr/bin/python + +import os +import sys + +sys.path.insert(0, os.environ['HOME'] + '/hg/conary') +sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-capsule-indexer') + +from conary.lib import util +sys.excepthook = util.genExcepthook() + +mbdir = os.path.abspath('../') +sys.path.insert(0, mbdir) + +confDir = os.path.join(mbdir, 'config', 'rhel4') + +from updatebot import log +from updatebot import bot +from updatebot import config + +import time +import logging + +slog = logging.getLogger('script') + +import rhnmirror + +log.addRootLogger() +cfg = config.UpdateBotConfig() +cfg.read(os.path.join(confDir, 'updatebotrc')) + +obj = bot.Bot(cfg) + +mcfg = rhnmirror.MirrorConfig() +mcfg.read(os.path.join(confDir, 'erratarc')) + +errata = rhnmirror.Errata(mcfg) +errata.fetch() + +pkgSource = obj._pkgSource +pkgSource.load() + +# get mapping of advisory to errata obj +advisories = dict((x.advisory, x) + for x in errata.iterByIssueDate(mcfg.channels)) + +# get mapping of nevra to pkg obj +nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) + for x in pkgSource.binPkgMap.keys() if x.arch != 'src') + +# pull nevras into errata sized buckets +buckets = {} +advMap = {} +nevraMap = {} + +arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') +for e in errata.iterByIssueDate(mcfg.channels): + bnevras = [] + bucket = [] + bucketId = None + slog.info('processing %s' % e.advisory) + for pkg in e.packages: + nevra = pkg.getNevra() + + # filter out channels we don't have indexed + channels = set([ x.label for x in pkg.channels ]) + if not set(mcfg.channels) & channels: + continue + + # ignore arches we don't know about. + if nevra[4] not in arches: + continue + + # convert rhn nevra to yum nevra + nevra = list(nevra) + if nevra[1] is None: + nevra[1] = '0' + if type(nevra[1]) == int: + nevra[1] = str(nevra[1]) + nevra = tuple(nevra) + + # move nevra to errata buckets + if nevra in nevras: + binPkg = nevras.pop(nevra) + bucket.append(binPkg) + bnevras.append(nevra) + + # nevra is already part of another bucket + elif nevra in nevraMap: + bucketId = nevraMap[nevra] + + # raise error if we can't find the required package + else: + raise KeyError + + if bucketId is None: + bucketId = int(time.mktime(time.strptime(e.issue_date, + '%Y-%m-%d %H:%M:%S'))) + buckets[bucketId] = bucket + else: + buckets[bucketId].extend(bucket) + + for nevra in bnevras: + nevraMap[nevra] = bucketId + + advMap[e.advisory] = bucketId + +# separate out golden bits +other = [] +golden = [] +firstErrata = sorted(buckets.keys())[0] +for nevra, pkg in nevras.iteritems(): + buildtime = int(pkg.buildTimestamp) + if buildtime < firstErrata: + golden.append(pkg) + else: + other.append(pkg) + +# sort by source package +srcMap = {} +for pkg in other: + src = pkgSource.binPkgMap[pkg] + if src not in srcMap: + srcMap[src] = [] + srcMap[src].append(pkg) + +# insert bins by buildstamp +for src, bins in srcMap.iteritems(): + buildstamp = int(sorted(bins)[0].buildTimestamp) + if buildstamp in buckets: + buckets[buildstamp].extend(bins) + else: + buckets[buildstamp] = bins + +# get sources to build +buildOrder = {0: set()} +for pkg in golden: + # lookup source package + src = pkgSource.binPkgMap[pkg] + buildOrder[0].add(src) + +for bucketId in sorted(buckets.keys()): + bucket = buckets[bucketId] + buildOrder[bucketId] = set() + for pkg in bucket: + src = pkgSource.binPkgMap[pkg] + buildOrder[bucketId].add(src) + +import epdb; epdb.st() diff --git a/scripts/sync-fedora.sh b/scripts/sync-fedora.sh --- a/scripts/sync-fedora.sh +++ b/scripts/sync-fedora.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,12 +13,24 @@ # full details. # -TARGET=$1 +if [ "$1" = "" ] ; then + # see if $0 has the target in the name + name=$(basename $0) + tmp=${name#sync-fedora-} + target=${tmp%.sh} +else + target=$1 +fi + +if [ "$target" = "" ] ; then + echo "Could not determine target" + exit 1 +fi SOURCE=rsync://mirror.linux.ncsu.edu/fedora-linux- DEST=/l/fedora/linux/ -CMD="rsync -arv --progress --bwlimit=800 --exclude ppc --exclude ppc64" +CMD="rsync -arv --progress --bwlimit=800 --exclude ppc --exclude ppc64 --exclude 9 --exclude 10 --exclude test --exclude testing" date @@ -30,5 +42,3 @@ # cleanup mirror $CMD --delete $SOURCE$target $DEST$target - -./hardlink.py $DEST diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -108,7 +108,7 @@ if len(toBuild): if not rebuild: # Build all newly imported packages. - trvMap, failed = self._builder.buildmany2(sorted(toBuild)) + trvMap, failed = self._builder.buildmany(sorted(toBuild)) log.info('failed to import %s packages' % len(failed)) if len(failed): for pkg in failed: diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -18,6 +18,7 @@ import time import logging +import tempfile import xml from Queue import Queue, Empty @@ -31,6 +32,7 @@ from rmake.cmdline import helper, monitor, commit from updatebot.lib import util +from updatebot import subscriber from updatebot.errors import JobFailedError, CommitFailedError log = logging.getLogger('updateBot.build') @@ -69,9 +71,15 @@ @param cfg: updateBot configuration object @type cfg: config.UpdateBotConfig + @param saveChangeSets: directory to save changesets to + @type saveChangeSets: str + @param sanityCheckCommits: get the changeset that was just committed from the + repository to verify everything made it into the + repository. + @type sanityCheckCommits: boolean """ - def __init__(self, cfg): + def __init__(self, cfg, saveChangeSets=None, sanityCheckCommits=False): self._cfg = cfg self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) @@ -81,6 +89,14 @@ self._client = conaryclient.ConaryClient(self._ccfg) + if saveChangeSets is None and self._cfg.saveChangeSets: + self._saveChangeSets=tempfile.mkdtemp(prefix=self._cfg.platformName, + suffix='-import-changesets') + else: + self._saveChangeSets = saveChangeSets + + self._sanityCheckCommits = self._cfg.sanityCheckCommits + # Get default pluginDirs from the rmake cfg object, setup the plugin # manager, then create a new rmake config object so that rmakeUser # will be parsed correctly. @@ -122,85 +138,13 @@ def buildmany(self, troveSpecs): """ - Build all packages in troveSpecs, 10 at a time, one per job. + Build many troves in separate jobs. @param troveSpecs: list of trove specs @type troveSpecs: [(name, versionObj, flavorObj), ...] @return troveMap: dictionary of troveSpecs to built troves """ - troveSpecs = list(troveSpecs) - def trvSort(a, b): - """ - Sort troves tuples based on the first element. - """ - - return cmp(a[0], b[0]) - troveSpecs.sort(trvSort) - - index = 0 - jobs = {} - for i, trv in enumerate(troveSpecs): - if index not in jobs: - jobs[index] = [] - - jobs[index].append(trv) - - if i % 20 == 0: - index += 1 - - failed = set() - results = {} - for job in jobs.itervalues(): - res, fail = self._buildmany(job) - failed.update(fail) - results.update(res) - - return results, failed - - def _buildmany(self, troveSpecs): - """ - Build a list of packages, one per job. - @param troveSpecs: list of trove specs - @type troveSpecs: [(name, versionObj, flavorObj), ...] - @return troveMap: dictionary of troveSpecs to built troves - """ - - jobs = {} - jobkeys = [] - for trv in troveSpecs: - jobkeys.append(trv) - jobs[trv] = self.start([trv, ]) - - for trv in jobkeys: - jobId = jobs[trv] - job = self._getJob(jobId) - self._wait(jobId) - - failed = set() - results = {} - for trv, jobId in jobs.iteritems(): - job = self._getJob(jobId) - if job.isFailed(): - failed.add((trv, jobId)) - elif job.isFinished(): - try: - res = self.commit(jobId) - results.update(res) - except JobFailedError: - failed.add((trv, jobId)) - - return results, failed - - def buildmany2(self, troveSpecs): - dispatcher = Dispatcher(self._cfg, 30) - return dispatcher.buildmany(troveSpecs) - - def buildmany3(self, troveSpecs): - dispatcher = Dispatcher2(self._cfg, 50) - return dispatcher.buildmany(troveSpecs) - - def buildmany4(self, troveSpecs): - dispatcher = Dispatcher3(self._cfg, 50) + dispatcher = subscriber.Dispatcher(self, 30) return dispatcher.buildmany(troveSpecs) def buildsplitarch(self, troveSpecs): @@ -234,7 +178,7 @@ # Wait for the jobs to finish. log.info('Waiting for jobs to complete') for jobId in jobIds.itervalues(): - self._wait(jobId) + self._monitorJob(jobId) # Sanity check all jobs. for jobId in jobIds.itervalues(): @@ -359,19 +303,6 @@ return jobId - def _wait(self, jobId): - """ - Wait for a job to complete. - @param jobId: rMake job ID - @type jobId: integer - """ - - log.info('waiting for job [%s] to complete' % jobId) - job = self._getJob(jobId) - while not job.isFinished() and not job.isFailed(): - time.sleep(5) - job = self._getJob(jobId) - @jobInfoExceptionHandler def _monitorJob(self, jobId): """ @@ -423,16 +354,44 @@ jobs = [ self._getJob(x) for x in jobIds ] log.info('Starting commit of job %s', jobIdsStr) + if self._saveChangeSets: + csfn = tempfile.mktemp(dir=self._saveChangeSets, suffix='.ccs') + + writeToFile = self._saveChangeSets and csfn or None + self._helper.client.startCommit(jobIds) succeeded, data = commit.commitJobs(self._helper.getConaryClient(), jobs, self._rmakeCfg.reposName, - self._cfg.commitMessage) + self._cfg.commitMessage, + writeToFile=writeToFile) if not succeeded: self._helper.client.commitFailed(jobIds, data) raise CommitFailedError(jobId=jobIdsStr, why=data) + if writeToFile: + log.info('changeset saved to %s' % writeToFile) + log.info('committing changeset to repository') + self._client.repos.commitChangeSetFile(writeToFile) + + if self._sanityCheckCommits: + # sanity check repository + log.info('checking repository for sanity') + jobList = [] + for job in data.itervalues(): + for arch in job.itervalues(): + for n, v, f in arch: + if n.startswith('group-'): continue + jobList.append((n, (None, None), (v, f), True)) + try: + cs = self._client.repos.createChangeSet(jobList, withFiles=True, + withFileContents=False) + except Exception, e: + self._errorEvent.setError(Exception, e) + if self._topLevel: + self._errorEvent.raiseError() + log.info('Commit of job %s completed in %.02f seconds', jobIdsStr, time.time() - startTime) @@ -500,310 +459,3 @@ """ Don't care about the build log """ - -## -# Experimental threaded builder, beware of dragons -## - -MESSAGE_TYPES = { - 0: 'log', - 'log': 0, - 1: 'results', - 'results': 1, - 2: 'error', - 'error': 2, -} - -class StatusMessage(object): - def __init__(self, name, trv, jobId, message, type=0): - assert type in MESSAGE_TYPES - self.name = name - self.trv = trv - self.jobId = jobId - self.message = message - self.type = type - - def __str__(self): - msg = '%(name)s: %(trv)s [%(jobId)s] - ' - if self.type == MESSAGE_TYPES['results']: - msg += 'done' - else: - msg += '%(message)s' - return msg % self.__dict__ - -class BuildWorker(Thread): - BuilderClass = Builder - - def __init__(self, cfg, toBuild, status, name=None, offset=0): - Thread.__init__(self, name=name) - - self.setDaemon(True) - - self.name = name - self.offset = offset - self.toBuild = toBuild - self.status = status - self.builder = self.BuilderClass(cfg) - - self.trv = None - self.jobId = None - - def run(self): - time.sleep(self.offset * 5) - while True: - self.trv = self.toBuild.get() - self.log('received trv') - - retries = 10 - built = False - while not built and retries: - retries -= 1 - try: - self._doBuild() - except Exception, e: - built = False - self.log('traceback while building %s, retrying' % e) - continue - built = True - - if not built: - self.error('job failed') - - self.toBuild.task_done() - - def _doBuild(self): - self.jobId = self.builder.start([self.trv, ]) - - self.builder._wait(self.jobId) - - job = self.builder._getJob(self.jobId) - if job.isFailed(): - self.error('job failed') - - else: - try: - res = self.builder.commit(self.jobId) - self.results(res) - except JobFailedError: - self.error('job failed') - - def _status(self, msg, type=0): - msg = StatusMessage(self.name, self.trv, self.jobId, msg, type) - self.status.put(msg) - - def error(self, msg): - self._status(msg, type=MESSAGE_TYPES['error']) - - def log(self, msg): - self._status(msg, type=MESSAGE_TYPES['log']) - - def results(self, res): - self._status(res, type=MESSAGE_TYPES['results']) - -class Dispatcher(object): - workerClass = BuildWorker - - def __init__(self, cfg, workerCount): - self._cfg = cfg - self._workerCount = workerCount - - self._workers = [] - self._started = False - - self._toBuild = Queue() - self._status = Queue() - - self._trvs = {} - - def provisionWorkers(self): - for i in range(self._workerCount): - worker = self.workerClass(self._cfg, self._toBuild, self._status, - name='Build Worker %s' % i, offset=i) - self._workers.append(worker) - - def start(self): - if self._started: - return - - for wkr in self._workers: - wkr.start() - self._started = True - - def buildmany(self, trvSpecs): - self.provisionWorkers() - self.start() - - for trv in trvSpecs: - self._trvs[trv] = [] - self._toBuild.put(trv) - - results, failed = self.monitorStatus() - return results, failed - - def monitorStatus(self): - done = False - while not done: - try: - log.debug('checking for status messages') - msg = self._status.get(timeout=5) - except Empty: - continue - - self._processMessage(msg) - done = self._buildDone() - - return self._getResultsAndErrors() - - def _processMessage(self, msg): - assert msg.trv in self._trvs - self._trvs[msg.trv].append(msg) - log.info(msg) - - def _buildDone(self): - for trv, msgs in self._trvs.iteritems(): - if len(msgs) == 0: - return False - elif msgs[-1].type not in (MESSAGE_TYPES['results'], MESSAGE_TYPES['error']): - return False - return True - - def _getResultsAndErrors(self): - errors = set() - results = [] - for trv, msgs in self._trvs.iteritems(): - msg = msgs[-1] - if msg.type == MESSAGE_TYPES['error']: - errors.add((trv, msg.jobId)) - elif msg.type == MESSAGE_TYPES['results']: - results.append(msg.message) - return results, errors - - -class Dispatcher2(object): - builderClass = Builder - - def __init__(self, cfg, workerCount): - self._cfg = cfg - self._workerCount = workerCount - - self._builder = self.builderClass(self._cfg) - - self._activeJobs = [] - self._commitJobs = [] - self._failedJobs = set() - self._completedJobs = [] - - self._results = {} - - def buildmany(self, troveSpecs): - troveSpecs = list(troveSpecs) - - while len(troveSpecs): - # Wait for some amount of jobs to complete. - while len(self._activeJobs) >= self._workerCount: - time.sleep(5) - self._checkStatus() - - # Populate build queue. - while len(self._activeJobs) < self._workerCount: - trvSpec = troveSpecs.pop() - self._start(trvSpec) - - # Commit completed jobs. - self._commit() - - return self._results, self._failedJobs - - @jobInfoExceptionHandler - def _start(self, troveSpec): - jobId = self._builder.start([troveSpec, ]) - self._activeJobs.append((troveSpec, jobId)) - - def _checkStatus(self): - for troveSpec, jobId in self._activeJobs: - log.info('Checking status of %s' % jobId) - job = self._builder._getJob(jobId, retry=10) - if job is None: - log.warn('Failed to retrieve job information for %s' % jobId) - import epdb; epdb.st() - elif job.isFailed(): - self._failedJobs.add((troveSpec, jobId)) - self._activeJobs.remove((troveSpec, jobId)) - elif job.isFinished(): - self._commitJobs.append((troveSpec, jobId)) - self._activeJobs.remove((troveSpec, jobId)) - - # Wait between each status check. - #time.sleep(1) - - def _commit(self): - for troveSpec, jobId in self._commitJobs: - try: - res = self._builder.commit(jobId) - self._results.update(res) - self._completedJobs.append(jobId) - except JobFailedError: - self._failedJobs.add((troveSpec, jobId)) - self._commitJobs.remove((troveSpec, jobId)) - - -class CommitWorker(Thread): - BuilderClass = Builder - - def __init__(self, cfg, toCommit, results, name=None): - Thread.__init__(self, name=name) - - self.name = name - self.toCommit = toCommit - self.results = results - self.builder = self.BuilderClass(cfg) - - self.setDaemon(True) - - def run(self): - while True: - trvSpec, jobId = self.toCommit.get() - try: - res = self.builder.commit(jobId) - self.msg(0, trvSpec, jobId, res) - except JobFailedError: - self.msg(1, trvSpec, jobId) - except Exception, e: - log.critical('%s received exception: %s' % (self.name, e)) - - def msg(self, rc, trvSpec, jobId, data=None): - self.results.put((rc, ((trvSpec, jobId), data))) - - -class Dispatcher3(Dispatcher2): - workerClass = CommitWorker - - def __init__(self, cfg, workerCount): - Dispatcher2.__init__(self, cfg, workerCount) - - self._commitQueue = Queue() - self._resultQueue = Queue() - - self._workers = [] - for i in range(10): - worker = self.workerClass(self._cfg, self._commitQueue, - self._resultQueue, name='Commit Worker %s' % i) - worker.start() - self._workers.append(worker) - - def _commit(self): - for trvSpec, jobId in self._commitJobs: - self._commitQueue.put((trvSpec, jobId)) - - try: - msg = self._resultQueue.get(False) - while msg: - rc, ((trvSpec, jobId), data) = msg - if rc == 0: - self._results.update(data) - self._completedJobs.append(jobId) - elif rc == 1: - self._failedJobs.add((trvSpec, jobId)) - msg = self._resultQueue.get(False) - except Empty: - pass diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -166,6 +166,13 @@ # flavors to build packages in for packages that need specific flavoring. packageFlavors = (CfgDict(CfgList(CfgContextFlavor)), {}) + # After committing a rMake job to the repository pull the changeset back out + # to make sure all of the contents made it into the repository. + sanityCheckCommits = (CfgBool, False) + + # Save all binary changesets to disk before committing them. + saveChangeSets = (CfgBool, False) + # email information for sending advisories emailFromName = CfgString emailFrom = CfgString diff --git a/updatebot/log.py b/updatebot/log.py --- a/updatebot/log.py +++ b/updatebot/log.py @@ -17,19 +17,36 @@ """ import sys +import time import logging +import tempfile +from logging import handlers -def addRootLogger(): +def addRootLogger(logFile=None): """ Setup the root logger that should be inherited by all other loggers. """ + logSize = 1024 * 1024 * 50 + logFile = logFile and logFile or tempfile.mktemp(prefix='updatebot-log-%s-' + % int(time.time())) + rootLog = logging.getLogger('') - handler = logging.StreamHandler(sys.stdout) + + streamHandler = logging.StreamHandler(sys.stdout) + logFileHandler = handlers.RotatingFileHandler(logFile, + maxBytes=logSize, + backupCount=5) + formatter = logging.Formatter('%(asctime)s %(levelname)s ' '%(name)s %(message)s') - handler.setFormatter(formatter) - rootLog.addHandler(handler) + + streamHandler.setFormatter(formatter) + logFileHandler.setFormatter(formatter) + + rootLog.addHandler(streamHandler) + rootLog.addHandler(logFileHandler) + rootLog.setLevel(logging.INFO) # Delete conary's log handler since it puts things on stderr and without diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py new file mode 100644 --- /dev/null +++ b/updatebot/subscriber.py @@ -0,0 +1,356 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +New implementation of builder module that uses rMake's message bus for job +monitoring. +""" + +import copy +import logging +from threading import Thread +from Queue import Queue, Empty + +from rmake.build import buildjob +from rmake.build import buildtrove +from rmake.cmdline import monitor + +log = logging.getLogger('updatebot.subscriber') + +class MessageTypes(object): + """ + Class for storing message type constants. + """ + + LOG = 0 + DATA = 1 + THREAD_DONE = 2 + THREAD_ERROR = 3 + + +class ThreadTypes(object): + """ + Class for storing thread types. + """ + + MONITOR = 0 + COMMIT = 1 + + names = { + MONITOR: 'Monitor', + COMMIT: 'Commit', + } + + +class JobMonitorCallback(monitor.JobLogDisplay): + """ + Monitor job status changes. + """ + + monitorStates = ( + buildjob.JOB_STATE_STARTED, + buildjob.JOB_STATE_BUILT, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_COMMITTING, + buildjob.JOB_STATE_COMMITTED, + ) + + def __init__(self, status, *args, **kwargs): + # override showBuildLogs since we don't handle writing to the out pipe + kwargs['showBuildLogs'] = False + + monitor.JobLogDisplay.__init__(self, *args, **kwargs) + self._status = status + + def _msg(self, msg, *args): + self._status.put((MessageTypes.LOG, msg)) + + def _data(self, data): + self._status.put((MessageTypes.DATA, data)) + + def _jobStateUpdated(self, jobId, state, status): + monitor.JobLogDisplay(self, jobId, state, None) + if state in self.monitorStates: + self._data((jobId, state)) + + def _troveLogUpdated(self, (jobId, troveTuple), state, status): + pass + + def _jobTrovesSet(self, jobId, troveData): + pass + + def _tailBuildLog(self, jobId, troveTuple): + pass + + def _primeOutput(self, jobId): + # Override parents _primeOutput to avoid sending output to stdout via + # print. + logMark = 0 + while True: + newLogs = self.client.getJobLogs(jobId, logMark) + if not newLogs: + break + logMark += len(newLogs) + for (timeStamp, message, args) in newLogs: + self._msg('[%s] - %s' % (jobId, message)) + + BUILDING = buildtrove.TROVE_STATE_BUILDING + troveTups = self.client.listTrovesByState(jobId, BUILDING).get(BUILDING, []) + for troveTuple in troveTups: + self._tailBuildLog(jobId, troveTuple) + + monitor._AbstractDisplay._primeOutput(self, jobId) + + +class MonitorWorker(Thread): + """ + Worker thread for monitoring jobs and reporting status. + """ + + threadType = ThreadTypes.MONITOR + displayClass = JobMonitorCallback + + def __init__(self, jobId, status, rmakeClient, name=None): + Thread.__init__(self, name=name) + + self.setDaemon(False) + + self.client = rmakeClient + self.jobId = jobId + self.status = status + self.name = name + + def run(self): + """ + Watch the monitor queue and monitor any available jobs. + """ + + # monitor job, job display class supplies status information back to + # the main thread via the status queue. + try: + self.monitorJob(self.jobId) + except Exception, e: + self.status.put((MessageTypes.THREAD_ERROR, + (ThreadTypes.MONITOR, self.jobId, e))) + + self.status.put((MessageTypes.THREAD_DONE, self.jobId)) + + def monitorJob(self, jobId): + """ + Monitor the specified job. + """ + + # FIXME: This is copied from rmake.cmdlin.monitor for the most part + # because I need to pass extra args to the display class. + + uri, tmpPath = monitor._getUri(self.client) + + try: + display = self.displayClass(self.status, self.client, + showBuildLogs=False, exitOnFinish=True) + client = self.client.listenToEvents(uri, jobId, display, + showTroveDetails=False, + serve=True) + return client + finally: + if tmpPath: + os.remove(tmpPath) + + +class CommitWorker(Thread): + """ + Worker thread for committing jobs. + """ + + threadType = ThreadTypes.COMMIT + + def __init__(self, jobId, status, builder, name=None): + Thread.__init__(self, name=name) + + self.builder = builder + self.jobId = jobId + self.status = status + self.name = name + + def run(self): + """ + Commit the specified job. + """ + + try: + result = self.builder.commit(self.jobId) + self.status.put((MessageTypes.DATA, (self.jobId, result))) + except Exception, e: + self.status.put((MessageTypes.THREAD_ERROR, + (ThreadTypes.COMMIT, self.jobId, e))) + + self.status.put((MessageTypes.THREAD_DONE, self.jobId)) + + +class AbstractStatusMonitor(object): + """ + Abstract class for implementing monitoring classes. + """ + + workerClass = None + + def __init__(self, threadArgs): + self._threadArgs = threadArgs + + self._status = Queue() + self._workers = {} + + def addJobId(self, jobId): + """ + Add a jobId to the worker pool. + """ + + threadName = ('%s Worker' + % ThreadTypes.names[self.workerClass.threadType]) + worker = self.workerClass(jobId, self._status, *self._threadArgs, + name=threadName) + self._workers[jobId] = worker + worker.daemon = True + worker.start() + + def getStatus(self): + """ + Process all messages in the status queue, returning any data messages. + """ + + data = [] + while True: + try: + msg = self._status.get_nowait() + except Empty: + break + + data.extend(self._processMessage(msg)) + + return data + + def _processMessage(self, msg): + """ + Handle messages. + """ + + data = [] + mtype, payload = msg + + if mtype == MessageTypes.LOG: + log.info(payload) + elif mtype == MessageTypes.DATA: + data.append(payload) + elif mtype == MessageTypes.THREAD_DONE: + jobId = payload + #assert not self._workers[jobId].isAlive() + del self._workers[jobId] + elif mtype == MessageTypes.THREAD_ERROR: + threadType, jobId, error = payload + #assert not self._workers[jobId].isAlive() + raise error + + return data + + +class JobMonitor(AbstractStatusMonitor): + """ + Abstraction around threaded monitoring model. + """ + + workerClass = MonitorWorker + monitorJob = AbstractStatusMonitor.addJobId + + +class JobCommitter(AbstractStatusMonitor): + """ + Abstraction around threaded commit model. + """ + + workerClass = CommitWorker + commitJob = AbstractStatusMonitor.addJobId + + +class Dispatcher(object): + """ + Manage building a list of troves in a way that doesn't bring rMake to its + knees. + """ + + _completed = (buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_COMMITTED) + + def __init__(self, builder, maxSlots): + self._builder = builder + self._maxSlots = maxSlots + self._slots = maxSlots + + self._monitor = JobMonitor((self._builder._helper.client, )) + self._committer = JobCommitter((self._builder, )) + + # jobId: (trv, status, commitData) + self._jobs = {} + + def buildmany(self, troveSpecs): + """ + Build as many packages as possible until we run out of slots. + """ + + troves = list(troveSpecs) + troves.reverse() + + while troves or not self._jobDone(): + # fill slots with available troves + while troves and self._slots: + # get trove to work on + trove = troves.pop() + + # start build job + jobId = self._builder.start((trove, )) + self._jobs[jobId] = [trove, -1, None] + self._slots -= 1 + self._monitor.monitorJob(jobId) + + # update job status changes + for jobId, status in self._monitor.getStatus(): + self._jobs[jobId][1] = status + if status in self._completed: + self._slots += 1 + assert self._slots <= self._maxSlots + + # commit any jobs that are complete + if status == buildjob.JOB_STATE_BUILT: + self._committer.commitJob(jobId) + + # check for commit status + for jobId, result in self._committer.getStatus(): + self._jobs[jobId][2] = result + + results = {} + for jobId, (trove, status, result) in self._jobs.iteritems(): + # log failed jobs + if status == buildjob.JOB_STATE_FAILED: + log.info('[%s] failed job: %s' % (jobId, trove)) + else: + results[trove] = result + + import epdb; epdb.st() + + return results + + def _jobDone(self): + for jobId, (trove, status, result) in self._jobs.iteritems(): + if status not in self._completed: + return False + return True From elliot at rpath.com Mon Mar 15 17:02:45 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:45 +0000 Subject: mirrorball: add fedora hardlink script Message-ID: <201003152102.o2FL2jw2005727@scc.eng.rpath.com> changeset: f9f53c94d217 user: Elliot Peele date: Wed, 21 Oct 2009 21:35:32 -0400 add fedora hardlink script diff --git a/scripts/hardlink-fedora.sh b/scripts/hardlink-fedora.sh new file mode 100755 --- /dev/null +++ b/scripts/hardlink-fedora.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +DEST=/l/fedora/linux/ + +date +./hardlink.py $DEST From elliot at rpath.com Mon Mar 15 17:02:47 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:47 +0000 Subject: mirrorball: initial support for: Message-ID: <201003152102.o2FL2lZA005775@scc.eng.rpath.com> changeset: df011b485194 user: Elliot Peele date: Sat, 24 Oct 2009 18:12:07 -0400 initial support for: 1. updating without advisories 2. allowing update to create packages 3. allowing consumers of the bot interface to specify srcPkgs to import diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -39,10 +39,12 @@ self._clients = {} self._pkgSource = pkgsource.PackageSource(self._cfg) self._updater = update.Updater(self._cfg, self._pkgSource) - self._advisor = advisories.Advisor(self._cfg, self._pkgSource, - self._cfg.platformName) self._builder = build.Builder(self._cfg) + if not self._cfg.disableAdvisories: + self._advisor = advisories.Advisor(self._cfg, self._pkgSource, + self._cfg.platformName) + @staticmethod def _flattenSetDict(setDict): """ @@ -57,9 +59,17 @@ lst.extend(list(trvSet)) return lst - def create(self, rebuild=False, recreate=None): + def create(self, rebuild=False, recreate=None, toCreate=None): """ Do initial imports. + + @param rebuild - rebuild all sources + @type rebuild - boolean + @param recreate - recreate all sources or a specific list of packages + @type recreate - boolean to recreate all sources or a list of specific + package names + @param toCreate - set of source package objects to create, implies recreate. + @type toCrate - iterable """ start = time.time() @@ -69,7 +79,9 @@ self._pkgSource.load() # Build list of packages - if type(recreate) == list: + if toCreate: + toPackage = None + elif type(recreate) == list: toPackage = set(recreate) elif self._cfg.packageAll: toPackage = set() @@ -99,7 +111,8 @@ # Import sources into repository. toBuild, fail = self._updater.create(toPackage, buildAll=rebuild, - recreate=bool(recreate)) + recreate=bool(recreate), + toCreate=toCreate) log.info('failed to create %s packages' % len(fail)) log.info('found %s packages to build' % len(toBuild)) @@ -125,17 +138,23 @@ return trvMap - def update(self, force=None): + def update(self, force=None, updatePkgs=None): """ Update the conary repository from the yum repositories. @param force: list of packages to update without exception @type force: list(pkgName, pkgName, ...) + @param updatePkgs: set of source package objects to update + @type updatePkgs: iterable of source package objects """ if force is not None: self._cfg.disableUpdateSanity = True assert isinstance(force, list) + updateTroves = None + if updatePkgs: + updateTroves = set(((x.name, None, None), x) for x in updatePkgs) + start = time.time() log.info('starting update') @@ -143,7 +162,7 @@ self._pkgSource.load() # Get troves to update and send advisories. - toAdvise, toUpdate = self._updater.getUpdates() + toAdvise, toUpdate = self._updater.getUpdates(updateTroves=updateTroves) # If forcing an update, make sure that all packages are listed in # toAdvise and toUpdate as needed. @@ -163,12 +182,13 @@ log.info('no updates available') return - # Populate patch source now that we know that there are updates - # available. - self._advisor.load() + if not self._cfg.disableAdvisories: + # Populate patch source now that we know that there are updates + # available. + self._advisor.load() - # Check to see if advisories exist for all required packages. - self._advisor.check(toAdvise) + # Check to see if advisories exist for all required packages. + self._advisor.check(toAdvise) # Update source for nvf, srcPkg in toUpdate: @@ -206,8 +226,9 @@ # Mirror out content self._updater.mirror() - # Send advisories. - self._advisor.send(toAdvise, newTroves) + if not self._cfg.disableAdvisories: + # Send advisories. + self._advisor.send(toAdvise, newTroves) log.info('update completed successfully') log.info('updated %s packages and sent %s advisories' diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -138,6 +138,9 @@ # Exclude these archs from the rpm source. excludeArch = (CfgList(CfgString), []) + # Disable advisories all together. + disableAdvisories = (CfgBool, False) + # Packages for which there might not reasonably be advisories. Define a # default advisory message to send with these packages. advisoryException = (CfgList(CfgList(CfgString)), []) diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -24,6 +24,7 @@ from updatebot.lib import util from updatebot import conaryhelper from updatebot.errors import GroupNotFound +from updatebot.errors import NoManifestFoundError from updatebot.errors import OldVersionNotFoundError from updatebot.errors import UpdateGoesBackwardsError from updatebot.errors import UpdateRemovesPackageError @@ -41,10 +42,12 @@ self._conaryhelper = conaryhelper.ConaryHelper(self._cfg) - def getUpdates(self): + def getUpdates(self, updateTroves=None): """ Find all packages that need updates and/or advisories from a top level binary group. + @param updateTroves: set of troves to update + @type updateTroves: iterable @return list of packages to send advisories for and list of packages to update """ @@ -53,7 +56,13 @@ toAdvise = [] toUpdate = [] - for nvf, srpm in self._findUpdatableTroves(self._cfg.topGroup): + + # If update set is not specified get the latest versions of packages to + # update. + if not updateTroves: + updateTroves = self._findUpdatableTroves(self._cfg.topGroup) + + for nvf, srpm in updateTroves: # Will raise exception if any errors are found, halting execution. if self._sanitizeTrove(nvf, srpm): toUpdate.append((nvf, srpm)) @@ -152,7 +161,15 @@ needsUpdate = False newNames = [ (x.name, x.arch) for x in self._pkgSource.srcPkgMap[srpm] ] metadata = None - manifest = self._conaryhelper.getManifest(nvf[0]) + + try: + manifest = self._conaryhelper.getManifest(nvf[0]) + except NoManifestFoundError, e: + # Create packages that do not have manifests. + # FIXME: might want to make this a config option? + log.info('no manifest found for %s, will create package' % nvf[0]) + return True + for line in manifest: # Some manifests were created with double slashes, need to # normalize the path to work around this problem. @@ -231,7 +248,7 @@ return ret - def create(self, pkgNames, buildAll=False, recreate=False): + def create(self, pkgNames=None, buildAll=False, recreate=False, toUpdate=None): """ Import a new package into the repository. @param pkgNames: list of packages to import @@ -241,13 +258,26 @@ @param recreate: a package manifest even if it already exists. @type recreate: boolean @return new source [(name, version, flavor), ... ] + + @param toUpdate: set of packages to update. If this is set all other + options are ignored. + @type toUpdate: set of source package objects. """ + assert pkgNames or toUpdate + + if pkgNames: + toUpdate = set() + else: + # Import very specific versions of packages, make sure to recreate + # them all. + pkgNames = [] + recreate = True + log.info('getting existing packages') pkgs = self._getExistingPackageNames() # Find all of the source to update. - toUpdate = set() for pkg in pkgNames: if pkg not in self._pkgSource.binNameMap: log.warn('no package named %s found in package source' % pkg) From elliot at rpath.com Mon Mar 15 17:02:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:49 +0000 Subject: mirrorball: more mirrorball location changes Message-ID: <201003152102.o2FL2n6c005827@scc.eng.rpath.com> changeset: 434a9d6996c3 user: Elliot Peele date: Sat, 24 Oct 2009 18:13:04 -0400 more mirrorball location changes diff --git a/scripts/buildautoloadrecipes b/scripts/buildautoloadrecipes --- a/scripts/buildautoloadrecipes +++ b/scripts/buildautoloadrecipes @@ -22,7 +22,7 @@ context="x86" fi -mirrorballpath="$HOME/hg/mirrorball" +mirrorballpath="$HOME/hg/mirrorball/mirrorball" platformConfig="$mirrorballpath/config/$platform/conaryrc" if [ ! -f $platformConfig ] ; then diff --git a/scripts/findbinaries b/scripts/findbinaries --- a/scripts/findbinaries +++ b/scripts/findbinaries @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -35,7 +36,7 @@ slog = logging.getLogger('findbinaries') cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) bot = bot.Bot(cfg) updater = bot._updater From elliot at rpath.com Mon Mar 15 17:02:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:52 +0000 Subject: mirrorball: sync centos 4 as well Message-ID: <201003152102.o2FL2qUn005877@scc.eng.rpath.com> changeset: 7e2788cbf937 user: Elliot Peele date: Sat, 24 Oct 2009 18:13:16 -0400 sync centos 4 as well diff --git a/scripts/sync-centos.sh b/scripts/sync-centos.sh --- a/scripts/sync-centos.sh +++ b/scripts/sync-centos.sh @@ -17,6 +17,6 @@ DEST=/l/CentOS/ date -rsync -arv --progress --bwlimit=700 --exclude 2* --exclude 3* --exclude 4* $SOURCE $DEST +rsync -arv --progress --bwlimit=700 --exclude 2* --exclude 3* $SOURCE $DEST ./hardlink.py $DEST From elliot at rpath.com Mon Mar 15 17:02:54 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:54 +0000 Subject: mirrorball: import public interfaces Message-ID: <201003152102.o2FL2t6F005925@scc.eng.rpath.com> changeset: 8b8e681a964e user: Elliot Peele date: Sat, 24 Oct 2009 18:19:26 -0400 import public interfaces diff --git a/updatebot/__init__.py b/updatebot/__init__.py --- a/updatebot/__init__.py +++ b/updatebot/__init__.py @@ -13,6 +13,9 @@ # """ -UpdateBot is a module for automated updating of a conary repository from a -SLES yum/rpm repository. +UpdateBot is a module for the automated creation and updating of a conary +packages from a yum or apt repository. """ + +from updatebot.bot import Bot +from updatebot.config import UpdateBotConfig From elliot at rpath.com Mon Mar 15 17:02:57 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:57 +0000 Subject: mirrorball: add initial support for ordered create/update Message-ID: <201003152102.o2FL2vwX005976@scc.eng.rpath.com> changeset: 95df3f4bae1c user: Elliot Peele date: Sun, 25 Oct 2009 18:00:55 -0400 add initial support for ordered create/update diff --git a/updatebot/errata.py b/updatebot/errata.py new file mode 100644 --- /dev/null +++ b/updatebot/errata.py @@ -0,0 +1,195 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for ordering errata. +""" + +import time +import logging + +log = logging.getLogger('updatebot.errata') + +def loadErrata(func): + def wrapper(self, *args, **kwargs): + if not self._order: + self._orderErrata() + return func(self, *args, **kwargs) + return wrapper + +class ErrataFilter(object): + """ + Filter data from a given errataSource in chronological order. + """ + + def __init__(self, pkgSource, errataSource): + self._pkgSource = pkgSource + self._errata = errataSource + + self._order = {} + self._advMap = {} + + @loadErrata + def getInitialPackages(self): + """ + Get the initial set of packages. + """ + + return self._order[0] + + @loadErrata + def lookupUpdateDetail(self, bucketId): + """ + Given a errata timestamp lookup the name and summary. + """ + + if bucketId in self._advMap: + return '%(name)s: %(summary)s' % self._advMap[bucketId] + else: + return '%s (no detail found)' % bucketId + + @loadErrata + def iterByIssueDate(self, start=None): + """ + Yield sets of srcPkgs by errata release date. + @param start: timestamp from which to start iterating. + @type start: int + """ + + for stamp in sorted(self._order.keys()): + if start > stamp: + continue + yield stamp, self._order[stamp] + + def _orderErrata(self): + """ + Order errata by timestamp. + """ + + # order packages by errata release + buckets, other = self._sortPackagesByErrataTimestamp() + + # insert packages that did not have errata and were not in the initial + # set of packages (golden bits) + srcMap = {} + for pkg in other: + src = self._pkgSource.binPkgMap[pkg] + if src not in srcMap: + srcMap[src] = [] + srcMap[src].append(pkg) + + # insert bins by buildstamp + for src, bins in srcMap.iteritems(): + buildstamp = int(sorted(bins)[0].buildTimestamp) + if buildstamp not in buckets: + buckets[buildstamp] = [] + buckets[buildstamp].extend(bins) + + # get sources to build + for bucketId in sorted(buckets.keys()): + bucket = buckets[bucketId] + self._order[bucketId] = set() + for pkg in bucket: + src = self._pkgSource.binPkgMap[pkg] + self._order[bucketId].add(src) + + def _getNevra(self, pkg): + """ + Get the NEVRA of a package object and do any transformation required. + """ + + # convert nevra to yum compatible nevra + nevra = list(pkg.getNevra()) + if nevra[1] is None: + nevra[1] = '0' + if type(nevra[1]) == int: + nevra[1] = str(nevra[1]) + nevra = tuple(nevra) + + return nevra + + def _sortPackagesByErrataTimestamp(self): + """ + Sort packages by errata release timestamp. + """ + + # get mapping of nevra to pkg obj + nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) + for x in self._pkgSource.binPkgMap.keys() + if x.arch != 'src' and '-debuginfo' not in x.name) + + # pull nevras into errata sized buckets + buckets = {} + nevraMap = {} + + indexedChannels = set(self._errata.getChannels()) + arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') + for e in self._errata.iterByIssueDate(): + bucket = [] + allocated = [] + bucketId = None + slog.info('processing %s' % e.advisory) + for pkg in e.packages: + nevra = self._getNevra(pkg) + + # ignore arches we don't know about. + if nevra[4] not in arches: + continue + + # filter out channels we don't have indexed + channels = set([ x.label for x in pkg.channels ]) + if not indexedChannels & channels: + continue + + # move nevra to errata buckets + if nevra in nevras: + binPkg = nevras.pop(nevra) + bucket.append(binPkg) + allocated.append(nevra) + + # nevra is already part of another bucket + elif nevra in nevraMap: + bucketId = nevraMap[nevra] + + # raise error if we can't find the required package + else: + raise ErrataPackageNotFoundError(pkg=nevra) + + if bucketId is None: + bucketId = int(time.mktime(time.strptime(e.issue_date, + '%Y-%m-%d %H:%M:%S'))) + buckets[bucketId] = bucket + else: + buckets[bucketId].extend(bucket) + + for nevra in allocated: + nevraMap[nevra] = bucketId + + self._advMap[bucketId] = {'name': e.advisory, + 'summary': e.synopsis} + + # separate out golden bits + other = [] + golden = [] + firstErrata = sorted(buckets.keys())[0] + for nevra, pkg in nevras.iteritems(): + buildtime = int(pkg.buildTimestamp) + if buildtime < firstErrata: + golden.append(pkg) + else: + other.append(pkg) + + buckets[0] = golden + + return buckets, other diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -237,3 +237,19 @@ _params = ['what', 'advisories'] _template = 'Found multiple advisories for %(what)s: %(advisories)s' + +class ErrataError(UpdateBotError): + """ + Base exception class for errata related errors. + """ + +class ErrataPackageNotFoundError(ErrataError): + """ + ErrataPackageNotFoundError, raised when a package can not be found in the + package source that matches a package in the errata source. + """ + + _params = ['pkg', ] + _templates = ('Could not find a matching package for %(pkg)s in the ' + 'configured repositories when attempting to map errata source to ' + 'package source.') diff --git a/updatebot/ordered.py b/updatebot/ordered.py new file mode 100644 --- /dev/null +++ b/updatebot/ordered.py @@ -0,0 +1,66 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for doing updates ordered by errata information. +""" + +import logging + +from updatebot import errata +from updatebot.bot import Bot as BotSuperClass + +log = logging.getLogger('updatebot.ordered') + +class Bot(BotSuperClass): + """ + Implement errata driven create/update interface. + """ + + _create = BotSuperClass.create + _update = BotSuperClass.update + + def __init__(self, cfg, errataSource): + BotSuperClass.__init__(self, cfg) + self._errata = errata.ErrataFilter(self._pkgSource, errataSource) + + def create(self, *args, **kwargs): + """ + Handle initial import case. + """ + + toCreate = self._errata.getInitialPackages() + return self._create(*args, toCreate=toCreate, **kwargs) + + def update(self, *args, **kwargs): + """ + Handle update case. + """ + + # Get current timestamp + # FIXME: Figure out where to store current errata level + raise NotImplementedError + + current = 0 + + for updateId, updates in self._errata.iterByIssueDate(start=current): + detail = self._errata.getUpdateDetail(updateId) + log.info('attempting to apply %s' % detail) + + # Update package set. + self._update(updatePkgs=updates) + + # Store current updateId. + # FIXME: figure out where/how to store the current updateId + From elliot at rpath.com Mon Mar 15 17:02:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:02:59 +0000 Subject: mirrorball: fixups Message-ID: <201003152102.o2FL2xn3006025@scc.eng.rpath.com> changeset: fba82d2bd450 user: Elliot Peele date: Sun, 25 Oct 2009 18:01:14 -0400 fixups diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py --- a/scripts/rhelorder.py +++ b/scripts/rhelorder.py @@ -17,8 +17,8 @@ confDir = os.path.join(mbdir, 'config', 'rhel4') from updatebot import log -from updatebot import bot -from updatebot import config +from updatebot import Bot +from updatebot import UpdateBotConfig import time import logging @@ -28,10 +28,10 @@ import rhnmirror log.addRootLogger() -cfg = config.UpdateBotConfig() +cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) -obj = bot.Bot(cfg) +bot = Bot(cfg) mcfg = rhnmirror.MirrorConfig() mcfg.read(os.path.join(confDir, 'erratarc')) @@ -39,7 +39,7 @@ errata = rhnmirror.Errata(mcfg) errata.fetch() -pkgSource = obj._pkgSource +pkgSource = bot._pkgSource pkgSource.load() # get mapping of advisory to errata obj From elliot at rpath.com Mon Mar 15 17:03:01 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:01 +0000 Subject: mirrorball: initial script for driving platform creation Message-ID: <201003152103.o2FL31to006072@scc.eng.rpath.com> changeset: 3086365d6bd9 user: Elliot Peele date: Sun, 25 Oct 2009 18:01:39 -0400 initial script for driving platform creation diff --git a/scripts/order_import.py b/scripts/order_import.py new file mode 100644 --- /dev/null +++ b/scripts/order_import.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +import os +import sys +import logging + +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) + +if 'CONARY_PATH' in os.environ: + sys.path.insert(0, os.environ['CONARY_PATH']) + +import conary +import updatebot + +print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__) +print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__) + +from conary.lib import util +sys.excepthook = util.genExcepthook() + +import rhnmirror + +from updatebot import config +from updatebot import ordered +from updatebot import log as logSetup + +logSetup.addRootLogger() +log = logging.getLogger('script') + +def usage(): + print 'usage: %s ' % sys.argv[0] + sys.exit(1) + +platform = sys.argv[1] +if platform not in os.listdir(mirrorballDir + '/config'): + usage() + +confDir = mirrorballDir + '/config/' + platform + +cfg = config.UpdateBotConfig() +cfg.read(confDir + '/updatebotrc') + +mcfg = rhnmirror.MirrorConfig() +mcfg.read(confDir + '/errata') + +errata = rhnmirror.Errata(mcfg) +errata.fetch() + +bot = ordered.Bot(cfg, errata) +bot.create() + +import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:03:03 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:03 +0000 Subject: mirrorball: don't load package source more than once Message-ID: <201003152103.o2FL34OY006121@scc.eng.rpath.com> changeset: ca26d9511768 user: Elliot Peele date: Mon, 26 Oct 2009 18:10:28 -0400 don't load package source more than once diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py --- a/updatebot/pkgsource/common.py +++ b/updatebot/pkgsource/common.py @@ -29,6 +29,8 @@ self._cfg = cfg self._excludeArch = self._cfg.excludeArch + self._loaded = False + # {repoShortUrl: clientObj} self._clients = dict() diff --git a/updatebot/pkgsource/debsource.py b/updatebot/pkgsource/debsource.py --- a/updatebot/pkgsource/debsource.py +++ b/updatebot/pkgsource/debsource.py @@ -39,6 +39,9 @@ Load repository metadata from a config object. """ + if self._loaded: + return + client = aptmd.Client(self._cfg.repositoryUrl) for repo in self._cfg.repositoryPaths: log.info('loading repository data %s' % repo) @@ -46,6 +49,7 @@ self._clients[repo] = client self.finalize() + self._loaded = True def loadFromClient(self, client, path): """ diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -47,6 +47,9 @@ Load package source based on config data. """ + if self._loaded: + return + for repo in self._cfg.repositoryPaths: log.info('loading repository data %s' % repo) client = repomd.Client(self._cfg.repositoryUrl + '/' + repo) @@ -54,6 +57,7 @@ self._clients[repo] = client self.finalize() + self._loaded = True def loadFromUrl(self, url, basePath=''): """ From elliot at rpath.com Mon Mar 15 17:03:05 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:05 +0000 Subject: mirrorball: fix typo, add missing import Message-ID: <201003152103.o2FL35kx006189@scc.eng.rpath.com> changeset: 1cefeda9ee17 user: Elliot Peele date: Mon, 26 Oct 2009 18:10:53 -0400 fix typo, add missing import diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -19,6 +19,8 @@ import time import logging +from updatebot.errors import ErrataPackageNotFoundError + log = logging.getLogger('updatebot.errata') def loadErrata(func): @@ -139,7 +141,7 @@ bucket = [] allocated = [] bucketId = None - slog.info('processing %s' % e.advisory) + log.info('processing %s' % e.advisory) for pkg in e.packages: nevra = self._getNevra(pkg) From elliot at rpath.com Mon Mar 15 17:03:07 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:07 +0000 Subject: mirrorball: load package source before handling errata Message-ID: <201003152103.o2FL37tH006240@scc.eng.rpath.com> changeset: 8899e73ec5d3 user: Elliot Peele date: Mon, 26 Oct 2009 18:11:12 -0400 load package source before handling errata diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -40,6 +40,7 @@ Handle initial import case. """ + self._pkgSource.load() toCreate = self._errata.getInitialPackages() return self._create(*args, toCreate=toCreate, **kwargs) @@ -54,6 +55,8 @@ current = 0 + self._pkgSource.load() + for updateId, updates in self._errata.iterByIssueDate(start=current): detail = self._errata.getUpdateDetail(updateId) log.info('attempting to apply %s' % detail) From elliot at rpath.com Mon Mar 15 17:03:09 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:09 +0000 Subject: mirrorball: toUpdate -> toCreate Message-ID: <201003152103.o2FL39ce006294@scc.eng.rpath.com> changeset: 4fd5ce28a2ef user: Elliot Peele date: Mon, 26 Oct 2009 18:11:32 -0400 toUpdate -> toCreate diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -248,7 +248,7 @@ return ret - def create(self, pkgNames=None, buildAll=False, recreate=False, toUpdate=None): + def create(self, pkgNames=None, buildAll=False, recreate=False, toCreate=None): """ Import a new package into the repository. @param pkgNames: list of packages to import @@ -259,20 +259,20 @@ @type recreate: boolean @return new source [(name, version, flavor), ... ] - @param toUpdate: set of packages to update. If this is set all other + @param toCreate: set of packages to update. If this is set all other options are ignored. - @type toUpdate: set of source package objects. + @type toCreate: set of source package objects. """ - assert pkgNames or toUpdate + assert pkgNames or toCreate if pkgNames: - toUpdate = set() + toCreate = set() else: # Import very specific versions of packages, make sure to recreate # them all. pkgNames = [] - recreate = True + recreate = False log.info('getting existing packages') pkgs = self._getExistingPackageNames() @@ -286,13 +286,13 @@ srcPkg = self._getPackagesToImport(pkg) if srcPkg.name not in pkgs or recreate: - toUpdate.add(srcPkg) + toCreate.add(srcPkg) # Update all of the unique sources. fail = set() toBuild = set() verCache = self._conaryhelper.getLatestVersions() - for pkg in sorted(toUpdate): + for pkg in sorted(toCreate): try: # Only import packages that haven't been imported before version = verCache.get('%s:source' % pkg.name) From elliot at rpath.com Mon Mar 15 17:03:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:10 +0000 Subject: mirrorball: add capsule recipe Message-ID: <201003152103.o2FL3Aos006354@scc.eng.rpath.com> changeset: 6b1819dc6a5a user: Elliot Peele date: Mon, 26 Oct 2009 18:12:10 -0400 add capsule recipe diff --git a/scripts/buildautoloadrecipes b/scripts/buildautoloadrecipes --- a/scripts/buildautoloadrecipes +++ b/scripts/buildautoloadrecipes @@ -45,6 +45,7 @@ derived fileset group + capsule groupinfo redirect userinfo From elliot at rpath.com Mon Mar 15 17:03:11 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:11 +0000 Subject: mirrorball: fix typo Message-ID: <201003152103.o2FL3CCV006401@scc.eng.rpath.com> changeset: d17ff81bf108 user: Elliot Peele date: Mon, 26 Oct 2009 18:12:21 -0400 fix typo diff --git a/scripts/order_import.py b/scripts/order_import.py old mode 100644 new mode 100755 --- a/scripts/order_import.py +++ b/scripts/order_import.py @@ -55,7 +55,7 @@ cfg.read(confDir + '/updatebotrc') mcfg = rhnmirror.MirrorConfig() -mcfg.read(confDir + '/errata') +mcfg.read(confDir + '/erratarc') errata = rhnmirror.Errata(mcfg) errata.fetch() From elliot at rpath.com Mon Mar 15 17:03:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:13 +0000 Subject: mirrorball: use unique lookaside dir for each use Message-ID: <201003152103.o2FL3E5F006455@scc.eng.rpath.com> changeset: d8db38a5774e user: Elliot Peele date: Wed, 28 Oct 2009 22:56:26 -0400 use unique lookaside dir for each use diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -59,6 +59,10 @@ # Have to initialize flavors to commit to the repository. self._ccfg.initializeFlavors() + # FIXME: CNY-3256 - use unique tmp directory for lookaside until + # this issue is fixed. + self._ccfg.lookaside = tempfile.mkdtemp(prefix=cfg.platformName + '-') + mirrorDir = util.join(cfg.configPath, 'mirrors') if os.path.exists(mirrorDir): self._ccfg.mirrorDirs.insert(0, mirrorDir) From elliot at rpath.com Mon Mar 15 17:03:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:15 +0000 Subject: mirrorball: use a unique lookaside for every run to work around a conary bug Message-ID: <201003152103.o2FL3FeG006504@scc.eng.rpath.com> changeset: 5be794ffcf0a user: Elliot Peele date: Thu, 29 Oct 2009 13:29:42 -0400 use a unique lookaside for every run to work around a conary bug diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -61,7 +61,9 @@ # FIXME: CNY-3256 - use unique tmp directory for lookaside until # this issue is fixed. - self._ccfg.lookaside = tempfile.mkdtemp(prefix=cfg.platformName + '-') + self._ccfg.lookaside = tempfile.mkdtemp( + prefix='%s-lookaside-' % cfg.platformName) + log.info('using lookaside %s' % self._ccfg.lookaside) mirrorDir = util.join(cfg.configPath, 'mirrors') if os.path.exists(mirrorDir): From elliot at rpath.com Mon Mar 15 17:03:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:18 +0000 Subject: mirrorball: don't log every errata Message-ID: <201003152103.o2FL3ImC006571@scc.eng.rpath.com> changeset: 0971c109430a user: Elliot Peele date: Thu, 29 Oct 2009 13:30:05 -0400 don't log every errata diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -135,13 +135,15 @@ buckets = {} nevraMap = {} + log.info('processing errata') + indexedChannels = set(self._errata.getChannels()) arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') for e in self._errata.iterByIssueDate(): bucket = [] allocated = [] bucketId = None - log.info('processing %s' % e.advisory) + #log.info('processing %s' % e.advisory) for pkg in e.packages: nevra = self._getNevra(pkg) From elliot at rpath.com Mon Mar 15 17:03:19 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:19 +0000 Subject: mirrorball: change FIXME to TODO Message-ID: <201003152103.o2FL3KJI006607@scc.eng.rpath.com> changeset: 0b1fd4daaadb user: Elliot Peele date: Thu, 29 Oct 2009 13:30:55 -0400 change FIXME to TODO diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -166,7 +166,7 @@ manifest = self._conaryhelper.getManifest(nvf[0]) except NoManifestFoundError, e: # Create packages that do not have manifests. - # FIXME: might want to make this a config option? + # TODO: might want to make this a config option? log.info('no manifest found for %s, will create package' % nvf[0]) return True From elliot at rpath.com Mon Mar 15 17:03:21 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:21 +0000 Subject: mirrorball: automatically build packages that we think are kernel modules as such Message-ID: <201003152103.o2FL3M2q006659@scc.eng.rpath.com> changeset: ab853cc8efb7 user: Elliot Peele date: Thu, 29 Oct 2009 13:32:03 -0400 automatically build packages that we think are kernel modules as such diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -246,7 +246,9 @@ troves.append((name, version, flavor)) # Kernels are special. - elif ((name == 'kernel' or name in self._cfg.kernelModules) + elif ((name == 'kernel' or + name in self._cfg.kernelModules or + util.isKernelModulePackage(name)) and self._cfg.kernelFlavors): for context, flavor in self._cfg.kernelFlavors: # Replace flag name to match package diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -123,3 +123,18 @@ for pkg in self.pkgs: self.binPkgMap[pkg] = src + +def isKernelModulePackage(paths): + """ + Check if a package file name or location is a kernel module. + """ + + if type(paths) == str: + paths = [ paths, ] + + for path in paths: + basePath = os.path.basename(path) + if (basePath.startswith('kmod-') or + basePath.startswith('kernel-module')): + return True + return False From elliot at rpath.com Mon Mar 15 17:03:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:25 +0000 Subject: mirrorball: just a little renaming to something more appropriate Message-ID: <201003152103.o2FL3Pbw006732@scc.eng.rpath.com> changeset: 19a40ff416d7 user: Whitney Battestilli date: Thu, 29 Oct 2009 23:12:44 -0400 just a little renaming to something more appropriate diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -16,16 +16,23 @@ Builder object implementation. """ +import stat import time import logging import tempfile +import itertools import xml from Queue import Queue, Empty from threading import Thread, RLock +from conary import conarycfg, conaryclient +from conary import rpmhelper +from conary import trove +from conary import files from conary.deps import deps -from conary import conarycfg, conaryclient +from conary.repository import changeset +from conary.repository.netrepos.proxy import ChangesetFilter from rmake import plugins from rmake.build import buildcfg @@ -334,6 +341,101 @@ log.error('Job %d has no built troves', jobId) raise JobFailedError(jobId=jobId, why='No troves found in job') + def _sanityCheckRPMCapsule(self, jobId, fileList, fileObjs, rpmFile): + """ + Compare an rpm capsule with the contents of a + trove to make sure that they agree. + """ + + h = rpmhelper.readHeader(rpmFile,fileIsStream=True) + rpmFileList = dict( + itertools.izip( h[rpmhelper.OLDFILENAMES], + itertools.izip( h[rpmhelper.FILEUSERNAME], + h[rpmhelper.FILEGROUPNAME], + h[rpmhelper.FILEMODES], + h[rpmhelper.FILESIZES], + h[rpmhelper.FILERDEVS], + h[rpmhelper.FILEFLAGS], + h[rpmhelper.FILEVERIFYFLAGS], + h[rpmhelper.FILELINKTOS], + ))) + + foundFiles = dict.fromkeys(rpmFileList) + + def fassert( test, path="", why=None ): + if not test: + if why: + raise CommitFailedError( jobId=jobId, why=why ) + else: + raise CommitFailedError(jobId=jobId, why="metadata in trove doesn't " + "agree with rpm header for file %s" % (path) ) + + for fileInfo, fileObj in zip(fileList, fileObjs): + fpath = fileInfo[1] + foundFiles[fpath] = True + rUser, rGroup, rMode, rSize, rDev, rFlags, rVflags, rLinkto = rpmFileList[fpath] + + # First, tests based on the Conary changeset + + # file metadata verification + if rUser != fileObj.inode.owner() or rGroup != fileObj.inode.group() \ + or stat.S_IMODE(rMode) != fileObj.inode.perms(): + fassert( False, fpath ) + + if isinstance(fileObj, files.RegularFile): + if not stat.S_ISREG( rMode ): + fassert( False, fpath ) + + # RPM config flag mapping + if rFlags & rpmhelper.RPMFILE_CONFIG: + if fileObj.contents.size(): + fassert(fileObj.flags.isConfig(), fpath) + else: + fassert(fileObj.flags.isInitialContents(), fpath) + + elif isinstance(fileObj, files.Directory): + fassert( stat.S_ISDIR( rMode ), fpath ) + fassert( not fileObj.flags.isPayload(), fpath ) + elif isinstance(fileObj, files.CharacterDevice): + fassert( stat.S_ISCHR( rMode ), fpath ) + + minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 + major = (rDev >> 8) & 0xfff + fassert( fileObj.devt.major() == major , fpath ) + fassert( fileObj.devt.minor() == minor , fpath ) + + fassert( not fileObj.flags.isPayload() ) + elif isinstance(fileObj, files.BlockDevice): + fassert( stat.S_ISBLK( rMode ), fpath ) + + minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 + major = (rDev >> 8) & 0xfff + fassert( fileObj.devt.major() == major, fpath ) + fassert( fileObj.devt.minor() == minor, fpath ) + + fassert( not fileObj.flags.isPayload(), fpath ) + elif isinstance(fileObj, files.NamedPipe): + fassert( stat.S_ISFIFO( rMode ), fpath ) + + fassert( not fileObj.flags.isPayload(), fpath ) + elif isinstance(fileObj, files.SymbolicLink): + fassert( stat.S_ISLNK( rMode ), fpath ) + fassert( fileObj.target() == rLinkto, fpath ) + + fassert( not fileObj.flags.isPayload(), fpath ) + else: + # unhandled file type + fassert( False, fpath ) + + # Now, some tests based on the contents of the RPM header + if (not stat.S_ISDIR(rMode)) and rFlags & rpmhelper.RPMFILE_GHOST: + fassert( fileObj.flags.isInitialContents(), fpath ) + + if not rVflags: + # %doc -- CNY-3254 + fassert( isinstance(fileObj, files.RegularFile), fpath ) + fassert( not fileObj.flags.isInitialContents(), fpath ) + def _commitJob(self, jobId): """ Commit completed job. @@ -372,6 +474,49 @@ if writeToFile: log.info('changeset saved to %s' % writeToFile) + newCs = changeset.ChangeSetFromFile( writeToFile ) + + log.info('comparing changeset to rpm capsules contained within it for changset %s' % writeToFile) + + oldCsJob = [] + for newTroveCs in newCs.iterNewTroveList(): + if newTroveCs.getTroveInfo().capsule.type() == 'rpm': + if newTroveCs.getOldVersion(): + name, version, flavor = newTroveCs.getOldNameVersionFlavor() + oldCsJob.append((name, (None, None), (version, flavor), True)) + + oldCs = None + if oldCsJob: + oldCs = self._client.repos.createChangeSet(oldCsJob, withFiles=True, withFileContents=False) + + fileObjs = [] + fileList = [] + capsuleFileContents = None + if newTroveCs.getOldVersion(): + oldTroveCs = oldCs.getNewTroveVersion(*newTroveCs.getOldNameVersionFlavor()) + assert oldTroveCs.getNewNameVersionFlavor() == newTroveCs.getOldNameVersionFlavor() + oldTrove = trove.Trove(oldTroveCs) + newTrove = oldTrove.copy() + newTrove.applyChangeSet(newTroveCs) + else: + oldTrove = None + newTrove = trove.Trove(newTroveCs) + + # get file streams for comparison + fileList = list(newTrove.iterFileList(capsules=False)) + for pathId, path, fileId, fileVer in fileList: + fileObjs.append( ChangesetFilter._getFileObject(pathId, fileId, oldTrove, oldCs, newCs) ) + + # get capsule file contents + capFileList = [ x[2:] for x in newTrove.iterFileList(capsules=True) ] + if len(capFileList) != 1: + raise CommitFailedError(jobId=jobId, why="More than 1 RPM capsule in trove %s" % newTroveCs.name() ) + fcList = self._client.repos.getFileContents(capFileList, compressed=False) + capsuleFileContents = fcList[0].get() + + # do the check + self._sanityCheckRPMCapsule( jobIdsStr, fileList, fileObjs, capsuleFileContents ) + log.info('committing changeset to repository') self._client.repos.commitChangeSetFile(writeToFile) From elliot at rpath.com Mon Mar 15 17:03:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:27 +0000 Subject: mirrorball: add rpm capsule verification during commit Message-ID: <201003152103.o2FL3RZA006776@scc.eng.rpath.com> changeset: d80d1aaa478b user: Whitney Battestilli date: Thu, 29 Oct 2009 23:19:42 -0400 add rpm capsule verification during commit diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -347,7 +347,7 @@ trove to make sure that they agree. """ - h = rpmhelper.readHeader(rpmFile,fileIsStream=True) + h = rpmhelper.readHeader(rpmFile,ignoreSize=True) rpmFileList = dict( itertools.izip( h[rpmhelper.OLDFILENAMES], itertools.izip( h[rpmhelper.FILEUSERNAME], From elliot at rpath.com Mon Mar 15 17:03:30 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:30 +0000 Subject: mirrorball: add support for specifieing rmake config name Message-ID: <201003152103.o2FL3Uo7006825@scc.eng.rpath.com> changeset: 6a546d53d710 user: Elliot Peele date: Thu, 29 Oct 2009 23:34:15 -0400 add support for specifieing rmake config name diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -73,13 +73,9 @@ @type cfg: config.UpdateBotConfig @param saveChangeSets: directory to save changesets to @type saveChangeSets: str - @param sanityCheckCommits: get the changeset that was just committed from the - repository to verify everything made it into the - repository. - @type sanityCheckCommits: boolean """ - def __init__(self, cfg, saveChangeSets=None, sanityCheckCommits=False): + def __init__(self, cfg, rmakeCfgFn=None, saveChangeSets=None): self._cfg = cfg self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) @@ -90,8 +86,9 @@ self._client = conaryclient.ConaryClient(self._ccfg) if saveChangeSets is None and self._cfg.saveChangeSets: - self._saveChangeSets=tempfile.mkdtemp(prefix=self._cfg.platformName, - suffix='-import-changesets') + self._saveChangeSets = tempfile.mkdtemp( + prefix=self._cfg.platformName, + suffix='-import-changesets') else: self._saveChangeSets = saveChangeSets @@ -108,8 +105,16 @@ pluginMgr.loadPlugins() pluginMgr.callClientHook('client_preInit', self, []) + rmakerc = 'rmakerc' + if rmakeCfgFn: + rmakeCfgPath = util.join(self._cfg.configPath, rmakeCfgFn) + if os.path.exists(rmakeCfgPath): + rmakerc = rmakeCfgFn + else: + log.warn('%s not found, falling back to rmakerc' % rmakeCfgFn) + self._rmakeCfg = buildcfg.BuildConfiguration(readConfigFiles=False) - self._rmakeCfg.read(util.join(self._cfg.configPath, 'rmakerc')) + self._rmakeCfg.read(util.join(self._cfg.configPath, rmakerc)) self._rmakeCfg.useConaryConfig(self._ccfg) self._rmakeCfg.copyInConfig = False self._rmakeCfg.strictMode = True From elliot at rpath.com Mon Mar 15 17:03:34 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:34 +0000 Subject: mirrorball: add xobjects for group management Message-ID: <201003152103.o2FL3ZUG006877@scc.eng.rpath.com> changeset: 89fc5aec5519 user: Elliot Peele date: Thu, 29 Oct 2009 23:34:38 -0400 add xobjects for group management diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -98,9 +98,10 @@ def __init__(self): self.items = [] + self._itemClass = self.__class__.__dict__['items'][0] def __setitem__(self, key, value): - item = XDictItem(key, value) + item = self._itemClass(key, value) if item in self.items: idx = self.items.index(item) self.items[idx] = item @@ -115,3 +116,68 @@ def __contains__(self, key): return key in self.items + + +class XItemList(XDocManager): + """ + List of items. + """ + + def __init__(self): + self.items = [] + self._itemClass = self.__class__.__dict__['items'][0] + XDocManager.__init__(self) + + +class XPackageItem(object): + """ + Object to represent package data required for group builds with the + managed group factory. + """ + + name = str + version = str + flavor = str + byDefault = int + use = str + source = str + originalName = str + + def __init__(self, name, version='', flavor='', byDefault=1, use=1, + source='', originalName=None): + self.name = name + self.version = version + self.flavor = flavor + self.byDefault = byDefault + self.use = use + self.source = source + self.originalName = originalName and originalName or name + + +class XPackageData(XItemList): + """ + Mapping of package name to package group data. + """ + + items = [ XPackageItem ] + + +class XGroup(object): + """ + Group file info. + """ + + name = str + filename = str + + def __init__(self, name, filename): + self.name = name + self.filename = filename + + +class XGroupList(XItemList): + """ + List of file names to load as groups. + """ + + items = [ XGroup ] From elliot at rpath.com Mon Mar 15 17:03:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:37 +0000 Subject: mirrorball: branch merge Message-ID: <201003152103.o2FL3cm3006907@scc.eng.rpath.com> changeset: c0554bcb61d9 user: Elliot Peele date: Thu, 29 Oct 2009 23:35:22 -0400 branch merge diff --git a/scripts/buildautoloadrecipes b/scripts/buildautoloadrecipes --- a/scripts/buildautoloadrecipes +++ b/scripts/buildautoloadrecipes @@ -45,6 +45,7 @@ derived fileset group + capsule groupinfo redirect userinfo diff --git a/scripts/order_import.py b/scripts/order_import.py old mode 100644 new mode 100755 --- a/scripts/order_import.py +++ b/scripts/order_import.py @@ -55,7 +55,7 @@ cfg.read(confDir + '/updatebotrc') mcfg = rhnmirror.MirrorConfig() -mcfg.read(confDir + '/errata') +mcfg.read(confDir + '/erratarc') errata = rhnmirror.Errata(mcfg) errata.fetch() diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -80,13 +80,9 @@ @type cfg: config.UpdateBotConfig @param saveChangeSets: directory to save changesets to @type saveChangeSets: str - @param sanityCheckCommits: get the changeset that was just committed from the - repository to verify everything made it into the - repository. - @type sanityCheckCommits: boolean """ - def __init__(self, cfg, saveChangeSets=None, sanityCheckCommits=False): + def __init__(self, cfg, rmakeCfgFn=None, saveChangeSets=None): self._cfg = cfg self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) @@ -97,8 +93,9 @@ self._client = conaryclient.ConaryClient(self._ccfg) if saveChangeSets is None and self._cfg.saveChangeSets: - self._saveChangeSets=tempfile.mkdtemp(prefix=self._cfg.platformName, - suffix='-import-changesets') + self._saveChangeSets = tempfile.mkdtemp( + prefix=self._cfg.platformName, + suffix='-import-changesets') else: self._saveChangeSets = saveChangeSets @@ -115,8 +112,16 @@ pluginMgr.loadPlugins() pluginMgr.callClientHook('client_preInit', self, []) + rmakerc = 'rmakerc' + if rmakeCfgFn: + rmakeCfgPath = util.join(self._cfg.configPath, rmakeCfgFn) + if os.path.exists(rmakeCfgPath): + rmakerc = rmakeCfgFn + else: + log.warn('%s not found, falling back to rmakerc' % rmakeCfgFn) + self._rmakeCfg = buildcfg.BuildConfiguration(readConfigFiles=False) - self._rmakeCfg.read(util.join(self._cfg.configPath, 'rmakerc')) + self._rmakeCfg.read(util.join(self._cfg.configPath, rmakerc)) self._rmakeCfg.useConaryConfig(self._ccfg) self._rmakeCfg.copyInConfig = False self._rmakeCfg.strictMode = True @@ -253,7 +258,9 @@ troves.append((name, version, flavor)) # Kernels are special. - elif ((name == 'kernel' or name in self._cfg.kernelModules) + elif ((name == 'kernel' or + name in self._cfg.kernelModules or + util.isKernelModulePackage(name)) and self._cfg.kernelFlavors): for context, flavor in self._cfg.kernelFlavors: # Replace flag name to match package diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -59,6 +59,12 @@ # Have to initialize flavors to commit to the repository. self._ccfg.initializeFlavors() + # FIXME: CNY-3256 - use unique tmp directory for lookaside until + # this issue is fixed. + self._ccfg.lookaside = tempfile.mkdtemp( + prefix='%s-lookaside-' % cfg.platformName) + log.info('using lookaside %s' % self._ccfg.lookaside) + mirrorDir = util.join(cfg.configPath, 'mirrors') if os.path.exists(mirrorDir): self._ccfg.mirrorDirs.insert(0, mirrorDir) diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -19,6 +19,8 @@ import time import logging +from updatebot.errors import ErrataPackageNotFoundError + log = logging.getLogger('updatebot.errata') def loadErrata(func): @@ -133,13 +135,15 @@ buckets = {} nevraMap = {} + log.info('processing errata') + indexedChannels = set(self._errata.getChannels()) arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') for e in self._errata.iterByIssueDate(): bucket = [] allocated = [] bucketId = None - slog.info('processing %s' % e.advisory) + #log.info('processing %s' % e.advisory) for pkg in e.packages: nevra = self._getNevra(pkg) diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -123,3 +123,18 @@ for pkg in self.pkgs: self.binPkgMap[pkg] = src + +def isKernelModulePackage(paths): + """ + Check if a package file name or location is a kernel module. + """ + + if type(paths) == str: + paths = [ paths, ] + + for path in paths: + basePath = os.path.basename(path) + if (basePath.startswith('kmod-') or + basePath.startswith('kernel-module')): + return True + return False diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -98,9 +98,10 @@ def __init__(self): self.items = [] + self._itemClass = self.__class__.__dict__['items'][0] def __setitem__(self, key, value): - item = XDictItem(key, value) + item = self._itemClass(key, value) if item in self.items: idx = self.items.index(item) self.items[idx] = item @@ -115,3 +116,68 @@ def __contains__(self, key): return key in self.items + + +class XItemList(XDocManager): + """ + List of items. + """ + + def __init__(self): + self.items = [] + self._itemClass = self.__class__.__dict__['items'][0] + XDocManager.__init__(self) + + +class XPackageItem(object): + """ + Object to represent package data required for group builds with the + managed group factory. + """ + + name = str + version = str + flavor = str + byDefault = int + use = str + source = str + originalName = str + + def __init__(self, name, version='', flavor='', byDefault=1, use=1, + source='', originalName=None): + self.name = name + self.version = version + self.flavor = flavor + self.byDefault = byDefault + self.use = use + self.source = source + self.originalName = originalName and originalName or name + + +class XPackageData(XItemList): + """ + Mapping of package name to package group data. + """ + + items = [ XPackageItem ] + + +class XGroup(object): + """ + Group file info. + """ + + name = str + filename = str + + def __init__(self, name, filename): + self.name = name + self.filename = filename + + +class XGroupList(XItemList): + """ + List of file names to load as groups. + """ + + items = [ XGroup ] diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -40,6 +40,7 @@ Handle initial import case. """ + self._pkgSource.load() toCreate = self._errata.getInitialPackages() return self._create(*args, toCreate=toCreate, **kwargs) @@ -54,6 +55,8 @@ current = 0 + self._pkgSource.load() + for updateId, updates in self._errata.iterByIssueDate(start=current): detail = self._errata.getUpdateDetail(updateId) log.info('attempting to apply %s' % detail) diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py --- a/updatebot/pkgsource/common.py +++ b/updatebot/pkgsource/common.py @@ -29,6 +29,8 @@ self._cfg = cfg self._excludeArch = self._cfg.excludeArch + self._loaded = False + # {repoShortUrl: clientObj} self._clients = dict() diff --git a/updatebot/pkgsource/debsource.py b/updatebot/pkgsource/debsource.py --- a/updatebot/pkgsource/debsource.py +++ b/updatebot/pkgsource/debsource.py @@ -39,6 +39,9 @@ Load repository metadata from a config object. """ + if self._loaded: + return + client = aptmd.Client(self._cfg.repositoryUrl) for repo in self._cfg.repositoryPaths: log.info('loading repository data %s' % repo) @@ -46,6 +49,7 @@ self._clients[repo] = client self.finalize() + self._loaded = True def loadFromClient(self, client, path): """ diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -47,6 +47,9 @@ Load package source based on config data. """ + if self._loaded: + return + for repo in self._cfg.repositoryPaths: log.info('loading repository data %s' % repo) client = repomd.Client(self._cfg.repositoryUrl + '/' + repo) @@ -54,6 +57,7 @@ self._clients[repo] = client self.finalize() + self._loaded = True def loadFromUrl(self, url, basePath=''): """ diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -166,7 +166,7 @@ manifest = self._conaryhelper.getManifest(nvf[0]) except NoManifestFoundError, e: # Create packages that do not have manifests. - # FIXME: might want to make this a config option? + # TODO: might want to make this a config option? log.info('no manifest found for %s, will create package' % nvf[0]) return True @@ -248,7 +248,7 @@ return ret - def create(self, pkgNames=None, buildAll=False, recreate=False, toUpdate=None): + def create(self, pkgNames=None, buildAll=False, recreate=False, toCreate=None): """ Import a new package into the repository. @param pkgNames: list of packages to import @@ -259,20 +259,20 @@ @type recreate: boolean @return new source [(name, version, flavor), ... ] - @param toUpdate: set of packages to update. If this is set all other + @param toCreate: set of packages to update. If this is set all other options are ignored. - @type toUpdate: set of source package objects. + @type toCreate: set of source package objects. """ - assert pkgNames or toUpdate + assert pkgNames or toCreate if pkgNames: - toUpdate = set() + toCreate = set() else: # Import very specific versions of packages, make sure to recreate # them all. pkgNames = [] - recreate = True + recreate = False log.info('getting existing packages') pkgs = self._getExistingPackageNames() @@ -286,13 +286,13 @@ srcPkg = self._getPackagesToImport(pkg) if srcPkg.name not in pkgs or recreate: - toUpdate.add(srcPkg) + toCreate.add(srcPkg) # Update all of the unique sources. fail = set() toBuild = set() verCache = self._conaryhelper.getLatestVersions() - for pkg in sorted(toUpdate): + for pkg in sorted(toCreate): try: # Only import packages that haven't been imported before version = verCache.get('%s:source' % pkg.name) From elliot at rpath.com Mon Mar 15 17:03:40 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:40 +0000 Subject: mirrorball: fixups Message-ID: <201003152103.o2FL3fcP006936@scc.eng.rpath.com> changeset: 66a3c7a7b742 user: Elliot Peele date: Thu, 29 Oct 2009 23:58:42 -0400 fixups diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -356,42 +356,44 @@ h = rpmhelper.readHeader(rpmFile,ignoreSize=True) rpmFileList = dict( - itertools.izip( h[rpmhelper.OLDFILENAMES], - itertools.izip( h[rpmhelper.FILEUSERNAME], - h[rpmhelper.FILEGROUPNAME], - h[rpmhelper.FILEMODES], - h[rpmhelper.FILESIZES], - h[rpmhelper.FILERDEVS], - h[rpmhelper.FILEFLAGS], - h[rpmhelper.FILEVERIFYFLAGS], - h[rpmhelper.FILELINKTOS], - ))) + itertools.izip(h[rpmhelper.OLDFILENAMES], + itertools.izip(h[rpmhelper.FILEUSERNAME], + h[rpmhelper.FILEGROUPNAME], + h[rpmhelper.FILEMODES], + h[rpmhelper.FILESIZES], + h[rpmhelper.FILERDEVS], + h[rpmhelper.FILEFLAGS], + h[rpmhelper.FILEVERIFYFLAGS], + h[rpmhelper.FILELINKTOS], + ))) foundFiles = dict.fromkeys(rpmFileList) - def fassert( test, path="", why=None ): + def fassert(test, path="", why=None): if not test: if why: - raise CommitFailedError( jobId=jobId, why=why ) + raise CommitFailedError(jobId=jobId, why=why) else: - raise CommitFailedError(jobId=jobId, why="metadata in trove doesn't " - "agree with rpm header for file %s" % (path) ) + raise CommitFailedError(jobId=jobId, why='metadata in trove' + ' does not agree with rpm header for file %s' % (path)) for fileInfo, fileObj in zip(fileList, fileObjs): fpath = fileInfo[1] foundFiles[fpath] = True - rUser, rGroup, rMode, rSize, rDev, rFlags, rVflags, rLinkto = rpmFileList[fpath] + rUser, rGroup, rMode, rSize, rDev, rFlags, rVflags, rLinkto = \ + rpmFileList[fpath] # First, tests based on the Conary changeset # file metadata verification - if rUser != fileObj.inode.owner() or rGroup != fileObj.inode.group() \ - or stat.S_IMODE(rMode) != fileObj.inode.perms(): - fassert( False, fpath ) + if (rUser != fileObj.inode.owner() or + rGroup != fileObj.inode.group() or + stat.S_IMODE(rMode) != fileObj.inode.perms()): + fassert(False, fpath) if isinstance(fileObj, files.RegularFile): - if not stat.S_ISREG( rMode ): - fassert( False, fpath ) + if not stat.S_ISREG(rMode): + fassert(False, fpath) # RPM config flag mapping if rFlags & rpmhelper.RPMFILE_CONFIG: @@ -401,47 +403,105 @@ fassert(fileObj.flags.isInitialContents(), fpath) elif isinstance(fileObj, files.Directory): - fassert( stat.S_ISDIR( rMode ), fpath ) - fassert( not fileObj.flags.isPayload(), fpath ) + fassert(stat.S_ISDIR(rMode), fpath) + fassert(not fileObj.flags.isPayload(), fpath) elif isinstance(fileObj, files.CharacterDevice): - fassert( stat.S_ISCHR( rMode ), fpath ) + fassert(stat.S_ISCHR(rMode), fpath) minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 major = (rDev >> 8) & 0xfff - fassert( fileObj.devt.major() == major , fpath ) - fassert( fileObj.devt.minor() == minor , fpath ) + fassert(fileObj.devt.major() == major , fpath) + fassert(fileObj.devt.minor() == minor , fpath) - fassert( not fileObj.flags.isPayload() ) + fassert(not fileObj.flags.isPayload()) elif isinstance(fileObj, files.BlockDevice): - fassert( stat.S_ISBLK( rMode ), fpath ) + fassert(stat.S_ISBLK(rMode), fpath) minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 major = (rDev >> 8) & 0xfff - fassert( fileObj.devt.major() == major, fpath ) - fassert( fileObj.devt.minor() == minor, fpath ) + fassert(fileObj.devt.major() == major, fpath) + fassert(fileObj.devt.minor() == minor, fpath) - fassert( not fileObj.flags.isPayload(), fpath ) + fassert(not fileObj.flags.isPayload(), fpath) elif isinstance(fileObj, files.NamedPipe): - fassert( stat.S_ISFIFO( rMode ), fpath ) + fassert(stat.S_ISFIFO(rMode), fpath) - fassert( not fileObj.flags.isPayload(), fpath ) + fassert(not fileObj.flags.isPayload(), fpath) elif isinstance(fileObj, files.SymbolicLink): - fassert( stat.S_ISLNK( rMode ), fpath ) - fassert( fileObj.target() == rLinkto, fpath ) + fassert(stat.S_ISLNK(rMode), fpath) + fassert(fileObj.target() == rLinkto, fpath) - fassert( not fileObj.flags.isPayload(), fpath ) + fassert(not fileObj.flags.isPayload(), fpath) else: # unhandled file type - fassert( False, fpath ) + fassert(False, fpath) # Now, some tests based on the contents of the RPM header if (not stat.S_ISDIR(rMode)) and rFlags & rpmhelper.RPMFILE_GHOST: - fassert( fileObj.flags.isInitialContents(), fpath ) + fassert(fileObj.flags.isInitialContents(), fpath) if not rVflags: # %doc -- CNY-3254 - fassert( isinstance(fileObj, files.RegularFile), fpath ) - fassert( not fileObj.flags.isInitialContents(), fpath ) + fassert(isinstance(fileObj, files.RegularFile), fpath) + fassert(not fileObj.flags.isInitialContents(), fpath) + + def _sanityCheckChangeSet(self, csFile, jobId): + """ + Sanity check changeset before commit. + """ + + newCs = changeset.ChangeSetFromFile(csFile) + log.info('comparing changeset to rpm capsules contained within it for + changset %s' % writeToFile) + + oldCsJob = [] + for newTroveCs in newCs.iterNewTroveList(): + if newTroveCs.getTroveInfo().capsule.type() == 'rpm': + if newTroveCs.getOldVersion(): + name, version, flavor = newTroveCs.getOldNameVersionFlavor() + oldCsJob.append((name, (None, None), + (version, flavor), True)) + + oldCs = None + if oldCsJob: + oldCs = self._client.repos.createChangeSet(oldCsJob, + withFiles=True, withFileContents=False) + + fileObjs = [] + fileList = [] + capsuleFileContents = None + if newTroveCs.getOldVersion(): + oldTroveCs = oldCs.getNewTroveVersion( + *newTroveCs.getOldNameVersionFlavor()) + assert (oldTroveCs.getNewNameVersionFlavor() == + newTroveCs.getOldNameVersionFlavor()) + oldTrove = trove.Trove(oldTroveCs) + newTrove = oldTrove.copy() + newTrove.applyChangeSet(newTroveCs) + else: + oldTrove = None + newTrove = trove.Trove(newTroveCs) + + # get file streams for comparison + fileList = list(newTrove.iterFileList(capsules=False)) + for pathId, path, fileId, fileVer in fileList: + fileObjs.append(ChangesetFilter._getFileObject( + pathId, fileId, oldTrove, oldCs, newCs)) + + # get capsule file contents + capFileList = [ x[2:] for x in + newTrove.iterFileList(capsules=True) ] + if len(capFileList) != 1: + raise CommitFailedError(jobId=jobId, why='More than 1 RPM ' + 'capsule in trove %s' % newTroveCs.name()) + fcList = self._client.repos.getFileContents(capFileList, + compressed=False) + capsuleFileContents = fcList[0].get() + + # do the check + self._sanityCheckRPMCapsule(jobIdsStr, fileList, fileObjs, + capsuleFileContents) + def _commitJob(self, jobId): """ @@ -481,48 +541,7 @@ if writeToFile: log.info('changeset saved to %s' % writeToFile) - newCs = changeset.ChangeSetFromFile( writeToFile ) - - log.info('comparing changeset to rpm capsules contained within it for changset %s' % writeToFile) - - oldCsJob = [] - for newTroveCs in newCs.iterNewTroveList(): - if newTroveCs.getTroveInfo().capsule.type() == 'rpm': - if newTroveCs.getOldVersion(): - name, version, flavor = newTroveCs.getOldNameVersionFlavor() - oldCsJob.append((name, (None, None), (version, flavor), True)) - - oldCs = None - if oldCsJob: - oldCs = self._client.repos.createChangeSet(oldCsJob, withFiles=True, withFileContents=False) - - fileObjs = [] - fileList = [] - capsuleFileContents = None - if newTroveCs.getOldVersion(): - oldTroveCs = oldCs.getNewTroveVersion(*newTroveCs.getOldNameVersionFlavor()) - assert oldTroveCs.getNewNameVersionFlavor() == newTroveCs.getOldNameVersionFlavor() - oldTrove = trove.Trove(oldTroveCs) - newTrove = oldTrove.copy() - newTrove.applyChangeSet(newTroveCs) - else: - oldTrove = None - newTrove = trove.Trove(newTroveCs) - - # get file streams for comparison - fileList = list(newTrove.iterFileList(capsules=False)) - for pathId, path, fileId, fileVer in fileList: - fileObjs.append( ChangesetFilter._getFileObject(pathId, fileId, oldTrove, oldCs, newCs) ) - - # get capsule file contents - capFileList = [ x[2:] for x in newTrove.iterFileList(capsules=True) ] - if len(capFileList) != 1: - raise CommitFailedError(jobId=jobId, why="More than 1 RPM capsule in trove %s" % newTroveCs.name() ) - fcList = self._client.repos.getFileContents(capFileList, compressed=False) - capsuleFileContents = fcList[0].get() - - # do the check - self._sanityCheckRPMCapsule( jobIdsStr, fileList, fileObjs, capsuleFileContents ) + self._sanityCheckChangeSet(writeToFile, jobIdsStr) log.info('committing changeset to repository') self._client.repos.commitChangeSetFile(writeToFile) From elliot at rpath.com Mon Mar 15 17:03:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:43 +0000 Subject: mirrorball: more fixups Message-ID: <201003152103.o2FL3hWx006968@scc.eng.rpath.com> changeset: 60c51a5155de user: Elliot Peele date: Fri, 30 Oct 2009 01:27:04 -0400 more fixups diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -354,7 +354,7 @@ trove to make sure that they agree. """ - h = rpmhelper.readHeader(rpmFile,ignoreSize=True) + h = rpmhelper.readHeader(rpmFile, ignoreSize=True) rpmFileList = dict( itertools.izip(h[rpmhelper.OLDFILENAMES], itertools.izip(h[rpmhelper.FILEUSERNAME], @@ -410,8 +410,8 @@ minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 major = (rDev >> 8) & 0xfff - fassert(fileObj.devt.major() == major , fpath) - fassert(fileObj.devt.minor() == minor , fpath) + fassert(fileObj.devt.major() == major, fpath) + fassert(fileObj.devt.minor() == minor, fpath) fassert(not fileObj.flags.isPayload()) elif isinstance(fileObj, files.BlockDevice): @@ -437,7 +437,7 @@ fassert(False, fpath) # Now, some tests based on the contents of the RPM header - if (not stat.S_ISDIR(rMode)) and rFlags & rpmhelper.RPMFILE_GHOST: + if not stat.S_ISDIR(rMode) and rFlags & rpmhelper.RPMFILE_GHOST: fassert(fileObj.flags.isInitialContents(), fpath) if not rVflags: @@ -451,8 +451,8 @@ """ newCs = changeset.ChangeSetFromFile(csFile) - log.info('comparing changeset to rpm capsules contained within it for - changset %s' % writeToFile) + log.info('comparing changeset to rpm capsules contained within it for' + 'changset %s' % csFile) oldCsJob = [] for newTroveCs in newCs.iterNewTroveList(): @@ -462,15 +462,9 @@ oldCsJob.append((name, (None, None), (version, flavor), True)) - oldCs = None - if oldCsJob: oldCs = self._client.repos.createChangeSet(oldCsJob, withFiles=True, withFileContents=False) - fileObjs = [] - fileList = [] - capsuleFileContents = None - if newTroveCs.getOldVersion(): oldTroveCs = oldCs.getNewTroveVersion( *newTroveCs.getOldNameVersionFlavor()) assert (oldTroveCs.getNewNameVersionFlavor() == @@ -478,10 +472,13 @@ oldTrove = trove.Trove(oldTroveCs) newTrove = oldTrove.copy() newTrove.applyChangeSet(newTroveCs) + else: + oldCs = None oldTrove = None newTrove = trove.Trove(newTroveCs) + fileObjs = [] # get file streams for comparison fileList = list(newTrove.iterFileList(capsules=False)) for pathId, path, fileId, fileVer in fileList: @@ -489,20 +486,30 @@ pathId, fileId, oldTrove, oldCs, newCs)) # get capsule file contents - capFileList = [ x[2:] for x in - newTrove.iterFileList(capsules=True) ] + if newTroveCs.getOldVersion(): + capFileList = [ x[2:] for x in + newTrove.iterFileList(capsules=True) ] + else: + capFileList = [ (x[0], x[2]) for x in + newTrove.iterFileList(capsules=True) ] + if len(capFileList) != 1: raise CommitFailedError(jobId=jobId, why='More than 1 RPM ' 'capsule in trove %s' % newTroveCs.name()) - fcList = self._client.repos.getFileContents(capFileList, - compressed=False) - capsuleFileContents = fcList[0].get() + + if newTroveCs.getOldVersion(): + getFileContents = self._client.repos.getFileContents + fcList = getFileContents(capFileList, compressed=False) + capsuleFileContents = fcList[0].get() + else: + getFileContents = newCs.getFileContents + fcList = getFileContents(*capFileList[0], compressed=False) + capsuleFileContents = fcList[1].get() # do the check - self._sanityCheckRPMCapsule(jobIdsStr, fileList, fileObjs, + self._sanityCheckRPMCapsule(jobId, fileList, fileObjs, capsuleFileContents) - def _commitJob(self, jobId): """ Commit completed job. From elliot at rpath.com Mon Mar 15 17:03:45 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:45 +0000 Subject: mirrorball: update to reflect a change in rpmhelper interface Message-ID: <201003152103.o2FL3k7V006998@scc.eng.rpath.com> changeset: 031a1bd46881 user: Whitney Battestilli date: Fri, 30 Oct 2009 01:53:47 -0400 update to reflect a change in rpmhelper interface diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -354,7 +354,7 @@ trove to make sure that they agree. """ - h = rpmhelper.readHeader(rpmFile, ignoreSize=True) + h = rpmhelper.readHeader(rpmFile, checkSize=False) rpmFileList = dict( itertools.izip(h[rpmhelper.OLDFILENAMES], itertools.izip(h[rpmhelper.FILEUSERNAME], From elliot at rpath.com Mon Mar 15 17:03:47 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:47 +0000 Subject: mirrorball: %doc can apply to directories as well as regular flies Message-ID: <201003152103.o2FL3lY8007026@scc.eng.rpath.com> changeset: 7c3223c755ee user: Michael K. Johnson date: Fri, 30 Oct 2009 10:34:44 -0400 %doc can apply to directories as well as regular flies diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -442,7 +442,6 @@ if not rVflags: # %doc -- CNY-3254 - fassert(isinstance(fileObj, files.RegularFile), fpath) fassert(not fileObj.flags.isInitialContents(), fpath) def _sanityCheckChangeSet(self, csFile, jobId): From elliot at rpath.com Mon Mar 15 17:03:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:49 +0000 Subject: mirrorball: adapt to new interface Message-ID: <201003152103.o2FL3nRN007050@scc.eng.rpath.com> changeset: 6ca4cf5a7e7c user: Elliot Peele date: Fri, 30 Oct 2009 11:16:53 -0400 adapt to new interface diff --git a/scripts/header.py b/scripts/header.py --- a/scripts/header.py +++ b/scripts/header.py @@ -46,7 +46,7 @@ cfg = config.UpdateBotConfig() cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) -builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp(), sanityCheckCommits=True) +builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp()) def displayTrove(nvf): flavor = '' From elliot at rpath.com Mon Mar 15 17:03:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:53 +0000 Subject: mirrorball: Improve handling of RPM config file validation Message-ID: <201003152103.o2FL3rGh007088@scc.eng.rpath.com> changeset: 47f8d43e00ad user: Michael K. Johnson date: Fri, 30 Oct 2009 11:22:13 -0400 Improve handling of RPM config file validation diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -397,10 +397,12 @@ # RPM config flag mapping if rFlags & rpmhelper.RPMFILE_CONFIG: - if fileObj.contents.size(): - fassert(fileObj.flags.isConfig(), fpath) + if fileObj.linkGroup() or not fileObj.contents.size(): + fassert(fileObj.flags.isInitialContents(), fpath) else: - fassert(fileObj.flags.isInitialContents(), fpath) + fassert(fileObj.flags.isConfig() or + fileObj.flags.isInitialContents(), + why='RPM config file %s is neither config file nor initialcontents' %fpath) elif isinstance(fileObj, files.Directory): fassert(stat.S_ISDIR(rMode), fpath) From elliot at rpath.com Mon Mar 15 17:03:56 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:56 +0000 Subject: mirrorball: code cleanups Message-ID: <201003152103.o2FL3ual007116@scc.eng.rpath.com> changeset: 5b129403cf99 user: Elliot Peele date: Fri, 30 Oct 2009 11:28:05 -0400 code cleanups diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -402,7 +402,8 @@ else: fassert(fileObj.flags.isConfig() or fileObj.flags.isInitialContents(), - why='RPM config file %s is neither config file nor initialcontents' %fpath) + why='RPM config file %s is neither config file ' + 'nor initialcontents' %fpath) elif isinstance(fileObj, files.Directory): fassert(stat.S_ISDIR(rMode), fpath) @@ -549,7 +550,9 @@ if writeToFile: log.info('changeset saved to %s' % writeToFile) - self._sanityCheckChangeSet(writeToFile, jobIdsStr) + + if self._sanityCheckCommits: + self._sanityCheckChangeSet(writeToFile, jobIdsStr) log.info('committing changeset to repository') self._client.repos.commitChangeSetFile(writeToFile) @@ -563,13 +566,9 @@ for n, v, f in arch: if n.startswith('group-'): continue jobList.append((n, (None, None), (v, f), True)) - try: - cs = self._client.repos.createChangeSet(jobList, withFiles=True, - withFileContents=False) - except Exception, e: - self._errorEvent.setError(Exception, e) - if self._topLevel: - self._errorEvent.raiseError() + + cs = self._client.repos.createChangeSet(jobList, withFiles=True, + withFileContents=False) log.info('Commit of job %s completed in %.02f seconds', jobIdsStr, time.time() - startTime) From elliot at rpath.com Mon Mar 15 17:03:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:03:58 +0000 Subject: mirrorball: fix changeset content ordering Message-ID: <201003152103.o2FL3w1D007146@scc.eng.rpath.com> changeset: df658bd1005f user: Elliot Peele date: Sat, 31 Oct 2009 13:39:08 -0400 fix changeset content ordering diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -354,6 +354,7 @@ trove to make sure that they agree. """ + rpmFile.seek(0) h = rpmhelper.readHeader(rpmFile, checkSize=False) rpmFileList = dict( itertools.izip(h[rpmhelper.OLDFILENAMES], @@ -452,17 +453,22 @@ Sanity check changeset before commit. """ + def idCmp(a, b): + apid, afid = a[0][0], a[0][2] + bpid, bfid = b[0][0], b[0][2] + + return cmp((apid, afid), (bpid, bfid)) + newCs = changeset.ChangeSetFromFile(csFile) - log.info('comparing changeset to rpm capsules contained within it for' - 'changset %s' % csFile) + log.info('comparing changeset to rpm capsules: %s' % csFile) - oldCsJob = [] + capsules = [] for newTroveCs in newCs.iterNewTroveList(): if newTroveCs.getTroveInfo().capsule.type() == 'rpm': if newTroveCs.getOldVersion(): name, version, flavor = newTroveCs.getOldNameVersionFlavor() - oldCsJob.append((name, (None, None), - (version, flavor), True)) + oldCsJob = [ (name, (None, None), + (version, flavor), True) ] oldCs = self._client.repos.createChangeSet(oldCsJob, withFiles=True, withFileContents=False) @@ -487,30 +493,33 @@ fileObjs.append(ChangesetFilter._getFileObject( pathId, fileId, oldTrove, oldCs, newCs)) - # get capsule file contents - if newTroveCs.getOldVersion(): - capFileList = [ x[2:] for x in - newTrove.iterFileList(capsules=True) ] - else: - capFileList = [ (x[0], x[2]) for x in - newTrove.iterFileList(capsules=True) ] + capFileList = [ x for x in + newTrove.iterFileList(capsules=True) ] if len(capFileList) != 1: raise CommitFailedError(jobId=jobId, why='More than 1 RPM ' 'capsule in trove %s' % newTroveCs.name()) - if newTroveCs.getOldVersion(): - getFileContents = self._client.repos.getFileContents - fcList = getFileContents(capFileList, compressed=False) - capsuleFileContents = fcList[0].get() - else: - getFileContents = newCs.getFileContents - fcList = getFileContents(*capFileList[0], compressed=False) - capsuleFileContents = fcList[1].get() + capsules.append((capFileList[0], fileList, fileObjs)) - # do the check - self._sanityCheckRPMCapsule(jobId, fileList, fileObjs, - capsuleFileContents) + contentsCache = {} + for capFile, fileList, fileObjs in sorted(capsules, cmp=idCmp): + if capFile[2] in contentsCache: + capsuleFileContents = contentsCache[capFile[2]] + elif newTroveCs.getOldVersion(): + getFileContents = self._client.repos.getFileContents + fcList = getFileContents(capFile[2:], compressed=False) + capsuleFileContents = fcList[0].get() + else: + getFileContents = newCs.getFileContents + fcList = getFileContents(capFile[0], capFile[2], + compressed=False) + capsuleFileContents = fcList[1].get() + contentsCache[capFile[2]] = capsuleFileContents + + # do the check + self._sanityCheckRPMCapsule(jobId, fileList, fileObjs, + capsuleFileContents) def _commitJob(self, jobId): """ From elliot at rpath.com Mon Mar 15 17:04:00 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:00 +0000 Subject: mirrorball: rework subscriber model to include threaded starting of jobs Message-ID: <201003152104.o2FL40a8007178@scc.eng.rpath.com> changeset: 3927575a5e00 user: Elliot Peele date: Sun, 01 Nov 2009 20:32:26 -0500 rework subscriber model to include threaded starting of jobs diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py --- a/updatebot/subscriber.py +++ b/updatebot/subscriber.py @@ -18,7 +18,9 @@ """ import copy +import time import logging +import itertools from threading import Thread from Queue import Queue, Empty @@ -44,10 +46,12 @@ Class for storing thread types. """ - MONITOR = 0 - COMMIT = 1 + START = 0 + MONITOR = 1 + COMMIT = 2 names = { + START: 'Start', MONITOR: 'Monitor', COMMIT: 'Commit', } @@ -113,7 +117,64 @@ monitor._AbstractDisplay._primeOutput(self, jobId) -class MonitorWorker(Thread): +class AbstractWorker(Thread): + """ + Abstract class for all worker nodes. + """ + + threadType = None + + def __init__(self, status, name=None): + Thread.__init__(self, name=name) + + self.status = status + self.workerId = None + + def run(self): + """ + Do work. + """ + + try: + self.work() + except Exception, e: + self.status.put((MessageTypes.THREAD_ERROR, + (self.threadType, self.workerId, e))) + + self.status.put((MessageTypes.THREAD_DONE, self.workerId)) + + def work(self): + """ + Stub for sub classes to implement. + """ + + raise NotImplementedError + + +class StartWorker(AbstractWorker): + """ + Worker thread for starting jobs and reporting status. + """ + + threadType = ThreadTypes.START + + def __init__(self, status, (builder, trove), name=None): + AbstractWorker.__init__(self, status, name=name) + + self.builder = builder + self.trove = trove + self.workerId = self.trove + + def work(self): + """ + Start the specified build and report jobId. + """ + + jobId = self.builder.start((self.trove, )) + self.status.put((MessageTypes.DATA, (jobId, self.trove))) + + +class MonitorWorker(AbstractWorker): """ Worker thread for monitoring jobs and reporting status. """ @@ -121,36 +182,18 @@ threadType = ThreadTypes.MONITOR displayClass = JobMonitorCallback - def __init__(self, jobId, status, rmakeClient, name=None): - Thread.__init__(self, name=name) - - self.setDaemon(False) + def __init__(self, status, (rmakeClient, jobId), name=None): + AbstractWorker.__init__(self, status, name=name) self.client = rmakeClient self.jobId = jobId - self.status = status - self.name = name + self.workerId = jobId - def run(self): + def work(self): """ Watch the monitor queue and monitor any available jobs. """ - # monitor job, job display class supplies status information back to - # the main thread via the status queue. - try: - self.monitorJob(self.jobId) - except Exception, e: - self.status.put((MessageTypes.THREAD_ERROR, - (ThreadTypes.MONITOR, self.jobId, e))) - - self.status.put((MessageTypes.THREAD_DONE, self.jobId)) - - def monitorJob(self, jobId): - """ - Monitor the specified job. - """ - # FIXME: This is copied from rmake.cmdlin.monitor for the most part # because I need to pass extra args to the display class. @@ -159,7 +202,7 @@ try: display = self.displayClass(self.status, self.client, showBuildLogs=False, exitOnFinish=True) - client = self.client.listenToEvents(uri, jobId, display, + client = self.client.listenToEvents(uri, self.jobId, display, showTroveDetails=False, serve=True) return client @@ -168,34 +211,27 @@ os.remove(tmpPath) -class CommitWorker(Thread): +class CommitWorker(AbstractWorker): """ Worker thread for committing jobs. """ threadType = ThreadTypes.COMMIT - def __init__(self, jobId, status, builder, name=None): - Thread.__init__(self, name=name) + def __init__(self, status, (builder, jobId), name=None): + AbstractWorker.__init__(self, status, name=name) self.builder = builder self.jobId = jobId - self.status = status - self.name = name + self.workerId = jobId - def run(self): + def work(self): """ Commit the specified job. """ - try: - result = self.builder.commit(self.jobId) - self.status.put((MessageTypes.DATA, (self.jobId, result))) - except Exception, e: - self.status.put((MessageTypes.THREAD_ERROR, - (ThreadTypes.COMMIT, self.jobId, e))) - - self.status.put((MessageTypes.THREAD_DONE, self.jobId)) + result = self.builder.commit(self.jobId) + self.status.put((MessageTypes.DATA, (self.jobId, result))) class AbstractStatusMonitor(object): @@ -206,21 +242,26 @@ workerClass = None def __init__(self, threadArgs): + if type(threadArgs) not in (list, tuple, set): + threadArgs = (threadArgs, ) self._threadArgs = threadArgs self._status = Queue() self._workers = {} + self._errors = [] - def addJobId(self, jobId): + def addJob(self, job): """ - Add a jobId to the worker pool. + Add a job to the worker pool. """ + args = list(self._threadArgs) + args.append(job) + threadName = ('%s Worker' % ThreadTypes.names[self.workerClass.threadType]) - worker = self.workerClass(jobId, self._status, *self._threadArgs, - name=threadName) - self._workers[jobId] = worker + worker = self.workerClass(self._status, args, name=threadName) + self._workers[job] = worker worker.daemon = True worker.start() @@ -240,6 +281,15 @@ return data + def getErrors(self): + """ + Return any errors found while status was being processed. + """ + + errors = self._errors + self._errors = [] + return errors + def _processMessage(self, msg): """ Handle messages. @@ -253,24 +303,35 @@ elif mtype == MessageTypes.DATA: data.append(payload) elif mtype == MessageTypes.THREAD_DONE: - jobId = payload - #assert not self._workers[jobId].isAlive() - del self._workers[jobId] + job = payload + #assert not self._workers[job].isAlive() + del self._workers[job] elif mtype == MessageTypes.THREAD_ERROR: - threadType, jobId, error = payload - #assert not self._workers[jobId].isAlive() - raise error + threadType, job, error = payload + #assert not self._workers[job].isAlive() + #raise error + log.error('[%s] FAILED with exception: %s' % (job, error)) + self._errors.append((job, error)) return data +class JobStarter(AbstractStatusMonitor): + """ + Abstraction around threaded starter model. + """ + + workerClass = StartWorker + startJob = AbstractStatusMonitor.addJob + + class JobMonitor(AbstractStatusMonitor): """ Abstraction around threaded monitoring model. """ workerClass = MonitorWorker - monitorJob = AbstractStatusMonitor.addJobId + monitorJob = AbstractStatusMonitor.addJob class JobCommitter(AbstractStatusMonitor): @@ -279,7 +340,7 @@ """ workerClass = CommitWorker - commitJob = AbstractStatusMonitor.addJobId + commitJob = AbstractStatusMonitor.addJob class Dispatcher(object): @@ -288,40 +349,68 @@ knees. """ - _completed = (buildjob.JOB_STATE_FAILED, - buildjob.JOB_STATE_COMMITTED) + _completed = ( + -1, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_BUILT, +# buildjob.JOB_STATE_COMMITTED + ) def __init__(self, builder, maxSlots): self._builder = builder self._maxSlots = maxSlots self._slots = maxSlots - self._monitor = JobMonitor((self._builder._helper.client, )) - self._committer = JobCommitter((self._builder, )) + self._maxStartSlots = 10 + self._startSlots = self._maxStartSlots + + self._starter = JobStarter(self._builder) + self._monitor = JobMonitor(self._builder._helper.client) + self._committer = JobCommitter(self._builder) # jobId: (trv, status, commitData) self._jobs = {} + self._failures = [] + def buildmany(self, troveSpecs): """ Build as many packages as possible until we run out of slots. """ + # Must have at least one trove to build, otherwise will end up in + # an infinite loop. + if not troveSpecs: + return {} + troves = list(troveSpecs) troves.reverse() while troves or not self._jobDone(): - # fill slots with available troves - while troves and self._slots: - # get trove to work on - trove = troves.pop() + # Only create more jobs once the last batch has been started. + if self._startSlots == self._maxStartSlots: + # fill slots with available troves + while troves and self._slots and self._startSlots: + # get trove to work on + trove = troves.pop() - # start build job - jobId = self._builder.start((trove, )) + # start build job + self._starter.startJob(trove) + self._slots -= 1 + self._startSlots -= 1 + + # get started status + for jobId, trove in self._starter.getStatus(): self._jobs[jobId] = [trove, -1, None] - self._slots -= 1 + self._startSlots += 1 self._monitor.monitorJob(jobId) + # process starter errors + for trove, error in self._starter.getErrors(): + self._startSlots += 1 + self._slots += 1 + self._failures.append((trove, error)) + # update job status changes for jobId, status in self._monitor.getStatus(): self._jobs[jobId][1] = status @@ -329,7 +418,7 @@ self._slots += 1 assert self._slots <= self._maxSlots - # commit any jobs that are complete + # commit any jobs that are done building if status == buildjob.JOB_STATE_BUILT: self._committer.commitJob(jobId) @@ -337,6 +426,16 @@ for jobId, result in self._committer.getStatus(): self._jobs[jobId][2] = result + # process monitor and commit errors + for jobId, error in itertools.chain(self._monitor.getErrors(), + self._committer.getErrors()): + self._slots += 1 + self._jobs[jobId][1] = -1 + self._failures.append((jobId, error)) + + # Wait for a bit before polling again. + time.sleep(3) + results = {} for jobId, (trove, status, result) in self._jobs.iteritems(): # log failed jobs @@ -345,11 +444,16 @@ else: results[trove] = result - import epdb; epdb.st() + # report failures + for job, error in self._failures: + log.error('[%s] failed with error: %s' % (job, error)) return results def _jobDone(self): + if not len(self._jobs): + return False + for jobId, (trove, status, result) in self._jobs.iteritems(): if status not in self._completed: return False From elliot at rpath.com Mon Mar 15 17:04:03 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:03 +0000 Subject: mirrorball: use different error code Message-ID: <201003152104.o2FL44ob007208@scc.eng.rpath.com> changeset: 33ba253b3eaa user: Elliot Peele date: Sun, 01 Nov 2009 22:03:51 -0500 use different error code diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py --- a/updatebot/subscriber.py +++ b/updatebot/subscriber.py @@ -350,7 +350,7 @@ """ _completed = ( - -1, + -1, -2, buildjob.JOB_STATE_FAILED, buildjob.JOB_STATE_BUILT, # buildjob.JOB_STATE_COMMITTED @@ -430,7 +430,7 @@ for jobId, error in itertools.chain(self._monitor.getErrors(), self._committer.getErrors()): self._slots += 1 - self._jobs[jobId][1] = -1 + self._jobs[jobId][1] = -2 self._failures.append((jobId, error)) # Wait for a bit before polling again. @@ -442,7 +442,7 @@ if status == buildjob.JOB_STATE_FAILED: log.info('[%s] failed job: %s' % (jobId, trove)) else: - results[trove] = result + results.update(result) # report failures for job, error in self._failures: From elliot at rpath.com Mon Mar 15 17:04:06 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:06 +0000 Subject: mirrorball: branch merge Message-ID: <201003152104.o2FL46Ma007234@scc.eng.rpath.com> changeset: da2c10f69513 user: Elliot Peele date: Sun, 01 Nov 2009 22:04:36 -0500 branch merge diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -20,7 +20,7 @@ import os -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from updatebot.lib import util diff --git a/repomd/patchesxml.py b/repomd/patchesxml.py --- a/repomd/patchesxml.py +++ b/repomd/patchesxml.py @@ -19,7 +19,7 @@ __all__ = ('PatchesXml', ) # import stable api -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.patchxml import PatchXml from repomd.xmlcommon import XmlFileParser, SlotNode diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -8,7 +8,7 @@ __all__ = ('PatchXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.xmlcommon import XmlFileParser, SlotNode from repomd.packagexml import PackageXmlMixIn diff --git a/repomd/primaryxml.py b/repomd/primaryxml.py --- a/repomd/primaryxml.py +++ b/repomd/primaryxml.py @@ -8,7 +8,7 @@ __all__ = ('PrimaryXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.xmlcommon import XmlFileParser from repomd.packagexml import PackageXmlMixIn diff --git a/repomd/repomdxml.py b/repomd/repomdxml.py --- a/repomd/repomdxml.py +++ b/repomd/repomdxml.py @@ -19,7 +19,7 @@ __all__ = ('RepoMdXml', ) # use stable api -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.primaryxml import PrimaryXml from repomd.patchesxml import PatchesXml @@ -89,7 +89,8 @@ Parser for repomd.xml data elements. """ __slots__ = ('location', 'checksum', 'checksumType', 'timestamp', - 'openChecksum', 'openChecksumType', 'databaseVersion', ) + 'openChecksum', 'openChecksumType', 'databaseVersion', + 'size', 'openSize') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. @@ -114,6 +115,10 @@ self.openChecksumType = child.getAttribute('type') elif name == 'database_version': self.databaseVersion = child.finalize() + elif name == 'size': + self.size = child.finalize() + elif name == 'open-size': + self.openSize = child.finalize() else: raise UnknownElementError(child) diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py --- a/repomd/updateinfoxml.py +++ b/repomd/updateinfoxml.py @@ -19,7 +19,7 @@ __all__ = ('UpdateInfoXml', ) -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib from repomd.packagexml import PackageCompare from repomd.xmlcommon import XmlFileParser, SlotNode diff --git a/repomd/xmlcommon.py b/repomd/xmlcommon.py --- a/repomd/xmlcommon.py +++ b/repomd/xmlcommon.py @@ -18,7 +18,7 @@ __all__ = ('XmlFileParser', 'SlotNode') -from rpath_common.xmllib import api1 as xmllib +from rpath_xmllib import api1 as xmllib class XmlFileParser(object): """ diff --git a/scripts/buildautoloadrecipes b/scripts/buildautoloadrecipes --- a/scripts/buildautoloadrecipes +++ b/scripts/buildautoloadrecipes @@ -22,7 +22,7 @@ context="x86" fi -mirrorballpath="$HOME/hg/mirrorball" +mirrorballpath="$HOME/hg/mirrorball/mirrorball" platformConfig="$mirrorballpath/config/$platform/conaryrc" if [ ! -f $platformConfig ] ; then @@ -45,6 +45,7 @@ derived fileset group + capsule groupinfo redirect userinfo diff --git a/scripts/buildmany b/scripts/buildmany --- a/scripts/buildmany +++ b/scripts/buildmany @@ -16,7 +16,7 @@ label = cfg.topSourceGroup[1] for pkg in sys.argv[2:]: trvs.add((pkg, label, None)) -trvMap = builder.buildmany2(trvs) +trvMap = builder.buildmany(trvs) print "built:\n" diff --git a/scripts/compare b/scripts/compare --- a/scripts/compare +++ b/scripts/compare @@ -143,7 +143,7 @@ for source in sorted(toBuild): jobSet.append((source, None, None)) - return self._builder.buildmany2(jobSet) + return self._builder.buildmany(jobSet) class CompareAndCopy(object): diff --git a/scripts/findbinaries b/scripts/findbinaries --- a/scripts/findbinaries +++ b/scripts/findbinaries @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -35,7 +36,7 @@ slog = logging.getLogger('findbinaries') cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) bot = bot.Bot(cfg) updater = bot._updater diff --git a/scripts/genmanifest b/scripts/genmanifest --- a/scripts/genmanifest +++ b/scripts/genmanifest @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -25,7 +26,7 @@ log.addRootLogger() cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) obj = bot.Bot(cfg) obj._pkgSource.load() diff --git a/scripts/hardlink-fedora.sh b/scripts/hardlink-fedora.sh new file mode 100755 --- /dev/null +++ b/scripts/hardlink-fedora.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +DEST=/l/fedora/linux/ + +date +./hardlink.py $DEST diff --git a/scripts/header.py b/scripts/header.py --- a/scripts/header.py +++ b/scripts/header.py @@ -14,6 +14,7 @@ import os import sys +import tempfile mirrorballDir = os.path.abspath('../') sys.path.insert(0, mirrorballDir) @@ -45,7 +46,7 @@ cfg = config.UpdateBotConfig() cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) -builder = build.Builder(cfg) +builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp()) def displayTrove(nvf): flavor = '' diff --git a/scripts/order_import.py b/scripts/order_import.py new file mode 100755 --- /dev/null +++ b/scripts/order_import.py @@ -0,0 +1,66 @@ +#!/usr/bin/python +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +import os +import sys +import logging + +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) + +if 'CONARY_PATH' in os.environ: + sys.path.insert(0, os.environ['CONARY_PATH']) + +import conary +import updatebot + +print >>sys.stderr, 'using conary from', os.path.dirname(conary.__file__) +print >>sys.stderr, 'using updatebot from', os.path.dirname(updatebot.__file__) + +from conary.lib import util +sys.excepthook = util.genExcepthook() + +import rhnmirror + +from updatebot import config +from updatebot import ordered +from updatebot import log as logSetup + +logSetup.addRootLogger() +log = logging.getLogger('script') + +def usage(): + print 'usage: %s ' % sys.argv[0] + sys.exit(1) + +platform = sys.argv[1] +if platform not in os.listdir(mirrorballDir + '/config'): + usage() + +confDir = mirrorballDir + '/config/' + platform + +cfg = config.UpdateBotConfig() +cfg.read(confDir + '/updatebotrc') + +mcfg = rhnmirror.MirrorConfig() +mcfg.read(confDir + '/erratarc') + +errata = rhnmirror.Errata(mcfg) +errata.fetch() + +bot = ordered.Bot(cfg, errata) +bot.create() + +import epdb; epdb.st() diff --git a/scripts/pkgsource b/scripts/pkgsource --- a/scripts/pkgsource +++ b/scripts/pkgsource @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -31,7 +32,7 @@ from updatebot import pkgsource cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1] ) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] ) pkgSource = pkgsource.PackageSource(cfg) pkgSource.load() diff --git a/scripts/recreate b/scripts/recreate --- a/scripts/recreate +++ b/scripts/recreate @@ -16,7 +16,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from updatebot import log from updatebot import bot @@ -25,7 +26,7 @@ log.addRootLogger() cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) obj = bot.Bot(cfg) diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py new file mode 100755 --- /dev/null +++ b/scripts/rhelorder.py @@ -0,0 +1,151 @@ +#!/usr/bin/python + +import os +import sys + +sys.path.insert(0, os.environ['HOME'] + '/hg/conary') +sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-capsule-indexer') + +from conary.lib import util +sys.excepthook = util.genExcepthook() + +mbdir = os.path.abspath('../') +sys.path.insert(0, mbdir) + +confDir = os.path.join(mbdir, 'config', 'rhel4') + +from updatebot import log +from updatebot import Bot +from updatebot import UpdateBotConfig + +import time +import logging + +slog = logging.getLogger('script') + +import rhnmirror + +log.addRootLogger() +cfg = UpdateBotConfig() +cfg.read(os.path.join(confDir, 'updatebotrc')) + +bot = Bot(cfg) + +mcfg = rhnmirror.MirrorConfig() +mcfg.read(os.path.join(confDir, 'erratarc')) + +errata = rhnmirror.Errata(mcfg) +errata.fetch() + +pkgSource = bot._pkgSource +pkgSource.load() + +# get mapping of advisory to errata obj +advisories = dict((x.advisory, x) + for x in errata.iterByIssueDate(mcfg.channels)) + +# get mapping of nevra to pkg obj +nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) + for x in pkgSource.binPkgMap.keys() if x.arch != 'src') + +# pull nevras into errata sized buckets +buckets = {} +advMap = {} +nevraMap = {} + +arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') +for e in errata.iterByIssueDate(mcfg.channels): + bnevras = [] + bucket = [] + bucketId = None + slog.info('processing %s' % e.advisory) + for pkg in e.packages: + nevra = pkg.getNevra() + + # filter out channels we don't have indexed + channels = set([ x.label for x in pkg.channels ]) + if not set(mcfg.channels) & channels: + continue + + # ignore arches we don't know about. + if nevra[4] not in arches: + continue + + # convert rhn nevra to yum nevra + nevra = list(nevra) + if nevra[1] is None: + nevra[1] = '0' + if type(nevra[1]) == int: + nevra[1] = str(nevra[1]) + nevra = tuple(nevra) + + # move nevra to errata buckets + if nevra in nevras: + binPkg = nevras.pop(nevra) + bucket.append(binPkg) + bnevras.append(nevra) + + # nevra is already part of another bucket + elif nevra in nevraMap: + bucketId = nevraMap[nevra] + + # raise error if we can't find the required package + else: + raise KeyError + + if bucketId is None: + bucketId = int(time.mktime(time.strptime(e.issue_date, + '%Y-%m-%d %H:%M:%S'))) + buckets[bucketId] = bucket + else: + buckets[bucketId].extend(bucket) + + for nevra in bnevras: + nevraMap[nevra] = bucketId + + advMap[e.advisory] = bucketId + +# separate out golden bits +other = [] +golden = [] +firstErrata = sorted(buckets.keys())[0] +for nevra, pkg in nevras.iteritems(): + buildtime = int(pkg.buildTimestamp) + if buildtime < firstErrata: + golden.append(pkg) + else: + other.append(pkg) + +# sort by source package +srcMap = {} +for pkg in other: + src = pkgSource.binPkgMap[pkg] + if src not in srcMap: + srcMap[src] = [] + srcMap[src].append(pkg) + +# insert bins by buildstamp +for src, bins in srcMap.iteritems(): + buildstamp = int(sorted(bins)[0].buildTimestamp) + if buildstamp in buckets: + buckets[buildstamp].extend(bins) + else: + buckets[buildstamp] = bins + +# get sources to build +buildOrder = {0: set()} +for pkg in golden: + # lookup source package + src = pkgSource.binPkgMap[pkg] + buildOrder[0].add(src) + +for bucketId in sorted(buckets.keys()): + bucket = buckets[bucketId] + buildOrder[bucketId] = set() + for pkg in bucket: + src = pkgSource.binPkgMap[pkg] + buildOrder[bucketId].add(src) + +import epdb; epdb.st() diff --git a/scripts/sync-centos.sh b/scripts/sync-centos.sh --- a/scripts/sync-centos.sh +++ b/scripts/sync-centos.sh @@ -17,6 +17,6 @@ DEST=/l/CentOS/ date -rsync -arv --progress --bwlimit=700 --exclude 2* --exclude 3* --exclude 4* $SOURCE $DEST +rsync -arv --progress --bwlimit=700 --exclude 2* --exclude 3* $SOURCE $DEST ./hardlink.py $DEST diff --git a/scripts/sync-fedora.sh b/scripts/sync-fedora.sh --- a/scripts/sync-fedora.sh +++ b/scripts/sync-fedora.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,12 +13,24 @@ # full details. # -TARGET=$1 +if [ "$1" = "" ] ; then + # see if $0 has the target in the name + name=$(basename $0) + tmp=${name#sync-fedora-} + target=${tmp%.sh} +else + target=$1 +fi + +if [ "$target" = "" ] ; then + echo "Could not determine target" + exit 1 +fi SOURCE=rsync://mirror.linux.ncsu.edu/fedora-linux- DEST=/l/fedora/linux/ -CMD="rsync -arv --progress --bwlimit=800 --exclude ppc --exclude ppc64" +CMD="rsync -arv --progress --bwlimit=800 --exclude ppc --exclude ppc64 --exclude 9 --exclude 10 --exclude test --exclude testing" date @@ -30,5 +42,3 @@ # cleanup mirror $CMD --delete $SOURCE$target $DEST$target - -./hardlink.py $DEST diff --git a/updatebot/__init__.py b/updatebot/__init__.py --- a/updatebot/__init__.py +++ b/updatebot/__init__.py @@ -13,6 +13,9 @@ # """ -UpdateBot is a module for automated updating of a conary repository from a -SLES yum/rpm repository. +UpdateBot is a module for the automated creation and updating of a conary +packages from a yum or apt repository. """ + +from updatebot.bot import Bot +from updatebot.config import UpdateBotConfig diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -39,10 +39,12 @@ self._clients = {} self._pkgSource = pkgsource.PackageSource(self._cfg) self._updater = update.Updater(self._cfg, self._pkgSource) - self._advisor = advisories.Advisor(self._cfg, self._pkgSource, - self._cfg.platformName) self._builder = build.Builder(self._cfg) + if not self._cfg.disableAdvisories: + self._advisor = advisories.Advisor(self._cfg, self._pkgSource, + self._cfg.platformName) + @staticmethod def _flattenSetDict(setDict): """ @@ -57,9 +59,17 @@ lst.extend(list(trvSet)) return lst - def create(self, rebuild=False, recreate=None): + def create(self, rebuild=False, recreate=None, toCreate=None): """ Do initial imports. + + @param rebuild - rebuild all sources + @type rebuild - boolean + @param recreate - recreate all sources or a specific list of packages + @type recreate - boolean to recreate all sources or a list of specific + package names + @param toCreate - set of source package objects to create, implies recreate. + @type toCrate - iterable """ start = time.time() @@ -69,7 +79,9 @@ self._pkgSource.load() # Build list of packages - if type(recreate) == list: + if toCreate: + toPackage = None + elif type(recreate) == list: toPackage = set(recreate) elif self._cfg.packageAll: toPackage = set() @@ -99,7 +111,8 @@ # Import sources into repository. toBuild, fail = self._updater.create(toPackage, buildAll=rebuild, - recreate=bool(recreate)) + recreate=bool(recreate), + toCreate=toCreate) log.info('failed to create %s packages' % len(fail)) log.info('found %s packages to build' % len(toBuild)) @@ -108,7 +121,7 @@ if len(toBuild): if not rebuild: # Build all newly imported packages. - trvMap, failed = self._builder.buildmany2(sorted(toBuild)) + trvMap, failed = self._builder.buildmany(sorted(toBuild)) log.info('failed to import %s packages' % len(failed)) if len(failed): for pkg in failed: @@ -125,17 +138,23 @@ return trvMap - def update(self, force=None): + def update(self, force=None, updatePkgs=None): """ Update the conary repository from the yum repositories. @param force: list of packages to update without exception @type force: list(pkgName, pkgName, ...) + @param updatePkgs: set of source package objects to update + @type updatePkgs: iterable of source package objects """ if force is not None: self._cfg.disableUpdateSanity = True assert isinstance(force, list) + updateTroves = None + if updatePkgs: + updateTroves = set(((x.name, None, None), x) for x in updatePkgs) + start = time.time() log.info('starting update') @@ -143,7 +162,7 @@ self._pkgSource.load() # Get troves to update and send advisories. - toAdvise, toUpdate = self._updater.getUpdates() + toAdvise, toUpdate = self._updater.getUpdates(updateTroves=updateTroves) # If forcing an update, make sure that all packages are listed in # toAdvise and toUpdate as needed. @@ -163,12 +182,13 @@ log.info('no updates available') return - # Populate patch source now that we know that there are updates - # available. - self._advisor.load() + if not self._cfg.disableAdvisories: + # Populate patch source now that we know that there are updates + # available. + self._advisor.load() - # Check to see if advisories exist for all required packages. - self._advisor.check(toAdvise) + # Check to see if advisories exist for all required packages. + self._advisor.check(toAdvise) # Update source for nvf, srcPkg in toUpdate: @@ -206,8 +226,9 @@ # Mirror out content self._updater.mirror() - # Send advisories. - self._advisor.send(toAdvise, newTroves) + if not self._cfg.disableAdvisories: + # Send advisories. + self._advisor.send(toAdvise, newTroves) log.info('update completed successfully') log.info('updated %s packages and sent %s advisories' diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -16,21 +16,30 @@ Builder object implementation. """ +import stat import time import logging +import tempfile +import itertools import xml from Queue import Queue, Empty from threading import Thread, RLock +from conary import conarycfg, conaryclient +from conary import rpmhelper +from conary import trove +from conary import files from conary.deps import deps -from conary import conarycfg, conaryclient +from conary.repository import changeset +from conary.repository.netrepos.proxy import ChangesetFilter from rmake import plugins from rmake.build import buildcfg from rmake.cmdline import helper, monitor, commit from updatebot.lib import util +from updatebot import subscriber from updatebot.errors import JobFailedError, CommitFailedError log = logging.getLogger('updateBot.build') @@ -69,9 +78,11 @@ @param cfg: updateBot configuration object @type cfg: config.UpdateBotConfig + @param saveChangeSets: directory to save changesets to + @type saveChangeSets: str """ - def __init__(self, cfg): + def __init__(self, cfg, rmakeCfgFn=None, saveChangeSets=None): self._cfg = cfg self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) @@ -81,6 +92,15 @@ self._client = conaryclient.ConaryClient(self._ccfg) + if saveChangeSets is None and self._cfg.saveChangeSets: + self._saveChangeSets = tempfile.mkdtemp( + prefix=self._cfg.platformName, + suffix='-import-changesets') + else: + self._saveChangeSets = saveChangeSets + + self._sanityCheckCommits = self._cfg.sanityCheckCommits + # Get default pluginDirs from the rmake cfg object, setup the plugin # manager, then create a new rmake config object so that rmakeUser # will be parsed correctly. @@ -92,8 +112,16 @@ pluginMgr.loadPlugins() pluginMgr.callClientHook('client_preInit', self, []) + rmakerc = 'rmakerc' + if rmakeCfgFn: + rmakeCfgPath = util.join(self._cfg.configPath, rmakeCfgFn) + if os.path.exists(rmakeCfgPath): + rmakerc = rmakeCfgFn + else: + log.warn('%s not found, falling back to rmakerc' % rmakeCfgFn) + self._rmakeCfg = buildcfg.BuildConfiguration(readConfigFiles=False) - self._rmakeCfg.read(util.join(self._cfg.configPath, 'rmakerc')) + self._rmakeCfg.read(util.join(self._cfg.configPath, rmakerc)) self._rmakeCfg.useConaryConfig(self._ccfg) self._rmakeCfg.copyInConfig = False self._rmakeCfg.strictMode = True @@ -122,85 +150,13 @@ def buildmany(self, troveSpecs): """ - Build all packages in troveSpecs, 10 at a time, one per job. + Build many troves in separate jobs. @param troveSpecs: list of trove specs @type troveSpecs: [(name, versionObj, flavorObj), ...] @return troveMap: dictionary of troveSpecs to built troves """ - troveSpecs = list(troveSpecs) - def trvSort(a, b): - """ - Sort troves tuples based on the first element. - """ - - return cmp(a[0], b[0]) - troveSpecs.sort(trvSort) - - index = 0 - jobs = {} - for i, trv in enumerate(troveSpecs): - if index not in jobs: - jobs[index] = [] - - jobs[index].append(trv) - - if i % 20 == 0: - index += 1 - - failed = set() - results = {} - for job in jobs.itervalues(): - res, fail = self._buildmany(job) - failed.update(fail) - results.update(res) - - return results, failed - - def _buildmany(self, troveSpecs): - """ - Build a list of packages, one per job. - @param troveSpecs: list of trove specs - @type troveSpecs: [(name, versionObj, flavorObj), ...] - @return troveMap: dictionary of troveSpecs to built troves - """ - - jobs = {} - jobkeys = [] - for trv in troveSpecs: - jobkeys.append(trv) - jobs[trv] = self.start([trv, ]) - - for trv in jobkeys: - jobId = jobs[trv] - job = self._getJob(jobId) - self._wait(jobId) - - failed = set() - results = {} - for trv, jobId in jobs.iteritems(): - job = self._getJob(jobId) - if job.isFailed(): - failed.add((trv, jobId)) - elif job.isFinished(): - try: - res = self.commit(jobId) - results.update(res) - except JobFailedError: - failed.add((trv, jobId)) - - return results, failed - - def buildmany2(self, troveSpecs): - dispatcher = Dispatcher(self._cfg, 30) - return dispatcher.buildmany(troveSpecs) - - def buildmany3(self, troveSpecs): - dispatcher = Dispatcher2(self._cfg, 50) - return dispatcher.buildmany(troveSpecs) - - def buildmany4(self, troveSpecs): - dispatcher = Dispatcher3(self._cfg, 50) + dispatcher = subscriber.Dispatcher(self, 30) return dispatcher.buildmany(troveSpecs) def buildsplitarch(self, troveSpecs): @@ -234,7 +190,7 @@ # Wait for the jobs to finish. log.info('Waiting for jobs to complete') for jobId in jobIds.itervalues(): - self._wait(jobId) + self._monitorJob(jobId) # Sanity check all jobs. for jobId in jobIds.itervalues(): @@ -302,7 +258,9 @@ troves.append((name, version, flavor)) # Kernels are special. - elif ((name == 'kernel' or name in self._cfg.kernelModules) + elif ((name == 'kernel' or + name in self._cfg.kernelModules or + util.isKernelModulePackage(name)) and self._cfg.kernelFlavors): for context, flavor in self._cfg.kernelFlavors: # Replace flag name to match package @@ -359,19 +317,6 @@ return jobId - def _wait(self, jobId): - """ - Wait for a job to complete. - @param jobId: rMake job ID - @type jobId: integer - """ - - log.info('waiting for job [%s] to complete' % jobId) - job = self._getJob(jobId) - while not job.isFinished() and not job.isFailed(): - time.sleep(5) - job = self._getJob(jobId) - @jobInfoExceptionHandler def _monitorJob(self, jobId): """ @@ -403,6 +348,179 @@ log.error('Job %d has no built troves', jobId) raise JobFailedError(jobId=jobId, why='No troves found in job') + def _sanityCheckRPMCapsule(self, jobId, fileList, fileObjs, rpmFile): + """ + Compare an rpm capsule with the contents of a + trove to make sure that they agree. + """ + + rpmFile.seek(0) + h = rpmhelper.readHeader(rpmFile, checkSize=False) + rpmFileList = dict( + itertools.izip(h[rpmhelper.OLDFILENAMES], + itertools.izip(h[rpmhelper.FILEUSERNAME], + h[rpmhelper.FILEGROUPNAME], + h[rpmhelper.FILEMODES], + h[rpmhelper.FILESIZES], + h[rpmhelper.FILERDEVS], + h[rpmhelper.FILEFLAGS], + h[rpmhelper.FILEVERIFYFLAGS], + h[rpmhelper.FILELINKTOS], + ))) + + foundFiles = dict.fromkeys(rpmFileList) + + def fassert(test, path="", why=None): + if not test: + if why: + raise CommitFailedError(jobId=jobId, why=why) + else: + raise CommitFailedError(jobId=jobId, why='metadata in trove' + ' does not agree with rpm header for file %s' % (path)) + + for fileInfo, fileObj in zip(fileList, fileObjs): + fpath = fileInfo[1] + foundFiles[fpath] = True + rUser, rGroup, rMode, rSize, rDev, rFlags, rVflags, rLinkto = \ + rpmFileList[fpath] + + # First, tests based on the Conary changeset + + # file metadata verification + if (rUser != fileObj.inode.owner() or + rGroup != fileObj.inode.group() or + stat.S_IMODE(rMode) != fileObj.inode.perms()): + fassert(False, fpath) + + if isinstance(fileObj, files.RegularFile): + if not stat.S_ISREG(rMode): + fassert(False, fpath) + + # RPM config flag mapping + if rFlags & rpmhelper.RPMFILE_CONFIG: + if fileObj.linkGroup() or not fileObj.contents.size(): + fassert(fileObj.flags.isInitialContents(), fpath) + else: + fassert(fileObj.flags.isConfig() or + fileObj.flags.isInitialContents(), + why='RPM config file %s is neither config file ' + 'nor initialcontents' %fpath) + + elif isinstance(fileObj, files.Directory): + fassert(stat.S_ISDIR(rMode), fpath) + fassert(not fileObj.flags.isPayload(), fpath) + elif isinstance(fileObj, files.CharacterDevice): + fassert(stat.S_ISCHR(rMode), fpath) + + minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 + major = (rDev >> 8) & 0xfff + fassert(fileObj.devt.major() == major, fpath) + fassert(fileObj.devt.minor() == minor, fpath) + + fassert(not fileObj.flags.isPayload()) + elif isinstance(fileObj, files.BlockDevice): + fassert(stat.S_ISBLK(rMode), fpath) + + minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 + major = (rDev >> 8) & 0xfff + fassert(fileObj.devt.major() == major, fpath) + fassert(fileObj.devt.minor() == minor, fpath) + + fassert(not fileObj.flags.isPayload(), fpath) + elif isinstance(fileObj, files.NamedPipe): + fassert(stat.S_ISFIFO(rMode), fpath) + + fassert(not fileObj.flags.isPayload(), fpath) + elif isinstance(fileObj, files.SymbolicLink): + fassert(stat.S_ISLNK(rMode), fpath) + fassert(fileObj.target() == rLinkto, fpath) + + fassert(not fileObj.flags.isPayload(), fpath) + else: + # unhandled file type + fassert(False, fpath) + + # Now, some tests based on the contents of the RPM header + if not stat.S_ISDIR(rMode) and rFlags & rpmhelper.RPMFILE_GHOST: + fassert(fileObj.flags.isInitialContents(), fpath) + + if not rVflags: + # %doc -- CNY-3254 + fassert(not fileObj.flags.isInitialContents(), fpath) + + def _sanityCheckChangeSet(self, csFile, jobId): + """ + Sanity check changeset before commit. + """ + + def idCmp(a, b): + apid, afid = a[0][0], a[0][2] + bpid, bfid = b[0][0], b[0][2] + + return cmp((apid, afid), (bpid, bfid)) + + newCs = changeset.ChangeSetFromFile(csFile) + log.info('comparing changeset to rpm capsules: %s' % csFile) + + capsules = [] + for newTroveCs in newCs.iterNewTroveList(): + if newTroveCs.getTroveInfo().capsule.type() == 'rpm': + if newTroveCs.getOldVersion(): + name, version, flavor = newTroveCs.getOldNameVersionFlavor() + oldCsJob = [ (name, (None, None), + (version, flavor), True) ] + + oldCs = self._client.repos.createChangeSet(oldCsJob, + withFiles=True, withFileContents=False) + + oldTroveCs = oldCs.getNewTroveVersion( + *newTroveCs.getOldNameVersionFlavor()) + assert (oldTroveCs.getNewNameVersionFlavor() == + newTroveCs.getOldNameVersionFlavor()) + oldTrove = trove.Trove(oldTroveCs) + newTrove = oldTrove.copy() + newTrove.applyChangeSet(newTroveCs) + + else: + oldCs = None + oldTrove = None + newTrove = trove.Trove(newTroveCs) + + fileObjs = [] + # get file streams for comparison + fileList = list(newTrove.iterFileList(capsules=False)) + for pathId, path, fileId, fileVer in fileList: + fileObjs.append(ChangesetFilter._getFileObject( + pathId, fileId, oldTrove, oldCs, newCs)) + + capFileList = [ x for x in + newTrove.iterFileList(capsules=True) ] + + if len(capFileList) != 1: + raise CommitFailedError(jobId=jobId, why='More than 1 RPM ' + 'capsule in trove %s' % newTroveCs.name()) + + capsules.append((capFileList[0], fileList, fileObjs)) + + contentsCache = {} + for capFile, fileList, fileObjs in sorted(capsules, cmp=idCmp): + if capFile[2] in contentsCache: + capsuleFileContents = contentsCache[capFile[2]] + elif newTroveCs.getOldVersion(): + getFileContents = self._client.repos.getFileContents + fcList = getFileContents(capFile[2:], compressed=False) + capsuleFileContents = fcList[0].get() + else: + getFileContents = newCs.getFileContents + fcList = getFileContents(capFile[0], capFile[2], + compressed=False) + capsuleFileContents = fcList[1].get() + contentsCache[capFile[2]] = capsuleFileContents + + # do the check + self._sanityCheckRPMCapsule(jobId, fileList, fileObjs, + capsuleFileContents) + def _commitJob(self, jobId): """ Commit completed job. @@ -423,16 +541,44 @@ jobs = [ self._getJob(x) for x in jobIds ] log.info('Starting commit of job %s', jobIdsStr) + if self._saveChangeSets: + csfn = tempfile.mktemp(dir=self._saveChangeSets, suffix='.ccs') + + writeToFile = self._saveChangeSets and csfn or None + self._helper.client.startCommit(jobIds) succeeded, data = commit.commitJobs(self._helper.getConaryClient(), jobs, self._rmakeCfg.reposName, - self._cfg.commitMessage) + self._cfg.commitMessage, + writeToFile=writeToFile) if not succeeded: self._helper.client.commitFailed(jobIds, data) raise CommitFailedError(jobId=jobIdsStr, why=data) + if writeToFile: + log.info('changeset saved to %s' % writeToFile) + + if self._sanityCheckCommits: + self._sanityCheckChangeSet(writeToFile, jobIdsStr) + + log.info('committing changeset to repository') + self._client.repos.commitChangeSetFile(writeToFile) + + if self._sanityCheckCommits: + # sanity check repository + log.info('checking repository for sanity') + jobList = [] + for job in data.itervalues(): + for arch in job.itervalues(): + for n, v, f in arch: + if n.startswith('group-'): continue + jobList.append((n, (None, None), (v, f), True)) + + cs = self._client.repos.createChangeSet(jobList, withFiles=True, + withFileContents=False) + log.info('Commit of job %s completed in %.02f seconds', jobIdsStr, time.time() - startTime) @@ -500,310 +646,3 @@ """ Don't care about the build log """ - -## -# Experimental threaded builder, beware of dragons -## - -MESSAGE_TYPES = { - 0: 'log', - 'log': 0, - 1: 'results', - 'results': 1, - 2: 'error', - 'error': 2, -} - -class StatusMessage(object): - def __init__(self, name, trv, jobId, message, type=0): - assert type in MESSAGE_TYPES - self.name = name - self.trv = trv - self.jobId = jobId - self.message = message - self.type = type - - def __str__(self): - msg = '%(name)s: %(trv)s [%(jobId)s] - ' - if self.type == MESSAGE_TYPES['results']: - msg += 'done' - else: - msg += '%(message)s' - return msg % self.__dict__ - -class BuildWorker(Thread): - BuilderClass = Builder - - def __init__(self, cfg, toBuild, status, name=None, offset=0): - Thread.__init__(self, name=name) - - self.setDaemon(True) - - self.name = name - self.offset = offset - self.toBuild = toBuild - self.status = status - self.builder = self.BuilderClass(cfg) - - self.trv = None - self.jobId = None - - def run(self): - time.sleep(self.offset * 5) - while True: - self.trv = self.toBuild.get() - self.log('received trv') - - retries = 10 - built = False - while not built and retries: - retries -= 1 - try: - self._doBuild() - except Exception, e: - built = False - self.log('traceback while building %s, retrying' % e) - continue - built = True - - if not built: - self.error('job failed') - - self.toBuild.task_done() - - def _doBuild(self): - self.jobId = self.builder.start([self.trv, ]) - - self.builder._wait(self.jobId) - - job = self.builder._getJob(self.jobId) - if job.isFailed(): - self.error('job failed') - - else: - try: - res = self.builder.commit(self.jobId) - self.results(res) - except JobFailedError: - self.error('job failed') - - def _status(self, msg, type=0): - msg = StatusMessage(self.name, self.trv, self.jobId, msg, type) - self.status.put(msg) - - def error(self, msg): - self._status(msg, type=MESSAGE_TYPES['error']) - - def log(self, msg): - self._status(msg, type=MESSAGE_TYPES['log']) - - def results(self, res): - self._status(res, type=MESSAGE_TYPES['results']) - -class Dispatcher(object): - workerClass = BuildWorker - - def __init__(self, cfg, workerCount): - self._cfg = cfg - self._workerCount = workerCount - - self._workers = [] - self._started = False - - self._toBuild = Queue() - self._status = Queue() - - self._trvs = {} - - def provisionWorkers(self): - for i in range(self._workerCount): - worker = self.workerClass(self._cfg, self._toBuild, self._status, - name='Build Worker %s' % i, offset=i) - self._workers.append(worker) - - def start(self): - if self._started: - return - - for wkr in self._workers: - wkr.start() - self._started = True - - def buildmany(self, trvSpecs): - self.provisionWorkers() - self.start() - - for trv in trvSpecs: - self._trvs[trv] = [] - self._toBuild.put(trv) - - results, failed = self.monitorStatus() - return results, failed - - def monitorStatus(self): - done = False - while not done: - try: - log.debug('checking for status messages') - msg = self._status.get(timeout=5) - except Empty: - continue - - self._processMessage(msg) - done = self._buildDone() - - return self._getResultsAndErrors() - - def _processMessage(self, msg): - assert msg.trv in self._trvs - self._trvs[msg.trv].append(msg) - log.info(msg) - - def _buildDone(self): - for trv, msgs in self._trvs.iteritems(): - if len(msgs) == 0: - return False - elif msgs[-1].type not in (MESSAGE_TYPES['results'], MESSAGE_TYPES['error']): - return False - return True - - def _getResultsAndErrors(self): - errors = set() - results = [] - for trv, msgs in self._trvs.iteritems(): - msg = msgs[-1] - if msg.type == MESSAGE_TYPES['error']: - errors.add((trv, msg.jobId)) - elif msg.type == MESSAGE_TYPES['results']: - results.append(msg.message) - return results, errors - - -class Dispatcher2(object): - builderClass = Builder - - def __init__(self, cfg, workerCount): - self._cfg = cfg - self._workerCount = workerCount - - self._builder = self.builderClass(self._cfg) - - self._activeJobs = [] - self._commitJobs = [] - self._failedJobs = set() - self._completedJobs = [] - - self._results = {} - - def buildmany(self, troveSpecs): - troveSpecs = list(troveSpecs) - - while len(troveSpecs): - # Wait for some amount of jobs to complete. - while len(self._activeJobs) >= self._workerCount: - time.sleep(5) - self._checkStatus() - - # Populate build queue. - while len(self._activeJobs) < self._workerCount: - trvSpec = troveSpecs.pop() - self._start(trvSpec) - - # Commit completed jobs. - self._commit() - - return self._results, self._failedJobs - - @jobInfoExceptionHandler - def _start(self, troveSpec): - jobId = self._builder.start([troveSpec, ]) - self._activeJobs.append((troveSpec, jobId)) - - def _checkStatus(self): - for troveSpec, jobId in self._activeJobs: - log.info('Checking status of %s' % jobId) - job = self._builder._getJob(jobId, retry=10) - if job is None: - log.warn('Failed to retrieve job information for %s' % jobId) - import epdb; epdb.st() - elif job.isFailed(): - self._failedJobs.add((troveSpec, jobId)) - self._activeJobs.remove((troveSpec, jobId)) - elif job.isFinished(): - self._commitJobs.append((troveSpec, jobId)) - self._activeJobs.remove((troveSpec, jobId)) - - # Wait between each status check. - #time.sleep(1) - - def _commit(self): - for troveSpec, jobId in self._commitJobs: - try: - res = self._builder.commit(jobId) - self._results.update(res) - self._completedJobs.append(jobId) - except JobFailedError: - self._failedJobs.add((troveSpec, jobId)) - self._commitJobs.remove((troveSpec, jobId)) - - -class CommitWorker(Thread): - BuilderClass = Builder - - def __init__(self, cfg, toCommit, results, name=None): - Thread.__init__(self, name=name) - - self.name = name - self.toCommit = toCommit - self.results = results - self.builder = self.BuilderClass(cfg) - - self.setDaemon(True) - - def run(self): - while True: - trvSpec, jobId = self.toCommit.get() - try: - res = self.builder.commit(jobId) - self.msg(0, trvSpec, jobId, res) - except JobFailedError: - self.msg(1, trvSpec, jobId) - except Exception, e: - log.critical('%s received exception: %s' % (self.name, e)) - - def msg(self, rc, trvSpec, jobId, data=None): - self.results.put((rc, ((trvSpec, jobId), data))) - - -class Dispatcher3(Dispatcher2): - workerClass = CommitWorker - - def __init__(self, cfg, workerCount): - Dispatcher2.__init__(self, cfg, workerCount) - - self._commitQueue = Queue() - self._resultQueue = Queue() - - self._workers = [] - for i in range(10): - worker = self.workerClass(self._cfg, self._commitQueue, - self._resultQueue, name='Commit Worker %s' % i) - worker.start() - self._workers.append(worker) - - def _commit(self): - for trvSpec, jobId in self._commitJobs: - self._commitQueue.put((trvSpec, jobId)) - - try: - msg = self._resultQueue.get(False) - while msg: - rc, ((trvSpec, jobId), data) = msg - if rc == 0: - self._results.update(data) - self._completedJobs.append(jobId) - elif rc == 1: - self._failedJobs.add((trvSpec, jobId)) - msg = self._resultQueue.get(False) - except Empty: - pass diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -59,6 +59,12 @@ # Have to initialize flavors to commit to the repository. self._ccfg.initializeFlavors() + # FIXME: CNY-3256 - use unique tmp directory for lookaside until + # this issue is fixed. + self._ccfg.lookaside = tempfile.mkdtemp( + prefix='%s-lookaside-' % cfg.platformName) + log.info('using lookaside %s' % self._ccfg.lookaside) + mirrorDir = util.join(cfg.configPath, 'mirrors') if os.path.exists(mirrorDir): self._ccfg.mirrorDirs.insert(0, mirrorDir) diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -138,6 +138,9 @@ # Exclude these archs from the rpm source. excludeArch = (CfgList(CfgString), []) + # Disable advisories all together. + disableAdvisories = (CfgBool, False) + # Packages for which there might not reasonably be advisories. Define a # default advisory message to send with these packages. advisoryException = (CfgList(CfgList(CfgString)), []) @@ -166,6 +169,13 @@ # flavors to build packages in for packages that need specific flavoring. packageFlavors = (CfgDict(CfgList(CfgContextFlavor)), {}) + # After committing a rMake job to the repository pull the changeset back out + # to make sure all of the contents made it into the repository. + sanityCheckCommits = (CfgBool, False) + + # Save all binary changesets to disk before committing them. + saveChangeSets = (CfgBool, False) + # email information for sending advisories emailFromName = CfgString emailFrom = CfgString diff --git a/updatebot/errata.py b/updatebot/errata.py new file mode 100644 --- /dev/null +++ b/updatebot/errata.py @@ -0,0 +1,199 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for ordering errata. +""" + +import time +import logging + +from updatebot.errors import ErrataPackageNotFoundError + +log = logging.getLogger('updatebot.errata') + +def loadErrata(func): + def wrapper(self, *args, **kwargs): + if not self._order: + self._orderErrata() + return func(self, *args, **kwargs) + return wrapper + +class ErrataFilter(object): + """ + Filter data from a given errataSource in chronological order. + """ + + def __init__(self, pkgSource, errataSource): + self._pkgSource = pkgSource + self._errata = errataSource + + self._order = {} + self._advMap = {} + + @loadErrata + def getInitialPackages(self): + """ + Get the initial set of packages. + """ + + return self._order[0] + + @loadErrata + def lookupUpdateDetail(self, bucketId): + """ + Given a errata timestamp lookup the name and summary. + """ + + if bucketId in self._advMap: + return '%(name)s: %(summary)s' % self._advMap[bucketId] + else: + return '%s (no detail found)' % bucketId + + @loadErrata + def iterByIssueDate(self, start=None): + """ + Yield sets of srcPkgs by errata release date. + @param start: timestamp from which to start iterating. + @type start: int + """ + + for stamp in sorted(self._order.keys()): + if start > stamp: + continue + yield stamp, self._order[stamp] + + def _orderErrata(self): + """ + Order errata by timestamp. + """ + + # order packages by errata release + buckets, other = self._sortPackagesByErrataTimestamp() + + # insert packages that did not have errata and were not in the initial + # set of packages (golden bits) + srcMap = {} + for pkg in other: + src = self._pkgSource.binPkgMap[pkg] + if src not in srcMap: + srcMap[src] = [] + srcMap[src].append(pkg) + + # insert bins by buildstamp + for src, bins in srcMap.iteritems(): + buildstamp = int(sorted(bins)[0].buildTimestamp) + if buildstamp not in buckets: + buckets[buildstamp] = [] + buckets[buildstamp].extend(bins) + + # get sources to build + for bucketId in sorted(buckets.keys()): + bucket = buckets[bucketId] + self._order[bucketId] = set() + for pkg in bucket: + src = self._pkgSource.binPkgMap[pkg] + self._order[bucketId].add(src) + + def _getNevra(self, pkg): + """ + Get the NEVRA of a package object and do any transformation required. + """ + + # convert nevra to yum compatible nevra + nevra = list(pkg.getNevra()) + if nevra[1] is None: + nevra[1] = '0' + if type(nevra[1]) == int: + nevra[1] = str(nevra[1]) + nevra = tuple(nevra) + + return nevra + + def _sortPackagesByErrataTimestamp(self): + """ + Sort packages by errata release timestamp. + """ + + # get mapping of nevra to pkg obj + nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) + for x in self._pkgSource.binPkgMap.keys() + if x.arch != 'src' and '-debuginfo' not in x.name) + + # pull nevras into errata sized buckets + buckets = {} + nevraMap = {} + + log.info('processing errata') + + indexedChannels = set(self._errata.getChannels()) + arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') + for e in self._errata.iterByIssueDate(): + bucket = [] + allocated = [] + bucketId = None + #log.info('processing %s' % e.advisory) + for pkg in e.packages: + nevra = self._getNevra(pkg) + + # ignore arches we don't know about. + if nevra[4] not in arches: + continue + + # filter out channels we don't have indexed + channels = set([ x.label for x in pkg.channels ]) + if not indexedChannels & channels: + continue + + # move nevra to errata buckets + if nevra in nevras: + binPkg = nevras.pop(nevra) + bucket.append(binPkg) + allocated.append(nevra) + + # nevra is already part of another bucket + elif nevra in nevraMap: + bucketId = nevraMap[nevra] + + # raise error if we can't find the required package + else: + raise ErrataPackageNotFoundError(pkg=nevra) + + if bucketId is None: + bucketId = int(time.mktime(time.strptime(e.issue_date, + '%Y-%m-%d %H:%M:%S'))) + buckets[bucketId] = bucket + else: + buckets[bucketId].extend(bucket) + + for nevra in allocated: + nevraMap[nevra] = bucketId + + self._advMap[bucketId] = {'name': e.advisory, + 'summary': e.synopsis} + + # separate out golden bits + other = [] + golden = [] + firstErrata = sorted(buckets.keys())[0] + for nevra, pkg in nevras.iteritems(): + buildtime = int(pkg.buildTimestamp) + if buildtime < firstErrata: + golden.append(pkg) + else: + other.append(pkg) + + buckets[0] = golden + + return buckets, other diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -237,3 +237,19 @@ _params = ['what', 'advisories'] _template = 'Found multiple advisories for %(what)s: %(advisories)s' + +class ErrataError(UpdateBotError): + """ + Base exception class for errata related errors. + """ + +class ErrataPackageNotFoundError(ErrataError): + """ + ErrataPackageNotFoundError, raised when a package can not be found in the + package source that matches a package in the errata source. + """ + + _params = ['pkg', ] + _templates = ('Could not find a matching package for %(pkg)s in the ' + 'configured repositories when attempting to map errata source to ' + 'package source.') diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -123,3 +123,18 @@ for pkg in self.pkgs: self.binPkgMap[pkg] = src + +def isKernelModulePackage(paths): + """ + Check if a package file name or location is a kernel module. + """ + + if type(paths) == str: + paths = [ paths, ] + + for path in paths: + basePath = os.path.basename(path) + if (basePath.startswith('kmod-') or + basePath.startswith('kernel-module')): + return True + return False diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -98,9 +98,10 @@ def __init__(self): self.items = [] + self._itemClass = self.__class__.__dict__['items'][0] def __setitem__(self, key, value): - item = XDictItem(key, value) + item = self._itemClass(key, value) if item in self.items: idx = self.items.index(item) self.items[idx] = item @@ -115,3 +116,68 @@ def __contains__(self, key): return key in self.items + + +class XItemList(XDocManager): + """ + List of items. + """ + + def __init__(self): + self.items = [] + self._itemClass = self.__class__.__dict__['items'][0] + XDocManager.__init__(self) + + +class XPackageItem(object): + """ + Object to represent package data required for group builds with the + managed group factory. + """ + + name = str + version = str + flavor = str + byDefault = int + use = str + source = str + originalName = str + + def __init__(self, name, version='', flavor='', byDefault=1, use=1, + source='', originalName=None): + self.name = name + self.version = version + self.flavor = flavor + self.byDefault = byDefault + self.use = use + self.source = source + self.originalName = originalName and originalName or name + + +class XPackageData(XItemList): + """ + Mapping of package name to package group data. + """ + + items = [ XPackageItem ] + + +class XGroup(object): + """ + Group file info. + """ + + name = str + filename = str + + def __init__(self, name, filename): + self.name = name + self.filename = filename + + +class XGroupList(XItemList): + """ + List of file names to load as groups. + """ + + items = [ XGroup ] diff --git a/updatebot/log.py b/updatebot/log.py --- a/updatebot/log.py +++ b/updatebot/log.py @@ -17,19 +17,36 @@ """ import sys +import time import logging +import tempfile +from logging import handlers -def addRootLogger(): +def addRootLogger(logFile=None): """ Setup the root logger that should be inherited by all other loggers. """ + logSize = 1024 * 1024 * 50 + logFile = logFile and logFile or tempfile.mktemp(prefix='updatebot-log-%s-' + % int(time.time())) + rootLog = logging.getLogger('') - handler = logging.StreamHandler(sys.stdout) + + streamHandler = logging.StreamHandler(sys.stdout) + logFileHandler = handlers.RotatingFileHandler(logFile, + maxBytes=logSize, + backupCount=5) + formatter = logging.Formatter('%(asctime)s %(levelname)s ' '%(name)s %(message)s') - handler.setFormatter(formatter) - rootLog.addHandler(handler) + + streamHandler.setFormatter(formatter) + logFileHandler.setFormatter(formatter) + + rootLog.addHandler(streamHandler) + rootLog.addHandler(logFileHandler) + rootLog.setLevel(logging.INFO) # Delete conary's log handler since it puts things on stderr and without diff --git a/updatebot/ordered.py b/updatebot/ordered.py new file mode 100644 --- /dev/null +++ b/updatebot/ordered.py @@ -0,0 +1,69 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for doing updates ordered by errata information. +""" + +import logging + +from updatebot import errata +from updatebot.bot import Bot as BotSuperClass + +log = logging.getLogger('updatebot.ordered') + +class Bot(BotSuperClass): + """ + Implement errata driven create/update interface. + """ + + _create = BotSuperClass.create + _update = BotSuperClass.update + + def __init__(self, cfg, errataSource): + BotSuperClass.__init__(self, cfg) + self._errata = errata.ErrataFilter(self._pkgSource, errataSource) + + def create(self, *args, **kwargs): + """ + Handle initial import case. + """ + + self._pkgSource.load() + toCreate = self._errata.getInitialPackages() + return self._create(*args, toCreate=toCreate, **kwargs) + + def update(self, *args, **kwargs): + """ + Handle update case. + """ + + # Get current timestamp + # FIXME: Figure out where to store current errata level + raise NotImplementedError + + current = 0 + + self._pkgSource.load() + + for updateId, updates in self._errata.iterByIssueDate(start=current): + detail = self._errata.getUpdateDetail(updateId) + log.info('attempting to apply %s' % detail) + + # Update package set. + self._update(updatePkgs=updates) + + # Store current updateId. + # FIXME: figure out where/how to store the current updateId + diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py --- a/updatebot/pkgsource/common.py +++ b/updatebot/pkgsource/common.py @@ -29,6 +29,8 @@ self._cfg = cfg self._excludeArch = self._cfg.excludeArch + self._loaded = False + # {repoShortUrl: clientObj} self._clients = dict() diff --git a/updatebot/pkgsource/debsource.py b/updatebot/pkgsource/debsource.py --- a/updatebot/pkgsource/debsource.py +++ b/updatebot/pkgsource/debsource.py @@ -39,6 +39,9 @@ Load repository metadata from a config object. """ + if self._loaded: + return + client = aptmd.Client(self._cfg.repositoryUrl) for repo in self._cfg.repositoryPaths: log.info('loading repository data %s' % repo) @@ -46,6 +49,7 @@ self._clients[repo] = client self.finalize() + self._loaded = True def loadFromClient(self, client, path): """ diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -47,6 +47,9 @@ Load package source based on config data. """ + if self._loaded: + return + for repo in self._cfg.repositoryPaths: log.info('loading repository data %s' % repo) client = repomd.Client(self._cfg.repositoryUrl + '/' + repo) @@ -54,6 +57,7 @@ self._clients[repo] = client self.finalize() + self._loaded = True def loadFromUrl(self, url, basePath=''): """ diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py new file mode 100644 --- /dev/null +++ b/updatebot/subscriber.py @@ -0,0 +1,460 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +New implementation of builder module that uses rMake's message bus for job +monitoring. +""" + +import copy +import time +import logging +import itertools +from threading import Thread +from Queue import Queue, Empty + +from rmake.build import buildjob +from rmake.build import buildtrove +from rmake.cmdline import monitor + +log = logging.getLogger('updatebot.subscriber') + +class MessageTypes(object): + """ + Class for storing message type constants. + """ + + LOG = 0 + DATA = 1 + THREAD_DONE = 2 + THREAD_ERROR = 3 + + +class ThreadTypes(object): + """ + Class for storing thread types. + """ + + START = 0 + MONITOR = 1 + COMMIT = 2 + + names = { + START: 'Start', + MONITOR: 'Monitor', + COMMIT: 'Commit', + } + + +class JobMonitorCallback(monitor.JobLogDisplay): + """ + Monitor job status changes. + """ + + monitorStates = ( + buildjob.JOB_STATE_STARTED, + buildjob.JOB_STATE_BUILT, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_COMMITTING, + buildjob.JOB_STATE_COMMITTED, + ) + + def __init__(self, status, *args, **kwargs): + # override showBuildLogs since we don't handle writing to the out pipe + kwargs['showBuildLogs'] = False + + monitor.JobLogDisplay.__init__(self, *args, **kwargs) + self._status = status + + def _msg(self, msg, *args): + self._status.put((MessageTypes.LOG, msg)) + + def _data(self, data): + self._status.put((MessageTypes.DATA, data)) + + def _jobStateUpdated(self, jobId, state, status): + monitor.JobLogDisplay(self, jobId, state, None) + if state in self.monitorStates: + self._data((jobId, state)) + + def _troveLogUpdated(self, (jobId, troveTuple), state, status): + pass + + def _jobTrovesSet(self, jobId, troveData): + pass + + def _tailBuildLog(self, jobId, troveTuple): + pass + + def _primeOutput(self, jobId): + # Override parents _primeOutput to avoid sending output to stdout via + # print. + logMark = 0 + while True: + newLogs = self.client.getJobLogs(jobId, logMark) + if not newLogs: + break + logMark += len(newLogs) + for (timeStamp, message, args) in newLogs: + self._msg('[%s] - %s' % (jobId, message)) + + BUILDING = buildtrove.TROVE_STATE_BUILDING + troveTups = self.client.listTrovesByState(jobId, BUILDING).get(BUILDING, []) + for troveTuple in troveTups: + self._tailBuildLog(jobId, troveTuple) + + monitor._AbstractDisplay._primeOutput(self, jobId) + + +class AbstractWorker(Thread): + """ + Abstract class for all worker nodes. + """ + + threadType = None + + def __init__(self, status, name=None): + Thread.__init__(self, name=name) + + self.status = status + self.workerId = None + + def run(self): + """ + Do work. + """ + + try: + self.work() + except Exception, e: + self.status.put((MessageTypes.THREAD_ERROR, + (self.threadType, self.workerId, e))) + + self.status.put((MessageTypes.THREAD_DONE, self.workerId)) + + def work(self): + """ + Stub for sub classes to implement. + """ + + raise NotImplementedError + + +class StartWorker(AbstractWorker): + """ + Worker thread for starting jobs and reporting status. + """ + + threadType = ThreadTypes.START + + def __init__(self, status, (builder, trove), name=None): + AbstractWorker.__init__(self, status, name=name) + + self.builder = builder + self.trove = trove + self.workerId = self.trove + + def work(self): + """ + Start the specified build and report jobId. + """ + + jobId = self.builder.start((self.trove, )) + self.status.put((MessageTypes.DATA, (jobId, self.trove))) + + +class MonitorWorker(AbstractWorker): + """ + Worker thread for monitoring jobs and reporting status. + """ + + threadType = ThreadTypes.MONITOR + displayClass = JobMonitorCallback + + def __init__(self, status, (rmakeClient, jobId), name=None): + AbstractWorker.__init__(self, status, name=name) + + self.client = rmakeClient + self.jobId = jobId + self.workerId = jobId + + def work(self): + """ + Watch the monitor queue and monitor any available jobs. + """ + + # FIXME: This is copied from rmake.cmdlin.monitor for the most part + # because I need to pass extra args to the display class. + + uri, tmpPath = monitor._getUri(self.client) + + try: + display = self.displayClass(self.status, self.client, + showBuildLogs=False, exitOnFinish=True) + client = self.client.listenToEvents(uri, self.jobId, display, + showTroveDetails=False, + serve=True) + return client + finally: + if tmpPath: + os.remove(tmpPath) + + +class CommitWorker(AbstractWorker): + """ + Worker thread for committing jobs. + """ + + threadType = ThreadTypes.COMMIT + + def __init__(self, status, (builder, jobId), name=None): + AbstractWorker.__init__(self, status, name=name) + + self.builder = builder + self.jobId = jobId + self.workerId = jobId + + def work(self): + """ + Commit the specified job. + """ + + result = self.builder.commit(self.jobId) + self.status.put((MessageTypes.DATA, (self.jobId, result))) + + +class AbstractStatusMonitor(object): + """ + Abstract class for implementing monitoring classes. + """ + + workerClass = None + + def __init__(self, threadArgs): + if type(threadArgs) not in (list, tuple, set): + threadArgs = (threadArgs, ) + self._threadArgs = threadArgs + + self._status = Queue() + self._workers = {} + self._errors = [] + + def addJob(self, job): + """ + Add a job to the worker pool. + """ + + args = list(self._threadArgs) + args.append(job) + + threadName = ('%s Worker' + % ThreadTypes.names[self.workerClass.threadType]) + worker = self.workerClass(self._status, args, name=threadName) + self._workers[job] = worker + worker.daemon = True + worker.start() + + def getStatus(self): + """ + Process all messages in the status queue, returning any data messages. + """ + + data = [] + while True: + try: + msg = self._status.get_nowait() + except Empty: + break + + data.extend(self._processMessage(msg)) + + return data + + def getErrors(self): + """ + Return any errors found while status was being processed. + """ + + errors = self._errors + self._errors = [] + return errors + + def _processMessage(self, msg): + """ + Handle messages. + """ + + data = [] + mtype, payload = msg + + if mtype == MessageTypes.LOG: + log.info(payload) + elif mtype == MessageTypes.DATA: + data.append(payload) + elif mtype == MessageTypes.THREAD_DONE: + job = payload + #assert not self._workers[job].isAlive() + del self._workers[job] + elif mtype == MessageTypes.THREAD_ERROR: + threadType, job, error = payload + #assert not self._workers[job].isAlive() + #raise error + log.error('[%s] FAILED with exception: %s' % (job, error)) + self._errors.append((job, error)) + + return data + + +class JobStarter(AbstractStatusMonitor): + """ + Abstraction around threaded starter model. + """ + + workerClass = StartWorker + startJob = AbstractStatusMonitor.addJob + + +class JobMonitor(AbstractStatusMonitor): + """ + Abstraction around threaded monitoring model. + """ + + workerClass = MonitorWorker + monitorJob = AbstractStatusMonitor.addJob + + +class JobCommitter(AbstractStatusMonitor): + """ + Abstraction around threaded commit model. + """ + + workerClass = CommitWorker + commitJob = AbstractStatusMonitor.addJob + + +class Dispatcher(object): + """ + Manage building a list of troves in a way that doesn't bring rMake to its + knees. + """ + + _completed = ( + -1, -2, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_BUILT, +# buildjob.JOB_STATE_COMMITTED + ) + + def __init__(self, builder, maxSlots): + self._builder = builder + self._maxSlots = maxSlots + self._slots = maxSlots + + self._maxStartSlots = 10 + self._startSlots = self._maxStartSlots + + self._starter = JobStarter(self._builder) + self._monitor = JobMonitor(self._builder._helper.client) + self._committer = JobCommitter(self._builder) + + # jobId: (trv, status, commitData) + self._jobs = {} + + self._failures = [] + + def buildmany(self, troveSpecs): + """ + Build as many packages as possible until we run out of slots. + """ + + # Must have at least one trove to build, otherwise will end up in + # an infinite loop. + if not troveSpecs: + return {} + + troves = list(troveSpecs) + troves.reverse() + + while troves or not self._jobDone(): + # Only create more jobs once the last batch has been started. + if self._startSlots == self._maxStartSlots: + # fill slots with available troves + while troves and self._slots and self._startSlots: + # get trove to work on + trove = troves.pop() + + # start build job + self._starter.startJob(trove) + self._slots -= 1 + self._startSlots -= 1 + + # get started status + for jobId, trove in self._starter.getStatus(): + self._jobs[jobId] = [trove, -1, None] + self._startSlots += 1 + self._monitor.monitorJob(jobId) + + # process starter errors + for trove, error in self._starter.getErrors(): + self._startSlots += 1 + self._slots += 1 + self._failures.append((trove, error)) + + # update job status changes + for jobId, status in self._monitor.getStatus(): + self._jobs[jobId][1] = status + if status in self._completed: + self._slots += 1 + assert self._slots <= self._maxSlots + + # commit any jobs that are done building + if status == buildjob.JOB_STATE_BUILT: + self._committer.commitJob(jobId) + + # check for commit status + for jobId, result in self._committer.getStatus(): + self._jobs[jobId][2] = result + + # process monitor and commit errors + for jobId, error in itertools.chain(self._monitor.getErrors(), + self._committer.getErrors()): + self._slots += 1 + self._jobs[jobId][1] = -2 + self._failures.append((jobId, error)) + + # Wait for a bit before polling again. + time.sleep(3) + + results = {} + for jobId, (trove, status, result) in self._jobs.iteritems(): + # log failed jobs + if status == buildjob.JOB_STATE_FAILED: + log.info('[%s] failed job: %s' % (jobId, trove)) + else: + results.update(result) + + # report failures + for job, error in self._failures: + log.error('[%s] failed with error: %s' % (job, error)) + + return results + + def _jobDone(self): + if not len(self._jobs): + return False + + for jobId, (trove, status, result) in self._jobs.iteritems(): + if status not in self._completed: + return False + return True diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -24,6 +24,7 @@ from updatebot.lib import util from updatebot import conaryhelper from updatebot.errors import GroupNotFound +from updatebot.errors import NoManifestFoundError from updatebot.errors import OldVersionNotFoundError from updatebot.errors import UpdateGoesBackwardsError from updatebot.errors import UpdateRemovesPackageError @@ -41,10 +42,12 @@ self._conaryhelper = conaryhelper.ConaryHelper(self._cfg) - def getUpdates(self): + def getUpdates(self, updateTroves=None): """ Find all packages that need updates and/or advisories from a top level binary group. + @param updateTroves: set of troves to update + @type updateTroves: iterable @return list of packages to send advisories for and list of packages to update """ @@ -53,7 +56,13 @@ toAdvise = [] toUpdate = [] - for nvf, srpm in self._findUpdatableTroves(self._cfg.topGroup): + + # If update set is not specified get the latest versions of packages to + # update. + if not updateTroves: + updateTroves = self._findUpdatableTroves(self._cfg.topGroup) + + for nvf, srpm in updateTroves: # Will raise exception if any errors are found, halting execution. if self._sanitizeTrove(nvf, srpm): toUpdate.append((nvf, srpm)) @@ -152,7 +161,15 @@ needsUpdate = False newNames = [ (x.name, x.arch) for x in self._pkgSource.srcPkgMap[srpm] ] metadata = None - manifest = self._conaryhelper.getManifest(nvf[0]) + + try: + manifest = self._conaryhelper.getManifest(nvf[0]) + except NoManifestFoundError, e: + # Create packages that do not have manifests. + # TODO: might want to make this a config option? + log.info('no manifest found for %s, will create package' % nvf[0]) + return True + for line in manifest: # Some manifests were created with double slashes, need to # normalize the path to work around this problem. @@ -231,7 +248,7 @@ return ret - def create(self, pkgNames, buildAll=False, recreate=False): + def create(self, pkgNames=None, buildAll=False, recreate=False, toCreate=None): """ Import a new package into the repository. @param pkgNames: list of packages to import @@ -241,13 +258,26 @@ @param recreate: a package manifest even if it already exists. @type recreate: boolean @return new source [(name, version, flavor), ... ] + + @param toCreate: set of packages to update. If this is set all other + options are ignored. + @type toCreate: set of source package objects. """ + assert pkgNames or toCreate + + if pkgNames: + toCreate = set() + else: + # Import very specific versions of packages, make sure to recreate + # them all. + pkgNames = [] + recreate = False + log.info('getting existing packages') pkgs = self._getExistingPackageNames() # Find all of the source to update. - toUpdate = set() for pkg in pkgNames: if pkg not in self._pkgSource.binNameMap: log.warn('no package named %s found in package source' % pkg) @@ -256,13 +286,13 @@ srcPkg = self._getPackagesToImport(pkg) if srcPkg.name not in pkgs or recreate: - toUpdate.add(srcPkg) + toCreate.add(srcPkg) # Update all of the unique sources. fail = set() toBuild = set() verCache = self._conaryhelper.getLatestVersions() - for pkg in sorted(toUpdate): + for pkg in sorted(toCreate): try: # Only import packages that haven't been imported before version = verCache.get('%s:source' % pkg.name) From elliot at rpath.com Mon Mar 15 17:04:08 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:08 +0000 Subject: mirrorball: fixup status reporting Message-ID: <201003152104.o2FL48xc007268@scc.eng.rpath.com> changeset: dc691a51505f user: Elliot Peele date: Sun, 01 Nov 2009 22:32:46 -0500 fixup status reporting diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py --- a/updatebot/subscriber.py +++ b/updatebot/subscriber.py @@ -350,10 +350,9 @@ """ _completed = ( - -1, -2, + -2, buildjob.JOB_STATE_FAILED, - buildjob.JOB_STATE_BUILT, -# buildjob.JOB_STATE_COMMITTED + buildjob.JOB_STATE_COMMITTED ) def __init__(self, builder, maxSlots): @@ -439,7 +438,7 @@ results = {} for jobId, (trove, status, result) in self._jobs.iteritems(): # log failed jobs - if status == buildjob.JOB_STATE_FAILED: + if status == buildjob.JOB_STATE_FAILED or not result: log.info('[%s] failed job: %s' % (jobId, trove)) else: results.update(result) From elliot at rpath.com Mon Mar 15 17:04:09 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:09 +0000 Subject: mirrorball: repos.getFileContents takes a list Message-ID: <201003152104.o2FL4A0M007295@scc.eng.rpath.com> changeset: cde8ab133655 user: Elliot Peele date: Sun, 01 Nov 2009 23:49:26 -0500 repos.getFileContents takes a list diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -508,7 +508,7 @@ capsuleFileContents = contentsCache[capFile[2]] elif newTroveCs.getOldVersion(): getFileContents = self._client.repos.getFileContents - fcList = getFileContents(capFile[2:], compressed=False) + fcList = getFileContents((capFile[2:], ), compressed=False) capsuleFileContents = fcList[0].get() else: getFileContents = newCs.getFileContents From elliot at rpath.com Mon Mar 15 17:04:11 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:11 +0000 Subject: mirrorball: add config option for group content definitions Message-ID: <201003152104.o2FL4Bjj007324@scc.eng.rpath.com> changeset: da002d5f7819 user: Elliot Peele date: Mon, 02 Nov 2009 00:18:16 -0500 add config option for group content definitions diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -110,6 +110,9 @@ # The top level source group. topSourceGroup = CfgTroveSpec + # Group contents info. + groupContents = (CfgList(CfgList(CfgString)), []) + # Other labels that are referenced in the group that need to be flattend # onto the targetLabel. sourceLabel = (CfgList(CfgBranch), []) From elliot at rpath.com Mon Mar 15 17:04:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:13 +0000 Subject: mirrorball: add initial group management bits, very untested Message-ID: <201003152104.o2FL4D2H007351@scc.eng.rpath.com> changeset: 49c17fe69aee user: Elliot Peele date: Mon, 02 Nov 2009 00:20:37 -0500 add initial group management bits, very untested diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr.py @@ -0,0 +1,197 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for managing conary groups. +""" + +import logging + +from updatebot.lib import util +from updatebot.build import Builder +from updatebot.conaryhelper import ConaryHelper + +from updatebot.lib.xobjects import XGroup +from updatebot.lib.xobjects import XGroupList +from updatebot.lib.xobjects import XPackageData +from updatebot.lib.xobjects import XPackageItem + +log = logging.getLogger('updatebot.groupmgr') + +class GroupManager(object): + """ + Manage group of all packages for a platform. + """ + + def __init__(self, cfg): + self._cfg = cfg + self._helper = GroupHelper(self._cfg) + self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') + + self._sourceName = self._cfg.topSourceGroup[0] + + self._dirty = False + + self._groups = {} + + def checkout(self): + """ + Get current group state from the repository. + """ + + # Checkout or create the source trove + self._groups = self._helper.getModel(sel._sourceName) + + def commit(self): + """ + Commit current changes to the group. + """ + + # 1. checkout current group + # 2. copy external group descriptions (group-$platform-platform) if + # description has changed. + # 3. freeze model + # 4. commit changes + + def build(self): + """ + Build all configured flavors of the group. + """ + + # build groups + + def update(self, trvList): + """ + Modify list of troves in a group. + """ + + # add/update version in model + + +class GroupHelper(ConaryHelper): + """ + Modified conary helper to deal with managing group sources. + """ + + def __init__(self, cfg): + ConaryHelper.__init__(self, cfg) + self._newPkgFactory = 'managed-group' + self._configDir = cfg.configPath + + def getModel(self, pkgName): + """ + Get a thawed data representation of the group xml data from the + repository. + """ + + log.info('loading model for %s' % pkgName) + recipeDir = self._edit(pkgName) + groupFileName = util.join(recipeDir, 'groups.xml') + + # load group model + groups = {} + if os.path.exists(groupFileName): + model = GroupModel.thaw(groupFileName) + for name, groupObj in model.iteritems(): + contentFileName = util.join(recipeDir, groupObj.filename) + contentsModel = GroupContentsModel.thaw(contentFileName) + contentsModel.groupName = groupObj.name + contentsModel.fileName = groupObj.filename + groups[groupObj.name] = contentsModel + + # copy in any group data + for gData in os.listdir(self._configDir): + if not gData.endswith('.xml'): + continue + + contentsModel = GroupContentsModel.thaw( + util.join(self._configDir, gData)) + + +class AbstractModel(object): + """ + Base object for models. + """ + + dataClass = None + + def __init__(self): + self._data = {} + self._hash = None + + def __hash__(self): + if not self._hash: + raise RuntimeError + return self._hash + + def __cmp__(self, other): + return cmp(self._hash, other._hash) + + @classmethod + def thaw(cls, xmlfn): + """ + Thaw the model from xml. + """ + + xml = open(xmlfn).read() + hash = hashlib.sha1(xml).hexdigest() + mode = self.dataClass.thaw(xml) + obj = cls() + obj._hash = hash + for item in model.items: + self._data[item.name] = item + + def freeze(self, toFile): + """ + Freeze the model to a given output file. + """ + + model = XGroupList() + model.items = self._data.values() + model.freeze(toFile) + + def iteritems(self): + """ + Iterate over the model data. + """ + + return self._data.iteritems() + +class GroupModel(AbstractModel): + """ + Model for representing group name and file name. + """ + + dataClass = XGroupList + + +class GroupContentsModel(AbstractModel): + """ + Model for representing group data. + """ + + dataClass = XPackageData + + def __init__(self, groupName): + AbstractModel.__init__(self) + self.groupName = groupName + + # figure out file name based on group name + name = ''.join([ x.capitalize() for x in self.groupName.split('-') ]) + self.fileName = name[0].lower() + name[1:] + + def update(self, trvList): + """ + Modify list of troves in a group. + """ From elliot at rpath.com Mon Mar 15 17:04:14 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:14 +0000 Subject: mirrorball: add missing import Message-ID: <201003152104.o2FL4EEm007383@scc.eng.rpath.com> changeset: 0cccb4f802bf user: Elliot Peele date: Mon, 02 Nov 2009 21:56:20 -0500 add missing import diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -16,6 +16,7 @@ Builder object implementation. """ +import os import stat import time import logging From elliot at rpath.com Mon Mar 15 17:04:16 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:16 +0000 Subject: mirrorball: improved group managment Message-ID: <201003152104.o2FL4GZc007411@scc.eng.rpath.com> changeset: 2c09b7dbbcb1 user: Elliot Peele date: Tue, 03 Nov 2009 18:03:57 -0500 improved group managment diff --git a/scripts/rhelorder.py b/scripts/gengroup.py copy from scripts/rhelorder.py copy to scripts/gengroup.py --- a/scripts/rhelorder.py +++ b/scripts/gengroup.py @@ -4,9 +4,8 @@ import sys sys.path.insert(0, os.environ['HOME'] + '/hg/conary') -sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') +sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py') sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-capsule-indexer') from conary.lib import util sys.excepthook = util.genExcepthook() @@ -14,7 +13,7 @@ mbdir = os.path.abspath('../') sys.path.insert(0, mbdir) -confDir = os.path.join(mbdir, 'config', 'rhel4') +confDir = os.path.join(mbdir, 'config', 'rhel5test') from updatebot import log from updatebot import Bot @@ -25,127 +24,33 @@ slog = logging.getLogger('script') -import rhnmirror - log.addRootLogger() cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) bot = Bot(cfg) -mcfg = rhnmirror.MirrorConfig() -mcfg.read(os.path.join(confDir, 'erratarc')) +from updatebot import groupmgr -errata = rhnmirror.Errata(mcfg) -errata.fetch() +mgr = groupmgr.GroupManager(cfg) -pkgSource = bot._pkgSource -pkgSource.load() +trvMap = mgr._helper._getLatestTroves() -# get mapping of advisory to errata obj -advisories = dict((x.advisory, x) - for x in errata.iterByIssueDate(mcfg.channels)) +from conary.deps import deps -# get mapping of nevra to pkg obj -nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) - for x in pkgSource.binPkgMap.keys() if x.arch != 'src') +#mgr.add('foo', byDefault=False, use=True, flavor=deps.parseFlavor('is:x86')) -# pull nevras into errata sized buckets -buckets = {} -advMap = {} -nevraMap = {} +troves = mgr._helper._getLatestTroves() +for name, vf in troves.iteritems(): + if ':' in name or bot._updater._fltrPkg(name): + continue -arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') -for e in errata.iterByIssueDate(mcfg.channels): - bnevras = [] - bucket = [] - bucketId = None - slog.info('processing %s' % e.advisory) - for pkg in e.packages: - nevra = pkg.getNevra() + assert len(vf.keys()) == 1 + version = vf.keys()[0] + flavors = vf[version] + mgr.addPackage(name, version, flavors) - # filter out channels we don't have indexed - channels = set([ x.label for x in pkg.channels ]) - if not set(mcfg.channels) & channels: - continue +mgr._commit() - # ignore arches we don't know about. - if nevra[4] not in arches: - continue - - # convert rhn nevra to yum nevra - nevra = list(nevra) - if nevra[1] is None: - nevra[1] = '0' - if type(nevra[1]) == int: - nevra[1] = str(nevra[1]) - nevra = tuple(nevra) - - # move nevra to errata buckets - if nevra in nevras: - binPkg = nevras.pop(nevra) - bucket.append(binPkg) - bnevras.append(nevra) - - # nevra is already part of another bucket - elif nevra in nevraMap: - bucketId = nevraMap[nevra] - - # raise error if we can't find the required package - else: - raise KeyError - - if bucketId is None: - bucketId = int(time.mktime(time.strptime(e.issue_date, - '%Y-%m-%d %H:%M:%S'))) - buckets[bucketId] = bucket - else: - buckets[bucketId].extend(bucket) - - for nevra in bnevras: - nevraMap[nevra] = bucketId - - advMap[e.advisory] = bucketId - -# separate out golden bits -other = [] -golden = [] -firstErrata = sorted(buckets.keys())[0] -for nevra, pkg in nevras.iteritems(): - buildtime = int(pkg.buildTimestamp) - if buildtime < firstErrata: - golden.append(pkg) - else: - other.append(pkg) - -# sort by source package -srcMap = {} -for pkg in other: - src = pkgSource.binPkgMap[pkg] - if src not in srcMap: - srcMap[src] = [] - srcMap[src].append(pkg) - -# insert bins by buildstamp -for src, bins in srcMap.iteritems(): - buildstamp = int(sorted(bins)[0].buildTimestamp) - if buildstamp in buckets: - buckets[buildstamp].extend(bins) - else: - buckets[buildstamp] = bins - -# get sources to build -buildOrder = {0: set()} -for pkg in golden: - # lookup source package - src = pkgSource.binPkgMap[pkg] - buildOrder[0].add(src) - -for bucketId in sorted(buckets.keys()): - bucket = buckets[bucketId] - buildOrder[bucketId] = set() - for pkg in bucket: - src = pkgSource.binPkgMap[pkg] - buildOrder[bucketId].add(src) import epdb; epdb.st() diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -551,15 +551,23 @@ return None + def _getLatestTroves(self): + """ + Get a dict of the latest troves on the buildLabel. + @return {name: {version: [flavor, ...]}}} + """ + + label = self._ccfg.buildLabel + trvMap = self._repos.getTroveLeavesByLabel({None: {label: None}}) + return trvMap + def getLatestVersions(self): """ Find all of the versions on the buildLabel. @return {trvName: trvVersion} """ - label = self._ccfg.buildLabel - - trvMap = self._repos.getTroveLeavesByLabel({None: {label: None}}) + trvMap = self._getLatestTroves() verMap = {} for name, verDict in trvMap.iteritems(): diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -111,7 +111,7 @@ topSourceGroup = CfgTroveSpec # Group contents info. - groupContents = (CfgList(CfgList(CfgString)), []) + groupContents = (CfgDict(CfgDict(CfgString)), {}) # Other labels that are referenced in the group that need to be flattend # onto the targetLabel. diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -250,6 +250,52 @@ """ _params = ['pkg', ] - _templates = ('Could not find a matching package for %(pkg)s in the ' + _template = ('Could not find a matching package for %(pkg)s in the ' 'configured repositories when attempting to map errata source to ' 'package source.') + +class GroupManagerError(UpdateBotError): + """ + GroupManagerError, generic error for group manager related errors. + """ + + _template = 'Group manager error' + +class UnsupportedTroveFlavorError(GroupManagerError): + """ + UnsupportedTroveError, raised when the group manager runs across a flavor it + does not know what to do with. + """ + + _params = ['name', 'flavor'] + _template = ('Do not know what to do with flavor %(flavor)s from trove ' + '%(name)s') + +class UnknownBuildContextError(GroupManagerError): + """ + UnknownBuildContextError, raised when the group manager finds a build + context on a special package that does not match either the x86 or + x86_64 filter. + """ + + _params = ['name', 'context'] + _template = ('Context does not fall into the x86 or x86_64 flavor sets.') + +class FlavorCountMismatchError(GroupManagerError): + """ + FlavorCountMismatchError, raised when the number of built package flavors + does not match the configured flavors. + """ + + _params = ['name', ] + _template = ('Could not find all built flavors for %(name)s. Maybe one of ' + 'the configured contexts resulted in an overlap.') + +class UnhandledPackageAdditionError(GroupManagerError): + """ + UnhandledPackageAdditionError, raised when the group manager just doesn't + know what to do when adding a package. + """ + + _params = ['name', ] + _templates = 'I don not know what to do with this package %(name)s.' diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -16,19 +16,43 @@ Module for managing conary groups. """ +import os import logging +from conary.deps import deps + from updatebot.lib import util from updatebot.build import Builder from updatebot.conaryhelper import ConaryHelper - from updatebot.lib.xobjects import XGroup +from updatebot.lib.xobjects import XGroupDoc from updatebot.lib.xobjects import XGroupList +from updatebot.lib.xobjects import XPackageDoc from updatebot.lib.xobjects import XPackageData from updatebot.lib.xobjects import XPackageItem +from updatebot.errors import FlavorCountMismatchError +from updatebot.errors import UnknownBuildContextError +from updatebot.errors import UnsupportedTroveFlavorError +from updatebot.errors import UnhandledPackageAdditionError log = logging.getLogger('updatebot.groupmgr') +def checkout(func): + def wrapper(self, *args, **kwargs): + if not self._checkedout: + self._checkout() + + return func(self, *args, **kwargs) + return wrapper + +def commit(func): + def wrapper(self, *args, **kwargs): + if self._checkedout: + self._commit() + + return func(self, *args, **kwargs) + return wrapper + class GroupManager(object): """ Manage group of all packages for a platform. @@ -40,43 +64,197 @@ self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') self._sourceName = self._cfg.topSourceGroup[0] + self._sourceVersion = self._cfg.topSourceGroup[1] - self._dirty = False + self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName - self._groups = {} + self._checkedout = False + self._groups = {} - def checkout(self): + def _checkout(self): """ Get current group state from the repository. """ - # Checkout or create the source trove - self._groups = self._helper.getModel(sel._sourceName) + self._groups = self._helper.getModel(self._sourceName) + self._checkedout = True - def commit(self): + def _commit(self): """ Commit current changes to the group. """ - # 1. checkout current group - # 2. copy external group descriptions (group-$platform-platform) if - # description has changed. - # 3. freeze model - # 4. commit changes + self._helper.setModel(self._sourceName, self._groups) + self._checkedout = False + @commit def build(self): """ Build all configured flavors of the group. """ - # build groups + # create list of trove specs to build + groupTroves = set() + for flavor in self._cfg.groupFlavors: + groupTroves.add((self._sourceName, self._sourceVersion, flavor)) - def update(self, trvList): + built = self._builder.build(groupTroves) + return built + + @checkout + def add(self, *args, **kwargs): """ - Modify list of troves in a group. + Add a trove to the package group contents. """ - # add/update version in model + # create package group model if it does not exist. + if self._pkgGroupName not in self._groups: + model = GroupContentsModel(self._pkgGroupName) + self._groups[self._pkgGroupName] = model + + add = self._groups[self._pkgGroupName].add(*args, **kwargs) + + @checkout + def remove(self, name): + """ + Remove a given trove from the package group contents. + """ + + return self._groups[self._pkgGroupName].remove(name) + + def addPackage(self, name, version, flavors): + """ + Add a package to the model. + @param name: name of the package + @type name: str + @param version: conary version from string object + @type version: conary.versions.VersionFromString + @param flavors: list of flavors + @type flavors: [conary.deps.deps.Flavor, ...] + """ + + assert len(flavors) + + plain = deps.parseFlavor('') + x86 = deps.parseFlavor('is: x86') + x86_64 = deps.parseFlavor('is: x86_64') + + if len(flavors) == 1: + flavor = flavors[0] + # noarch package, add unconditionally + if flavor == plain: + self.add(name) + + # x86, add with use=x86 + elif flavor.satisfies(x86): + self.add(name, flavor=flavor, use='x86') + + # x86_64, add with use=x86_64 + elif flavor.satisfies(x86_64): + self.add(name, flavor=flavor, use='x86_64') + + else: + raise UnsupportedTroveFlavorError(name=name, flavor=flavor) + + return + + elif len(flavors) == 2: + # This is most likely a normal package with both x86 and x86_64, but + # lets make sure anyway. + flvCount = {x86: 0, x86_64: 0, plain: 0} + for flavor in flavors: + if flavor.satisfies(x86): + flvCount[x86] += 1 + elif flavor.satisfies(x86_64): + flvCount[x86_64] += 1 + elif flavor.freeze() == '': + flvCount[plain] += 1 + else: + raise UnsupportedTroveFlavorError(name=name, flavor=flavor) + + # make sure there is only one instance of x86 and once instance of + # x86_64 in the flavor list. + assert (flvCount[x86_64] > 0) ^ (flvCount[plain] > 0) + assert len([ x for x, y in flvCount.iteritems() if y != 1 ]) == 1 + + # In this case just add the package unconditionally + self.add(name) + + return + + # These are special cases. + else: + # The way I see it there are a few ways you could end up here. + # 1. this is a kernel + # 2. this is a kernel module + # 3. this is a special package like glibc or openssl where there + # are i386, i686, and x86_64 varients. + # 4. this is a package with package flags that I don't know + # about. + # Lets see if we know about this package and think it should have + # more than two flavors. + + + # Get source trove name. + log.info('retrieving trove info for %s' % name) + srcTroveMap = self._helper._getSourceTroves( + (name, version, flavors[0]) + ) + srcTroveName = srcTroveMap.keys()[0][0].split(':')[0] + + # handle kernels. + if srcTroveName == 'kernel' or util.isKernelModulePackage(name): + # add all x86ish flavors with use=x86 and all x86_64ish flavors + # with use=x86_64 + for flavor in flavors: + if flavor.satisfies(x86): + self.add(name, flavor=flavor, use='x86') + elif flavor.satisfies(x86_64): + self.add(name, flavor=flavor, use='x86_64') + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + + # maybe this is one of the special flavors we know about. + elif srcTroveName in self._cfg.packageFlavors: + # separate packages into x86 and x86_64 by context name + # TODO: If we were really smart we would load the conary + # contexts and see what buildFlavors they contained. + flavorCount = {'x86': 0, 'x86_64': 0} + for context, bldflv in self._cfg.packageFlavors[srcTroveName]: + if context in ('i386', 'i486', 'i586', 'i686', 'x86'): + flavorCount['x86'] += 1 + elif context in ('x86_64', ): + flavorCount['x86_64'] += 1 + else: + raise UnknownBuildContextError(name=name, + flavor=context) + + for flavor in flavors: + if flavor.satisfies(x86): + flavorCount['x86'] -= 1 + elif flavor.satisfies(x86_64): + flavorCount['x86_64'] -= 1 + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + + errors = [ x for x, y in flavorCount.iteritems() if y != 0 ] + if errors: + raise FlavorCountMismatchError(name=name) + + for flavor in flavors: + if flavor.satisfies(x86): + self.add(name, flavor=flavor, use='x86') + elif flavor.satisfies(x86_64): + self.add(name, flavor=flavor, use='x86_64') + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + return + + # Unknown state. + raise UnhandledPackageAdditionError(name=name) class GroupHelper(ConaryHelper): @@ -86,8 +264,9 @@ def __init__(self, cfg): ConaryHelper.__init__(self, cfg) + self._configDir = cfg.configPath self._newPkgFactory = 'managed-group' - self._configDir = cfg.configPath + self._groupContents = cfg.groupContents def getModel(self, pkgName): """ @@ -104,62 +283,112 @@ if os.path.exists(groupFileName): model = GroupModel.thaw(groupFileName) for name, groupObj in model.iteritems(): - contentFileName = util.join(recipeDir, groupObj.filename) - contentsModel = GroupContentsModel.thaw(contentFileName) - contentsModel.groupName = groupObj.name - contentsModel.fileName = groupObj.filename + contentFileName = util.join(recipeDir, groupObj.fileName) + contentsModel = GroupContentsModel.thaw(contentFileName, + (name, groupObj.byDefault, groupObj.depCheck)) + contentsModel.fileName = groupObj.fileName groups[groupObj.name] = contentsModel # copy in any group data - for gData in os.listdir(self._configDir): - if not gData.endswith('.xml'): - continue + for name, data in self._groupContents.iteritems(): + newGroups = [ x for x in groups.itervalues() + if x.groupName == name and + x.fileName == data['filename'] ] + assert len(newGroups) in (0, 1) + + byDefault = data['byDefault'] == 'True' and True or False + depCheck = data['depCheck'] == 'True' and True or False + + # load model contentsModel = GroupContentsModel.thaw( - util.join(self._configDir, gData)) - + util.join(self._configDir, data['filename']), + (name, byDefault, depCheck) + ) + + # override anything from the repo + groups[name] = contentsModel + + return groups + + def setModel(self, pkgName, groups): + """ + Freeze group model and save to the repository. + """ + + log.info('saving model for %s' % pkgName) + recipeDir = self._edit(pkgName) + groupFileName = util.join(recipeDir, 'groups.xml') + + groupModel = GroupModel() + for name, model in groups.iteritems(): + groupfn = util.join(recipeDir, model.fileName) + + model.freeze(groupfn) + groupModel.add(name, model.fileName) + self._addFile(recipeDir, model.fileName) + + groupModel.freeze(groupFileName) + self._addFile(recipeDir, 'groups.xml') + + #self._commit(recipeDir, commitMessage='automated group update') + class AbstractModel(object): """ Base object for models. """ + docClass = None dataClass = None + elementClass = None def __init__(self): self._data = {} - self._hash = None + self._nameMap = {} - def __hash__(self): - if not self._hash: - raise RuntimeError - return self._hash + def _addItem(self, item): + """ + Add an item to the appropriate structures. + """ - def __cmp__(self, other): - return cmp(self._hash, other._hash) + self._data[item.key] = item + if item.name not in self._nameMap: + self._nameMap[item.name] = set() + self._nameMap[item.name].add(item.key) + + def _removeItem(self, name): + """ + Remove an item from the appropriate structures. + """ + + keys = self._nameMap.pop(name) + for key in keys: + self._data.pop(key) @classmethod - def thaw(cls, xmlfn): + def thaw(cls, xmlfn, args=None): """ Thaw the model from xml. """ - xml = open(xmlfn).read() - hash = hashlib.sha1(xml).hexdigest() - mode = self.dataClass.thaw(xml) - obj = cls() - obj._hash = hash - for item in model.items: - self._data[item.name] = item + model = cls.docClass.fromfile(xmlfn) + obj = args and cls(*args) or cls() + for item in model.data.items: + obj._addItem(item) + return obj def freeze(self, toFile): """ Freeze the model to a given output file. """ - model = XGroupList() + model = self.dataClass() model.items = self._data.values() - model.freeze(toFile) + + doc = self.docClass() + doc.data = model + doc.tofile(toFile) def iteritems(self): """ @@ -168,12 +397,29 @@ return self._data.iteritems() + def add(self, *args, **kwargs): + """ + Add an data element. + """ + + obj = self.elementClass(*args, **kwargs) + self._addItem(obj) + + def remove(self, name): + """ + Remove data element. + """ + + self._removeItem(name) + class GroupModel(AbstractModel): """ Model for representing group name and file name. """ + docClass = XGroupDoc dataClass = XGroupList + elementClass = XGroup class GroupContentsModel(AbstractModel): @@ -181,17 +427,16 @@ Model for representing group data. """ + docClass = XPackageDoc dataClass = XPackageData + elementClass = XPackageItem - def __init__(self, groupName): + def __init__(self, groupName, byDefault=True, depCheck=True): AbstractModel.__init__(self) self.groupName = groupName + self.byDefault = byDefault + self.depCheck = depCheck # figure out file name based on group name name = ''.join([ x.capitalize() for x in self.groupName.split('-') ]) - self.fileName = name[0].lower() + name[1:] - - def update(self, trvList): - """ - Modify list of troves in a group. - """ + self.fileName = name[0].lower() + name[1:] + '.xml' diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -18,6 +18,8 @@ from xobj import xobj +import conary + from aptmd.packages import _Package from aptmd.sources import _SourcePackage @@ -37,6 +39,24 @@ return xobj.parse(xml, documentClass=cls) + @classmethod + def fromfile(cls, fn): + """ + Deserialize from file. + """ + + return xobj.parsef(fn, documentClass=cls) + + def tofile(self, fn): + """ + Save model to file name. + """ + + fObj = open(fn, 'w') + xml = self.toxml() + fObj.write(xml) + fObj.close() + class XMetadata(object): """ @@ -118,18 +138,35 @@ return key in self.items -class XItemList(XDocManager): +class XItemList(object): """ List of items. """ + items = None + def __init__(self): self.items = [] self._itemClass = self.__class__.__dict__['items'][0] - XDocManager.__init__(self) -class XPackageItem(object): +class XHashableItem(object): + """ + Base class for hashable items. + """ + + @property + def key(self): + raise NotImplementedError + + def __hash__(self): + return hash(self.key) + + def __cmp__(self, other): + return cmp(self.key, other.key) + + +class XPackageItem(XHashableItem): """ Object to represent package data required for group builds with the managed group factory. @@ -141,18 +178,29 @@ byDefault = int use = str source = str - originalName = str - def __init__(self, name, version='', flavor='', byDefault=1, use=1, - source='', originalName=None): + def __init__(self, name=None, version=None, flavor=None, byDefault=True, + use=True, source=None): + + self.name = name self.version = version - self.flavor = flavor - self.byDefault = byDefault - self.use = use + self.byDefault = byDefault and 1 or 0 self.source = source - self.originalName = originalName and originalName or name + if use in (True, False): + self.use = int(use) + else: + self.use = use + + if type(flavor) == conary.deps.deps.Flavor: + self.flavor = flavor.freeze() + else: + self.flavor = flavor + + @property + def key(self): + return (self.name, self.flavor) class XPackageData(XItemList): """ @@ -162,17 +210,33 @@ items = [ XPackageItem ] -class XGroup(object): +class XPackageDoc(XDocManager): + """ + Document class for group data. + """ + + data = XPackageData + + +class XGroup(XHashableItem): """ Group file info. """ name = str filename = str + byDefault = int + depCheck = int - def __init__(self, name, filename): + def __init__(self, name=None, filename=None, byDefault=True, depCheck=True): self.name = name self.filename = filename + self.byDefault = byDefault and 1 or 0 + self.depCheck = depCheck and 1 or 0 + + @property + def key(self): + return self.name class XGroupList(XItemList): @@ -181,3 +245,11 @@ """ items = [ XGroup ] + + +class XGroupDoc(XDocManager): + """ + Document for managing group.xml. + """ + + data = XGroupList From elliot at rpath.com Mon Mar 15 17:04:17 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:17 +0000 Subject: mirrorball: verify that every file in the RPM is checked Message-ID: <201003152104.o2FL4H30007439@scc.eng.rpath.com> changeset: 184c529e3195 user: Whitney Battestilli date: Tue, 03 Nov 2009 15:53:47 -0500 verify that every file in the RPM is checked diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -448,6 +448,11 @@ # %doc -- CNY-3254 fassert(not fileObj.flags.isInitialContents(), fpath) + # Make sure we have explicitly checked every file in the RPM + uncheckedFiles = [x[0] for x in foundFiles.iteritems() if not x[1]] + fassert( not uncheckedFiles, str(uncheckedFiles) ) + + def _sanityCheckChangeSet(self, csFile, jobId): """ Sanity check changeset before commit. From elliot at rpath.com Mon Mar 15 17:04:19 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:19 +0000 Subject: mirrorball: branch merge Message-ID: <201003152104.o2FL4JAD007467@scc.eng.rpath.com> changeset: 9218507172ea user: Elliot Peele date: Tue, 03 Nov 2009 18:04:23 -0500 branch merge diff --git a/scripts/rhelorder.py b/scripts/gengroup.py copy from scripts/rhelorder.py copy to scripts/gengroup.py --- a/scripts/rhelorder.py +++ b/scripts/gengroup.py @@ -4,9 +4,8 @@ import sys sys.path.insert(0, os.environ['HOME'] + '/hg/conary') -sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') +sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py') sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-capsule-indexer') from conary.lib import util sys.excepthook = util.genExcepthook() @@ -14,7 +13,7 @@ mbdir = os.path.abspath('../') sys.path.insert(0, mbdir) -confDir = os.path.join(mbdir, 'config', 'rhel4') +confDir = os.path.join(mbdir, 'config', 'rhel5test') from updatebot import log from updatebot import Bot @@ -25,127 +24,33 @@ slog = logging.getLogger('script') -import rhnmirror - log.addRootLogger() cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) bot = Bot(cfg) -mcfg = rhnmirror.MirrorConfig() -mcfg.read(os.path.join(confDir, 'erratarc')) +from updatebot import groupmgr -errata = rhnmirror.Errata(mcfg) -errata.fetch() +mgr = groupmgr.GroupManager(cfg) -pkgSource = bot._pkgSource -pkgSource.load() +trvMap = mgr._helper._getLatestTroves() -# get mapping of advisory to errata obj -advisories = dict((x.advisory, x) - for x in errata.iterByIssueDate(mcfg.channels)) +from conary.deps import deps -# get mapping of nevra to pkg obj -nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) - for x in pkgSource.binPkgMap.keys() if x.arch != 'src') +#mgr.add('foo', byDefault=False, use=True, flavor=deps.parseFlavor('is:x86')) -# pull nevras into errata sized buckets -buckets = {} -advMap = {} -nevraMap = {} +troves = mgr._helper._getLatestTroves() +for name, vf in troves.iteritems(): + if ':' in name or bot._updater._fltrPkg(name): + continue -arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') -for e in errata.iterByIssueDate(mcfg.channels): - bnevras = [] - bucket = [] - bucketId = None - slog.info('processing %s' % e.advisory) - for pkg in e.packages: - nevra = pkg.getNevra() + assert len(vf.keys()) == 1 + version = vf.keys()[0] + flavors = vf[version] + mgr.addPackage(name, version, flavors) - # filter out channels we don't have indexed - channels = set([ x.label for x in pkg.channels ]) - if not set(mcfg.channels) & channels: - continue +mgr._commit() - # ignore arches we don't know about. - if nevra[4] not in arches: - continue - - # convert rhn nevra to yum nevra - nevra = list(nevra) - if nevra[1] is None: - nevra[1] = '0' - if type(nevra[1]) == int: - nevra[1] = str(nevra[1]) - nevra = tuple(nevra) - - # move nevra to errata buckets - if nevra in nevras: - binPkg = nevras.pop(nevra) - bucket.append(binPkg) - bnevras.append(nevra) - - # nevra is already part of another bucket - elif nevra in nevraMap: - bucketId = nevraMap[nevra] - - # raise error if we can't find the required package - else: - raise KeyError - - if bucketId is None: - bucketId = int(time.mktime(time.strptime(e.issue_date, - '%Y-%m-%d %H:%M:%S'))) - buckets[bucketId] = bucket - else: - buckets[bucketId].extend(bucket) - - for nevra in bnevras: - nevraMap[nevra] = bucketId - - advMap[e.advisory] = bucketId - -# separate out golden bits -other = [] -golden = [] -firstErrata = sorted(buckets.keys())[0] -for nevra, pkg in nevras.iteritems(): - buildtime = int(pkg.buildTimestamp) - if buildtime < firstErrata: - golden.append(pkg) - else: - other.append(pkg) - -# sort by source package -srcMap = {} -for pkg in other: - src = pkgSource.binPkgMap[pkg] - if src not in srcMap: - srcMap[src] = [] - srcMap[src].append(pkg) - -# insert bins by buildstamp -for src, bins in srcMap.iteritems(): - buildstamp = int(sorted(bins)[0].buildTimestamp) - if buildstamp in buckets: - buckets[buildstamp].extend(bins) - else: - buckets[buildstamp] = bins - -# get sources to build -buildOrder = {0: set()} -for pkg in golden: - # lookup source package - src = pkgSource.binPkgMap[pkg] - buildOrder[0].add(src) - -for bucketId in sorted(buckets.keys()): - bucket = buckets[bucketId] - buildOrder[bucketId] = set() - for pkg in bucket: - src = pkgSource.binPkgMap[pkg] - buildOrder[bucketId].add(src) import epdb; epdb.st() diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -16,6 +16,7 @@ Builder object implementation. """ +import os import stat import time import logging diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -551,15 +551,23 @@ return None + def _getLatestTroves(self): + """ + Get a dict of the latest troves on the buildLabel. + @return {name: {version: [flavor, ...]}}} + """ + + label = self._ccfg.buildLabel + trvMap = self._repos.getTroveLeavesByLabel({None: {label: None}}) + return trvMap + def getLatestVersions(self): """ Find all of the versions on the buildLabel. @return {trvName: trvVersion} """ - label = self._ccfg.buildLabel - - trvMap = self._repos.getTroveLeavesByLabel({None: {label: None}}) + trvMap = self._getLatestTroves() verMap = {} for name, verDict in trvMap.iteritems(): diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -111,7 +111,7 @@ topSourceGroup = CfgTroveSpec # Group contents info. - groupContents = (CfgList(CfgList(CfgString)), []) + groupContents = (CfgDict(CfgDict(CfgString)), {}) # Other labels that are referenced in the group that need to be flattend # onto the targetLabel. diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -250,6 +250,52 @@ """ _params = ['pkg', ] - _templates = ('Could not find a matching package for %(pkg)s in the ' + _template = ('Could not find a matching package for %(pkg)s in the ' 'configured repositories when attempting to map errata source to ' 'package source.') + +class GroupManagerError(UpdateBotError): + """ + GroupManagerError, generic error for group manager related errors. + """ + + _template = 'Group manager error' + +class UnsupportedTroveFlavorError(GroupManagerError): + """ + UnsupportedTroveError, raised when the group manager runs across a flavor it + does not know what to do with. + """ + + _params = ['name', 'flavor'] + _template = ('Do not know what to do with flavor %(flavor)s from trove ' + '%(name)s') + +class UnknownBuildContextError(GroupManagerError): + """ + UnknownBuildContextError, raised when the group manager finds a build + context on a special package that does not match either the x86 or + x86_64 filter. + """ + + _params = ['name', 'context'] + _template = ('Context does not fall into the x86 or x86_64 flavor sets.') + +class FlavorCountMismatchError(GroupManagerError): + """ + FlavorCountMismatchError, raised when the number of built package flavors + does not match the configured flavors. + """ + + _params = ['name', ] + _template = ('Could not find all built flavors for %(name)s. Maybe one of ' + 'the configured contexts resulted in an overlap.') + +class UnhandledPackageAdditionError(GroupManagerError): + """ + UnhandledPackageAdditionError, raised when the group manager just doesn't + know what to do when adding a package. + """ + + _params = ['name', ] + _templates = 'I don not know what to do with this package %(name)s.' diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -16,19 +16,43 @@ Module for managing conary groups. """ +import os import logging +from conary.deps import deps + from updatebot.lib import util from updatebot.build import Builder from updatebot.conaryhelper import ConaryHelper - from updatebot.lib.xobjects import XGroup +from updatebot.lib.xobjects import XGroupDoc from updatebot.lib.xobjects import XGroupList +from updatebot.lib.xobjects import XPackageDoc from updatebot.lib.xobjects import XPackageData from updatebot.lib.xobjects import XPackageItem +from updatebot.errors import FlavorCountMismatchError +from updatebot.errors import UnknownBuildContextError +from updatebot.errors import UnsupportedTroveFlavorError +from updatebot.errors import UnhandledPackageAdditionError log = logging.getLogger('updatebot.groupmgr') +def checkout(func): + def wrapper(self, *args, **kwargs): + if not self._checkedout: + self._checkout() + + return func(self, *args, **kwargs) + return wrapper + +def commit(func): + def wrapper(self, *args, **kwargs): + if self._checkedout: + self._commit() + + return func(self, *args, **kwargs) + return wrapper + class GroupManager(object): """ Manage group of all packages for a platform. @@ -40,43 +64,197 @@ self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') self._sourceName = self._cfg.topSourceGroup[0] + self._sourceVersion = self._cfg.topSourceGroup[1] - self._dirty = False + self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName - self._groups = {} + self._checkedout = False + self._groups = {} - def checkout(self): + def _checkout(self): """ Get current group state from the repository. """ - # Checkout or create the source trove - self._groups = self._helper.getModel(sel._sourceName) + self._groups = self._helper.getModel(self._sourceName) + self._checkedout = True - def commit(self): + def _commit(self): """ Commit current changes to the group. """ - # 1. checkout current group - # 2. copy external group descriptions (group-$platform-platform) if - # description has changed. - # 3. freeze model - # 4. commit changes + self._helper.setModel(self._sourceName, self._groups) + self._checkedout = False + @commit def build(self): """ Build all configured flavors of the group. """ - # build groups + # create list of trove specs to build + groupTroves = set() + for flavor in self._cfg.groupFlavors: + groupTroves.add((self._sourceName, self._sourceVersion, flavor)) - def update(self, trvList): + built = self._builder.build(groupTroves) + return built + + @checkout + def add(self, *args, **kwargs): """ - Modify list of troves in a group. + Add a trove to the package group contents. """ - # add/update version in model + # create package group model if it does not exist. + if self._pkgGroupName not in self._groups: + model = GroupContentsModel(self._pkgGroupName) + self._groups[self._pkgGroupName] = model + + add = self._groups[self._pkgGroupName].add(*args, **kwargs) + + @checkout + def remove(self, name): + """ + Remove a given trove from the package group contents. + """ + + return self._groups[self._pkgGroupName].remove(name) + + def addPackage(self, name, version, flavors): + """ + Add a package to the model. + @param name: name of the package + @type name: str + @param version: conary version from string object + @type version: conary.versions.VersionFromString + @param flavors: list of flavors + @type flavors: [conary.deps.deps.Flavor, ...] + """ + + assert len(flavors) + + plain = deps.parseFlavor('') + x86 = deps.parseFlavor('is: x86') + x86_64 = deps.parseFlavor('is: x86_64') + + if len(flavors) == 1: + flavor = flavors[0] + # noarch package, add unconditionally + if flavor == plain: + self.add(name) + + # x86, add with use=x86 + elif flavor.satisfies(x86): + self.add(name, flavor=flavor, use='x86') + + # x86_64, add with use=x86_64 + elif flavor.satisfies(x86_64): + self.add(name, flavor=flavor, use='x86_64') + + else: + raise UnsupportedTroveFlavorError(name=name, flavor=flavor) + + return + + elif len(flavors) == 2: + # This is most likely a normal package with both x86 and x86_64, but + # lets make sure anyway. + flvCount = {x86: 0, x86_64: 0, plain: 0} + for flavor in flavors: + if flavor.satisfies(x86): + flvCount[x86] += 1 + elif flavor.satisfies(x86_64): + flvCount[x86_64] += 1 + elif flavor.freeze() == '': + flvCount[plain] += 1 + else: + raise UnsupportedTroveFlavorError(name=name, flavor=flavor) + + # make sure there is only one instance of x86 and once instance of + # x86_64 in the flavor list. + assert (flvCount[x86_64] > 0) ^ (flvCount[plain] > 0) + assert len([ x for x, y in flvCount.iteritems() if y != 1 ]) == 1 + + # In this case just add the package unconditionally + self.add(name) + + return + + # These are special cases. + else: + # The way I see it there are a few ways you could end up here. + # 1. this is a kernel + # 2. this is a kernel module + # 3. this is a special package like glibc or openssl where there + # are i386, i686, and x86_64 varients. + # 4. this is a package with package flags that I don't know + # about. + # Lets see if we know about this package and think it should have + # more than two flavors. + + + # Get source trove name. + log.info('retrieving trove info for %s' % name) + srcTroveMap = self._helper._getSourceTroves( + (name, version, flavors[0]) + ) + srcTroveName = srcTroveMap.keys()[0][0].split(':')[0] + + # handle kernels. + if srcTroveName == 'kernel' or util.isKernelModulePackage(name): + # add all x86ish flavors with use=x86 and all x86_64ish flavors + # with use=x86_64 + for flavor in flavors: + if flavor.satisfies(x86): + self.add(name, flavor=flavor, use='x86') + elif flavor.satisfies(x86_64): + self.add(name, flavor=flavor, use='x86_64') + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + + # maybe this is one of the special flavors we know about. + elif srcTroveName in self._cfg.packageFlavors: + # separate packages into x86 and x86_64 by context name + # TODO: If we were really smart we would load the conary + # contexts and see what buildFlavors they contained. + flavorCount = {'x86': 0, 'x86_64': 0} + for context, bldflv in self._cfg.packageFlavors[srcTroveName]: + if context in ('i386', 'i486', 'i586', 'i686', 'x86'): + flavorCount['x86'] += 1 + elif context in ('x86_64', ): + flavorCount['x86_64'] += 1 + else: + raise UnknownBuildContextError(name=name, + flavor=context) + + for flavor in flavors: + if flavor.satisfies(x86): + flavorCount['x86'] -= 1 + elif flavor.satisfies(x86_64): + flavorCount['x86_64'] -= 1 + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + + errors = [ x for x, y in flavorCount.iteritems() if y != 0 ] + if errors: + raise FlavorCountMismatchError(name=name) + + for flavor in flavors: + if flavor.satisfies(x86): + self.add(name, flavor=flavor, use='x86') + elif flavor.satisfies(x86_64): + self.add(name, flavor=flavor, use='x86_64') + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + return + + # Unknown state. + raise UnhandledPackageAdditionError(name=name) class GroupHelper(ConaryHelper): @@ -86,8 +264,9 @@ def __init__(self, cfg): ConaryHelper.__init__(self, cfg) + self._configDir = cfg.configPath self._newPkgFactory = 'managed-group' - self._configDir = cfg.configPath + self._groupContents = cfg.groupContents def getModel(self, pkgName): """ @@ -104,62 +283,112 @@ if os.path.exists(groupFileName): model = GroupModel.thaw(groupFileName) for name, groupObj in model.iteritems(): - contentFileName = util.join(recipeDir, groupObj.filename) - contentsModel = GroupContentsModel.thaw(contentFileName) - contentsModel.groupName = groupObj.name - contentsModel.fileName = groupObj.filename + contentFileName = util.join(recipeDir, groupObj.fileName) + contentsModel = GroupContentsModel.thaw(contentFileName, + (name, groupObj.byDefault, groupObj.depCheck)) + contentsModel.fileName = groupObj.fileName groups[groupObj.name] = contentsModel # copy in any group data - for gData in os.listdir(self._configDir): - if not gData.endswith('.xml'): - continue + for name, data in self._groupContents.iteritems(): + newGroups = [ x for x in groups.itervalues() + if x.groupName == name and + x.fileName == data['filename'] ] + assert len(newGroups) in (0, 1) + + byDefault = data['byDefault'] == 'True' and True or False + depCheck = data['depCheck'] == 'True' and True or False + + # load model contentsModel = GroupContentsModel.thaw( - util.join(self._configDir, gData)) - + util.join(self._configDir, data['filename']), + (name, byDefault, depCheck) + ) + + # override anything from the repo + groups[name] = contentsModel + + return groups + + def setModel(self, pkgName, groups): + """ + Freeze group model and save to the repository. + """ + + log.info('saving model for %s' % pkgName) + recipeDir = self._edit(pkgName) + groupFileName = util.join(recipeDir, 'groups.xml') + + groupModel = GroupModel() + for name, model in groups.iteritems(): + groupfn = util.join(recipeDir, model.fileName) + + model.freeze(groupfn) + groupModel.add(name, model.fileName) + self._addFile(recipeDir, model.fileName) + + groupModel.freeze(groupFileName) + self._addFile(recipeDir, 'groups.xml') + + #self._commit(recipeDir, commitMessage='automated group update') + class AbstractModel(object): """ Base object for models. """ + docClass = None dataClass = None + elementClass = None def __init__(self): self._data = {} - self._hash = None + self._nameMap = {} - def __hash__(self): - if not self._hash: - raise RuntimeError - return self._hash + def _addItem(self, item): + """ + Add an item to the appropriate structures. + """ - def __cmp__(self, other): - return cmp(self._hash, other._hash) + self._data[item.key] = item + if item.name not in self._nameMap: + self._nameMap[item.name] = set() + self._nameMap[item.name].add(item.key) + + def _removeItem(self, name): + """ + Remove an item from the appropriate structures. + """ + + keys = self._nameMap.pop(name) + for key in keys: + self._data.pop(key) @classmethod - def thaw(cls, xmlfn): + def thaw(cls, xmlfn, args=None): """ Thaw the model from xml. """ - xml = open(xmlfn).read() - hash = hashlib.sha1(xml).hexdigest() - mode = self.dataClass.thaw(xml) - obj = cls() - obj._hash = hash - for item in model.items: - self._data[item.name] = item + model = cls.docClass.fromfile(xmlfn) + obj = args and cls(*args) or cls() + for item in model.data.items: + obj._addItem(item) + return obj def freeze(self, toFile): """ Freeze the model to a given output file. """ - model = XGroupList() + model = self.dataClass() model.items = self._data.values() - model.freeze(toFile) + + doc = self.docClass() + doc.data = model + doc.tofile(toFile) def iteritems(self): """ @@ -168,12 +397,29 @@ return self._data.iteritems() + def add(self, *args, **kwargs): + """ + Add an data element. + """ + + obj = self.elementClass(*args, **kwargs) + self._addItem(obj) + + def remove(self, name): + """ + Remove data element. + """ + + self._removeItem(name) + class GroupModel(AbstractModel): """ Model for representing group name and file name. """ + docClass = XGroupDoc dataClass = XGroupList + elementClass = XGroup class GroupContentsModel(AbstractModel): @@ -181,17 +427,16 @@ Model for representing group data. """ + docClass = XPackageDoc dataClass = XPackageData + elementClass = XPackageItem - def __init__(self, groupName): + def __init__(self, groupName, byDefault=True, depCheck=True): AbstractModel.__init__(self) self.groupName = groupName + self.byDefault = byDefault + self.depCheck = depCheck # figure out file name based on group name name = ''.join([ x.capitalize() for x in self.groupName.split('-') ]) - self.fileName = name[0].lower() + name[1:] - - def update(self, trvList): - """ - Modify list of troves in a group. - """ + self.fileName = name[0].lower() + name[1:] + '.xml' diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -18,6 +18,8 @@ from xobj import xobj +import conary + from aptmd.packages import _Package from aptmd.sources import _SourcePackage @@ -37,6 +39,24 @@ return xobj.parse(xml, documentClass=cls) + @classmethod + def fromfile(cls, fn): + """ + Deserialize from file. + """ + + return xobj.parsef(fn, documentClass=cls) + + def tofile(self, fn): + """ + Save model to file name. + """ + + fObj = open(fn, 'w') + xml = self.toxml() + fObj.write(xml) + fObj.close() + class XMetadata(object): """ @@ -118,18 +138,35 @@ return key in self.items -class XItemList(XDocManager): +class XItemList(object): """ List of items. """ + items = None + def __init__(self): self.items = [] self._itemClass = self.__class__.__dict__['items'][0] - XDocManager.__init__(self) -class XPackageItem(object): +class XHashableItem(object): + """ + Base class for hashable items. + """ + + @property + def key(self): + raise NotImplementedError + + def __hash__(self): + return hash(self.key) + + def __cmp__(self, other): + return cmp(self.key, other.key) + + +class XPackageItem(XHashableItem): """ Object to represent package data required for group builds with the managed group factory. @@ -141,18 +178,29 @@ byDefault = int use = str source = str - originalName = str - def __init__(self, name, version='', flavor='', byDefault=1, use=1, - source='', originalName=None): + def __init__(self, name=None, version=None, flavor=None, byDefault=True, + use=True, source=None): + + self.name = name self.version = version - self.flavor = flavor - self.byDefault = byDefault - self.use = use + self.byDefault = byDefault and 1 or 0 self.source = source - self.originalName = originalName and originalName or name + if use in (True, False): + self.use = int(use) + else: + self.use = use + + if type(flavor) == conary.deps.deps.Flavor: + self.flavor = flavor.freeze() + else: + self.flavor = flavor + + @property + def key(self): + return (self.name, self.flavor) class XPackageData(XItemList): """ @@ -162,17 +210,33 @@ items = [ XPackageItem ] -class XGroup(object): +class XPackageDoc(XDocManager): + """ + Document class for group data. + """ + + data = XPackageData + + +class XGroup(XHashableItem): """ Group file info. """ name = str filename = str + byDefault = int + depCheck = int - def __init__(self, name, filename): + def __init__(self, name=None, filename=None, byDefault=True, depCheck=True): self.name = name self.filename = filename + self.byDefault = byDefault and 1 or 0 + self.depCheck = depCheck and 1 or 0 + + @property + def key(self): + return self.name class XGroupList(XItemList): @@ -181,3 +245,11 @@ """ items = [ XGroup ] + + +class XGroupDoc(XDocManager): + """ + Document for managing group.xml. + """ + + data = XGroupList From elliot at rpath.com Mon Mar 15 17:04:21 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:21 +0000 Subject: mirrorball: free slots once the job is built, but not committed Message-ID: <201003152104.o2FL4ML9007489@scc.eng.rpath.com> changeset: 43d6815dbbfb user: Elliot Peele date: Tue, 03 Nov 2009 23:51:10 -0500 free slots once the job is built, but not committed diff --git a/updatebot/subscriber.py b/updatebot/subscriber.py --- a/updatebot/subscriber.py +++ b/updatebot/subscriber.py @@ -57,6 +57,16 @@ } +class JobStatus(object): + """ + Job states in dispatcher. + """ + + JOB_NOT_STARTED = -1 + ERROR_MONITOR_FAILURE = -2 + ERROR_COMITTER_FAILURE = -3 + + class JobMonitorCallback(monitor.JobLogDisplay): """ Monitor job status changes. @@ -352,9 +362,16 @@ _completed = ( -2, buildjob.JOB_STATE_FAILED, - buildjob.JOB_STATE_COMMITTED + buildjob.JOB_STATE_COMMITTED, ) + _slotdone = ( + -2, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_BUILT, + ) + + def __init__(self, builder, maxSlots): self._builder = builder self._maxSlots = maxSlots @@ -400,7 +417,7 @@ # get started status for jobId, trove in self._starter.getStatus(): - self._jobs[jobId] = [trove, -1, None] + self._jobs[jobId] = [trove, JobStatus.JOB_NOT_STARTED, None] self._startSlots += 1 self._monitor.monitorJob(jobId) @@ -413,7 +430,8 @@ # update job status changes for jobId, status in self._monitor.getStatus(): self._jobs[jobId][1] = status - if status in self._completed: + # free up the slot once the job is built + if status in self._slotdone: self._slots += 1 assert self._slots <= self._maxSlots @@ -425,11 +443,15 @@ for jobId, result in self._committer.getStatus(): self._jobs[jobId][2] = result - # process monitor and commit errors - for jobId, error in itertools.chain(self._monitor.getErrors(), - self._committer.getErrors()): + # process monitor errors + for jobId, error in self._monitor.getErrors(): self._slots += 1 - self._jobs[jobId][1] = -2 + self._jobs[jobId][1] = JobStatus.ERROR_MONITOR_FAILURE + self._failures.append((jobId, error)) + + # process committer errors + for jobId, error in self._committer.getErrors(): + self._jobs[jobId][1] = JobStatus.ERROR_COMITTER_FAILURE self._failures.append((jobId, error)) # Wait for a bit before polling again. From elliot at rpath.com Mon Mar 15 17:04:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:24 +0000 Subject: mirrorball: add sanityCheckChangesets Message-ID: <201003152104.o2FL4OjH007526@scc.eng.rpath.com> changeset: e304540993e9 user: Elliot Peele date: Wed, 04 Nov 2009 00:10:55 -0500 add sanityCheckChangesets diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -79,11 +79,9 @@ @param cfg: updateBot configuration object @type cfg: config.UpdateBotConfig - @param saveChangeSets: directory to save changesets to - @type saveChangeSets: str """ - def __init__(self, cfg, rmakeCfgFn=None, saveChangeSets=None): + def __init__(self, cfg, rmakeCfgFn=None): self._cfg = cfg self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) @@ -93,13 +91,14 @@ self._client = conaryclient.ConaryClient(self._ccfg) - if saveChangeSets is None and self._cfg.saveChangeSets: + if self._cfg.saveChangeSets or self._cfg.sanityCheckChangesets: self._saveChangeSets = tempfile.mkdtemp( prefix=self._cfg.platformName, suffix='-import-changesets') else: - self._saveChangeSets = saveChangeSets + self._saveChangeSets = False + self._santiyCheckChangesets = self._cfg.sanityCheckChangesets self._sanityCheckCommits = self._cfg.sanityCheckCommits # Get default pluginDirs from the rmake cfg object, setup the plugin @@ -566,7 +565,7 @@ if writeToFile: log.info('changeset saved to %s' % writeToFile) - if self._sanityCheckCommits: + if self._sanityCheckChangesets: self._sanityCheckChangeSet(writeToFile, jobIdsStr) log.info('committing changeset to repository') diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -176,6 +176,10 @@ # to make sure all of the contents made it into the repository. sanityCheckCommits = (CfgBool, False) + # Check the changeset for any rpm capsules and validate that the changeset + # contents match the rpm header. Implies saveChangeSets. + sanityCheckChangesets = (CfgBool, False) + # Save all binary changesets to disk before committing them. saveChangeSets = (CfgBool, False) From elliot at rpath.com Mon Mar 15 17:04:26 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:26 +0000 Subject: mirrorball: remove committed add Message-ID: <201003152104.o2FL4QTt007555@scc.eng.rpath.com> changeset: b8d6332aec78 user: Elliot Peele date: Wed, 04 Nov 2009 00:11:15 -0500 remove committed add diff --git a/scripts/gengroup.py b/scripts/gengroup.py --- a/scripts/gengroup.py +++ b/scripts/gengroup.py @@ -38,8 +38,6 @@ from conary.deps import deps -#mgr.add('foo', byDefault=False, use=True, flavor=deps.parseFlavor('is:x86')) - troves = mgr._helper._getLatestTroves() for name, vf in troves.iteritems(): if ':' in name or bot._updater._fltrPkg(name): From elliot at rpath.com Mon Mar 15 17:04:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:27 +0000 Subject: mirrorball: add more specific errors for changeset validation Message-ID: <201003152104.o2FL4SB5007583@scc.eng.rpath.com> changeset: b561ffa7c325 user: Elliot Peele date: Wed, 04 Nov 2009 00:33:13 -0500 add more specific errors for changeset validation diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -17,31 +17,34 @@ """ import os +import xml import stat import time import logging import tempfile import itertools -import xml -from Queue import Queue, Empty -from threading import Thread, RLock - -from conary import conarycfg, conaryclient -from conary import rpmhelper from conary import trove from conary import files +from conary import rpmhelper +from conary import conarycfg from conary.deps import deps +from conary import conaryclient from conary.repository import changeset from conary.repository.netrepos.proxy import ChangesetFilter from rmake import plugins from rmake.build import buildcfg -from rmake.cmdline import helper, monitor, commit +from rmake.cmdline import commit +from rmake.cmdline import helper +from rmake.cmdline import monitor from updatebot.lib import util from updatebot import subscriber -from updatebot.errors import JobFailedError, CommitFailedError +from updatebot.errors import JobFailedError +from updatebot.errors import CommitFailedError +from updatebot.errors import FailedToRetrieveChangesetError +from updatebot.errors import ChangesetValidationFailedError log = logging.getLogger('updateBot.build') @@ -267,9 +270,11 @@ if name != 'kernel': # Don't build kernel modules with a .debug flag, that # is only for kernels. - if flavor.stronglySatisfies(deps.parseFlavor('kernel.debug')): + if flavor.stronglySatisfies( + deps.parseFlavor('kernel.debug')): continue - flavor = deps.parseFlavor(str(flavor).replace('kernel', name)) + flavor = deps.parseFlavor( + str(flavor).replace('kernel', name)) troves.append((name, version, flavor, context)) # Handle special package flavors when specified. @@ -370,13 +375,10 @@ foundFiles = dict.fromkeys(rpmFileList) - def fassert(test, path="", why=None): + def fassert(test, path='', why=''): if not test: - if why: - raise CommitFailedError(jobId=jobId, why=why) - else: - raise CommitFailedError(jobId=jobId, why='metadata in trove' - ' does not agree with rpm header for file %s' % (path)) + raise ChangesetValidationFailedError(jobId=jobId, why=why, + path=path) for fileInfo, fileObj in zip(fileList, fileObjs): fpath = fileInfo[1] @@ -502,8 +504,8 @@ newTrove.iterFileList(capsules=True) ] if len(capFileList) != 1: - raise CommitFailedError(jobId=jobId, why='More than 1 RPM ' - 'capsule in trove %s' % newTroveCs.name()) + raise FailedToRetrieveChangesetError(jobId=jobId, why='More' + ' than 1 RPM capsule in trove %s' % newTroveCs.name()) capsules.append((capFileList[0], fileList, fileObjs)) diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -50,6 +50,25 @@ _template = 'rMake job %(jobId)s failed to commit: %(why)s' +class FailedToRetrieveChangesetError(CommitFailedError): + """ + FailedToRetrieveChangesetError, + """ + + _template = 'Failed to fetch changeset contents for job %(jobId)s: %(why)s' + + +class ChangesetValidationFailedError(CommitFailedError): + """ + ChangesetValidationFailedError, raised when the builder fails to validate + the built changeset. + """ + + _params = ['jobId', 'why', 'path'] + _template = ('Changeset from rmake job %(jobId)s failed to pass ' + 'validation for path %(path)s: %(why)s') + + class JobFailedError(UpdateBotError): """ JobFailedError, raised when an rMake job fails. From elliot at rpath.com Mon Mar 15 17:04:30 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:30 +0000 Subject: mirrorball: use a different formatter for stream logger Message-ID: <201003152104.o2FL4VJl007613@scc.eng.rpath.com> changeset: 5b6449ab8be7 user: Elliot Peele date: Wed, 04 Nov 2009 01:14:09 -0500 use a different formatter for stream logger diff --git a/updatebot/log.py b/updatebot/log.py --- a/updatebot/log.py +++ b/updatebot/log.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -38,11 +38,12 @@ maxBytes=logSize, backupCount=5) - formatter = logging.Formatter('%(asctime)s %(levelname)s ' + streamFomatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + fileFormatter = logging.Formatter('%(asctime)s %(levelname)s ' '%(name)s %(message)s') - streamHandler.setFormatter(formatter) - logFileHandler.setFormatter(formatter) + streamHandler.setFormatter(streamFormatter) + logFileHandler.setFormatter(fileFormatter) rootLog.addHandler(streamHandler) rootLog.addHandler(logFileHandler) From elliot at rpath.com Mon Mar 15 17:04:33 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:33 +0000 Subject: mirrorball: fix log name Message-ID: <201003152104.o2FL4Zcw007643@scc.eng.rpath.com> changeset: 17c4055c303c user: Elliot Peele date: Wed, 04 Nov 2009 01:14:25 -0500 fix log name diff --git a/updatebot/build.py b/updatebot/build.py --- a/updatebot/build.py +++ b/updatebot/build.py @@ -46,7 +46,7 @@ from updatebot.errors import FailedToRetrieveChangesetError from updatebot.errors import ChangesetValidationFailedError -log = logging.getLogger('updateBot.build') +log = logging.getLogger('updatebot.build') def jobInfoExceptionHandler(func): def deco(self, *args, **kwargs): From elliot at rpath.com Mon Mar 15 17:04:36 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:36 +0000 Subject: mirrorball: fixup license and copyright Message-ID: <201003152104.o2FL4bmA007673@scc.eng.rpath.com> changeset: 875d5250b177 user: Elliot Peele date: Wed, 04 Nov 2009 01:19:55 -0500 fixup license and copyright diff --git a/aptmd/__init__.py b/aptmd/__init__.py --- a/aptmd/__init__.py +++ b/aptmd/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/aptmd/errors.py b/aptmd/errors.py --- a/aptmd/errors.py +++ b/aptmd/errors.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/aptmd/parser.py b/aptmd/parser.py --- a/aptmd/parser.py +++ b/aptmd/parser.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/pmap/centos.py b/pmap/centos.py --- a/pmap/centos.py +++ b/pmap/centos.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/repomd/__init__.py b/repomd/__init__.py --- a/repomd/__init__.py +++ b/repomd/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/repomd/errors.py b/repomd/errors.py --- a/repomd/errors.py +++ b/repomd/errors.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/repomd/filelistsxml.py b/repomd/filelistsxml.py --- a/repomd/filelistsxml.py +++ b/repomd/filelistsxml.py @@ -1,5 +1,15 @@ # -# Copryright (c) 2008 rPath, Inc. +# Copryright (c) 2008-2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. # """ diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/repomd/patchesxml.py b/repomd/patchesxml.py --- a/repomd/patchesxml.py +++ b/repomd/patchesxml.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -1,5 +1,15 @@ # -# Copryright (c) 2008 rPath, Inc. +# Copryright (c) 2008-2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. # """ diff --git a/repomd/primaryxml.py b/repomd/primaryxml.py --- a/repomd/primaryxml.py +++ b/repomd/primaryxml.py @@ -1,5 +1,15 @@ # -# Copryright (c) 2008 rPath, Inc. +# Copryright (c) 2008-2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. # """ diff --git a/repomd/repomdxml.py b/repomd/repomdxml.py --- a/repomd/repomdxml.py +++ b/repomd/repomdxml.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/repomd/repository.py b/repomd/repository.py --- a/repomd/repository.py +++ b/repomd/repository.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/repomd/xmlcommon.py b/repomd/xmlcommon.py --- a/repomd/xmlcommon.py +++ b/repomd/xmlcommon.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/rpmutils/__init__.py b/rpmutils/__init__.py --- a/rpmutils/__init__.py +++ b/rpmutils/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/rpmutils/header.py b/rpmutils/header.py --- a/rpmutils/header.py +++ b/rpmutils/header.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/rpmutils/vercmp.py b/rpmutils/vercmp.py --- a/rpmutils/vercmp.py +++ b/rpmutils/vercmp.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/updatebot/__init__.py b/updatebot/__init__.py --- a/updatebot/__init__.py +++ b/updatebot/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this From elliot at rpath.com Mon Mar 15 17:04:38 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:38 +0000 Subject: mirrorball: more copyright updates Message-ID: <201003152104.o2FL4dra007702@scc.eng.rpath.com> changeset: 42bc1b643c24 user: Elliot Peele date: Wed, 04 Nov 2009 01:23:17 -0500 more copyright updates diff --git a/updatebot/cmdline/__init__.py b/updatebot/cmdline/__init__.py --- a/updatebot/cmdline/__init__.py +++ b/updatebot/cmdline/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/updatebot/cmdline/clientcfg.py b/updatebot/cmdline/clientcfg.py --- a/updatebot/cmdline/clientcfg.py +++ b/updatebot/cmdline/clientcfg.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/updatebot/cmdline/command.py b/updatebot/cmdline/command.py --- a/updatebot/cmdline/command.py +++ b/updatebot/cmdline/command.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this diff --git a/updatebot/cmdline/main.py b/updatebot/cmdline/main.py --- a/updatebot/cmdline/main.py +++ b/updatebot/cmdline/main.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2009 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this From elliot at rpath.com Mon Mar 15 17:04:40 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:40 +0000 Subject: mirrorball: refactor build modules a bit Message-ID: <201003152104.o2FL4fr1007732@scc.eng.rpath.com> changeset: 846817e8dc8d user: Elliot Peele date: Wed, 04 Nov 2009 01:54:27 -0500 refactor build modules a bit diff --git a/updatebot/build/__init__.py b/updatebot/build/__init__.py new file mode 100644 --- /dev/null +++ b/updatebot/build/__init__.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for building packages. +""" + +from updatebot.build.build import Builder diff --git a/updatebot/build.py b/updatebot/build/build.py rename from updatebot/build.py rename to updatebot/build/build.py --- a/updatebot/build.py +++ b/updatebot/build/build.py @@ -40,12 +40,14 @@ from rmake.cmdline import monitor from updatebot.lib import util -from updatebot import subscriber from updatebot.errors import JobFailedError from updatebot.errors import CommitFailedError from updatebot.errors import FailedToRetrieveChangesetError from updatebot.errors import ChangesetValidationFailedError +from updatebot.build.subscriber import Dispatcher +from updatebot.build.callbacks import StatusOnlyDisplay + log = logging.getLogger('updatebot.build') def jobInfoExceptionHandler(func): @@ -159,7 +161,7 @@ @return troveMap: dictionary of troveSpecs to built troves """ - dispatcher = subscriber.Dispatcher(self, 30) + dispatcher = Dispatcher(self, 30) return dispatcher.buildmany(troveSpecs) def buildsplitarch(self, troveSpecs): @@ -332,7 +334,7 @@ # Watch build, wait for completion monitor.monitorJob(self._helper.client, jobId, - exitOnFinish=True, displayClass=_StatusOnlyDisplay) + exitOnFinish=True, displayClass=StatusOnlyDisplay) def _sanityCheckJob(self, jobId): """ @@ -622,34 +624,3 @@ def _registerCommand(self, *args, **kwargs): 'Fake rMake hook' - - -class _StatusOnlyDisplay(monitor.JobLogDisplay): - """ - Display only job and trove status. No log output. - - Copied from bob3 - """ - - # R0901 - Too many ancestors - # pylint: disable-msg=R0901 - - def _troveLogUpdated(self, (jobId, troveTuple), state, status): - """ - Don't care about trove logs - """ - - def _trovePreparingChroot(self, (jobId, troveTuple), host, path): - """ - Don't care about resolving/installing chroot - """ - - def _tailBuildLog(self, jobId, troveTuple): - """ - Don't care about the build log - """ - - def _stopTailing(self, jobId, troveTuple): - """ - Don't care about the build log - """ diff --git a/updatebot/build/callbacks.py b/updatebot/build/callbacks.py new file mode 100644 --- /dev/null +++ b/updatebot/build/callbacks.py @@ -0,0 +1,107 @@ +# +# Copyright (c) 2008-2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +from rmake.build import buildjob +from rmake.build import buildtrove +from rmake.cmdline import monitor + +class StatusOnlyDisplay(monitor.JobLogDisplay): + """ + Display only job and trove status. No log output. + + Copied from bob3 + """ + + # R0901 - Too many ancestors + # pylint: disable-msg=R0901 + + def _troveLogUpdated(self, (jobId, troveTuple), state, status): + """ + Don't care about trove logs + """ + + def _trovePreparingChroot(self, (jobId, troveTuple), host, path): + """ + Don't care about resolving/installing chroot + """ + + def _tailBuildLog(self, jobId, troveTuple): + """ + Don't care about the build log + """ + + def _stopTailing(self, jobId, troveTuple): + """ + Don't care about the build log + """ + + +class JobMonitorCallback(monitor.JobLogDisplay): + """ + Monitor job status changes. + """ + + monitorStates = ( + buildjob.JOB_STATE_STARTED, + buildjob.JOB_STATE_BUILT, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_COMMITTING, + buildjob.JOB_STATE_COMMITTED, + ) + + def __init__(self, status, *args, **kwargs): + # override showBuildLogs since we don't handle writing to the out pipe + kwargs['showBuildLogs'] = False + + monitor.JobLogDisplay.__init__(self, *args, **kwargs) + self._status = status + + def _msg(self, msg, *args): + self._status.put((MessageTypes.LOG, msg)) + + def _data(self, data): + self._status.put((MessageTypes.DATA, data)) + + def _jobStateUpdated(self, jobId, state, status): + monitor.JobLogDisplay(self, jobId, state, None) + if state in self.monitorStates: + self._data((jobId, state)) + + def _troveLogUpdated(self, (jobId, troveTuple), state, status): + pass + + def _jobTrovesSet(self, jobId, troveData): + pass + + def _tailBuildLog(self, jobId, troveTuple): + pass + + def _primeOutput(self, jobId): + # Override parents _primeOutput to avoid sending output to stdout via + # print. + logMark = 0 + while True: + newLogs = self.client.getJobLogs(jobId, logMark) + if not newLogs: + break + logMark += len(newLogs) + for (timeStamp, message, args) in newLogs: + self._msg('[%s] - %s' % (jobId, message)) + + BUILDING = buildtrove.TROVE_STATE_BUILDING + troveTups = self.client.listTrovesByState(jobId, BUILDING).get(BUILDING, []) + for troveTuple in troveTups: + self._tailBuildLog(jobId, troveTuple) + + monitor._AbstractDisplay._primeOutput(self, jobId) diff --git a/updatebot/build/common.py b/updatebot/build/common.py new file mode 100644 --- /dev/null +++ b/updatebot/build/common.py @@ -0,0 +1,142 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for common abstract classes. +""" + +import logging +from Queue import Queue +from Queue import Empty +from threading import Thread + +from updatebot.build.constants import ThreadTypes +from updatebot.build.constants import MessageTypes + +log = logging.getLogger('updatebot.build') + +class AbstractWorker(Thread): + """ + Abstract class for all worker nodes. + """ + + threadType = None + + def __init__(self, status, name=None): + Thread.__init__(self, name=name) + + self.status = status + self.workerId = None + + def run(self): + """ + Do work. + """ + + try: + self.work() + except Exception, e: + self.status.put((MessageTypes.THREAD_ERROR, + (self.threadType, self.workerId, e))) + + self.status.put((MessageTypes.THREAD_DONE, self.workerId)) + + def work(self): + """ + Stub for sub classes to implement. + """ + + raise NotImplementedError + + +class AbstractStatusMonitor(object): + """ + Abstract class for implementing monitoring classes. + """ + + workerClass = None + + def __init__(self, threadArgs): + if type(threadArgs) not in (list, tuple, set): + threadArgs = (threadArgs, ) + self._threadArgs = threadArgs + + self._status = Queue() + self._workers = {} + self._errors = [] + + def addJob(self, job): + """ + Add a job to the worker pool. + """ + + args = list(self._threadArgs) + args.append(job) + + threadName = ('%s Worker' + % ThreadTypes.names[self.workerClass.threadType]) + worker = self.workerClass(self._status, args, name=threadName) + self._workers[job] = worker + worker.daemon = True + worker.start() + + def getStatus(self): + """ + Process all messages in the status queue, returning any data messages. + """ + + data = [] + while True: + try: + msg = self._status.get_nowait() + except Empty: + break + + data.extend(self._processMessage(msg)) + + return data + + def getErrors(self): + """ + Return any errors found while status was being processed. + """ + + errors = self._errors + self._errors = [] + return errors + + def _processMessage(self, msg): + """ + Handle messages. + """ + + data = [] + mtype, payload = msg + + if mtype == MessageTypes.LOG: + log.info(payload) + elif mtype == MessageTypes.DATA: + data.append(payload) + elif mtype == MessageTypes.THREAD_DONE: + job = payload + #assert not self._workers[job].isAlive() + del self._workers[job] + elif mtype == MessageTypes.THREAD_ERROR: + threadType, job, error = payload + #assert not self._workers[job].isAlive() + #raise error + log.error('[%s] FAILED with exception: %s' % (job, error)) + self._errors.append((job, error)) + + return data diff --git a/updatebot/build/constants.py b/updatebot/build/constants.py new file mode 100644 --- /dev/null +++ b/updatebot/build/constants.py @@ -0,0 +1,53 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Build system contants. +""" + +class MessageTypes(object): + """ + Class for storing message type constants. + """ + + LOG = 0 + DATA = 1 + THREAD_DONE = 2 + THREAD_ERROR = 3 + + +class ThreadTypes(object): + """ + Class for storing thread types. + """ + + START = 0 + MONITOR = 1 + COMMIT = 2 + + names = { + START: 'Start', + MONITOR: 'Monitor', + COMMIT: 'Commit', + } + + +class JobStatus(object): + """ + Job states in dispatcher. + """ + + JOB_NOT_STARTED = -1 + ERROR_MONITOR_FAILURE = -2 + ERROR_COMITTER_FAILURE = -3 diff --git a/updatebot/subscriber.py b/updatebot/build/dispatcher.py rename from updatebot/subscriber.py rename to updatebot/build/dispatcher.py --- a/updatebot/subscriber.py +++ b/updatebot/build/dispatcher.py @@ -17,341 +17,17 @@ monitoring. """ -import copy import time import logging -import itertools -from threading import Thread -from Queue import Queue, Empty from rmake.build import buildjob -from rmake.build import buildtrove -from rmake.cmdline import monitor -log = logging.getLogger('updatebot.subscriber') +from updatebot.build.monitor import JobStarter +from updatebot.build.monitor import JobMonitor +from updatebot.build.monitor import Jobcommitter +from updatebot.build.constants import JobStatus -class MessageTypes(object): - """ - Class for storing message type constants. - """ - - LOG = 0 - DATA = 1 - THREAD_DONE = 2 - THREAD_ERROR = 3 - - -class ThreadTypes(object): - """ - Class for storing thread types. - """ - - START = 0 - MONITOR = 1 - COMMIT = 2 - - names = { - START: 'Start', - MONITOR: 'Monitor', - COMMIT: 'Commit', - } - - -class JobStatus(object): - """ - Job states in dispatcher. - """ - - JOB_NOT_STARTED = -1 - ERROR_MONITOR_FAILURE = -2 - ERROR_COMITTER_FAILURE = -3 - - -class JobMonitorCallback(monitor.JobLogDisplay): - """ - Monitor job status changes. - """ - - monitorStates = ( - buildjob.JOB_STATE_STARTED, - buildjob.JOB_STATE_BUILT, - buildjob.JOB_STATE_FAILED, - buildjob.JOB_STATE_COMMITTING, - buildjob.JOB_STATE_COMMITTED, - ) - - def __init__(self, status, *args, **kwargs): - # override showBuildLogs since we don't handle writing to the out pipe - kwargs['showBuildLogs'] = False - - monitor.JobLogDisplay.__init__(self, *args, **kwargs) - self._status = status - - def _msg(self, msg, *args): - self._status.put((MessageTypes.LOG, msg)) - - def _data(self, data): - self._status.put((MessageTypes.DATA, data)) - - def _jobStateUpdated(self, jobId, state, status): - monitor.JobLogDisplay(self, jobId, state, None) - if state in self.monitorStates: - self._data((jobId, state)) - - def _troveLogUpdated(self, (jobId, troveTuple), state, status): - pass - - def _jobTrovesSet(self, jobId, troveData): - pass - - def _tailBuildLog(self, jobId, troveTuple): - pass - - def _primeOutput(self, jobId): - # Override parents _primeOutput to avoid sending output to stdout via - # print. - logMark = 0 - while True: - newLogs = self.client.getJobLogs(jobId, logMark) - if not newLogs: - break - logMark += len(newLogs) - for (timeStamp, message, args) in newLogs: - self._msg('[%s] - %s' % (jobId, message)) - - BUILDING = buildtrove.TROVE_STATE_BUILDING - troveTups = self.client.listTrovesByState(jobId, BUILDING).get(BUILDING, []) - for troveTuple in troveTups: - self._tailBuildLog(jobId, troveTuple) - - monitor._AbstractDisplay._primeOutput(self, jobId) - - -class AbstractWorker(Thread): - """ - Abstract class for all worker nodes. - """ - - threadType = None - - def __init__(self, status, name=None): - Thread.__init__(self, name=name) - - self.status = status - self.workerId = None - - def run(self): - """ - Do work. - """ - - try: - self.work() - except Exception, e: - self.status.put((MessageTypes.THREAD_ERROR, - (self.threadType, self.workerId, e))) - - self.status.put((MessageTypes.THREAD_DONE, self.workerId)) - - def work(self): - """ - Stub for sub classes to implement. - """ - - raise NotImplementedError - - -class StartWorker(AbstractWorker): - """ - Worker thread for starting jobs and reporting status. - """ - - threadType = ThreadTypes.START - - def __init__(self, status, (builder, trove), name=None): - AbstractWorker.__init__(self, status, name=name) - - self.builder = builder - self.trove = trove - self.workerId = self.trove - - def work(self): - """ - Start the specified build and report jobId. - """ - - jobId = self.builder.start((self.trove, )) - self.status.put((MessageTypes.DATA, (jobId, self.trove))) - - -class MonitorWorker(AbstractWorker): - """ - Worker thread for monitoring jobs and reporting status. - """ - - threadType = ThreadTypes.MONITOR - displayClass = JobMonitorCallback - - def __init__(self, status, (rmakeClient, jobId), name=None): - AbstractWorker.__init__(self, status, name=name) - - self.client = rmakeClient - self.jobId = jobId - self.workerId = jobId - - def work(self): - """ - Watch the monitor queue and monitor any available jobs. - """ - - # FIXME: This is copied from rmake.cmdlin.monitor for the most part - # because I need to pass extra args to the display class. - - uri, tmpPath = monitor._getUri(self.client) - - try: - display = self.displayClass(self.status, self.client, - showBuildLogs=False, exitOnFinish=True) - client = self.client.listenToEvents(uri, self.jobId, display, - showTroveDetails=False, - serve=True) - return client - finally: - if tmpPath: - os.remove(tmpPath) - - -class CommitWorker(AbstractWorker): - """ - Worker thread for committing jobs. - """ - - threadType = ThreadTypes.COMMIT - - def __init__(self, status, (builder, jobId), name=None): - AbstractWorker.__init__(self, status, name=name) - - self.builder = builder - self.jobId = jobId - self.workerId = jobId - - def work(self): - """ - Commit the specified job. - """ - - result = self.builder.commit(self.jobId) - self.status.put((MessageTypes.DATA, (self.jobId, result))) - - -class AbstractStatusMonitor(object): - """ - Abstract class for implementing monitoring classes. - """ - - workerClass = None - - def __init__(self, threadArgs): - if type(threadArgs) not in (list, tuple, set): - threadArgs = (threadArgs, ) - self._threadArgs = threadArgs - - self._status = Queue() - self._workers = {} - self._errors = [] - - def addJob(self, job): - """ - Add a job to the worker pool. - """ - - args = list(self._threadArgs) - args.append(job) - - threadName = ('%s Worker' - % ThreadTypes.names[self.workerClass.threadType]) - worker = self.workerClass(self._status, args, name=threadName) - self._workers[job] = worker - worker.daemon = True - worker.start() - - def getStatus(self): - """ - Process all messages in the status queue, returning any data messages. - """ - - data = [] - while True: - try: - msg = self._status.get_nowait() - except Empty: - break - - data.extend(self._processMessage(msg)) - - return data - - def getErrors(self): - """ - Return any errors found while status was being processed. - """ - - errors = self._errors - self._errors = [] - return errors - - def _processMessage(self, msg): - """ - Handle messages. - """ - - data = [] - mtype, payload = msg - - if mtype == MessageTypes.LOG: - log.info(payload) - elif mtype == MessageTypes.DATA: - data.append(payload) - elif mtype == MessageTypes.THREAD_DONE: - job = payload - #assert not self._workers[job].isAlive() - del self._workers[job] - elif mtype == MessageTypes.THREAD_ERROR: - threadType, job, error = payload - #assert not self._workers[job].isAlive() - #raise error - log.error('[%s] FAILED with exception: %s' % (job, error)) - self._errors.append((job, error)) - - return data - - -class JobStarter(AbstractStatusMonitor): - """ - Abstraction around threaded starter model. - """ - - workerClass = StartWorker - startJob = AbstractStatusMonitor.addJob - - -class JobMonitor(AbstractStatusMonitor): - """ - Abstraction around threaded monitoring model. - """ - - workerClass = MonitorWorker - monitorJob = AbstractStatusMonitor.addJob - - -class JobCommitter(AbstractStatusMonitor): - """ - Abstraction around threaded commit model. - """ - - workerClass = CommitWorker - commitJob = AbstractStatusMonitor.addJob - +log = logging.getLogger('updatebot.build') class Dispatcher(object): """ diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py new file mode 100644 --- /dev/null +++ b/updatebot/build/monitor.py @@ -0,0 +1,133 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for managing monitors. +""" + +from updatebot.build.common import AbstractWorker +from updatebot.build.common import AbstractStatusMonitor + +from updatebot.build.constants import ThreadTypes +from updatebot.build.constants import MessageTypes +from updatebot.build.callaback import JobMonitorCallback + +class StartWorker(AbstractWorker): + """ + Worker thread for starting jobs and reporting status. + """ + + threadType = ThreadTypes.START + + def __init__(self, status, (builder, trove), name=None): + AbstractWorker.__init__(self, status, name=name) + + self.builder = builder + self.trove = trove + self.workerId = self.trove + + def work(self): + """ + Start the specified build and report jobId. + """ + + jobId = self.builder.start((self.trove, )) + self.status.put((MessageTypes.DATA, (jobId, self.trove))) + + +class MonitorWorker(AbstractWorker): + """ + Worker thread for monitoring jobs and reporting status. + """ + + threadType = ThreadTypes.MONITOR + displayClass = JobMonitorCallback + + def __init__(self, status, (rmakeClient, jobId), name=None): + AbstractWorker.__init__(self, status, name=name) + + self.client = rmakeClient + self.jobId = jobId + self.workerId = jobId + + def work(self): + """ + Watch the monitor queue and monitor any available jobs. + """ + + # FIXME: This is copied from rmake.cmdlin.monitor for the most part + # because I need to pass extra args to the display class. + + uri, tmpPath = monitor._getUri(self.client) + + try: + display = self.displayClass(self.status, self.client, + showBuildLogs=False, exitOnFinish=True) + client = self.client.listenToEvents(uri, self.jobId, display, + showTroveDetails=False, + serve=True) + return client + finally: + if tmpPath: + os.remove(tmpPath) + + +class CommitWorker(AbstractWorker): + """ + Worker thread for committing jobs. + """ + + threadType = ThreadTypes.COMMIT + + def __init__(self, status, (builder, jobId), name=None): + AbstractWorker.__init__(self, status, name=name) + + self.builder = builder + self.jobId = jobId + self.workerId = jobId + + def work(self): + """ + Commit the specified job. + """ + + result = self.builder.commit(self.jobId) + self.status.put((MessageTypes.DATA, (self.jobId, result))) + + +class JobStarter(AbstractStatusMonitor): + """ + Abstraction around threaded starter model. + """ + + workerClass = StartWorker + startJob = AbstractStatusMonitor.addJob + + +class JobMonitor(AbstractStatusMonitor): + """ + Abstraction around threaded monitoring model. + """ + + workerClass = MonitorWorker + monitorJob = AbstractStatusMonitor.addJob + + +class JobCommitter(AbstractStatusMonitor): + """ + Abstraction around threaded commit model. + """ + + workerClass = CommitWorker + commitJob = AbstractStatusMonitor.addJob From elliot at rpath.com Mon Mar 15 17:04:42 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:42 +0000 Subject: mirrorball: fix incorrect import Message-ID: <201003152104.o2FL4hee007761@scc.eng.rpath.com> changeset: dca3f4f5514e user: Elliot Peele date: Wed, 04 Nov 2009 01:55:07 -0500 fix incorrect import diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -45,7 +45,7 @@ from updatebot.errors import FailedToRetrieveChangesetError from updatebot.errors import ChangesetValidationFailedError -from updatebot.build.subscriber import Dispatcher +from updatebot.build.dispatcher import Dispatcher from updatebot.build.callbacks import StatusOnlyDisplay log = logging.getLogger('updatebot.build') From elliot at rpath.com Mon Mar 15 17:04:44 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:44 +0000 Subject: mirrorball: fix incorrect imports Message-ID: <201003152104.o2FL4jjT007788@scc.eng.rpath.com> changeset: 0414928c29c4 user: Elliot Peele date: Wed, 04 Nov 2009 09:28:40 -0500 fix incorrect imports diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -24,7 +24,7 @@ from updatebot.build.monitor import JobStarter from updatebot.build.monitor import JobMonitor -from updatebot.build.monitor import Jobcommitter +from updatebot.build.monitor import JobCommitter from updatebot.build.constants import JobStatus log = logging.getLogger('updatebot.build') diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py --- a/updatebot/build/monitor.py +++ b/updatebot/build/monitor.py @@ -21,7 +21,7 @@ from updatebot.build.constants import ThreadTypes from updatebot.build.constants import MessageTypes -from updatebot.build.callaback import JobMonitorCallback +from updatebot.build.callbacks import JobMonitorCallback class StartWorker(AbstractWorker): """ From elliot at rpath.com Mon Mar 15 17:04:46 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:46 +0000 Subject: mirrorball: fix typo Message-ID: <201003152104.o2FL4llI007820@scc.eng.rpath.com> changeset: 8f05655f735a user: Elliot Peele date: Wed, 04 Nov 2009 09:29:12 -0500 fix typo diff --git a/updatebot/log.py b/updatebot/log.py --- a/updatebot/log.py +++ b/updatebot/log.py @@ -38,7 +38,7 @@ maxBytes=logSize, backupCount=5) - streamFomatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') + streamFormatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') fileFormatter = logging.Formatter('%(asctime)s %(levelname)s ' '%(name)s %(message)s') From elliot at rpath.com Mon Mar 15 17:04:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:49 +0000 Subject: mirrorball: fix typo Message-ID: <201003152104.o2FL4nFj007847@scc.eng.rpath.com> changeset: 135d8f9ed65b user: Elliot Peele date: Wed, 04 Nov 2009 09:45:22 -0500 fix typo diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -103,7 +103,7 @@ else: self._saveChangeSets = False - self._santiyCheckChangesets = self._cfg.sanityCheckChangesets + self._sanityCheckChangesets = self._cfg.sanityCheckChangesets self._sanityCheckCommits = self._cfg.sanityCheckCommits # Get default pluginDirs from the rmake cfg object, setup the plugin From elliot at rpath.com Mon Mar 15 17:04:50 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:50 +0000 Subject: mirrorball: add missing imports Message-ID: <201003152104.o2FL4p2G007876@scc.eng.rpath.com> changeset: b25a95cdd6a6 user: Elliot Peele date: Wed, 04 Nov 2009 09:45:34 -0500 add missing imports diff --git a/updatebot/build/callbacks.py b/updatebot/build/callbacks.py --- a/updatebot/build/callbacks.py +++ b/updatebot/build/callbacks.py @@ -16,6 +16,8 @@ from rmake.build import buildtrove from rmake.cmdline import monitor +from updatebot.build.constants import MessageTypes + class StatusOnlyDisplay(monitor.JobLogDisplay): """ Display only job and trove status. No log output. diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py --- a/updatebot/build/monitor.py +++ b/updatebot/build/monitor.py @@ -16,6 +16,8 @@ Module for managing monitors. """ +from rmake.cmdline import monitor + from updatebot.build.common import AbstractWorker from updatebot.build.common import AbstractStatusMonitor From elliot at rpath.com Mon Mar 15 17:04:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:53 +0000 Subject: mirrorball: save dir has been pushed down into the builder class Message-ID: <201003152104.o2FL4rUq007903@scc.eng.rpath.com> changeset: f5add4e64a5a user: Elliot Peele date: Wed, 04 Nov 2009 09:46:17 -0500 save dir has been pushed down into the builder class diff --git a/scripts/header.py b/scripts/header.py --- a/scripts/header.py +++ b/scripts/header.py @@ -46,7 +46,7 @@ cfg = config.UpdateBotConfig() cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) -builder = build.Builder(cfg, saveChangeSets=tempfile.mkdtemp()) +builder = build.Builder(cfg) def displayTrove(nvf): flavor = '' From elliot at rpath.com Mon Mar 15 17:04:55 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:55 +0000 Subject: mirrorball: fix spacing Message-ID: <201003152104.o2FL4t9e007935@scc.eng.rpath.com> changeset: 37e3a7d55997 user: Elliot Peele date: Wed, 04 Nov 2009 11:12:35 -0500 fix spacing diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -408,7 +408,7 @@ fassert(fileObj.flags.isConfig() or fileObj.flags.isInitialContents(), why='RPM config file %s is neither config file ' - 'nor initialcontents' %fpath) + 'nor initialcontents' % fpath) elif isinstance(fileObj, files.Directory): fassert(stat.S_ISDIR(rMode), fpath) From elliot at rpath.com Mon Mar 15 17:04:57 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:57 +0000 Subject: mirrorball: Add specific information to changeset validation failure errors Message-ID: <201003152104.o2FL4vVr007963@scc.eng.rpath.com> changeset: 90176bc488ac user: Michael K. Johnson date: Wed, 04 Nov 2009 11:42:38 -0500 Add specific information to changeset validation failure errors diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -382,6 +382,19 @@ raise ChangesetValidationFailedError(jobId=jobId, why=why, path=path) + def devassert(path, rDev, fileObj): + minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 + major = (rDev >> 8) & 0xfff + fassert(fileObj.devt.major() == major, path, + 'Device major mismatch: RPM %d != Conary %d' + %(major, fileObj.devt.major())) + fassert(fileObj.devt.minor() == minor, path, + 'Device minor mismatch: RPM %d != Conary %d' + %(minor, fileObj.devt.minor())) + fassert(not fileObj.flags.isPayload(), path, + 'Device file is marked as payload') + + for fileInfo, fileObj in zip(fileList, fileObjs): fpath = fileInfo[1] foundFiles[fpath] = True @@ -392,69 +405,91 @@ # file metadata verification if (rUser != fileObj.inode.owner() or - rGroup != fileObj.inode.group() or - stat.S_IMODE(rMode) != fileObj.inode.perms()): - fassert(False, fpath) + rGroup != fileObj.inode.group()): + fassert(False, fpath, + 'User/Group mismatch: RPM %s:%s != Conary %s:%s' + %(rUser, rGroup, + fileObj.inode.owner(), fileObj.inode.group())) + + if stat.S_IMODE(rMode) != fileObj.inode.perms(): + fassert(False, fpath, + 'Mode mismatch: RPM 0%o != Conary 0%o' + %(rMode, fileObj.inode.perms())) if isinstance(fileObj, files.RegularFile): if not stat.S_ISREG(rMode): - fassert(False, fpath) + fassert(False, fpath, + 'Conary Regular file has non-regular mode 0%o' + %rMode) # RPM config flag mapping if rFlags & rpmhelper.RPMFILE_CONFIG: if fileObj.linkGroup() or not fileObj.contents.size(): - fassert(fileObj.flags.isInitialContents(), fpath) + fassert(fileObj.flags.isInitialContents(), fpath, + 'RPM config file without size or' + ' hardlinked is not InitialContents') else: fassert(fileObj.flags.isConfig() or - fileObj.flags.isInitialContents(), - why='RPM config file %s is neither config file ' - 'nor initialcontents' % fpath) + fileObj.flags.isInitialContents(), fpath, + 'RPM config file is neither Config file ' + 'nor InitialContents') elif isinstance(fileObj, files.Directory): - fassert(stat.S_ISDIR(rMode), fpath) - fassert(not fileObj.flags.isPayload(), fpath) + fassert(stat.S_ISDIR(rMode), fpath, + 'Conary directory has non-directory RPM mode 0%o' + %rMode) + fassert(not fileObj.flags.isPayload(), fpath, + 'Conary directory marked as payload') + elif isinstance(fileObj, files.CharacterDevice): - fassert(stat.S_ISCHR(rMode), fpath) + fassert(stat.S_ISCHR(rMode), fpath, + 'Conary CharacterDevice has RPM non-character-device' + ' mode 0%o' %rMode) + devassert(fpath, rDev, fileObj) - minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 - major = (rDev >> 8) & 0xfff - fassert(fileObj.devt.major() == major, fpath) - fassert(fileObj.devt.minor() == minor, fpath) + elif isinstance(fileObj, files.BlockDevice): + fassert(stat.S_ISBLK(rMode), fpath, + 'Conary BlockDevice has RPM non-block-device' + ' mode 0%o' %rMode) + devassert(fpath, rDev, fileObj) - fassert(not fileObj.flags.isPayload()) - elif isinstance(fileObj, files.BlockDevice): - fassert(stat.S_ISBLK(rMode), fpath) - - minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 - major = (rDev >> 8) & 0xfff - fassert(fileObj.devt.major() == major, fpath) - fassert(fileObj.devt.minor() == minor, fpath) - - fassert(not fileObj.flags.isPayload(), fpath) elif isinstance(fileObj, files.NamedPipe): fassert(stat.S_ISFIFO(rMode), fpath) + 'Conary NamedPipe has RPM non-named-pipe' + ' mode 0%o' %rMode) + fassert(not fileObj.flags.isPayload(), fpath, + 'NamedPipe file is marked as payload') - fassert(not fileObj.flags.isPayload(), fpath) elif isinstance(fileObj, files.SymbolicLink): fassert(stat.S_ISLNK(rMode), fpath) - fassert(fileObj.target() == rLinkto, fpath) + 'Conary SymbolicLink has RPM non-symlink' + ' mode 0%o' %rMode) + fassert(fileObj.target() == rLinkto, fpath, + 'Symlink target mismatch:' + ' RPM %s != Conary %s' + %(rLinkto, fileObj.target())) + fassert(not fileObj.flags.isPayload(), fpath, + 'SymbolicLink file is marked as payload') - fassert(not fileObj.flags.isPayload(), fpath) else: # unhandled file type - fassert(False, fpath) + fassert(False, fpath, + 'Unknown Conary file type %r' %fileObj) # Now, some tests based on the contents of the RPM header if not stat.S_ISDIR(rMode) and rFlags & rpmhelper.RPMFILE_GHOST: - fassert(fileObj.flags.isInitialContents(), fpath) + fassert(fileObj.flags.isInitialContents(), fpath, + 'RPM ghost non-directory is not InitialContents') if not rVflags: # %doc -- CNY-3254 - fassert(not fileObj.flags.isInitialContents(), fpath) + fassert(not fileObj.flags.isInitialContents(), fpath, + 'RPM %%doc file is InitialContents') # Make sure we have explicitly checked every file in the RPM uncheckedFiles = [x[0] for x in foundFiles.iteritems() if not x[1]] - fassert( not uncheckedFiles, str(uncheckedFiles) ) + fassert(not uncheckedFiles, str(uncheckedFiles), + 'Files contained in RPM not contained in Conary changeset') def _sanityCheckChangeSet(self, csFile, jobId): From elliot at rpath.com Mon Mar 15 17:04:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:04:59 +0000 Subject: mirrorball: collect all validation errors for a changeset rather than halting on first validation error Message-ID: <201003152104.o2FL4xsg007991@scc.eng.rpath.com> changeset: 2e91fa836c5a user: Michael K. Johnson date: Wed, 04 Nov 2009 12:19:32 -0500 collect all validation errors for a changeset rather than halting on first validation error diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -375,12 +375,13 @@ h[rpmhelper.FILELINKTOS], ))) + errors = [] + foundFiles = dict.fromkeys(rpmFileList) def fassert(test, path='', why=''): if not test: - raise ChangesetValidationFailedError(jobId=jobId, why=why, - path=path) + errors.append((path, why)) def devassert(path, rDev, fileObj): minor = rDev & 0xff | (rDev >> 12) & 0xffffff00 @@ -491,6 +492,12 @@ fassert(not uncheckedFiles, str(uncheckedFiles), 'Files contained in RPM not contained in Conary changeset') + if errors: + raise ChangesetValidationFailedError(jobId=jobId, + reason='\n'.join([ + '%s: %s' %(x, y) for x, y in errors + ])) + def _sanityCheckChangeSet(self, csFile, jobId): """ diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -64,9 +64,9 @@ the built changeset. """ - _params = ['jobId', 'why', 'path'] + _params = ['jobId', 'reason'] _template = ('Changeset from rmake job %(jobId)s failed to pass ' - 'validation for path %(path)s: %(why)s') + 'validation because:\n%(reason)s') class JobFailedError(UpdateBotError): From elliot at rpath.com Mon Mar 15 17:05:01 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:01 +0000 Subject: mirrorball: Fix two syntax errors Message-ID: <201003152105.o2FL51VR008019@scc.eng.rpath.com> changeset: c85a452275fb user: Michael K. Johnson date: Wed, 04 Nov 2009 12:45:28 -0500 Fix two syntax errors diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -455,14 +455,14 @@ devassert(fpath, rDev, fileObj) elif isinstance(fileObj, files.NamedPipe): - fassert(stat.S_ISFIFO(rMode), fpath) + fassert(stat.S_ISFIFO(rMode), fpath, 'Conary NamedPipe has RPM non-named-pipe' ' mode 0%o' %rMode) fassert(not fileObj.flags.isPayload(), fpath, 'NamedPipe file is marked as payload') elif isinstance(fileObj, files.SymbolicLink): - fassert(stat.S_ISLNK(rMode), fpath) + fassert(stat.S_ISLNK(rMode), fpath, 'Conary SymbolicLink has RPM non-symlink' ' mode 0%o' %rMode) fassert(fileObj.target() == rLinkto, fpath, From elliot at rpath.com Mon Mar 15 17:05:03 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:03 +0000 Subject: mirrorball: skip validation if the path in question is '/' Message-ID: <201003152105.o2FL54St008050@scc.eng.rpath.com> changeset: 5ba17a503e56 user: Elliot Peele date: Wed, 04 Nov 2009 14:52:06 -0500 skip validation if the path in question is '/' diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -394,7 +394,6 @@ %(minor, fileObj.devt.minor())) fassert(not fileObj.flags.isPayload(), path, 'Device file is marked as payload') - for fileInfo, fileObj in zip(fileList, fileObjs): fpath = fileInfo[1] @@ -488,7 +487,8 @@ 'RPM %%doc file is InitialContents') # Make sure we have explicitly checked every file in the RPM - uncheckedFiles = [x[0] for x in foundFiles.iteritems() if not x[1]] + uncheckedFiles = [x[0] for x in foundFiles.iteritems() + if not x[1] and x[0] != '/' ] fassert(not uncheckedFiles, str(uncheckedFiles), 'Files contained in RPM not contained in Conary changeset') From elliot at rpath.com Mon Mar 15 17:05:06 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:06 +0000 Subject: mirrorball: add methods for managing versions and errata state info Message-ID: <201003152105.o2FL56sd008079@scc.eng.rpath.com> changeset: caea5a634574 user: Elliot Peele date: Wed, 04 Nov 2009 15:28:57 -0500 add methods for managing versions and errata state info diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -356,6 +356,49 @@ # add file to the source compoent self._addFile(recipeDir, 'buildrequires') + def getVersion(self, pkgname): + """ + Get the version of the specified package if this package has a version + file in the source component, otherwise return None. + @param pkgname: name of hte package to edit + @type pkgname: string + """ + + log.info('getting version info for %s' % pkgname) + + recipeDir = self._edit(pkgname) + versionFileName = util.join(recipeDir, 'version') + + if not os.path.exists(versionFileName): + return None + + version = open(versionFileName).read().strip() + return version + + def setVersion(self, pkgname, version): + """ + Set the version of the specified package, for this to be meaningful + there must be a factory that consumes this data. + @param pkgname: name of hte package to edit + @type pkgname: string + @param version: upstream version of the package, required to be a valid + conary version. + @type version: string + """ + + log.info('setting version info for %s' % pkgname) + + recipeDir = self._edit(pkgname) + versionFileName = util.join(recipeDir, 'version') + + # write version info + versionfh = open(versionFileName, 'w') + versionfh.write(version) + versionfh.close() + + # make sure version file has been added to package + self._addFile(recipeDir, 'version') + def commit(self, pkgname, commitMessage=''): """ Commit the cached checkout of a source component. diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -333,6 +333,41 @@ #self._commit(recipeDir, commitMessage='automated group update') + def getErrataState(self, pkgname): + """ + Get the contents of the errata state file from the specified package, + if file does not exist, return None. + """ + + log.info('getting errata state information from %s' % pkgname) + + recipeDir = self._edit(pkgname) + stateFileName = util.join(recipeDir, 'erratastate') + + if not os.path.exists(stateFileName): + return None + + state = open(stateFileName).read().strip() + return state + + def setErrataState(self, pkgname, state): + """ + Set the current errata state for the given package. + """ + + log.info('storing errata state information in %s' % pkgname) + + recipeDir = self._edit(pkgname) + stateFileName = util.join(recipeDir, 'erratastate') + + # write state info + statefh = open(stateFileName, 'w') + statefh.write(state) + statefh.close() + + # make sure state file is part of source trove + self._adFile('erratastate') + class AbstractModel(object): """ From elliot at rpath.com Mon Mar 15 17:05:08 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:08 +0000 Subject: mirrorball: handle new states Message-ID: <201003152105.o2FL58qC008106@scc.eng.rpath.com> changeset: 977ab5bd2573 user: Elliot Peele date: Wed, 04 Nov 2009 15:30:25 -0500 handle new states diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -36,13 +36,15 @@ """ _completed = ( - -2, + JobStatus.ERROR_MONITOR_FAILURE, + JobStatus.ERROR_COMITTER_FAILURE, buildjob.JOB_STATE_FAILED, buildjob.JOB_STATE_COMMITTED, ) _slotdone = ( - -2, + JobStatus.ERROR_MONITOR_FAILURE, + JobStatus.ERROR_COMITTER_FAILURE, buildjob.JOB_STATE_FAILED, buildjob.JOB_STATE_BUILT, ) From elliot at rpath.com Mon Mar 15 17:05:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:10 +0000 Subject: mirrorball: remove '/' workaround Message-ID: <201003152105.o2FL5Ae1008138@scc.eng.rpath.com> changeset: 9e5367ef32a5 user: Elliot Peele date: Wed, 04 Nov 2009 15:31:16 -0500 remove '/' workaround diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -487,8 +487,7 @@ 'RPM %%doc file is InitialContents') # Make sure we have explicitly checked every file in the RPM - uncheckedFiles = [x[0] for x in foundFiles.iteritems() - if not x[1] and x[0] != '/' ] + uncheckedFiles = [x[0] for x in foundFiles.iteritems() if not x[1] ] fassert(not uncheckedFiles, str(uncheckedFiles), 'Files contained in RPM not contained in Conary changeset') From elliot at rpath.com Mon Mar 15 17:05:12 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:12 +0000 Subject: mirrorball: add methods to the group manager class for managing version and errata info Message-ID: <201003152105.o2FL5DZS008165@scc.eng.rpath.com> changeset: ee43c05fc68f user: Elliot Peele date: Wed, 04 Nov 2009 15:57:27 -0500 add methods to the group manager class for managing version and errata info diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -256,6 +256,30 @@ # Unknown state. raise UnhandledPackageAdditionError(name=name) + @checkout + def setVersion(self, version): + """ + Set the version of the managed group. + """ + + self._helper.setVersion(self._sourceName, version) + + @checkout + def setErrataState(self, state): + """ + Set errata state info for the managed platform. + """ + + self._helper.setErrataState(self._sourceName, state) + + @checkout + def getErrataState(self): + """ + Get the errata state info. + """ + + self._helper.getErrataState(self._sourceName) + class GroupHelper(ConaryHelper): """ From elliot at rpath.com Mon Mar 15 17:05:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:15 +0000 Subject: mirrorball: small fixups Message-ID: <201003152105.o2FL5GNU008214@scc.eng.rpath.com> changeset: 7a854088a42a user: Elliot Peele date: Thu, 05 Nov 2009 00:30:48 -0500 small fixups diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -492,11 +492,10 @@ 'Files contained in RPM not contained in Conary changeset') if errors: - raise ChangesetValidationFailedError(jobId=jobId, - reason='\n'.join([ - '%s: %s' %(x, y) for x, y in errors - ])) - + raise ChangesetValidationFailedError(jobId=jobId, + reason='\n'.join([ + '%s: %s' %(x, y) for x, y in errors + ])) def _sanityCheckChangeSet(self, csFile, jobId): """ @@ -510,7 +509,7 @@ return cmp((apid, afid), (bpid, bfid)) newCs = changeset.ChangeSetFromFile(csFile) - log.info('comparing changeset to rpm capsules: %s' % csFile) + log.info('[%s] comparing changeset to rpm capsules' % jobId) capsules = [] for newTroveCs in newCs.iterNewTroveList(): @@ -571,6 +570,10 @@ self._sanityCheckRPMCapsule(jobId, fileList, fileObjs, capsuleFileContents) + # Make sure the changeset gets closed so that we don't run out of + # file descriptors. + del newCs + def _commitJob(self, jobId): """ Commit completed job. @@ -589,7 +592,7 @@ # Do the commit startTime = time.time() jobs = [ self._getJob(x) for x in jobIds ] - log.info('Starting commit of job %s', jobIdsStr) + log.info('[%s] starting commit' % jobIdsStr) if self._saveChangeSets: csfn = tempfile.mktemp(dir=self._saveChangeSets, suffix='.ccs') @@ -608,17 +611,17 @@ raise CommitFailedError(jobId=jobIdsStr, why=data) if writeToFile: - log.info('changeset saved to %s' % writeToFile) + log.info('[%s] changeset saved to %s' % (jobIdsStr, writeToFile)) if self._sanityCheckChangesets: self._sanityCheckChangeSet(writeToFile, jobIdsStr) - log.info('committing changeset to repository') + log.info('[%s] committing changeset to repository' % jobIdsStr) self._client.repos.commitChangeSetFile(writeToFile) if self._sanityCheckCommits: # sanity check repository - log.info('checking repository for sanity') + log.info('[%s] checking repository for sanity' % jobIdsStr) jobList = [] for job in data.itervalues(): for arch in job.itervalues(): @@ -629,7 +632,7 @@ cs = self._client.repos.createChangeSet(jobList, withFiles=True, withFileContents=False) - log.info('Commit of job %s completed in %.02f seconds', + log.info('[%s] commit completed in %.02f seconds', jobIdsStr, time.time() - startTime) troveMap = {} @@ -664,4 +667,6 @@ return ret def _registerCommand(self, *args, **kwargs): - 'Fake rMake hook' + """ + Fake rMake hook + """ diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -111,7 +111,9 @@ # free up the slot once the job is built if status in self._slotdone: self._slots += 1 - assert self._slots <= self._maxSlots + + if self._slots > self._maxSlots: + log.critical('slots is greater than maxSlots') # commit any jobs that are done building if status == buildjob.JOB_STATE_BUILT: @@ -135,19 +137,20 @@ # Wait for a bit before polling again. time.sleep(3) + # report failures + for job, error in self._failures: + log.error('[%s] failed with error: %s' % (job, error)) + results = {} for jobId, (trove, status, result) in self._jobs.iteritems(): # log failed jobs if status == buildjob.JOB_STATE_FAILED or not result: log.info('[%s] failed job: %s' % (jobId, trove)) + self._failures.append((jobId, status)) else: results.update(result) - # report failures - for job, error in self._failures: - log.error('[%s] failed with error: %s' % (job, error)) - - return results + return results, self._failures def _jobDone(self): if not len(self._jobs): From elliot at rpath.com Mon Mar 15 17:05:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:18 +0000 Subject: mirrorball: add save for just committing chnages Message-ID: <201003152105.o2FL5IFZ008242@scc.eng.rpath.com> changeset: af0a306ae49a user: Elliot Peele date: Thu, 05 Nov 2009 00:31:19 -0500 add save for just committing chnages diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -87,6 +87,8 @@ self._helper.setModel(self._sourceName, self._groups) self._checkedout = False + save = _commit + @commit def build(self): """ From elliot at rpath.com Mon Mar 15 17:05:21 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:21 +0000 Subject: mirrorball: ensure that monitor threads with properly quit when a job as reached the Message-ID: <201003152105.o2FL5LNP008282@scc.eng.rpath.com> changeset: 7fd3c17d8d4c user: Elliot Peele date: Thu, 05 Nov 2009 14:56:45 -0500 ensure that monitor threads with properly quit when a job as reached the correct state diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -242,6 +242,15 @@ ret = self._formatOutput(trvMap) return ret + def setCommitFailed(self, jobId): + """ + Sets the job as failed in rmake. + @param jobId: id of the build job to commit + @type jobId: integer + """ + + self._helper.client.commitFailed([jobId, ], {}) + def _formatInput(self, troveSpecs): """ Formats the list of troves provided into a job list for rMake. @@ -635,12 +644,13 @@ log.info('[%s] commit completed in %.02f seconds', jobIdsStr, time.time() - startTime) + self._helper.client.commitSucceeded(data) + troveMap = {} for troveTupleDict in data.itervalues(): for buildTroveTuple, committedList in troveTupleDict.iteritems(): troveMap[buildTroveTuple] = committedList - self._helper.client.commitSucceeded(data) return troveMap diff --git a/updatebot/build/callbacks.py b/updatebot/build/callbacks.py --- a/updatebot/build/callbacks.py +++ b/updatebot/build/callbacks.py @@ -62,6 +62,11 @@ buildjob.JOB_STATE_COMMITTED, ) + doneStates = ( + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_COMMITTED, + ) + def __init__(self, status, *args, **kwargs): # override showBuildLogs since we don't handle writing to the out pipe kwargs['showBuildLogs'] = False @@ -76,9 +81,11 @@ self._status.put((MessageTypes.DATA, data)) def _jobStateUpdated(self, jobId, state, status): - monitor.JobLogDisplay(self, jobId, state, None) + monitor.JobLogDisplay._jobStateUpdated(self, jobId, state, None) if state in self.monitorStates: self._data((jobId, state)) + if state in self.doneStates: + self.finished = True def _troveLogUpdated(self, (jobId, troveTuple), state, status): pass @@ -89,6 +96,12 @@ def _tailBuildLog(self, jobId, troveTuple): pass + def _serveLoopHook(self): + pass + + def _setFinished(self): + pass + def _primeOutput(self, jobId): # Override parents _primeOutput to avoid sending output to stdout via # print. diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -43,8 +43,6 @@ ) _slotdone = ( - JobStatus.ERROR_MONITOR_FAILURE, - JobStatus.ERROR_COMITTER_FAILURE, buildjob.JOB_STATE_FAILED, buildjob.JOB_STATE_BUILT, ) @@ -134,6 +132,9 @@ self._jobs[jobId][1] = JobStatus.ERROR_COMITTER_FAILURE self._failures.append((jobId, error)) + # Flag job as failed so that monitor worker will exit properly. + self._builder.setCommitFailed(jobId) + # Wait for a bit before polling again. time.sleep(3) From elliot at rpath.com Mon Mar 15 17:05:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:24 +0000 Subject: mirrorball: remove name that is identical to class name Message-ID: <201003152105.o2FL5PZF008315@scc.eng.rpath.com> changeset: b4c3b31b5cc5 user: Elliot Peele date: Tue, 10 Nov 2009 14:56:50 -0500 remove name that is identical to class name diff --git a/updatebot/build/common.py b/updatebot/build/common.py --- a/updatebot/build/common.py +++ b/updatebot/build/common.py @@ -33,8 +33,8 @@ threadType = None - def __init__(self, status, name=None): - Thread.__init__(self, name=name) + def __init__(self, status): + Thread.__init__(self) self.status = status self.workerId = None @@ -81,12 +81,15 @@ Add a job to the worker pool. """ + if job in self._workers: + log.critical('job already being monitored: %s' % (job, )) + import epdb; epdb.st() + args = list(self._threadArgs) args.append(job) - threadName = ('%s Worker' - % ThreadTypes.names[self.workerClass.threadType]) - worker = self.workerClass(self._status, args, name=threadName) + worker = self.workerClass(self._status, args) + self._workers[job] = worker worker.daemon = True worker.start() diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py --- a/updatebot/build/monitor.py +++ b/updatebot/build/monitor.py @@ -32,8 +32,8 @@ threadType = ThreadTypes.START - def __init__(self, status, (builder, trove), name=None): - AbstractWorker.__init__(self, status, name=name) + def __init__(self, status, (builder, trove)): + AbstractWorker.__init__(self, status) self.builder = builder self.trove = trove @@ -56,8 +56,8 @@ threadType = ThreadTypes.MONITOR displayClass = JobMonitorCallback - def __init__(self, status, (rmakeClient, jobId), name=None): - AbstractWorker.__init__(self, status, name=name) + def __init__(self, status, (rmakeClient, jobId)): + AbstractWorker.__init__(self, status) self.client = rmakeClient self.jobId = jobId @@ -92,8 +92,8 @@ threadType = ThreadTypes.COMMIT - def __init__(self, status, (builder, jobId), name=None): - AbstractWorker.__init__(self, status, name=name) + def __init__(self, status, (builder, jobId)): + AbstractWorker.__init__(self, status) self.builder = builder self.jobId = jobId From elliot at rpath.com Mon Mar 15 17:05:28 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:28 +0000 Subject: mirrorball: add reason argument Message-ID: <201003152105.o2FL5TVx008363@scc.eng.rpath.com> changeset: a5ce015d8952 user: Elliot Peele date: Tue, 10 Nov 2009 14:57:06 -0500 add reason argument diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -242,14 +242,17 @@ ret = self._formatOutput(trvMap) return ret - def setCommitFailed(self, jobId): + def setCommitFailed(self, jobId, reason=None): """ Sets the job as failed in rmake. @param jobId: id of the build job to commit @type jobId: integer + @param reason: message to be stored on the rmake server + @type resaon: str """ - self._helper.client.commitFailed([jobId, ], {}) + reason = reason and reason or 'none specified' + self._helper.client.commitFailed([jobId, ], reason) def _formatInput(self, troveSpecs): """ From elliot at rpath.com Mon Mar 15 17:05:33 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:33 +0000 Subject: mirrorball: on create always report failed jobs Message-ID: <201003152105.o2FL5Yc9008401@scc.eng.rpath.com> changeset: e077b9f25111 user: Elliot Peele date: Tue, 10 Nov 2009 15:01:06 -0500 on create always report failed jobs diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -128,6 +128,7 @@ log.warn('%s' % (pkg, )) else: # ReBuild all packages. + failed = () trvMap = self._builder.buildsplitarch(sorted(toBuild)) log.info('import completed successfully') log.info('imported %s source packages' % (len(toBuild), )) @@ -136,7 +137,7 @@ log.info('elapsed time %s' % (time.time() - start, )) - return trvMap + return trvMap, failed def update(self, force=None, updatePkgs=None): """ From elliot at rpath.com Mon Mar 15 17:05:36 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:36 +0000 Subject: mirrorball: add util functions for dealing with filedescriptors Message-ID: <201003152105.o2FL5ac3008430@scc.eng.rpath.com> changeset: 0b6272376e07 user: Elliot Peele date: Tue, 10 Nov 2009 15:01:55 -0500 add util functions for dealing with filedescriptors diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -20,6 +20,7 @@ # pylint: disable-msg=W0611 import os +import resource from conary.lib.util import rmtree from rpmutils import rpmvercmp @@ -138,3 +139,31 @@ basePath.startswith('kernel-module')): return True return False + +def setMaxRLimit(): + """ + Set the max file descriptors to the maximum allowed. + """ + + cur, max = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (max, max)) + return max + +def getRLimit(): + """ + Get the current number of file descriptors. + """ + + cur, max = resource.getrlimit(resource.RLIMIT_NOFILE) + return cur + +def getAvailableFileDescriptors(setMax=False): + """ + Get the number of available file descriptors. + """ + + openfds = len(os.listdir('/proc/self/fd')) + if setMax: + setMaxRLimit() + limit = getRLimit() + return limit - openfds From elliot at rpath.com Mon Mar 15 17:05:38 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:38 +0000 Subject: mirrorball: conary requires that files end in a newline Message-ID: <201003152105.o2FL5crp008480@scc.eng.rpath.com> changeset: 5e6b8b4c6789 user: Elliot Peele date: Tue, 10 Nov 2009 15:02:18 -0500 conary requires that files end in a newline diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -394,6 +394,10 @@ # write version info versionfh = open(versionFileName, 'w') versionfh.write(version) + + # source files must end in a trailing newline + versionfh.write('\n') + versionfh.close() # make sure version file has been added to package From elliot at rpath.com Mon Mar 15 17:05:41 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:41 +0000 Subject: mirrorball: set default byDefault to Non Message-ID: <201003152105.o2FL5fPi008522@scc.eng.rpath.com> changeset: 17b04be78162 user: Elliot Peele date: Tue, 10 Nov 2009 15:03:06 -0500 set default byDefault to Non diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -179,15 +179,18 @@ use = str source = str - def __init__(self, name=None, version=None, flavor=None, byDefault=True, - use=True, source=None): - + def __init__(self, name=None, version=None, flavor=None, byDefault=None, + use=None, source=None): self.name = name self.version = version - self.byDefault = byDefault and 1 or 0 self.source = source + if byDefault in (True, False): + self.byDefault = int(byDefault) + else: + self.byDefault = byDefault + if use in (True, False): self.use = int(use) else: From elliot at rpath.com Mon Mar 15 17:05:44 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:44 +0000 Subject: mirrorball: adapt script to new interface Message-ID: <201003152105.o2FL5i9O008564@scc.eng.rpath.com> changeset: 22bc8665656a user: Elliot Peele date: Tue, 10 Nov 2009 15:23:01 -0500 adapt script to new interface diff --git a/scripts/buildmany b/scripts/buildmany --- a/scripts/buildmany +++ b/scripts/buildmany @@ -16,7 +16,7 @@ label = cfg.topSourceGroup[1] for pkg in sys.argv[2:]: trvs.add((pkg, label, None)) -trvMap = builder.buildmany(trvs) +trvMap, failed = builder.buildmany(trvs) print "built:\n" From elliot at rpath.com Mon Mar 15 17:05:46 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:46 +0000 Subject: mirrorball: 1. batch commits if more than one job is ready to be committed Message-ID: <201003152105.o2FL5kG4008633@scc.eng.rpath.com> changeset: 9cf22b21de06 user: Elliot Peele date: Thu, 12 Nov 2009 13:47:19 -0500 1. batch commits if more than one job is ready to be committed 2. only try to commit 5 changesets at once rather than letting them pile up on the repository. 3. add checks in an attempt to not run out of file descriptors diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -314,11 +314,14 @@ @param retry: information about retrying the get job, if retry is None then retry forever, if retry is an integer retry n times. @type retry: None - @type retry: integet + @type retry: integer @return rmake job instance """ - return self._helper.getJob(jobId) + if not isinstance(jobId, (list, tuple, set)): + return self._helper.client.getJob(jobId) + else: + return self._helper.client.getJobs(jobId) def _startJob(self, troveSpecs): """ @@ -348,24 +351,29 @@ monitor.monitorJob(self._helper.client, jobId, exitOnFinish=True, displayClass=StatusOnlyDisplay) - def _sanityCheckJob(self, jobId): + def _sanityCheckJob(self, jobIds): """ Verify the status of a job. - @param jobId: rMake job ID - @type jobId: integer + @param jobIds: rMake job ID, or list of jobIds + @type jobIds: integer or iterable """ + if not isinstance(jobIds, (tuple, list, set)): + jobIds = [ jobIds, ] + # Check for errors - job = self._getJob(jobId) - if job.isFailed(): - log.error('Job %d failed', jobId) - raise JobFailedError(jobId=jobId, why=job.status) - elif not job.isFinished(): - log.error('Job %d is not done, yet watch returned early!', jobId) - raise JobFailedError(jobId=jobId, why=job.status) - elif not list(job.iterBuiltTroves()): - log.error('Job %d has no built troves', jobId) - raise JobFailedError(jobId=jobId, why='No troves found in job') + for job in self._getJob(jobIds): + jobId = job.jobId + if job.isFailed(): + log.error('Job %d failed', jobId) + raise JobFailedError(jobId=jobId, why=job.status) + elif not job.isFinished(): + log.error('Job %d is not done, yet watch returned early!', + jobId) + raise JobFailedError(jobId=jobId, why=job.status) + elif not list(job.iterBuiltTroves()): + log.error('Job %d has no built troves', jobId) + raise JobFailedError(jobId=jobId, why='No troves found in job') def _sanityCheckRPMCapsule(self, jobId, fileList, fileObjs, rpmFile): """ @@ -594,7 +602,7 @@ @return troveMap: dictionary of troveSpecs to built troves """ - if type(jobId) != list: + if not isinstance(jobId, (list, tuple, set)): jobIds = [ jobId, ] else: jobIds = jobId @@ -603,7 +611,7 @@ # Do the commit startTime = time.time() - jobs = [ self._getJob(x) for x in jobIds ] + jobs = self._getJob(jobIds) log.info('[%s] starting commit' % jobIdsStr) if self._saveChangeSets: @@ -654,7 +662,6 @@ for buildTroveTuple, committedList in troveTupleDict.iteritems(): troveMap[buildTroveTuple] = committedList - return troveMap @staticmethod diff --git a/updatebot/build/constants.py b/updatebot/build/constants.py --- a/updatebot/build/constants.py +++ b/updatebot/build/constants.py @@ -50,4 +50,5 @@ JOB_NOT_STARTED = -1 ERROR_MONITOR_FAILURE = -2 - ERROR_COMITTER_FAILURE = -3 + ERROR_COMMITTER_FAILURE = -3 + JOB_COMMITTING = -4 diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -17,11 +17,13 @@ monitoring. """ +import math import time import logging from rmake.build import buildjob +from updatebot.lib import util from updatebot.build.monitor import JobStarter from updatebot.build.monitor import JobMonitor from updatebot.build.monitor import JobCommitter @@ -37,7 +39,7 @@ _completed = ( JobStatus.ERROR_MONITOR_FAILURE, - JobStatus.ERROR_COMITTER_FAILURE, + JobStatus.ERROR_COMMITTER_FAILURE, buildjob.JOB_STATE_FAILED, buildjob.JOB_STATE_COMMITTED, ) @@ -56,6 +58,9 @@ self._maxStartSlots = 10 self._startSlots = self._maxStartSlots + self._maxCommitSlots = 5 + self._commitSlots = self._maxCommitSlots + self._starter = JobStarter(self._builder) self._monitor = JobMonitor(self._builder._helper.client) self._committer = JobCommitter(self._builder) @@ -82,7 +87,8 @@ # Only create more jobs once the last batch has been started. if self._startSlots == self._maxStartSlots: # fill slots with available troves - while troves and self._slots and self._startSlots: + while (troves and self._slots and self._startSlots and + self._availableFDs()): # get trove to work on trove = troves.pop() @@ -113,13 +119,25 @@ if self._slots > self._maxSlots: log.critical('slots is greater than maxSlots') - # commit any jobs that are done building + # submit any jobs that are ready to commit as long as there are + # commit slots + toCommit = set() + for jobId, (trove, status, result) in self._jobs.iteritems(): + # don't try to commit anything if there are no free commit + # slots. + if not self._commitSlots: + break + # batch up all jobs that are ready to be committed if status == buildjob.JOB_STATE_BUILT: - self._committer.commitJob(jobId) + # update status to !BUILT so that we don't try to commit + # this job more than once. + self._jobs[jobId][1] = JobStatus.JOB_COMMITTING + toCommit.add(jobId) - # check for commit status - for jobId, result in self._committer.getStatus(): - self._jobs[jobId][2] = result + # commit all available jobs at one time. + if toCommit: + self._committer.commitJob(tuple(toCommit)) + self._commitSlots -= 1 # process monitor errors for jobId, error in self._monitor.getErrors(): @@ -127,13 +145,27 @@ self._jobs[jobId][1] = JobStatus.ERROR_MONITOR_FAILURE self._failures.append((jobId, error)) + # check for commit status + for jobId, result in self._committer.getStatus(): + # unbatch commit jobs + if not isinstance(jobId, tuple): + jobId = (jobId, ) + for jobId in jobId: + self._jobs[jobId][2] = result + self._commitSlots += 1 + # process committer errors for jobId, error in self._committer.getErrors(): - self._jobs[jobId][1] = JobStatus.ERROR_COMITTER_FAILURE - self._failures.append((jobId, error)) + # unbatch commit jobs + if not isinstance(jobId, tuple): + jobId = (jobId, ) + for jobId in jobId: + self._jobs[jobId][1] = JobStatus.ERROR_COMMITTER_FAILURE + self._failures.append((jobId, error)) + self._commitSlots += 1 - # Flag job as failed so that monitor worker will exit properly. - self._builder.setCommitFailed(jobId) + # Flag job as failed so that monitor worker will exit properly. + self._builder.setCommitFailed(jobId, reason=str(error)) # Wait for a bit before polling again. time.sleep(3) @@ -154,6 +186,10 @@ return results, self._failures def _jobDone(self): + """ + Check if all jobs are complete. + """ + if not len(self._jobs): return False @@ -161,3 +197,11 @@ if status not in self._completed: return False return True + + def _availableFDs(self, setMax=False): + """ + Get 80% of available file descriptors in hopes of not running out. + """ + + avail = util.getAvailableFileDescriptors(setMax=setMax) + return math.floor(0.8 * avail) From elliot at rpath.com Mon Mar 15 17:05:50 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:50 +0000 Subject: mirrorball: move package class into a class variable so that it can be overridden in a sub Message-ID: <201003152105.o2FL5os9008702@scc.eng.rpath.com> changeset: 68ccc12889fe user: Elliot Peele date: Thu, 12 Nov 2009 13:48:11 -0500 move package class into a class variable so that it can be overridden in a sub class diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -30,6 +30,8 @@ Class that builds maps of packages from multiple yum repositories. """ + PkgClass = repomd.packagexml._Package + def __init__(self, cfg): BasePackageSource.__init__(self, cfg) @@ -256,11 +258,9 @@ if not self._cfg.synthesizeSources: return - PkgClass = repomd.packagexml._Package - # Create a fake source rpm object for each key in the rpmMap. for (name, epoch, version, release, arch), bins in self._rpmMap.iteritems(): - srcPkg = PkgClass() + srcPkg = self.PkgClass() srcPkg.name = name srcPkg.epoch = epoch srcPkg.version = version From elliot at rpath.com Mon Mar 15 17:05:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:53 +0000 Subject: mirrorball: filter troves by build flavor Message-ID: <201003152105.o2FL5r7f008749@scc.eng.rpath.com> changeset: dad54db8f117 user: Elliot Peele date: Fri, 13 Nov 2009 11:36:33 -0500 filter troves by build flavor diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -605,7 +605,15 @@ """ label = self._ccfg.buildLabel - trvMap = self._repos.getTroveLeavesByLabel({None: {label: None}}) + + # Filter by buildFlavor to handle mutli stage bootstrap cases where you + # want to build all packages that haven't yet been built !bootstrap. + flavors = set() + for section in self._ccfg._sections.itervalues(): + if hasattr(section, 'buildFlavor'): + flavors.add(section.buildFlavor) + + trvMap = self._repos.getTroveLeavesByLabel({None: {label: flavors}}) return trvMap def getLatestVersions(self): From elliot at rpath.com Mon Mar 15 17:05:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:05:58 +0000 Subject: mirrorball: fix typo Message-ID: <201003152105.o2FL5wiW008805@scc.eng.rpath.com> changeset: cc14678c80be user: Elliot Peele date: Fri, 13 Nov 2009 11:36:48 -0500 fix typo diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -69,7 +69,7 @@ @type recreate - boolean to recreate all sources or a list of specific package names @param toCreate - set of source package objects to create, implies recreate. - @type toCrate - iterable + @type toCreate - iterable """ start = time.time() From elliot at rpath.com Mon Mar 15 17:06:01 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:06:01 +0000 Subject: mirrorball: clear autoLoadRecipes when building groups with a custom rmakerc, since this Message-ID: <201003152106.o2FL62lD008853@scc.eng.rpath.com> changeset: 0c8816dea086 user: Elliot Peele date: Mon, 16 Nov 2009 13:09:02 -0500 clear autoLoadRecipes when building groups with a custom rmakerc, since this is most likely done when switching to an rPL based platform. diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -122,6 +122,10 @@ rmakeCfgPath = util.join(self._cfg.configPath, rmakeCfgFn) if os.path.exists(rmakeCfgPath): rmakerc = rmakeCfgFn + # FIXME: This is a hack to work around having two conaryrc + # files, one for building groups and one for building + # packages. + self._ccfg.autoLoadRecipes = [] else: log.warn('%s not found, falling back to rmakerc' % rmakeCfgFn) From elliot at rpath.com Mon Mar 15 17:06:04 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:06:04 +0000 Subject: mirrorball: add url walker Message-ID: <201003152106.o2FL65SZ008903@scc.eng.rpath.com> changeset: 8957da925ef3 user: Elliot Peele date: Mon, 16 Nov 2009 13:56:58 -0500 add url walker diff --git a/updatebot/lib/urlwalker.py b/updatebot/lib/urlwalker.py new file mode 100644 --- /dev/null +++ b/updatebot/lib/urlwalker.py @@ -0,0 +1,181 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for walking url trees much like os.walk +""" + +import os +import urllib2 +import urlparse +from HTMLParser import HTMLParser + +from updatebot.lib import util + +class Parser(HTMLParser): + """ + Parse hrefs out of html documents. + """ + + def __init__(self, callback): + HTMLParser.__init__(self) + self._callback = callback + + def handle_starttag(self, tag, attrs): + """ + Parse "a" tags. + """ + + if tag == 'a': + d = dict(attrs) + if 'href' in d: + self._callback._regref(d['href']) + + +class UrlWalker(object): + """ + Class to implement walker interface for urls, similar to os.walk. + """ + + _ignore = ('/', ) + + def __init__(self, baseUrl): + url = urlparse.urlparse(baseUrl) + self._baseUrl = url.geturl()[:-len(url.path)] + self._cur = os.path.abspath(url.path) + + self._prev = None + self._parser = Parser(self) + self._refs = [] + + @classmethod + def walk(cls, baseUrl): + """ + Walk a url path. + """ + + obj = cls(baseUrl) + return obj._walk() + + def _walk(self, prev=None): + """ + Walk url path. + """ + + files = [] + directories = [] + + for path in self._read(): + if self._isDirectory(path): + if not self._filterDirectory(path, prev): + directories.append(os.path.normpath(path)) + elif not self._filterFile(path): + files.append(path) + + yield (self._getCurrentUrl(), directories, files) + + cur = self._cur + for directory in directories: + self._cur = os.path.normpath(os.path.join(cur, directory)) + for res in self._walk(prev=cur): + yield res + + def _read(self): + """ + Read a url path and parse the html. + """ + + self._refs = [] + url = self._getCurrentUrl() + doc = urllib2.urlopen(url).read() + self._parser.feed(doc) + self._parser.close() + return self._refs + + def _regref(self, ref): + """ + Callback for the html parser to register hrefs. + """ + + self._refs.append(ref) + + def _getCurrentUrl(self): + """ + Return the current url. + """ + + if self._cur == '/': + return self._baseUrl + + base = self._baseUrl.endswith('/') + cur = self._cur.startswith('/') + + if (base and not cur) or (not base and cur): + return self._baseUrl + self._cur + elif not base and not cur: + return self._baseUrl + '/' + self._cur + elif base and cur: + return self._baseUrl + self._cur[1:] + + @staticmethod + def _isDirectory(name): + """ + Check if an href is a directory. + """ + + return name.endswith('/') + + def _filterDirectory(self, name, prev): + """ + Check if a directory should be filtered out. + """ + + name = os.path.normpath(name) + + if prev is None and self._cur.startswith(name): + return True + if name == self._cur: + return True + if name == prev: + return True + if name in self._ignore: + return True + + return False + + @staticmethod + def _filterFile(name): + """ + Check if a file should be filtered out. + """ + + # looks like an apache internal file. + if name.startswith('?'): + return True + + return False + +walk = UrlWalker.walk + +if __name__ == '__main__': + import sys + from conary.lib import util as cutil + sys.excepthook = cutil.genExcepthook() + + for cur, dirs, files in UrlWalker.walk(sys.argv[1]): + print 'cur :', cur + for d in dirs: + print 'dir :', d + #for f in files: + # print 'file:', f From elliot at rpath.com Mon Mar 15 17:06:08 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:06:08 +0000 Subject: mirrorball: fixups Message-ID: <201003152106.o2FL69XE008961@scc.eng.rpath.com> changeset: 779bbeb6f03e user: Elliot Peele date: Tue, 17 Nov 2009 21:22:41 -0500 fixups diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -85,6 +85,7 @@ """ self._helper.setModel(self._sourceName, self._groups) + self._helper.commit(self._sourceName, commitMessage='automated commit') self._checkedout = False save = _commit @@ -114,7 +115,7 @@ model = GroupContentsModel(self._pkgGroupName) self._groups[self._pkgGroupName] = model - add = self._groups[self._pkgGroupName].add(*args, **kwargs) + self._groups[self._pkgGroupName].add(*args, **kwargs) @checkout def remove(self, name): @@ -309,10 +310,10 @@ if os.path.exists(groupFileName): model = GroupModel.thaw(groupFileName) for name, groupObj in model.iteritems(): - contentFileName = util.join(recipeDir, groupObj.fileName) + contentFileName = util.join(recipeDir, groupObj.filename) contentsModel = GroupContentsModel.thaw(contentFileName, (name, groupObj.byDefault, groupObj.depCheck)) - contentsModel.fileName = groupObj.fileName + contentsModel.fileName = groupObj.filename groups[groupObj.name] = contentsModel # copy in any group data @@ -351,14 +352,15 @@ groupfn = util.join(recipeDir, model.fileName) model.freeze(groupfn) - groupModel.add(name, model.fileName) + groupModel.add(name=name, + filename=model.fileName, + byDefault=model.byDefault, + depCheck=model.depCheck) self._addFile(recipeDir, model.fileName) groupModel.freeze(groupFileName) self._addFile(recipeDir, 'groups.xml') - #self._commit(recipeDir, commitMessage='automated group update') - def getErrataState(self, pkgname): """ Get the contents of the errata state file from the specified package, @@ -389,10 +391,14 @@ # write state info statefh = open(stateFileName, 'w') statefh.write(state) + + # source files must end in a trailing newline + statefh.write('\n') + statefh.close() # make sure state file is part of source trove - self._adFile('erratastate') + self._addFile(recipeDir, 'erratastate') class AbstractModel(object): From elliot at rpath.com Mon Mar 15 17:06:16 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:06:16 +0000 Subject: mirrorball: 1. Change name of yum based repository handling. Message-ID: <201003152106.o2FL6Hp9009046@scc.eng.rpath.com> changeset: 5dfe75524e26 user: Elliot Peele date: Tue, 17 Nov 2009 21:52:00 -0500 1. Change name of yum based repository handling. 2. Add new rpm source module for dealing with directories of rpms. diff --git a/updatebot/pkgsource/__init__.py b/updatebot/pkgsource/__init__.py --- a/updatebot/pkgsource/__init__.py +++ b/updatebot/pkgsource/__init__.py @@ -17,6 +17,7 @@ """ from updatebot.pkgsource.rpmsource import RpmSource +from updatebot.pkgsource.yumsource import YumSource from updatebot.pkgsource.debsource import DebSource from updatebot.pkgsource.errors import UnsupportedRepositoryError @@ -29,7 +30,7 @@ if cfg.repositoryFormat == 'apt': return DebSource(cfg) elif cfg.repositoryFormat == 'yum': - return RpmSource(cfg) + return YumSource(cfg) else: raise UnsupportedRepositoryError(repo=cfg.repositoryFormat, supported=['apt', 'yum']) diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -14,278 +14,159 @@ # """ -Module for interacting with packages in multiple yum repositories. +Module for interacting with a directory of packages. """ import logging -import repomd -from updatebot.lib import util -from updatebot.pkgsource.common import BasePackageSource +from conary import rpmhelper log = logging.getLogger('updatebot.pkgsource') -class RpmSource(BasePackageSource): +from repomd.packagexml import _Package +from rpmutils import header as rpmheader + +from updatebot.lib import urlwalker +from updatebot.pkgsource.yumsource import YumSource as PackageSource + +class Package(_Package): """ - Class that builds maps of packages from multiple yum repositories. + Class for represnting package data. """ - PkgClass = repomd.packagexml._Package + def __init__(self, *args, **kwargs): + _Package.__init__(self, *args) + for key, val in kwargs.iteritems(): + setattr(self, key, val) - def __init__(self, cfg): - BasePackageSource.__init__(self, cfg) + def getNevra(self): + """ + Return the name, epoch, version, release, and arch the package. + """ - # {srcTup: srpm} - self._srcMap = dict() + return (self.name, self.epoch, self.version, self.release, self.arch) - # {srcTup: {rpm: path} - self._rpmMap = dict() + def getConaryVersion(self): + """ + Get the conary version of a source package. + """ - # set of all src pkg objects - self._srcPkgs = set() + assert self.arch == 'src' + filename = os.path.basename(self.location) + nameVerRelease = ".".join(filename.split(".")[:-2]) + ver = "_".join(nameVerRelease.split("-")[-2:]) + return ver + + +class Client(object): + """ + Client class for walking package tree. + """ + + walkMethod = os.walk + + def __init__(self, path): + self._path = path + + def getPackageDetail(self): + """ + Walk the specified path to find rpms. + """ + + for path, dirs, files in self.walkMethod(self._path): + for f in files: + if f.endswith('.rpm'): + yield self._index(os.path.join(path, f)) + + def _index(self, rpm): + """ + Index an individual rpm. + """ + + log.info(os.path.basename(rpm)) + + fh = rpmheader.SeekableStream(rpm) + h = rpmhelper.readHeader(fh) + name = h[rpmhelper.NAME] + epoch = h.get(rpmhelper.EPOCH, None) + if isinstance(epoch, (list, tuple)): + assert len(epoch) == 1 + epoch = str(epoch[0]) + version = h[rpmhelper.VERSION] + release = h[rpmhelper.RELEASE] + arch = h.isSource and 'src' or h[rpmhelper.ARCH] + sourcename = h.get(rpmhelper.SOURCERPM, None) + + basename = os.path.basename(rpm) + pkg = Package(name=name, + epoch=epoch, + version=version, + release=release, + arch=arch, + sourcerpm=sourcename, + location=basename) + return pkg + + +class UrlClient(Client): + """ + Url based client. + """ + + walkMethod = urlwalker.walk + + +class RpmSource(PackageSource): + """ + Walk a directory, find rpms, index them. + """ + + PkgClass = Package + + def __init__(self, cfg, path): + PackageSource.__init__(self, cfg) + self._path = path def load(self): """ - Load package source based on config data. + load package source. """ if self._loaded: return - for repo in self._cfg.repositoryPaths: - log.info('loading repository data %s' % repo) - client = repomd.Client(self._cfg.repositoryUrl + '/' + repo) - self.loadFromClient(client, repo) - self._clients[repo] = client + log.info('loading %s' % self._path) + + client = Client(self._path) + self.loadFromClient(client) self.finalize() self._loaded = True def loadFromUrl(self, url, basePath=''): """ - Walk the yum repository rooted at url/basePath and collect information - about rpms found. - @param url: url to common directory on host where all repositories resides. - @type url: string - @param basePath: directory where repository resides. - @type basePath: string + This method is not supported for indexing, raise a decent error. """ - client = repomd.Client(url + '/' + basePath) + if self._loaded: + return + + fullUrl = url + '/' + basePath + + log.info('loading %s' % fullUrl) + + client = UrlClient(fullUrl) self.loadFromClient(client, basePath=basePath) - def loadFromClient(self, client, basePath=''): + self.finalize() + self._loaded = True + + def iterPackageSet(self): """ - Walk the yum repository rooted at url/basePath and collect information - about rpms found. - @param client: client object for extracting data from the repo metadata - @type client: repomd.Client - @param basePath: path to prefix location metadata with - @type basePath: string + Iterate over the set of source packages. """ - for pkg in client.getPackageDetail(): - # ignore the 32-bit compatibility libs - we will - # simply use the 32-bit components from the repository - if '32bit' in pkg.name: + for srcPkg, binPkgs in self.srcPkgMap.iteritems(): + if not len(binPkgs): continue - - # Don't use all arches. - if pkg.arch in self._excludeArch: - continue - - assert '-' not in pkg.version - assert '-' not in pkg.release - - pkg.location = basePath + '/' + pkg.location - - # ignore 32bit rpms in a 64bit repo. - if (pkg.arch in ('i386', 'i586', 'i686') and - 'x86_64' in pkg.location): - continue - - if pkg.sourcerpm == '' or pkg.sourcerpm is None: - self._procSrc(pkg) - else: - self._procBin(pkg) - - def _procSrc(self, package): - """ - Process source rpms. - @param package: package object - @type package: repomd.packagexml._Package - """ - - if package.name not in self.srcNameMap: - self.srcNameMap[package.name] = set() - self.srcNameMap[package.name].add(package) - - self.locationMap[package.location] = package - - self._srcPkgs.add(package) - self._srcMap[(package.name, package.epoch, package.version, - package.release, package.arch)] = package - - def _procBin(self, package): - """ - Process binary rpms. - @param package: package object - @type package: repomd.packagexml._Package - """ - - # FIXME: There should be a better way to figure out the tuple that - # represents the hash of the srcPkg. - srcParts = package.sourcerpm.split('-') - srcName = '-'.join(srcParts[:-2]) - srcVersion = srcParts[-2] - if srcParts[-1].endswith('.src.rpm'): - srcRelease = srcParts[-1][:-8] # remove '.src.rpm' - elif srcParts[-1].endswith('.nosrc.rpm'): - srcRelease = srcParts[-1][:-10] - rpmMapKey = (srcName, package.epoch, srcVersion, srcRelease, 'src') - if rpmMapKey not in self._rpmMap: - self._rpmMap[rpmMapKey] = set() - self._rpmMap[rpmMapKey].add(package) - - if package.name not in self.binNameMap: - self.binNameMap[package.name] = set() - self.binNameMap[package.name].add(package) - - self.locationMap[package.location] = package - - def finalize(self): - """ - Make some final datastructures now that we are done populating object. - """ - - # Build source structures from binaries if no sources are available from - # the repository. - if self._cfg.synthesizeSources: - self._createSrcMap() - - # Now that we have processed all of the rpms, build some more data - # structures. - count = 0 - toDelete = set() - for pkg in self._srcPkgs: - key = (pkg.name, pkg.epoch, pkg.version, pkg.release, pkg.arch) - if pkg in self.srcPkgMap: - continue - - if key not in self._rpmMap: - #log.warn('found source without binary rpms: %s' % pkg) - #log.debug(key) - #log.debug([ x for x in self._rpmMap if x[0] == key[0] ]) - - count += 1 - if pkg in self.srcNameMap[pkg.name]: - self.srcNameMap[pkg.name].remove(pkg) - continue - - self.srcPkgMap[pkg] = self._rpmMap[key] - self.srcPkgMap[pkg].add(pkg) - toDelete.add(key) - - for binPkg in self.srcPkgMap[pkg]: - self.binPkgMap[binPkg] = pkg - - if count > 0: - log.warn('found %s source rpms without matching binary ' - 'rpms' % count) - - # Defer deletes, contents of rpmMap are used more than once. - for key in toDelete: - del self._rpmMap[key] - - # Attempt to match up remaining binaries with srpms. - for srcTup in self._rpmMap.keys(): - srcKey = list(srcTup) - epoch = int(srcKey[1]) - while epoch >= 0: - srcKey[1] = str(epoch) - key = tuple(srcKey) - if key in self._srcMap: - srcPkg = self._srcMap[key] - for binPkg in self._rpmMap[srcTup]: - self.srcPkgMap[srcPkg].add(binPkg) - self.binPkgMap[binPkg] = srcPkg - del self._rpmMap[srcTup] - break - epoch -= 1 - - if self._rpmMap: - count = sum([ len(x) for x in self._rpmMap.itervalues() ]) - log.warn('found %s binary rpms without matching srpms' % count) - - srcs = {} - for x in self._rpmMap.itervalues(): - for y in x: - # skip debuginfo rpms - if 'debuginfo' in y.location or 'debugsource' in y.location: - continue - - # skip rpms built from nosrc rpms - if 'nosrc' in y.sourcerpm: - continue - - if y.sourcerpm not in srcs: - srcs[y.sourcerpm] = set() - srcs[y.sourcerpm].add(y.location) - - for src, locs in srcs.iteritems(): - log.warn('missing srpm: %s' % src) - log.warn('for rpm(s):') - for loc in sorted(locs): - log.warn('\t%s' % loc) - - def loadFileLists(self, client, basePath): - """ - Parse file information. - """ - - for pkg in client.getFileLists(): - for binPkg in self.binPkgMap.iterkeys(): - if util.packageCompare(pkg, binPkg) == 0: - binPkg.files = pkg.files - break - - def _createSrcMap(self): - """ - Create a source map from the binary map if no sources are available. - """ - - # Return if sources should be available in repos. - if not self._cfg.synthesizeSources: - return - - # Create a fake source rpm object for each key in the rpmMap. - for (name, epoch, version, release, arch), bins in self._rpmMap.iteritems(): - srcPkg = self.PkgClass() - srcPkg.name = name - srcPkg.epoch = epoch - srcPkg.version = version - srcPkg.release = release - srcPkg.arch = arch - - # grab the first binary package - pkg = sorted(bins)[0] - - # Set the location of the fake source package to just be the name - # of the file. The factory will take care of the rest. - srcPkg.location = pkg.sourcerpm - - # Handle sub packages with different epochs that should be taken - # care of with the epoch fuzzing that happens in finalize. This - # should only happen with differently named packages. - if name not in [ x.name for x in bins ]: - # Find sources that match on all cases except epoch. - sources = [ x for x in self._srcMap.iterkeys() - if (name, version, release, arch) == (x[0], x[2], x[3], x[4]) ] - - # leave it up to fuzzing - if sources: continue - - # add source to structures - if (name, epoch, version, release, arch) not in self._srcMap: - log.warn('synthesizing source package %s' % srcPkg) - self._procSrc(srcPkg) + yield (srcPkg.name, srcPkg.getConaryVersion()) diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/yumsource.py copy from updatebot/pkgsource/rpmsource.py copy to updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -25,7 +25,7 @@ log = logging.getLogger('updatebot.pkgsource') -class RpmSource(BasePackageSource): +class YumSource(BasePackageSource): """ Class that builds maps of packages from multiple yum repositories. """ From elliot at rpath.com Mon Mar 15 17:06:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:06:25 +0000 Subject: mirrorball: make sure failed jobs are reported consistently Message-ID: <201003152106.o2FL6QNp009137@scc.eng.rpath.com> changeset: 1f89d807f63e user: Elliot Peele date: Wed, 18 Nov 2009 10:12:24 -0500 make sure failed jobs are reported consistently diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -118,6 +118,7 @@ log.info('found %s packages to build' % len(toBuild)) trvMap = [] + failed = () if len(toBuild): if not rebuild: # Build all newly imported packages. @@ -128,12 +129,12 @@ log.warn('%s' % (pkg, )) else: # ReBuild all packages. - failed = () trvMap = self._builder.buildsplitarch(sorted(toBuild)) log.info('import completed successfully') log.info('imported %s source packages' % (len(toBuild), )) else: - log.info('no packages found to build') + log.info('no packages found to build, maybe there is a flavor ' + 'configuration issue') log.info('elapsed time %s' % (time.time() - start, )) From elliot at rpath.com Mon Mar 15 17:06:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:06:39 +0000 Subject: mirrorball: add missing import Message-ID: <201003152106.o2FL6fqO009226@scc.eng.rpath.com> changeset: 0bfc8d64275c user: Elliot Peele date: Wed, 18 Nov 2009 13:37:21 -0500 add missing import diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2006,2008-2009 rPath, Inc. +# Copyright (c) 2009 rPath, Inc. # # # This program is distributed under the terms of the Common Public License, @@ -17,6 +17,7 @@ Module for interacting with a directory of packages. """ +import os import logging from conary import rpmhelper From elliot at rpath.com Mon Mar 15 17:06:55 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:06:55 +0000 Subject: mirrorball: make class external, add method to get size Message-ID: <201003152106.o2FL6ueZ009282@scc.eng.rpath.com> changeset: 9ccf2650ff73 user: Elliot Peele date: Wed, 18 Nov 2009 13:37:57 -0500 make class external, add method to get size diff --git a/rpmutils/header.py b/rpmutils/header.py --- a/rpmutils/header.py +++ b/rpmutils/header.py @@ -20,7 +20,7 @@ from conary import rpmhelper -class _SeekableStream(object): +class SeekableStream(object): """ File like object that can be seeked forward. """ @@ -61,6 +61,13 @@ return self._pos + def getTotalSize(self): + """ + Return the size in the header. + @return content length + """ + + return int(self._fh.headers.getheader('content-length')) def readHeader(url): """ @@ -70,7 +77,7 @@ @return conary.rpmhelper.RpmHeader object """ - fh = _SeekableStream(url) + fh = SeekableStream(url) # Have to read into the file a bit to get to the begining of the header # that we care about. From elliot at rpath.com Mon Mar 15 17:07:07 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:07 +0000 Subject: mirrorball: add debug handler Message-ID: <201003152107.o2FL77Zw009358@scc.eng.rpath.com> changeset: addfb47e9abb user: Elliot Peele date: Wed, 18 Nov 2009 19:51:16 -0500 add debug handler diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -20,6 +20,8 @@ # pylint: disable-msg=W0611 import os +import epdb +import signal import resource from conary.lib.util import rmtree @@ -167,3 +169,13 @@ setMaxRLimit() limit = getRLimit() return limit - openfds + +def setupDebugHandler(): + """ + Sets up a USR1 signal handler to trigger epdb.serv(). + """ + + def handler(signum, sigtb): + epdb.serve() + + signal.signal(signal.SIGUSR1, handler) From elliot at rpath.com Mon Mar 15 17:07:14 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:14 +0000 Subject: mirrorball: add group versions Message-ID: <201003152107.o2FL7Fo3009430@scc.eng.rpath.com> changeset: 66a09d44e0f8 user: Elliot Peele date: Wed, 18 Nov 2009 20:20:43 -0500 add group versions diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -53,6 +53,7 @@ return func(self, *args, **kwargs) return wrapper + class GroupManager(object): """ Manage group of all packages for a platform. @@ -479,6 +480,7 @@ self._removeItem(name) + class GroupModel(AbstractModel): """ Model for representing group name and file name. @@ -507,3 +509,53 @@ # figure out file name based on group name name = ''.join([ x.capitalize() for x in self.groupName.split('-') ]) self.fileName = name[0].lower() + name[1:] + '.xml' + + +class URLVersionSource(object): + """ + Class for handling the indexing and comparison of ISO contents. + """ + + def __init__(self, cfg, version, url): + self._cfg = cfg + self._version = version + self._url = url + + self._pkgSource = pkgsource.rpmSource(cfg) + + def areWeHereYet(self, pkgSet): + """ + Figure out if the given package set is this version. + @param pkgSet: set of source names and source versions + @type pkgSet: set([(conarySourceName, conarySourceVersion)]) + @return boolean + """ + + self._pkgSource.loadFromUrl(self._url) + + sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ] + return sourceSet == pkgSet + + +class VersionFactory(object): + """ + Class for generating group verisons from a variety of sources. + """ + + def __init__(self, cfg): + self._cfg = cfg + self._sources = {} + + for version, url in cfg.versionSources: + self._sources[version] = URLVersionSource(self._cfg, version, url) + + def getVersions(self, pkgSet): + """ + Given the available package sources find the current versions. + """ + + versions = [] + for version, source in self._sources.iteritems(): + if sources.areWeHereYet(pkgSet): + versions.append(version) + return versions From elliot at rpath.com Mon Mar 15 17:07:20 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:20 +0000 Subject: mirrorball: 1. disable group build, promote, and mirror if not handling advisories. Message-ID: <201003152107.o2FL7LxZ009489@scc.eng.rpath.com> changeset: a5547dab69df user: Elliot Peele date: Thu, 19 Nov 2009 10:57:14 -0500 1. disable group build, promote, and mirror if not handling advisories. 2. refactory a bit to reuse some code 3. return a package list to be consistent and provide data to subclasses diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -59,6 +59,18 @@ lst.extend(list(trvSet)) return lst + def _getGroupBuildTroves(self): + """ + Get the list of name, version, and flavor of the groups to build. + """ + + grpTrvs = set() + for flavor in self._cfg.groupFlavors: + grpTrvs.add((self._cfg.topSourceGroup[0], + self._cfg.topSourceGroup[1], + flavor)) + return grpTrvs + def create(self, rebuild=False, recreate=None, toCreate=None): """ Do initial imports. @@ -210,25 +222,21 @@ else: trvMap = self._builder.buildsplitarch(buildTroves) - # Build group. - grpTrvs = set() - for flavor in self._cfg.groupFlavors: - grpTrvs.add((self._cfg.topSourceGroup[0], - self._cfg.topSourceGroup[1], - flavor)) - grpTrvMap = self._builder.build(grpTrvs) + if not self._cfg.disableAdvisories: + # Build group. + grpTrvs = self._getGroupBuildTroves() + grpTrvMap = self._builder.build(grpTrvs) - # Promote group. - # We expect that everything that was built will be published. - expected = self._flattenSetDict(trvMap) - toPublish = self._flattenSetDict(grpTrvMap) - newTroves = self._updater.publish(toPublish, expected, - self._cfg.targetLabel) + # Promote group. + # We expect that everything that was built will be published. + expected = self._flattenSetDict(trvMap) + toPublish = self._flattenSetDict(grpTrvMap) + newTroves = self._updater.publish(toPublish, expected, + self._cfg.targetLabel) - # Mirror out content - self._updater.mirror() + # Mirror out content + self._updater.mirror() - if not self._cfg.disableAdvisories: # Send advisories. self._advisor.send(toAdvise, newTroves) @@ -236,3 +244,5 @@ log.info('updated %s packages and sent %s advisories' % (len(toUpdate), len(toAdvise))) log.info('elapsed time %s' % (time.time() - start, )) + + return trvMap From elliot at rpath.com Mon Mar 15 17:07:28 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:28 +0000 Subject: mirrorball: be more resilient to odd behavior Message-ID: <201003152107.o2FL7Tjr009597@scc.eng.rpath.com> changeset: ca887b50a527 user: Elliot Peele date: Fri, 20 Nov 2009 12:59:23 -0500 be more resilient to odd behavior diff --git a/updatebot/build/common.py b/updatebot/build/common.py --- a/updatebot/build/common.py +++ b/updatebot/build/common.py @@ -84,6 +84,7 @@ if job in self._workers: log.critical('job already being monitored: %s' % (job, )) import epdb; epdb.st() + return args = list(self._threadArgs) args.append(job) @@ -134,7 +135,10 @@ elif mtype == MessageTypes.THREAD_DONE: job = payload #assert not self._workers[job].isAlive() - del self._workers[job] + if job in self._workers: + del self._workers[job] + else: + log.critical('JOB NOT FOUND %s' % (job, )) elif mtype == MessageTypes.THREAD_ERROR: threadType, job, error = payload #assert not self._workers[job].isAlive() From elliot at rpath.com Mon Mar 15 17:07:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:37 +0000 Subject: mirrorball: don't report the same state consecutively Message-ID: <201003152107.o2FL7cvG009669@scc.eng.rpath.com> changeset: 346d5030f59c user: Elliot Peele date: Fri, 20 Nov 2009 14:19:36 -0500 don't report the same state consecutively diff --git a/updatebot/build/callbacks.py b/updatebot/build/callbacks.py --- a/updatebot/build/callbacks.py +++ b/updatebot/build/callbacks.py @@ -73,6 +73,7 @@ monitor.JobLogDisplay.__init__(self, *args, **kwargs) self._status = status + self._laststate = None def _msg(self, msg, *args): self._status.put((MessageTypes.LOG, msg)) @@ -82,6 +83,9 @@ def _jobStateUpdated(self, jobId, state, status): monitor.JobLogDisplay._jobStateUpdated(self, jobId, state, None) + if state == self._laststate + return + self._laststate = state if state in self.monitorStates: self._data((jobId, state)) if state in self.doneStates: From elliot at rpath.com Mon Mar 15 17:07:45 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:45 +0000 Subject: mirrorball: fix typo Message-ID: <201003152107.o2FL7kYB009731@scc.eng.rpath.com> changeset: 75d3f77ca32e user: Elliot Peele date: Fri, 20 Nov 2009 14:22:34 -0500 fix typo diff --git a/updatebot/build/callbacks.py b/updatebot/build/callbacks.py --- a/updatebot/build/callbacks.py +++ b/updatebot/build/callbacks.py @@ -83,7 +83,7 @@ def _jobStateUpdated(self, jobId, state, status): monitor.JobLogDisplay._jobStateUpdated(self, jobId, state, None) - if state == self._laststate + if state == self._laststate: return self._laststate = state if state in self.monitorStates: From elliot at rpath.com Mon Mar 15 17:07:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:53 +0000 Subject: mirrorball: make sure all status messages are unique Message-ID: <201003152107.o2FL7sdr009815@scc.eng.rpath.com> changeset: a022fe914ba2 user: Elliot Peele date: Sat, 21 Nov 2009 12:17:56 -0500 make sure all status messages are unique diff --git a/updatebot/build/callbacks.py b/updatebot/build/callbacks.py --- a/updatebot/build/callbacks.py +++ b/updatebot/build/callbacks.py @@ -73,7 +73,7 @@ monitor.JobLogDisplay.__init__(self, *args, **kwargs) self._status = status - self._laststate = None + self._states = [] def _msg(self, msg, *args): self._status.put((MessageTypes.LOG, msg)) @@ -83,9 +83,9 @@ def _jobStateUpdated(self, jobId, state, status): monitor.JobLogDisplay._jobStateUpdated(self, jobId, state, None) - if state == self._laststate: + if state in self._states: return - self._laststate = state + self._states.append(state) if state in self.monitorStates: self._data((jobId, state)) if state in self.doneStates: From elliot at rpath.com Mon Mar 15 17:07:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:07:59 +0000 Subject: mirrorball: use the alternate group rmakerc if it exists Message-ID: <201003152107.o2FL7x05009896@scc.eng.rpath.com> changeset: ee4c4611c381 user: Elliot Peele date: Mon, 23 Nov 2009 01:01:59 -0500 use the alternate group rmakerc if it exists diff --git a/scripts/buildgroups b/scripts/buildgroups --- a/scripts/buildgroups +++ b/scripts/buildgroups @@ -9,6 +9,8 @@ from header import * +builder = build.Builder(cfg, rmakeCfgFn='rmakerc-groups') + grpTrvs = set() for flavor in cfg.groupFlavors: grpTrvs.add((cfg.topSourceGroup[0], cfg.topSourceGroup[1], flavor)) From elliot at rpath.com Mon Mar 15 17:08:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:02 +0000 Subject: mirrorball: setup usr1 debug handler Message-ID: <201003152108.o2FL83AH009945@scc.eng.rpath.com> changeset: d7f6e72609ff user: Elliot Peele date: Mon, 23 Nov 2009 01:02:51 -0500 setup usr1 debug handler diff --git a/scripts/buildmany b/scripts/buildmany --- a/scripts/buildmany +++ b/scripts/buildmany @@ -9,6 +9,9 @@ from header import * +from updatebot.lib import util +util.setupDebugHandler() + if len(sys.argv) < 3: usage() From elliot at rpath.com Mon Mar 15 17:08:05 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:05 +0000 Subject: mirrorball: store create output for reporting and inspection Message-ID: <201003152108.o2FL85b1009997@scc.eng.rpath.com> changeset: d004bc3281a4 user: Elliot Peele date: Mon, 23 Nov 2009 01:03:45 -0500 store create output for reporting and inspection diff --git a/scripts/order_import.py b/scripts/order_import.py --- a/scripts/order_import.py +++ b/scripts/order_import.py @@ -61,6 +61,6 @@ errata.fetch() bot = ordered.Bot(cfg, errata) -bot.create() +pkgMap, failures = bot.create() import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:08:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:10 +0000 Subject: mirrorball: fixup paths Message-ID: <201003152108.o2FL8AZl010050@scc.eng.rpath.com> changeset: 29013b47f2f5 user: Elliot Peele date: Mon, 23 Nov 2009 01:04:28 -0500 fixup paths diff --git a/scripts/findprefixpackages b/scripts/findprefixpackages --- a/scripts/findprefixpackages +++ b/scripts/findprefixpackages @@ -20,7 +20,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mbdir = os.path.abspath('../') +sys.path.insert(0, mbdir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -41,7 +42,7 @@ slog = logging.getLogger('findprefixpackages') cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mbdir + '/config/%s/updatebotrc' % sys.argv[1]) prefixes = set(sys.argv[2:]) From elliot at rpath.com Mon Mar 15 17:08:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:13 +0000 Subject: mirrorball: testing changes Message-ID: <201003152108.o2FL8DLA010105@scc.eng.rpath.com> changeset: 8a1e4f671915 user: Elliot Peele date: Mon, 23 Nov 2009 01:05:15 -0500 testing changes diff --git a/scripts/gengroup.py b/scripts/gengroup.py --- a/scripts/gengroup.py +++ b/scripts/gengroup.py @@ -13,7 +13,7 @@ mbdir = os.path.abspath('../') sys.path.insert(0, mbdir) -confDir = os.path.join(mbdir, 'config', 'rhel5test') +confDir = os.path.join(mbdir, 'config', sys.argv[1]) from updatebot import log from updatebot import Bot @@ -34,21 +34,24 @@ mgr = groupmgr.GroupManager(cfg) -trvMap = mgr._helper._getLatestTroves() +troves = mgr._helper._getLatestTroves() +mgr._checkout() -from conary.deps import deps +import epdb; epdb.st() -troves = mgr._helper._getLatestTroves() for name, vf in troves.iteritems(): if ':' in name or bot._updater._fltrPkg(name): continue - assert len(vf.keys()) == 1 - version = vf.keys()[0] + versions = vf.keys() + versions.sort() + version = versions[-1] flavors = vf[version] - mgr.addPackage(name, version, flavors) +# mgr.addPackage(name, version, flavors) +mgr.setVersion('0') +mgr.setErrataState('0') mgr._commit() - +#mgr.build() import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:08:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:15 +0000 Subject: mirrorball: add helper methods for getting advisory information Message-ID: <201003152108.o2FL8Ghn010156@scc.eng.rpath.com> changeset: 9175156fbae8 user: Elliot Peele date: Mon, 23 Nov 2009 01:06:21 -0500 add helper methods for getting advisory information diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -51,13 +51,24 @@ return self._order[0] @loadErrata - def lookupUpdateDetail(self, bucketId): + def getUpdateDetail(self, bucketId): """ Given a errata timestamp lookup the name and summary. """ + return self._advMap.get(bucketId, None) + + @loadErrata + def getUpdateDetailMessage(self, bucketId): + """ + Given a errata timestamp create a name and summary message. + """ + if bucketId in self._advMap: - return '%(name)s: %(summary)s' % self._advMap[bucketId] + msg = '' + for adv in self._advMap[bucketId]: + msg += '(%(name)s: %(summary)s) ' % adv + return msg else: return '%s (no detail found)' % bucketId @@ -180,8 +191,10 @@ for nevra in allocated: nevraMap[nevra] = bucketId - self._advMap[bucketId] = {'name': e.advisory, - 'summary': e.synopsis} + if bucketId not in self._advMap: + self._advMap[bucketId] = set() + self._advMap[bucketId].add({'name': e.advisory, + 'summary': e.synopsis}) # separate out golden bits other = [] From elliot at rpath.com Mon Mar 15 17:08:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:18 +0000 Subject: mirrorball: unique version list Message-ID: <201003152108.o2FL8IGq010205@scc.eng.rpath.com> changeset: 6421f2ce0131 user: Elliot Peele date: Mon, 23 Nov 2009 01:10:13 -0500 unique version list diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -558,4 +558,4 @@ for version, source in self._sources.iteritems(): if sources.areWeHereYet(pkgSet): versions.append(version) - return versions + return set(versions) From elliot at rpath.com Mon Mar 15 17:08:20 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:20 +0000 Subject: mirrorball: add script for testing iso indexing Message-ID: <201003152108.o2FL8KNA010232@scc.eng.rpath.com> changeset: df1bffadfc53 user: Elliot Peele date: Mon, 23 Nov 2009 01:10:47 -0500 add script for testing iso indexing diff --git a/scripts/parsepkgdata.py b/scripts/parsepkgdata.py new file mode 100755 --- /dev/null +++ b/scripts/parsepkgdata.py @@ -0,0 +1,35 @@ +#!/usr/bin/python +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Script for walking a tree of rpms and building the metadata needed to detect +repository state based on the contents of such a directory. +""" + +from header import * + +from updatebot import log as logger +from updatebot.pkgsource import rpmSource + +log = logger.addRootLogger() + +path = sys.argv[2] + +obj = RpmDirectoryIndexer(cfg, path) +obj.loadFromUrl(sys.argv[2], basePath=sys.argv[3]) + +nv = [ x for x in obj.iterPackageSet() ] + +import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:08:22 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:22 +0000 Subject: mirrorball: add config option for version sources Message-ID: <201003152108.o2FL8M8d010263@scc.eng.rpath.com> changeset: fd0836252a47 user: Elliot Peele date: Mon, 23 Nov 2009 01:11:09 -0500 add config option for version sources diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -104,6 +104,10 @@ # Paths based off of the repositoryUrl to get to individual repositories. repositoryPaths = (CfgList(CfgString), ['/']) + # Data source for determining platform version information, only used for + # group versioning. + versionSources = (CfgList(CfgList(CfgString)), []) + # The top level binary group, this may be the same as topSourceGroup. topGroup = CfgTroveSpec From elliot at rpath.com Mon Mar 15 17:08:23 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:23 +0000 Subject: mirrorball: add a method for retrieving source name/version map Message-ID: <201003152108.o2FL8NtD010294@scc.eng.rpath.com> changeset: 52db7605117f user: Elliot Peele date: Mon, 23 Nov 2009 01:12:06 -0500 add a method for retrieving source name/version map diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -127,6 +127,16 @@ log.info('found %s protentially updatable troves' % len(troves)) return troves + def getSourceVersionMap(self): + """ + Query the repository for a list of the latest source names and versions. + """ + + return dict([ (x.split(':')[0], y) for x, y, z in + self._conaryhelper.getSourceTroves(self._cfg.topGroup).iterkeys() + if not self._fltrPkg(x.split(':')[0]) + ]) + def _getLatestSource(self, name): """ Get the latest src package for a given package name. From elliot at rpath.com Mon Mar 15 17:08:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:25 +0000 Subject: mirrorball: current state of ordred update handling Message-ID: <201003152108.o2FL8QUF010322@scc.eng.rpath.com> changeset: 01f7c11548ae user: Elliot Peele date: Mon, 23 Nov 2009 01:14:07 -0500 current state of ordred update handling diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -317,4 +317,28 @@ """ _params = ['name', ] - _templates = 'I don not know what to do with this package %(name)s.' + _template = 'I don not know what to do with this package %(name)s.' + +class ImportError(UpdateBotError): + """ + General purpose error for all import related issues. + """ + +class PlatformAlreadyImportedError(ImportError): + """ + PlatformAlreadyImportedError, raised when a platform is being created in + ordered mode that already exists. + """ + + _params = [] + _template = ('This platform has already been imported, you probably meant ' + 'to run an update.') + +class PlatformNotImportedError(ImportError): + """ + PlatformNotImportedError, raised when a platform is being updated, but has + not yet been created. + """ + + _params = [] + _template = 'This platform has not yet been created.' diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -19,8 +19,11 @@ import logging from updatebot import errata +from updatebot import groupmgr from updatebot.bot import Bot as BotSuperClass +from updatebot.errors import PlatformAlreadyImportedError + log = logging.getLogger('updatebot.ordered') class Bot(BotSuperClass): @@ -34,15 +37,59 @@ def __init__(self, cfg, errataSource): BotSuperClass.__init__(self, cfg) self._errata = errata.ErrataFilter(self._pkgSource, errataSource) + self._groupmgr = groupmgr.GroupManager(self._cfg) + self._versionFactory = groupmgr.VersionFactory(self._cfg) + + def _addPackages(self, pkgMap): + """ + Add pkgMap to group. + """ + + for binSet in pkgMap.itervalues(): + pkgs = {} + for n, v, f in binSet: + if n not in pkgs: + pkgs[n] = {v: set([f, ])} + elif v not in pkgs[n]: + pkgs[n][v] = set([f, ]) + else: + pkgs[n][v].add(f) + + for name, vf in pkgs.iteritems(): + if ':' in name: + continue + assert len(vf) == 1 + version = vf.keys()[0] + flavors = list(vf[version]) + self._groupmgr.addPackage(name, version, flavors) def create(self, *args, **kwargs): """ Handle initial import case. """ + # Make sure this platform has not already been imported. + if self._groupmgr.getErrataState(): + raise PlatformAlreadyImportedError + self._pkgSource.load() toCreate = self._errata.getInitialPackages() - return self._create(*args, toCreate=toCreate, **kwargs) + pkgMap, failures = self._create(*args, toCreate=toCreate, **kwargs) + + # Insert package map into group. + self._addPackages(pkgMap) + + # Save group changes if there are any failures. + if failures: + self._groupmgr.save() + + # Try to build the group if everything imported. + else: + self._groupmgr.setErrataState('0') + self._groupmgr.setVersion('0') + self._groupmgr.build() + + return pkgMap, failures def update(self, *args, **kwargs): """ @@ -50,20 +97,50 @@ """ # Get current timestamp - # FIXME: Figure out where to store current errata level - raise NotImplementedError + current = self._groupmgr.getErrataState() + if current is None: + raise PlatformNotImportedError - current = 0 - + # Load package source. self._pkgSource.load() + updateSet = {} for updateId, updates in self._errata.iterByIssueDate(start=current): - detail = self._errata.getUpdateDetail(updateId) + detail = self._errata.getUpdateDetailMessage(updateId) log.info('attempting to apply %s' % detail) # Update package set. - self._update(updatePkgs=updates) + pkgMap = self._update(updatePkgs=updates) # Store current updateId. - # FIXME: figure out where/how to store the current updateId + self._groupmgr.setErrataState(updateId) + # Find errata group versions. + errataVersions = set() + for advisory in self._errata.getUpdateDetail(updateId): + # advisory names are in the form RHSA-2009:1234 + # transform to a conary version. + name = advisory['name'] + name = name.replace('-', '_') + name = name.replace(':', '.') + name += '_rolling' + errataVersions.add(name) + + # FIXME: this are probably not the versions that we need + # Get current set of source names and versions. + nvMap = self._updater.getSourceVersionMap() + for n, v, f in pkgMap.iterkeys(): + n = n.split(':')[0] + nvMap[n] = v + pkgSet = set(nvMap.items()) + + # Build various group verisons. + for version in (errataVersions + + self._groupmgr.getVersions(pkgSet)): + log.info('setting version %s' % version) + self._groupmgr.setVersion(version) + self._groupmgr.build() + + updateSet.update(pkgMap) + + return updateSet From elliot at rpath.com Mon Mar 15 17:08:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:27 +0000 Subject: mirrorball: use a hashable type when adding to a set Message-ID: <201003152108.o2FL8STX010351@scc.eng.rpath.com> changeset: 77b50bb15a46 user: Elliot Peele date: Mon, 23 Nov 2009 13:29:11 -0500 use a hashable type when adding to a set diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -56,7 +56,7 @@ Given a errata timestamp lookup the name and summary. """ - return self._advMap.get(bucketId, None) + return dict(self._advMap.get(bucketId, tuple())) @loadErrata def getUpdateDetailMessage(self, bucketId): @@ -193,8 +193,8 @@ if bucketId not in self._advMap: self._advMap[bucketId] = set() - self._advMap[bucketId].add({'name': e.advisory, - 'summary': e.synopsis}) + self._advMap[bucketId].add((('name', e.advisory), + ('summary', e.synopsis))) # separate out golden bits other = [] From elliot at rpath.com Mon Mar 15 17:08:29 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:29 +0000 Subject: mirrorball: fix syntax error Message-ID: <201003152108.o2FL8Udd010378@scc.eng.rpath.com> changeset: 473784684e17 user: Elliot Peele date: Mon, 23 Nov 2009 13:29:21 -0500 fix syntax error diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -533,7 +533,7 @@ self._pkgSource.loadFromUrl(self._url) - sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ] + sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ]) return sourceSet == pkgSet From elliot at rpath.com Mon Mar 15 17:08:34 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:34 +0000 Subject: mirrorball: change size of commit queue to 2 so that more jobs are bunched together under Message-ID: <201003152108.o2FL8YgD010411@scc.eng.rpath.com> changeset: 360c806b11f5 user: Elliot Peele date: Mon, 30 Nov 2009 13:03:37 -0500 change size of commit queue to 2 so that more jobs are bunched together under pressure diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -58,7 +58,7 @@ self._maxStartSlots = 10 self._startSlots = self._maxStartSlots - self._maxCommitSlots = 5 + self._maxCommitSlots = 2 self._commitSlots = self._maxCommitSlots self._starter = JobStarter(self._builder) From elliot at rpath.com Mon Mar 15 17:08:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:35 +0000 Subject: mirrorball: fix indent Message-ID: <201003152108.o2FL8a5l010440@scc.eng.rpath.com> changeset: 701133d43560 user: Elliot Peele date: Wed, 02 Dec 2009 22:58:10 -0500 fix indent diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -195,7 +195,7 @@ list(topTrove.iterTroveList(weakRefs=True, strongRefs=True))) for i, (n, v, f) in enumerate(topTrove.iterTroveList(weakRefs=True, - strongRefs=True)): + strongRefs=True)): src = (sources[i](), v.getSourceVersion(), None) if src not in srcTrvs: srcTrvs[src] = set() From elliot at rpath.com Mon Mar 15 17:08:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:37 +0000 Subject: mirrorball: always synth sources for indexed directories Message-ID: <201003152108.o2FL8bOR010467@scc.eng.rpath.com> changeset: 34bb9af95c3e user: Elliot Peele date: Wed, 02 Dec 2009 22:58:40 -0500 always synth sources for indexed directories diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -17,6 +17,7 @@ """ import os +import copy import logging from conary.deps import deps @@ -543,7 +544,8 @@ """ def __init__(self, cfg): - self._cfg = cfg + self._cfg = copy.deepcopy(cfg) + self._cfg.synthesizeSources = True self._sources = {} for version, url in cfg.versionSources: From elliot at rpath.com Mon Mar 15 17:08:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:39 +0000 Subject: mirrorball: move group flavoring down into the build module Message-ID: <201003152108.o2FL8eAq010499@scc.eng.rpath.com> changeset: 3163cf51f1af user: Elliot Peele date: Thu, 03 Dec 2009 14:25:06 -0500 move group flavoring down into the build module diff --git a/scripts/buildgroups b/scripts/buildgroups --- a/scripts/buildgroups +++ b/scripts/buildgroups @@ -11,10 +11,7 @@ builder = build.Builder(cfg, rmakeCfgFn='rmakerc-groups') -grpTrvs = set() -for flavor in cfg.groupFlavors: - grpTrvs.add((cfg.topSourceGroup[0], cfg.topSourceGroup[1], flavor)) -grpTrvMap = builder.build(grpTrvs) +grpTrvMap = builder.build((cfg.topSourceGroup, )) print "built:\n" diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -59,18 +59,6 @@ lst.extend(list(trvSet)) return lst - def _getGroupBuildTroves(self): - """ - Get the list of name, version, and flavor of the groups to build. - """ - - grpTrvs = set() - for flavor in self._cfg.groupFlavors: - grpTrvs.add((self._cfg.topSourceGroup[0], - self._cfg.topSourceGroup[1], - flavor)) - return grpTrvs - def create(self, rebuild=False, recreate=None, toCreate=None): """ Do initial imports. @@ -224,7 +212,7 @@ if not self._cfg.disableAdvisories: # Build group. - grpTrvs = self._getGroupBuildTroves() + grpTrvs = (self._cfg.topSourceGroup, ) grpTrvMap = self._builder.build(grpTrvs) # Promote group. diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -273,10 +273,11 @@ # the deps modules in conary. name = name.encode() - # Don't set context for groups, they will already have the - # correct flavors. + # Build groups in all of the defined falvors. We don't need a + # context here since groups are all built in a single job. if name.startswith('group-'): - troves.append((name, version, flavor)) + for flv in self._cfg.groupFlavors: + troves.append((name, version, flv)) # Kernels are special. elif ((name == 'kernel' or From elliot at rpath.com Mon Mar 15 17:08:41 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:41 +0000 Subject: mirrorball: fix a few typos Message-ID: <201003152108.o2FL8fKn010526@scc.eng.rpath.com> changeset: f3a200ce7d79 user: Michael K. Johnson date: Wed, 25 Nov 2009 19:08:24 -0500 fix a few typos diff --git a/scripts/buildpackages b/scripts/buildpackages --- a/scripts/buildpackages +++ b/scripts/buildpackages @@ -4,7 +4,7 @@ # """ -Script for cooking groups defined in the updatebot config. +Script for cooking packages with updatebot config. """ from header import * diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -317,7 +317,7 @@ """ _params = ['name', ] - _template = 'I don not know what to do with this package %(name)s.' + _template = 'I do not know what to do with this package %(name)s.' class ImportError(UpdateBotError): """ From elliot at rpath.com Mon Mar 15 17:08:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:43 +0000 Subject: mirrorball: change mode validation for symlinks (CNY-3304) Message-ID: <201003152108.o2FL8h9c010555@scc.eng.rpath.com> changeset: 5c12071615b6 user: Michael K. Johnson date: Mon, 07 Dec 2009 14:40:43 -0500 change mode validation for symlinks (CNY-3304) diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -435,10 +435,14 @@ %(rUser, rGroup, fileObj.inode.owner(), fileObj.inode.group())) - if stat.S_IMODE(rMode) != fileObj.inode.perms(): + if isinstance(fileObj, files.SymbolicLink): + expectedMode = 0777 # CNY-3304 + else: + expectedMode = stat.S_IMODE(rMode) + if fileObj.inode.perms() != expectedMode: fassert(False, fpath, 'Mode mismatch: RPM 0%o != Conary 0%o' - %(rMode, fileObj.inode.perms())) + %(expectedMode, fileObj.inode.perms())) if isinstance(fileObj, files.RegularFile): if not stat.S_ISREG(rMode): From elliot at rpath.com Mon Mar 15 17:08:44 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:44 +0000 Subject: mirrorball: added group editing Message-ID: <201003152108.o2FL8iVo010583@scc.eng.rpath.com> changeset: f338f6bbb604 user: Elliot Peele date: Tue, 08 Dec 2009 11:08:17 -0500 added group editing diff --git a/scripts/gengroup.py b/scripts/gengroup.py --- a/scripts/gengroup.py +++ b/scripts/gengroup.py @@ -34,9 +34,21 @@ mgr = groupmgr.GroupManager(cfg) +slog.info('retrieving trove information') troves = mgr._helper._getLatestTroves() +label = mgr._helper._ccfg.buildLabel +allTroves = mgr._helper._repos.getTroveLeavesByLabel({None: {label: None}}) mgr._checkout() +import itertools +for k1, k2 in itertools.izip(sorted(troves), sorted(allTroves)): + assert k1 == k2 + a = troves[k1] + b = allTroves[k2] + if not k1.startswith('group-') and len(a.values()[0]) != len(b.values()[0]): + slog.error('unhandled flavor found %s' % k1) + raise RuntimeError + import epdb; epdb.st() for name, vf in troves.iteritems(): @@ -47,11 +59,17 @@ versions.sort() version = versions[-1] flavors = vf[version] -# mgr.addPackage(name, version, flavors) + + if name == 'kernel': + import epdb; epdb.st() + + mgr.addPackage(name, version, flavors) + +import epdb; epdb.st() mgr.setVersion('0') mgr.setErrataState('0') -mgr._commit() +#mgr._commit() #mgr.build() import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:08:47 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:47 +0000 Subject: mirrorball: script for testing versioning Message-ID: <201003152108.o2FL8lee010616@scc.eng.rpath.com> changeset: 3bb10205fb41 user: Elliot Peele date: Tue, 08 Dec 2009 11:08:42 -0500 script for testing versioning diff --git a/scripts/gengroup.py b/scripts/vercheck.py copy from scripts/gengroup.py copy to scripts/vercheck.py --- a/scripts/gengroup.py +++ b/scripts/vercheck.py @@ -22,9 +22,7 @@ import time import logging -slog = logging.getLogger('script') - -log.addRootLogger() +slog = log.addRootLogger() cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) @@ -32,44 +30,11 @@ from updatebot import groupmgr -mgr = groupmgr.GroupManager(cfg) +versionFactory = groupmgr.VersionFactory(cfg) +pkgset = bot._updater.getSourceVersionMap() +sources = set([ (x, y.trailingRevision().getVersion()) + for x, y in pkgset.iteritems() ]) -slog.info('retrieving trove information') -troves = mgr._helper._getLatestTroves() -label = mgr._helper._ccfg.buildLabel -allTroves = mgr._helper._repos.getTroveLeavesByLabel({None: {label: None}}) -mgr._checkout() - -import itertools -for k1, k2 in itertools.izip(sorted(troves), sorted(allTroves)): - assert k1 == k2 - a = troves[k1] - b = allTroves[k2] - if not k1.startswith('group-') and len(a.values()[0]) != len(b.values()[0]): - slog.error('unhandled flavor found %s' % k1) - raise RuntimeError +match = versionFactory.getVersions(sources) import epdb; epdb.st() - -for name, vf in troves.iteritems(): - if ':' in name or bot._updater._fltrPkg(name): - continue - - versions = vf.keys() - versions.sort() - version = versions[-1] - flavors = vf[version] - - if name == 'kernel': - import epdb; epdb.st() - - mgr.addPackage(name, version, flavors) - -import epdb; epdb.st() - -mgr.setVersion('0') -mgr.setErrataState('0') -#mgr._commit() -#mgr.build() - -import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:08:50 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:50 +0000 Subject: mirrorball: fix formating Message-ID: <201003152108.o2FL8o0R010646@scc.eng.rpath.com> changeset: 3f1ce0934854 user: Elliot Peele date: Tue, 08 Dec 2009 11:09:28 -0500 fix formating diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -205,6 +205,7 @@ def key(self): return (self.name, self.flavor) + class XPackageData(XItemList): """ Mapping of package name to package group data. From elliot at rpath.com Mon Mar 15 17:08:51 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:51 +0000 Subject: mirrorball: improved indexing and logging Message-ID: <201003152108.o2FL8pFw010674@scc.eng.rpath.com> changeset: e4b24b5cddbf user: Elliot Peele date: Tue, 08 Dec 2009 11:10:42 -0500 improved indexing and logging diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -74,9 +74,13 @@ Walk the specified path to find rpms. """ + idx = 0 for path, dirs, files in self.walkMethod(self._path): for f in files: if f.endswith('.rpm'): + idx += 1 + if idx % 50 == 0: + log.info('indexing %s' % idx) yield self._index(os.path.join(path, f)) def _index(self, rpm): @@ -84,8 +88,6 @@ Index an individual rpm. """ - log.info(os.path.basename(rpm)) - fh = rpmheader.SeekableStream(rpm) h = rpmhelper.readHeader(fh) name = h[rpmhelper.NAME] @@ -124,7 +126,7 @@ PkgClass = Package - def __init__(self, cfg, path): + def __init__(self, cfg, path=None): PackageSource.__init__(self, cfg) self._path = path @@ -136,6 +138,9 @@ if self._loaded: return + if not self.path: + return + log.info('loading %s' % self._path) client = Client(self._path) From elliot at rpath.com Mon Mar 15 17:08:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:53 +0000 Subject: mirrorball: more changes towards auto versioned groups Message-ID: <201003152108.o2FL8r65010705@scc.eng.rpath.com> changeset: 92d5c51544ba user: Elliot Peele date: Tue, 08 Dec 2009 11:11:14 -0500 more changes towards auto versioned groups diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -106,7 +106,7 @@ # Data source for determining platform version information, only used for # group versioning. - versionSources = (CfgList(CfgList(CfgString)), []) + versionSources = (CfgDict(CfgString), {}) # The top level binary group, this may be the same as topSourceGroup. topGroup = CfgTroveSpec diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -24,6 +24,7 @@ from updatebot.lib import util from updatebot.build import Builder +from updatebot.pkgsource import RpmSource from updatebot.conaryhelper import ConaryHelper from updatebot.lib.xobjects import XGroup from updatebot.lib.xobjects import XGroupDoc @@ -31,6 +32,7 @@ from updatebot.lib.xobjects import XPackageDoc from updatebot.lib.xobjects import XPackageData from updatebot.lib.xobjects import XPackageItem + from updatebot.errors import FlavorCountMismatchError from updatebot.errors import UnknownBuildContextError from updatebot.errors import UnsupportedTroveFlavorError @@ -98,11 +100,7 @@ Build all configured flavors of the group. """ - # create list of trove specs to build - groupTroves = set() - for flavor in self._cfg.groupFlavors: - groupTroves.add((self._sourceName, self._sourceVersion, flavor)) - + groupTroves = ((self._sourceName, self._sourceVersion, None), ) built = self._builder.build(groupTroves) return built @@ -452,8 +450,11 @@ Freeze the model to a given output file. """ + def _srtByKey(a, b): + return cmp(a.key, b.key) + model = self.dataClass() - model.items = self._data.values() + model.items = sorted(self._data.values(), cmp=_srtByKey) doc = self.docClass() doc.data = model @@ -522,7 +523,7 @@ self._version = version self._url = url - self._pkgSource = pkgsource.rpmSource(cfg) + self._pkgSource = RpmSource(cfg, ) def areWeHereYet(self, pkgSet): """ @@ -535,6 +536,7 @@ self._pkgSource.loadFromUrl(self._url) sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ]) + import epdb; epdb.st() return sourceSet == pkgSet @@ -548,7 +550,7 @@ self._cfg.synthesizeSources = True self._sources = {} - for version, url in cfg.versionSources: + for version, url in sorted(cfg.versionSources.iteritems()): self._sources[version] = URLVersionSource(self._cfg, version, url) def getVersions(self, pkgSet): @@ -558,6 +560,6 @@ versions = [] for version, source in self._sources.iteritems(): - if sources.areWeHereYet(pkgSet): + if source.areWeHereYet(pkgSet): versions.append(version) return set(versions) diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -48,7 +48,9 @@ for binSet in pkgMap.itervalues(): pkgs = {} for n, v, f in binSet: - if n not in pkgs: + if ':' in n: + continue + elif n not in pkgs: pkgs[n] = {v: set([f, ])} elif v not in pkgs[n]: pkgs[n][v] = set([f, ]) @@ -56,8 +58,6 @@ pkgs[n][v].add(f) for name, vf in pkgs.iteritems(): - if ':' in name: - continue assert len(vf) == 1 version = vf.keys()[0] flavors = list(vf[version]) @@ -74,6 +74,7 @@ self._pkgSource.load() toCreate = self._errata.getInitialPackages() + pkgMap, failures = self._create(*args, toCreate=toCreate, **kwargs) # Insert package map into group. From elliot at rpath.com Mon Mar 15 17:08:55 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:55 +0000 Subject: mirrorball: branch merge Message-ID: <201003152108.o2FL8tlt010734@scc.eng.rpath.com> changeset: 58a3c34d6667 user: Elliot Peele date: Tue, 08 Dec 2009 11:11:24 -0500 branch merge diff --git a/scripts/buildgroups b/scripts/buildgroups --- a/scripts/buildgroups +++ b/scripts/buildgroups @@ -11,10 +11,7 @@ builder = build.Builder(cfg, rmakeCfgFn='rmakerc-groups') -grpTrvs = set() -for flavor in cfg.groupFlavors: - grpTrvs.add((cfg.topSourceGroup[0], cfg.topSourceGroup[1], flavor)) -grpTrvMap = builder.build(grpTrvs) +grpTrvMap = builder.build((cfg.topSourceGroup, )) print "built:\n" diff --git a/scripts/gengroup.py b/scripts/gengroup.py --- a/scripts/gengroup.py +++ b/scripts/gengroup.py @@ -34,9 +34,21 @@ mgr = groupmgr.GroupManager(cfg) +slog.info('retrieving trove information') troves = mgr._helper._getLatestTroves() +label = mgr._helper._ccfg.buildLabel +allTroves = mgr._helper._repos.getTroveLeavesByLabel({None: {label: None}}) mgr._checkout() +import itertools +for k1, k2 in itertools.izip(sorted(troves), sorted(allTroves)): + assert k1 == k2 + a = troves[k1] + b = allTroves[k2] + if not k1.startswith('group-') and len(a.values()[0]) != len(b.values()[0]): + slog.error('unhandled flavor found %s' % k1) + raise RuntimeError + import epdb; epdb.st() for name, vf in troves.iteritems(): @@ -47,11 +59,17 @@ versions.sort() version = versions[-1] flavors = vf[version] -# mgr.addPackage(name, version, flavors) + + if name == 'kernel': + import epdb; epdb.st() + + mgr.addPackage(name, version, flavors) + +import epdb; epdb.st() mgr.setVersion('0') mgr.setErrataState('0') -mgr._commit() +#mgr._commit() #mgr.build() import epdb; epdb.st() diff --git a/scripts/gengroup.py b/scripts/vercheck.py copy from scripts/gengroup.py copy to scripts/vercheck.py --- a/scripts/gengroup.py +++ b/scripts/vercheck.py @@ -22,9 +22,7 @@ import time import logging -slog = logging.getLogger('script') - -log.addRootLogger() +slog = log.addRootLogger() cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) @@ -32,26 +30,11 @@ from updatebot import groupmgr -mgr = groupmgr.GroupManager(cfg) +versionFactory = groupmgr.VersionFactory(cfg) +pkgset = bot._updater.getSourceVersionMap() +sources = set([ (x, y.trailingRevision().getVersion()) + for x, y in pkgset.iteritems() ]) -troves = mgr._helper._getLatestTroves() -mgr._checkout() +match = versionFactory.getVersions(sources) import epdb; epdb.st() - -for name, vf in troves.iteritems(): - if ':' in name or bot._updater._fltrPkg(name): - continue - - versions = vf.keys() - versions.sort() - version = versions[-1] - flavors = vf[version] -# mgr.addPackage(name, version, flavors) - -mgr.setVersion('0') -mgr.setErrataState('0') -mgr._commit() -#mgr.build() - -import epdb; epdb.st() diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -59,18 +59,6 @@ lst.extend(list(trvSet)) return lst - def _getGroupBuildTroves(self): - """ - Get the list of name, version, and flavor of the groups to build. - """ - - grpTrvs = set() - for flavor in self._cfg.groupFlavors: - grpTrvs.add((self._cfg.topSourceGroup[0], - self._cfg.topSourceGroup[1], - flavor)) - return grpTrvs - def create(self, rebuild=False, recreate=None, toCreate=None): """ Do initial imports. @@ -224,7 +212,7 @@ if not self._cfg.disableAdvisories: # Build group. - grpTrvs = self._getGroupBuildTroves() + grpTrvs = (self._cfg.topSourceGroup, ) grpTrvMap = self._builder.build(grpTrvs) # Promote group. diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -273,10 +273,11 @@ # the deps modules in conary. name = name.encode() - # Don't set context for groups, they will already have the - # correct flavors. + # Build groups in all of the defined falvors. We don't need a + # context here since groups are all built in a single job. if name.startswith('group-'): - troves.append((name, version, flavor)) + for flv in self._cfg.groupFlavors: + troves.append((name, version, flv)) # Kernels are special. elif ((name == 'kernel' or diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -58,7 +58,7 @@ self._maxStartSlots = 10 self._startSlots = self._maxStartSlots - self._maxCommitSlots = 5 + self._maxCommitSlots = 2 self._commitSlots = self._maxCommitSlots self._starter = JobStarter(self._builder) diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -195,7 +195,7 @@ list(topTrove.iterTroveList(weakRefs=True, strongRefs=True))) for i, (n, v, f) in enumerate(topTrove.iterTroveList(weakRefs=True, - strongRefs=True)): + strongRefs=True)): src = (sources[i](), v.getSourceVersion(), None) if src not in srcTrvs: srcTrvs[src] = set() diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -106,7 +106,7 @@ # Data source for determining platform version information, only used for # group versioning. - versionSources = (CfgList(CfgList(CfgString)), []) + versionSources = (CfgDict(CfgString), {}) # The top level binary group, this may be the same as topSourceGroup. topGroup = CfgTroveSpec diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -17,12 +17,14 @@ """ import os +import copy import logging from conary.deps import deps from updatebot.lib import util from updatebot.build import Builder +from updatebot.pkgsource import RpmSource from updatebot.conaryhelper import ConaryHelper from updatebot.lib.xobjects import XGroup from updatebot.lib.xobjects import XGroupDoc @@ -30,6 +32,7 @@ from updatebot.lib.xobjects import XPackageDoc from updatebot.lib.xobjects import XPackageData from updatebot.lib.xobjects import XPackageItem + from updatebot.errors import FlavorCountMismatchError from updatebot.errors import UnknownBuildContextError from updatebot.errors import UnsupportedTroveFlavorError @@ -97,11 +100,7 @@ Build all configured flavors of the group. """ - # create list of trove specs to build - groupTroves = set() - for flavor in self._cfg.groupFlavors: - groupTroves.add((self._sourceName, self._sourceVersion, flavor)) - + groupTroves = ((self._sourceName, self._sourceVersion, None), ) built = self._builder.build(groupTroves) return built @@ -451,8 +450,11 @@ Freeze the model to a given output file. """ + def _srtByKey(a, b): + return cmp(a.key, b.key) + model = self.dataClass() - model.items = self._data.values() + model.items = sorted(self._data.values(), cmp=_srtByKey) doc = self.docClass() doc.data = model @@ -521,7 +523,7 @@ self._version = version self._url = url - self._pkgSource = pkgsource.rpmSource(cfg) + self._pkgSource = RpmSource(cfg, ) def areWeHereYet(self, pkgSet): """ @@ -534,6 +536,7 @@ self._pkgSource.loadFromUrl(self._url) sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ]) + import epdb; epdb.st() return sourceSet == pkgSet @@ -543,10 +546,11 @@ """ def __init__(self, cfg): - self._cfg = cfg + self._cfg = copy.deepcopy(cfg) + self._cfg.synthesizeSources = True self._sources = {} - for version, url in cfg.versionSources: + for version, url in sorted(cfg.versionSources.iteritems()): self._sources[version] = URLVersionSource(self._cfg, version, url) def getVersions(self, pkgSet): @@ -556,6 +560,6 @@ versions = [] for version, source in self._sources.iteritems(): - if sources.areWeHereYet(pkgSet): + if source.areWeHereYet(pkgSet): versions.append(version) return set(versions) diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -205,6 +205,7 @@ def key(self): return (self.name, self.flavor) + class XPackageData(XItemList): """ Mapping of package name to package group data. diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -48,7 +48,9 @@ for binSet in pkgMap.itervalues(): pkgs = {} for n, v, f in binSet: - if n not in pkgs: + if ':' in n: + continue + elif n not in pkgs: pkgs[n] = {v: set([f, ])} elif v not in pkgs[n]: pkgs[n][v] = set([f, ]) @@ -56,8 +58,6 @@ pkgs[n][v].add(f) for name, vf in pkgs.iteritems(): - if ':' in name: - continue assert len(vf) == 1 version = vf.keys()[0] flavors = list(vf[version]) @@ -74,6 +74,7 @@ self._pkgSource.load() toCreate = self._errata.getInitialPackages() + pkgMap, failures = self._create(*args, toCreate=toCreate, **kwargs) # Insert package map into group. diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -74,9 +74,13 @@ Walk the specified path to find rpms. """ + idx = 0 for path, dirs, files in self.walkMethod(self._path): for f in files: if f.endswith('.rpm'): + idx += 1 + if idx % 50 == 0: + log.info('indexing %s' % idx) yield self._index(os.path.join(path, f)) def _index(self, rpm): @@ -84,8 +88,6 @@ Index an individual rpm. """ - log.info(os.path.basename(rpm)) - fh = rpmheader.SeekableStream(rpm) h = rpmhelper.readHeader(fh) name = h[rpmhelper.NAME] @@ -124,7 +126,7 @@ PkgClass = Package - def __init__(self, cfg, path): + def __init__(self, cfg, path=None): PackageSource.__init__(self, cfg) self._path = path @@ -136,6 +138,9 @@ if self._loaded: return + if not self.path: + return + log.info('loading %s' % self._path) client = Client(self._path) From elliot at rpath.com Mon Mar 15 17:08:57 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:57 +0000 Subject: mirrorball: test missingok mapping (CNY-3306) Message-ID: <201003152108.o2FL8v4k010763@scc.eng.rpath.com> changeset: 38ed43e474fc user: Michael K. Johnson date: Tue, 08 Dec 2009 15:47:24 -0500 test missingok mapping (CNY-3306) diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -510,6 +510,13 @@ fassert(fileObj.flags.isInitialContents(), fpath, 'RPM ghost non-directory is not InitialContents') + if rFlags & rpmhelper.RPMFILE_MISSINGOK: + fassert(fileObj.flags.isMissingOkay(), fpath, + 'RPM missingok file does not have missingOkay flag') + if fileObj.flags.isMissingOkay(): + fassert(rFlags & rpmhelper.RPMFILE_MISSINGOK, fpath, + 'missingOkay file does not have RPM missingok flag') + if not rVflags: # %doc -- CNY-3254 fassert(not fileObj.flags.isInitialContents(), fpath, From elliot at rpath.com Mon Mar 15 17:08:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:08:59 +0000 Subject: mirrorball: turn down number of concurent builds Message-ID: <201003152108.o2FL8xFu010829@scc.eng.rpath.com> changeset: 15dfbaffb462 user: Elliot Peele date: Sat, 12 Dec 2009 22:25:30 -0500 turn down number of concurent builds diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -165,7 +165,7 @@ @return troveMap: dictionary of troveSpecs to built troves """ - dispatcher = Dispatcher(self, 30) + dispatcher = Dispatcher(self, 15) return dispatcher.buildmany(troveSpecs) def buildsplitarch(self, troveSpecs): From elliot at rpath.com Mon Mar 15 17:09:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:02 +0000 Subject: mirrorball: always using buildmany when rebuilding in ordred mode Message-ID: <201003152109.o2FL93Oo010869@scc.eng.rpath.com> changeset: 28eb321ba973 user: Elliot Peele date: Sat, 12 Dec 2009 22:26:40 -0500 always using buildmany when rebuilding in ordred mode diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -120,7 +120,7 @@ trvMap = [] failed = () if len(toBuild): - if not rebuild: + if not rebuild or (rebuild and toCreate): # Build all newly imported packages. trvMap, failed = self._builder.buildmany(sorted(toBuild)) log.info('failed to import %s packages' % len(failed)) From elliot at rpath.com Mon Mar 15 17:09:07 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:07 +0000 Subject: mirrorball: Handle restarts when rebuildAll is set by looking for the latest binary Message-ID: <201003152109.o2FL98Hq010930@scc.eng.rpath.com> changeset: 7450b1ce8186 user: Elliot Peele date: Sat, 12 Dec 2009 22:31:06 -0500 Handle restarts when rebuildAll is set by looking for the latest binary grouppp and latest versions on the buildLabel and comparing versions. Only the packages that do not have newer versions will be rebuilt. diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -279,8 +279,7 @@ if pkgNames: toCreate = set() else: - # Import very specific versions of packages, make sure to recreate - # them all. + # Import very specific versions of packages. pkgNames = [] recreate = False @@ -298,10 +297,32 @@ if srcPkg.name not in pkgs or recreate: toCreate.add(srcPkg) + verCache = self._conaryhelper.getLatestVersions() + # In the case that we are rebuilding and already have groups available, + # try to only build packages that have not been rebuilt. + if buildAll and pkgs and not pkgNames: + nv = {} + for sn, pkgs in pkgs.iteritems(): + for n, v, f in pkgs: + if n not in nv: + nv[n] = (v, sn) + + create = set() + toCreateMap = dict([ (x.name, x) for x in toCreate ]) + for n, v in verCache.iteritems(): + # Skip all components, including sources + if len(n.split(':')) > 1: + continue + # If binary is in the group and the verison on the label is the + # same it needs to be updated. + if n in nv and v == nv[n][0] and nv[n][1] in toCreateMap: + create.add(toCreateMap[nv[n][1]]) + + toCreate = create + # Update all of the unique sources. fail = set() toBuild = set() - verCache = self._conaryhelper.getLatestVersions() for pkg in sorted(toCreate): try: # Only import packages that haven't been imported before @@ -318,7 +339,7 @@ log.error('failed to import %s: %s' % (pkg, e)) fail.add((pkg, e)) - if buildAll and pkgs: + if buildAll and pkgs and pkgNames: toBuild.update( [ (x, self._conaryhelper.getLatestSourceVersion(x), None) for x in pkgs if not self._fltrPkg(x) ] @@ -335,10 +356,10 @@ # pylint: disable-msg=W0612 try: - return [ n.split(':')[0] for n, v, f in - self._conaryhelper.getSourceTroves(self._cfg.topGroup).iterkeys() ] + return dict([(n.split(':')[0], pkgs) for (n, v, f), pkgs in + self._conaryhelper.getSourceTroves(self._cfg.topGroup).iteritems()]) except GroupNotFound: - return [] + return {} def _getPackagesToImport(self, name): """ From elliot at rpath.com Mon Mar 15 17:09:12 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:12 +0000 Subject: mirrorball: Report errors when the errataSource produces errata without packages Message-ID: <201003152109.o2FL9C8r010969@scc.eng.rpath.com> changeset: 6fab677d41fb user: Elliot Peele date: Sat, 12 Dec 2009 22:34:48 -0500 Report errors when the errataSource produces errata without packages diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -20,6 +20,7 @@ import logging from updatebot.errors import ErrataPackageNotFoundError +from updatebot.errors import ErrataSourceDataMissingError log = logging.getLogger('updatebot.errata') @@ -112,7 +113,8 @@ # get sources to build for bucketId in sorted(buckets.keys()): bucket = buckets[bucketId] - self._order[bucketId] = set() + if bucketId not in self._order: + self._order[bucketId] = set() for pkg in bucket: src = self._pkgSource.binPkgMap[pkg] self._order[bucketId].add(src) @@ -145,6 +147,7 @@ # pull nevras into errata sized buckets buckets = {} nevraMap = {} + broken = [] log.info('processing errata') @@ -181,6 +184,13 @@ else: raise ErrataPackageNotFoundError(pkg=nevra) + # There should be packages in the bucket, if there aren't the errata + # store is probably broken. + if not bucket: + broken.append(e.advisory) + log.critical('broken advisory: %s' % e.advisory) + continue + if bucketId is None: bucketId = int(time.mktime(time.strptime(e.issue_date, '%Y-%m-%d %H:%M:%S'))) @@ -196,6 +206,9 @@ self._advMap[bucketId].add((('name', e.advisory), ('summary', e.synopsis))) + if broken: + raise ErrataSourceDataMissingError(broken=broken) + # separate out golden bits other = [] golden = [] diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -273,6 +273,18 @@ 'configured repositories when attempting to map errata source to ' 'package source.') +class ErrataSourceDataMissingError(ErrataError): + """ + ErrataSourceDataMissingError, raised when missing package or channel data is + detected. This normally means that the errata source is corupt or missing + data. + """ + + _params = ['broken', ] + _tempalte = ('Found missing information when parsing errata source. This ' + 'normally means that the errat source is corrupt or incorect. ' + '%(broken)s') + class GroupManagerError(UpdateBotError): """ GroupManagerError, generic error for group manager related errors. From elliot at rpath.com Mon Mar 15 17:09:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:15 +0000 Subject: mirrorball: coment fixups and formating Message-ID: <201003152109.o2FL9Glc011000@scc.eng.rpath.com> changeset: 4e41f1653297 user: Elliot Peele date: Sun, 13 Dec 2009 20:24:01 -0500 coment fixups and formating diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -145,13 +145,14 @@ if x.arch != 'src' and '-debuginfo' not in x.name) # pull nevras into errata sized buckets + broken = [] buckets = {} nevraMap = {} - broken = [] log.info('processing errata') indexedChannels = set(self._errata.getChannels()) + # FIXME: This should not be a hard coded set of arches. arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') for e in self._errata.iterByIssueDate(): bucket = [] @@ -184,9 +185,10 @@ else: raise ErrataPackageNotFoundError(pkg=nevra) - # There should be packages in the bucket, if there aren't the errata - # store is probably broken. - if not bucket: + # There should be packages in the bucket or the packages should + # already be in an existing bucket (bucketId != None), if there + # aren't the errata store is probably broken. + if not bucket and bucketId is None: broken.append(e.advisory) log.critical('broken advisory: %s' % e.advisory) continue From elliot at rpath.com Mon Mar 15 17:09:19 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:19 +0000 Subject: mirrorball: use the builtin ordering Message-ID: <201003152109.o2FL9LBd011034@scc.eng.rpath.com> changeset: 1da69dfb492f user: Elliot Peele date: Mon, 14 Dec 2009 18:24:36 -0500 use the builtin ordering diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py --- a/scripts/rhelorder.py +++ b/scripts/rhelorder.py @@ -2,11 +2,12 @@ import os import sys +import tempfile sys.path.insert(0, os.environ['HOME'] + '/hg/conary') sys.path.insert(0, os.environ['HOME'] + '/hg/rhnmirror') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') -sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-capsule-indexer') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-xmllib') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-5.5/rpath-capsule-indexer') from conary.lib import util sys.excepthook = util.genExcepthook() @@ -14,138 +15,38 @@ mbdir = os.path.abspath('../') sys.path.insert(0, mbdir) -confDir = os.path.join(mbdir, 'config', 'rhel4') +confDir = os.path.join(mbdir, 'config', 'rhel5') from updatebot import log -from updatebot import Bot +from updatebot.ordered import Bot from updatebot import UpdateBotConfig -import time -import logging - -slog = logging.getLogger('script') - import rhnmirror -log.addRootLogger() -cfg = UpdateBotConfig() -cfg.read(os.path.join(confDir, 'updatebotrc')) - -bot = Bot(cfg) +slog = log.addRootLogger() mcfg = rhnmirror.MirrorConfig() -mcfg.read(os.path.join(confDir, 'erratarc')) +mcfg.read(confDir + '/erratarc') +#mcfg.indexerDb += '5' +#mcfg.indexerDb = 'sqlite:///%s' % tempfile.mktemp(suffix='.db', prefix='order-') + +slog.info('db = %s' % mcfg.indexerDb) + +#mcfg.channels = [ +# 'rhel-x86_64-server-5', +# 'rhel-i386-server-5', +# 'rhel-x86_64-as-4', +# 'rhel-i386-as-4', +#] errata = rhnmirror.Errata(mcfg) errata.fetch() -pkgSource = bot._pkgSource -pkgSource.load() +cfg = UpdateBotConfig() +cfg.read(os.path.join(confDir, 'updatebotrc')) -# get mapping of advisory to errata obj -advisories = dict((x.advisory, x) - for x in errata.iterByIssueDate(mcfg.channels)) - -# get mapping of nevra to pkg obj -nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) - for x in pkgSource.binPkgMap.keys() if x.arch != 'src') - -# pull nevras into errata sized buckets -buckets = {} -advMap = {} -nevraMap = {} - -arches = ('i386', 'i486', 'i586', 'i686', 'x86_64', 'noarch') -for e in errata.iterByIssueDate(mcfg.channels): - bnevras = [] - bucket = [] - bucketId = None - slog.info('processing %s' % e.advisory) - for pkg in e.packages: - nevra = pkg.getNevra() - - # filter out channels we don't have indexed - channels = set([ x.label for x in pkg.channels ]) - if not set(mcfg.channels) & channels: - continue - - # ignore arches we don't know about. - if nevra[4] not in arches: - continue - - # convert rhn nevra to yum nevra - nevra = list(nevra) - if nevra[1] is None: - nevra[1] = '0' - if type(nevra[1]) == int: - nevra[1] = str(nevra[1]) - nevra = tuple(nevra) - - # move nevra to errata buckets - if nevra in nevras: - binPkg = nevras.pop(nevra) - bucket.append(binPkg) - bnevras.append(nevra) - - # nevra is already part of another bucket - elif nevra in nevraMap: - bucketId = nevraMap[nevra] - - # raise error if we can't find the required package - else: - raise KeyError - - if bucketId is None: - bucketId = int(time.mktime(time.strptime(e.issue_date, - '%Y-%m-%d %H:%M:%S'))) - buckets[bucketId] = bucket - else: - buckets[bucketId].extend(bucket) - - for nevra in bnevras: - nevraMap[nevra] = bucketId - - advMap[e.advisory] = bucketId - -# separate out golden bits -other = [] -golden = [] -firstErrata = sorted(buckets.keys())[0] -for nevra, pkg in nevras.iteritems(): - buildtime = int(pkg.buildTimestamp) - if buildtime < firstErrata: - golden.append(pkg) - else: - other.append(pkg) - -# sort by source package -srcMap = {} -for pkg in other: - src = pkgSource.binPkgMap[pkg] - if src not in srcMap: - srcMap[src] = [] - srcMap[src].append(pkg) - -# insert bins by buildstamp -for src, bins in srcMap.iteritems(): - buildstamp = int(sorted(bins)[0].buildTimestamp) - if buildstamp in buckets: - buckets[buildstamp].extend(bins) - else: - buckets[buildstamp] = bins - -# get sources to build -buildOrder = {0: set()} -for pkg in golden: - # lookup source package - src = pkgSource.binPkgMap[pkg] - buildOrder[0].add(src) - -for bucketId in sorted(buckets.keys()): - bucket = buckets[bucketId] - buildOrder[bucketId] = set() - for pkg in bucket: - src = pkgSource.binPkgMap[pkg] - buildOrder[bucketId].add(src) +bot = Bot(cfg, errata) +bot._pkgSource.load() +bot._errata._orderErrata() import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:09:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:24 +0000 Subject: mirrorball: add helper methods to package objects Message-ID: <201003152109.o2FL9OiT011064@scc.eng.rpath.com> changeset: c2726e4f4968 user: Elliot Peele date: Mon, 14 Dec 2009 18:26:10 -0500 add helper methods to package objects diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -157,6 +157,24 @@ return cmp(self.location, other.location) + def getNevra(self): + """ + Return the name, epoch, version, release, and arch the package. + """ + + return (self.name, self.epoch, self.version, self.release, self.arch) + + def getConaryVersion(self): + """ + Get the conary version of a source package. + """ + + assert self.arch == 'src' + filename = os.path.basename(self.location) + nameVerRelease = ".".join(filename.split(".")[:-2]) + ver = "_".join(nameVerRelease.split("-")[-2:]) + return ver + class _RpmRequires(xmllib.BaseNode): """ From elliot at rpath.com Mon Mar 15 17:09:26 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:26 +0000 Subject: mirrorball: split out getSourceVersions Message-ID: <201003152109.o2FL9Qeb011098@scc.eng.rpath.com> changeset: 1cbf1ebe32d0 user: Elliot Peele date: Mon, 14 Dec 2009 18:29:07 -0500 split out getSourceVersions diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -187,15 +187,25 @@ recurse=False) topTrove = self._getTrove(cs, name, version, flavor) + troves = topTrove.iterTroveList(weakRefs=True, strongRefs=True) + + return self.getSourceVersions(list(troves)) + + def getSourceVersions(self, troveSpecs): + """ + Given a list of trove specs, query the repository for all of the related + source versions. + @param troveSpecs: list of troves to query for. + @type troveSpecs: [(name, versionObj, flavObj), ... ] + @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ...]} + """ # Iterate over both strong and weak refs because msw said it was a # good idea. srcTrvs = {} - sources = self._repos.getTroveInfo(trove._TROVEINFO_TAG_SOURCENAME, - list(topTrove.iterTroveList(weakRefs=True, - strongRefs=True))) - for i, (n, v, f) in enumerate(topTrove.iterTroveList(weakRefs=True, - strongRefs=True)): + tiSourceName = trove._TROVEINFO_TAG_SOURCENAME + sources = self._repos.getTroveInfo(tiSourceName, troveSpecs) + for i, (n, v, f) in enumerate(troveSpecs): src = (sources[i](), v.getSourceVersion(), None) if src not in srcTrvs: srcTrvs[src] = set() From elliot at rpath.com Mon Mar 15 17:09:30 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:30 +0000 Subject: mirrorball: extend getSourceVersionMap to get the source versions for all binaries that Message-ID: <201003152109.o2FL9VGT011153@scc.eng.rpath.com> changeset: 1d9ad7592c80 user: Elliot Peele date: Mon, 14 Dec 2009 18:29:49 -0500 extend getSourceVersionMap to get the source versions for all binaries that would otherwise be included in a group cook. diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -129,13 +129,32 @@ def getSourceVersionMap(self): """ - Query the repository for a list of the latest source names and versions. + Query the repository for a list of the latest source names and versions + that have binary versions. + @return {sourceName: sourceVersion} """ - return dict([ (x.split(':')[0], y) for x, y, z in - self._conaryhelper.getSourceTroves(self._cfg.topGroup).iterkeys() - if not self._fltrPkg(x.split(':')[0]) - ]) + # Get the latest versions from repository + troves = self._conaryhelper._getLatestTroves() + + # Convert to a n, v, f list excluding components and sources. + troveSpecs = [] + for name, verDict in troves.iteritems(): + if ':' in name: + continue + assert len(verDict) == 1 + version = verDict.keys()[0] + flavor = list(verDict[version])[0] + troveSpecs.append((name, version, flavor)) + + # Get the sources for all binary packages. + sourceVersions = self._conaryhelper.getSourceVersions(troveSpecs) + + # Convert to sourceName: version map. + return dict([ (x.split(':')[0], y) + for x, y, z in sourceVersions.iterkeys() + if not self._fltrPkg(x.split(':')[0]) + ]) def _getLatestSource(self, name): """ From elliot at rpath.com Mon Mar 15 17:09:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:35 +0000 Subject: mirrorball: hook up version factory Message-ID: <201003152109.o2FL9Zv0011197@scc.eng.rpath.com> changeset: 124e5827da08 user: Elliot Peele date: Mon, 14 Dec 2009 18:33:17 -0500 hook up version factory diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -67,6 +67,8 @@ self._helper = GroupHelper(self._cfg) self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') + self._versionFactory = VersionFactory(cfg) + self._sourceName = self._cfg.topSourceGroup[0] self._sourceVersion = self._cfg.topSourceGroup[1] @@ -283,6 +285,14 @@ self._helper.getErrataState(self._sourceName) + def getVersions(self, pkgSet): + """ + Get the set of versions that are represented by the given set of + packages from the version factory. + """ + + return self._versionFactory.getVersions(pkgSet) + class GroupHelper(ConaryHelper): """ @@ -534,10 +544,14 @@ """ self._pkgSource.loadFromUrl(self._url) + self._pkgSource.finalize() sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ]) - import epdb; epdb.st() - return sourceSet == pkgSet + + # Make sure that all versions that are on the ISO are in the conary + # repository. It appears to be a common case that the ISO is missing + # content from RHN. + return not sourceSet.difference(pkgSet) class VersionFactory(object): @@ -559,7 +573,7 @@ """ versions = [] - for version, source in self._sources.iteritems(): + for version, source in sorted(self._sources.iteritems()): if source.areWeHereYet(pkgSet): versions.append(version) return set(versions) From elliot at rpath.com Mon Mar 15 17:09:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:37 +0000 Subject: mirrorball: make sure to promote built groups Message-ID: <201003152109.o2FL9bWw011229@scc.eng.rpath.com> changeset: ea9d43f2e01e user: Elliot Peele date: Mon, 14 Dec 2009 18:35:23 -0500 make sure to promote built groups diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -130,6 +130,7 @@ # FIXME: this are probably not the versions that we need # Get current set of source names and versions. nvMap = self._updater.getSourceVersionMap() + # Add in new names and versions that have just been built. for n, v, f in pkgMap.iterkeys(): n = n.split(':')[0] nvMap[n] = v @@ -140,7 +141,13 @@ self._groupmgr.getVersions(pkgSet)): log.info('setting version %s' % version) self._groupmgr.setVersion(version) - self._groupmgr.build() + grpTrvMap = self._groupmgr.build() + + log.info('promoting version %s' % version) + expected = self._flattenSetDict(pkgMap) + toPublish = self._flattenSetDict(grpTrvMap) + newTroves = self._updater.publish(toPublish, expected, + self._cfg.targetLabel) updateSet.update(pkgMap) From elliot at rpath.com Mon Mar 15 17:09:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:43 +0000 Subject: mirrorball: add hook for excluding packages Message-ID: <201003152109.o2FL9hNQ011262@scc.eng.rpath.com> changeset: 2d10e62be7ff user: Elliot Peele date: Mon, 14 Dec 2009 18:36:18 -0500 add hook for excluding packages diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -71,6 +71,7 @@ @type basePath: string """ + log.info('loading repository data %s/%s' % (url, basePath)) client = repomd.Client(url + '/' + basePath) self.loadFromClient(client, basePath=basePath) @@ -99,6 +100,9 @@ pkg.location = basePath + '/' + pkg.location + if self._excludeLocation(pkg.location): + continue + # ignore 32bit rpms in a 64bit repo. if (pkg.arch in ('i386', 'i586', 'i686') and 'x86_64' in pkg.location): @@ -153,6 +157,13 @@ self.locationMap[package.location] = package + def _excludeLocation(self, location): + """ + Method for filtering packages based on locaiton. + """ + + return False + def finalize(self): """ Make some final datastructures now that we are done populating object. From elliot at rpath.com Mon Mar 15 17:09:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:49 +0000 Subject: mirrorball: switch to using a yum backed source Message-ID: <201003152109.o2FL9nCh011299@scc.eng.rpath.com> changeset: 7a901a77c7e2 user: Elliot Peele date: Mon, 14 Dec 2009 18:42:24 -0500 switch to using a yum backed source diff --git a/updatebot/pkgsource/rpmsource.py b/updatebot/pkgsource/rpmsource.py --- a/updatebot/pkgsource/rpmsource.py +++ b/updatebot/pkgsource/rpmsource.py @@ -120,6 +120,26 @@ class RpmSource(PackageSource): + PkgClass = Package + + def iterPackageSet(self): + """ + Iterate over the set of source packages. + """ + + for srcPkg, binPkgs in self.srcPkgMap.iteritems(): + if not len(binPkgs): + continue + yield (srcPkg.name, srcPkg.getConaryVersion()) + + def _excludeLocation(self, location): + # FIXME: This should not be hard coded. There needs to be a config + # option for exclusions. + path = os.path.dirname(location) + if 'VT' in path or 'Cluster' in path: + return True + +class _RpmSource(PackageSource): """ Walk a directory, find rpms, index them. """ From elliot at rpath.com Mon Mar 15 17:09:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:52 +0000 Subject: mirrorball: updates don't have failures Message-ID: <201003152109.o2FL9qF5011347@scc.eng.rpath.com> changeset: 2e0cac1f7cd1 user: Elliot Peele date: Tue, 15 Dec 2009 14:09:07 -0500 updates don't have failures diff --git a/scripts/order_import.py b/scripts/order_update.py copy from scripts/order_import.py copy to scripts/order_update.py --- a/scripts/order_import.py +++ b/scripts/order_update.py @@ -61,6 +61,6 @@ errata.fetch() bot = ordered.Bot(cfg, errata) -pkgMap, failures = bot.create() +pkgMap = bot.update() import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:09:55 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:55 +0000 Subject: mirrorball: correctly convert _advMap entries to dicts Message-ID: <201003152109.o2FL9umQ011397@scc.eng.rpath.com> changeset: ffeca99c2f18 user: Elliot Peele date: Tue, 15 Dec 2009 14:12:59 -0500 correctly convert _advMap entries to dicts switch to current instead of start correctly add packages to buckets diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -57,7 +57,7 @@ Given a errata timestamp lookup the name and summary. """ - return dict(self._advMap.get(bucketId, tuple())) + return [ dict(x) for x in self._advMap.get(bucketId, tuple()) ] @loadErrata def getUpdateDetailMessage(self, bucketId): @@ -68,21 +68,22 @@ if bucketId in self._advMap: msg = '' for adv in self._advMap[bucketId]: - msg += '(%(name)s: %(summary)s) ' % adv + msg += '(%(name)s: %(summary)s) ' % dict(adv) return msg else: return '%s (no detail found)' % bucketId @loadErrata - def iterByIssueDate(self, start=None): + def iterByIssueDate(self, current=None): """ Yield sets of srcPkgs by errata release date. - @param start: timestamp from which to start iterating. - @type start: int + @param current: the current state, start iterating after this state has + been reached. + @type current: int """ for stamp in sorted(self._order.keys()): - if start > stamp: + if current >= stamp: continue yield stamp, self._order[stamp] @@ -113,8 +114,7 @@ # get sources to build for bucketId in sorted(buckets.keys()): bucket = buckets[bucketId] - if bucketId not in self._order: - self._order[bucketId] = set() + self._order[bucketId] = set() for pkg in bucket: src = self._pkgSource.binPkgMap[pkg] self._order[bucketId].add(src) @@ -196,9 +196,10 @@ if bucketId is None: bucketId = int(time.mktime(time.strptime(e.issue_date, '%Y-%m-%d %H:%M:%S'))) - buckets[bucketId] = bucket - else: - buckets[bucketId].extend(bucket) + + if bucketId not in buckets: + buckets[bucketId] = set() + buckets[bucketId].update(bucket) for nevra in allocated: nevraMap[nevra] = bucketId From elliot at rpath.com Mon Mar 15 17:09:57 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:09:57 +0000 Subject: mirrorball: try to pull contents from the changeset before going to the repository Message-ID: <201003152109.o2FL9vZi011466@scc.eng.rpath.com> changeset: e33737c9773f user: Elliot Peele date: Tue, 15 Dec 2009 14:13:47 -0500 try to pull contents from the changeset before going to the repository diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -591,16 +591,17 @@ for capFile, fileList, fileObjs in sorted(capsules, cmp=idCmp): if capFile[2] in contentsCache: capsuleFileContents = contentsCache[capFile[2]] - elif newTroveCs.getOldVersion(): - getFileContents = self._client.repos.getFileContents - fcList = getFileContents((capFile[2:], ), compressed=False) - capsuleFileContents = fcList[0].get() else: - getFileContents = newCs.getFileContents - fcList = getFileContents(capFile[0], capFile[2], - compressed=False) - capsuleFileContents = fcList[1].get() - contentsCache[capFile[2]] = capsuleFileContents + try: + getFileContents = newCs.getFileContents + fcList = getFileContents(capFile[0], capFile[2], + compressed=False) + capsuleFileContents = fcList[1].get() + except KeyError, e: + getFileContents = self._client.repos.getFileContents + fcList = getFileContents((capFile[2:], ), compressed=False) + capsuleFileContents = fcList[0].get() + contentsCache[capFile[2]] = capsuleFileContents # do the check self._sanityCheckRPMCapsule(jobId, fileList, fileObjs, From elliot at rpath.com Mon Mar 15 17:10:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:10:02 +0000 Subject: mirrorball: minor fixups Message-ID: <201003152110.o2FLA28N011504@scc.eng.rpath.com> changeset: ebcf4078dc45 user: Elliot Peele date: Tue, 15 Dec 2009 14:14:21 -0500 minor fixups don't use autoLoadRecipes when dealing with groups, they don't play well with factory-version diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -283,7 +283,7 @@ Get the errata state info. """ - self._helper.getErrataState(self._sourceName) + return self._helper.getErrataState(self._sourceName) def getVersions(self, pkgSet): """ @@ -305,6 +305,13 @@ self._newPkgFactory = 'managed-group' self._groupContents = cfg.groupContents + # FIXME: autoLoadRecipes causes group versioning to go sideways + # The group super class in the repository has a version defined, which + # overrides the version from factory-version. This should probably be + # considered a bug in factory-managed-group, but we don't need + # autoLoadRecipes here anyway. + self._ccfg.autoLoadRecipes = [] + def getModel(self, pkgName): """ Get a thawed data representation of the group xml data from the @@ -386,6 +393,8 @@ return None state = open(stateFileName).read().strip() + if state.isdigit(): + state = int(state) return state def setErrataState(self, pkgname, state): @@ -400,7 +409,7 @@ # write state info statefh = open(stateFileName, 'w') - statefh.write(state) + statefh.write(str(state)) # source files must end in a trailing newline statefh.write('\n') From elliot at rpath.com Mon Mar 15 17:10:09 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:10:09 +0000 Subject: mirrorball: fix update path Message-ID: <201003152110.o2FLA9UP011537@scc.eng.rpath.com> changeset: a5be214ca1e1 user: Elliot Peele date: Tue, 15 Dec 2009 16:57:10 -0500 fix update path diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -16,12 +16,14 @@ Module for doing updates ordered by errata information. """ +import time import logging from updatebot import errata from updatebot import groupmgr from updatebot.bot import Bot as BotSuperClass +from updatebot.errors import PlatformNotImportedError from updatebot.errors import PlatformAlreadyImportedError log = logging.getLogger('updatebot.ordered') @@ -106,15 +108,13 @@ self._pkgSource.load() updateSet = {} - for updateId, updates in self._errata.iterByIssueDate(start=current): + for updateId, updates in self._errata.iterByIssueDate(current=current): + start = time.time() detail = self._errata.getUpdateDetailMessage(updateId) log.info('attempting to apply %s' % detail) # Update package set. - pkgMap = self._update(updatePkgs=updates) - - # Store current updateId. - self._groupmgr.setErrataState(updateId) + pkgMap = self._update(*args, updatePkgs=updates, **kwargs) # Find errata group versions. errataVersions = set() @@ -127,7 +127,6 @@ name += '_rolling' errataVersions.add(name) - # FIXME: this are probably not the versions that we need # Get current set of source names and versions. nvMap = self._updater.getSourceVersionMap() # Add in new names and versions that have just been built. @@ -135,20 +134,39 @@ n = n.split(':')[0] nvMap[n] = v pkgSet = set(nvMap.items()) + # Get the major distro verisons from the group manager. + majorVersions = self._groupmgr.getVersions(pkgSet) + + # Store current updateId. + self._groupmgr.setErrataState(updateId) + + # Make sure built troves are part of group. + self._addPackages(pkgMap) # Build various group verisons. - for version in (errataVersions + - self._groupmgr.getVersions(pkgSet)): + expected = self._flattenSetDict(pkgMap) + for version in errataVersions | majorVersions: log.info('setting version %s' % version) self._groupmgr.setVersion(version) grpTrvMap = self._groupmgr.build() log.info('promoting version %s' % version) - expected = self._flattenSetDict(pkgMap) toPublish = self._flattenSetDict(grpTrvMap) - newTroves = self._updater.publish(toPublish, expected, - self._cfg.targetLabel) + newTroves = self._updater.publish( + toPublish, + expected, + self._cfg.targetLabel + ) + + # After the first promote, packages should not be repromoted. + expected = set() updateSet.update(pkgMap) + # Report timings + advTime = time.strftime('%m-%d-%Y %H:%M:%S', + time.localtime(updateId)) + totalTime = time.time() - start + log.info('published update %s in %s seconds' % (advTime, totalTime)) + return updateSet From elliot at rpath.com Mon Mar 15 17:10:17 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:10:17 +0000 Subject: mirrorball: only call download callback once Message-ID: <201003152110.o2FLAIlI011573@scc.eng.rpath.com> changeset: 63e147f7b504 user: Elliot Peele date: Tue, 15 Dec 2009 16:57:25 -0500 only call download callback once diff --git a/updatebot/lib/conarycallbacks.py b/updatebot/lib/conarycallbacks.py --- a/updatebot/lib/conarycallbacks.py +++ b/updatebot/lib/conarycallbacks.py @@ -79,3 +79,6 @@ def gettingCloneData(self): CloneCallback.gettingCloneData(self) + @callonce + def sendingChangset(self, got, need): + self._message('uploading changeset') From elliot at rpath.com Mon Mar 15 17:10:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:10:27 +0000 Subject: mirrorball: check isinstance instead of type to cover conary.streams.FlavorStream as well Message-ID: <201003152110.o2FLAR3F011609@scc.eng.rpath.com> changeset: 9174dd603816 user: Elliot Peele date: Tue, 15 Dec 2009 18:58:36 -0500 check isinstance instead of type to cover conary.streams.FlavorStream as well as any other subclasses of conary.deps.deps.Flavor diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -196,7 +196,7 @@ else: self.use = use - if type(flavor) == conary.deps.deps.Flavor: + if isinstance(flavor, conary.deps.deps.Flavor): self.flavor = flavor.freeze() else: self.flavor = flavor From elliot at rpath.com Mon Mar 15 17:10:34 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:10:34 +0000 Subject: mirrorball: add hooks to make sure data doesn't get loaded twice Message-ID: <201003152110.o2FLAYUT011644@scc.eng.rpath.com> changeset: b6ebb8218792 user: Elliot Peele date: Wed, 16 Dec 2009 11:30:08 -0500 add hooks to make sure data doesn't get loaded twice diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -25,6 +25,14 @@ log = logging.getLogger('updatebot.pkgsource') +def loaded(func): + def wrapper(self, *args, **kwargs): + if self._loaded: + return + return func(self, *args, **kwargs) + return wrapper + + class YumSource(BasePackageSource): """ Class that builds maps of packages from multiple yum repositories. @@ -44,14 +52,15 @@ # set of all src pkg objects self._srcPkgs = set() + def setLoaded(self): + self._loaded = True + + @loaded def load(self): """ Load package source based on config data. """ - if self._loaded: - return - for repo in self._cfg.repositoryPaths: log.info('loading repository data %s' % repo) client = repomd.Client(self._cfg.repositoryUrl + '/' + repo) @@ -61,6 +70,7 @@ self.finalize() self._loaded = True + @loaded def loadFromUrl(self, url, basePath=''): """ Walk the yum repository rooted at url/basePath and collect information @@ -75,6 +85,7 @@ client = repomd.Client(url + '/' + basePath) self.loadFromClient(client, basePath=basePath) + @loaded def loadFromClient(self, client, basePath=''): """ Walk the yum repository rooted at url/basePath and collect information @@ -164,6 +175,7 @@ return False + @loaded def finalize(self): """ Make some final datastructures now that we are done populating object. From elliot at rpath.com Mon Mar 15 17:10:41 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:10:41 +0000 Subject: mirrorball: set the pkgsource as loaded Message-ID: <201003152110.o2FLAfUu011676@scc.eng.rpath.com> changeset: c2a247faac29 user: Elliot Peele date: Wed, 16 Dec 2009 11:30:31 -0500 set the pkgsource as loaded diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -554,6 +554,7 @@ self._pkgSource.loadFromUrl(self._url) self._pkgSource.finalize() + self._pkgSource.setLoaded() sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ]) From elliot at rpath.com Mon Mar 15 17:10:44 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:10:44 +0000 Subject: mirrorball: sort versions Message-ID: <201003152110.o2FLAjGU011711@scc.eng.rpath.com> changeset: 9c207136c67e user: Elliot Peele date: Wed, 16 Dec 2009 11:30:48 -0500 sort versions diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -99,6 +99,18 @@ Handle update case. """ + # FIXME: this should probably be provided by the errata object. + # Method for sorting versions. + def verCmp(a, b): + if a.startswith('RH') and b.startswith('RH'): + return cmp(a.split('-')[1], b.split('-')[1]) + elif a.startswith('RH') and not b.startswith('RH'): + return 1 + elif not a.startswith('RH') and b.startswith('RH'): + return -1 + else: + return cmp(a, b) + # Get current timestamp current = self._groupmgr.getErrataState() if current is None: @@ -145,7 +157,7 @@ # Build various group verisons. expected = self._flattenSetDict(pkgMap) - for version in errataVersions | majorVersions: + for version in sorted(errataVersions | majorVersions, cmp=verCmp): log.info('setting version %s' % version) self._groupmgr.setVersion(version) grpTrvMap = self._groupmgr.build() From elliot at rpath.com Mon Mar 15 17:11:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:11:02 +0000 Subject: mirrorball: preload version source data and avoid keeping refrences to pkgsource that we Message-ID: <201003152111.o2FLB3JJ011756@scc.eng.rpath.com> changeset: 8b391a10f77a user: Elliot Peele date: Wed, 16 Dec 2009 13:03:19 -0500 preload version source data and avoid keeping refrences to pkgsource that we don't use diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -542,7 +542,12 @@ self._version = version self._url = url - self._pkgSource = RpmSource(cfg, ) + pkgSource = RpmSource(cfg, ) + pkgSource.loadFromUrl(url) + pkgSource.finalize() + pkgSource.setLoaded() + + self._sourceSet = set([ x for x in pkgSource.iterPackageSet() ]) def areWeHereYet(self, pkgSet): """ @@ -552,16 +557,10 @@ @return boolean """ - self._pkgSource.loadFromUrl(self._url) - self._pkgSource.finalize() - self._pkgSource.setLoaded() - - sourceSet = set([ x for x in self._pkgSource.iterPackageSet() ]) - # Make sure that all versions that are on the ISO are in the conary # repository. It appears to be a common case that the ISO is missing # content from RHN. - return not sourceSet.difference(pkgSet) + return not self._sourceSet.difference(pkgSet) class VersionFactory(object): From elliot at rpath.com Mon Mar 15 17:11:19 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:11:19 +0000 Subject: mirrorball: at the time of sorting dashes have been replaced by underscores Message-ID: <201003152111.o2FLBK4Z011803@scc.eng.rpath.com> changeset: bb19513fa312 user: Elliot Peele date: Wed, 16 Dec 2009 13:03:51 -0500 at the time of sorting dashes have been replaced by underscores diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -103,7 +103,7 @@ # Method for sorting versions. def verCmp(a, b): if a.startswith('RH') and b.startswith('RH'): - return cmp(a.split('-')[1], b.split('-')[1]) + return cmp(a.split('_')[1], b.split('_')[1]) elif a.startswith('RH') and not b.startswith('RH'): return 1 elif not a.startswith('RH') and b.startswith('RH'): From elliot at rpath.com Mon Mar 15 17:11:33 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:11:33 +0000 Subject: mirrorball: add synthesized paths to the location map so that old manifest files can be Message-ID: <201003152111.o2FLBYZk011849@scc.eng.rpath.com> changeset: 2dc4363e6f9f user: Elliot Peele date: Wed, 16 Dec 2009 17:57:36 -0500 add synthesized paths to the location map so that old manifest files can be parsed diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -312,3 +312,9 @@ if (name, epoch, version, release, arch) not in self._srcMap: log.warn('synthesizing source package %s' % srcPkg) self._procSrc(srcPkg) + + # Add location mappings for packages that may have once been + # synthesized so that parsing old manifest files still works. + elif srcPkg.location not in self.locationMap: + pkg = self._srcMap[(name, epoch, version, release, arch)] + self.locationMap[srcPkg.location] = pkg From elliot at rpath.com Mon Mar 15 17:11:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:11:43 +0000 Subject: mirrorball: don't load the version factory twice Message-ID: <201003152111.o2FLBiGn011888@scc.eng.rpath.com> changeset: 71a14bf178e2 user: Elliot Peele date: Wed, 16 Dec 2009 17:57:57 -0500 don't load the version factory twice disable promote for now diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -40,7 +40,6 @@ BotSuperClass.__init__(self, cfg) self._errata = errata.ErrataFilter(self._pkgSource, errataSource) self._groupmgr = groupmgr.GroupManager(self._cfg) - self._versionFactory = groupmgr.VersionFactory(self._cfg) def _addPackages(self, pkgMap): """ @@ -156,22 +155,25 @@ self._addPackages(pkgMap) # Build various group verisons. - expected = self._flattenSetDict(pkgMap) - for version in sorted(errataVersions | majorVersions, cmp=verCmp): + #expected = self._flattenSetDict(pkgMap) + versions = sorted(errataVersions | majorVersions, cmp=verCmp) + if not versions: + versions = set(['unknown.%s' % updateId, ]) + for version in versions: log.info('setting version %s' % version) self._groupmgr.setVersion(version) grpTrvMap = self._groupmgr.build() - log.info('promoting version %s' % version) - toPublish = self._flattenSetDict(grpTrvMap) - newTroves = self._updater.publish( - toPublish, - expected, - self._cfg.targetLabel - ) + #log.info('promoting version %s' % version) + #toPublish = self._flattenSetDict(grpTrvMap) + #newTroves = self._updater.publish( + # toPublish, + # expected, + # self._cfg.targetLabel + #) # After the first promote, packages should not be repromoted. - expected = set() + #expected = set() updateSet.update(pkgMap) From elliot at rpath.com Mon Mar 15 17:11:55 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:11:55 +0000 Subject: mirrorball: cook groups localy Message-ID: <201003152111.o2FLBt9d011924@scc.eng.rpath.com> changeset: 718be6607457 user: Elliot Peele date: Wed, 16 Dec 2009 17:58:18 -0500 cook groups localy diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -45,6 +45,7 @@ from updatebot.errors import FailedToRetrieveChangesetError from updatebot.errors import ChangesetValidationFailedError +from updatebot.build.cvc import Cvc from updatebot.build.dispatcher import Dispatcher from updatebot.build.callbacks import StatusOnlyDisplay @@ -141,6 +142,8 @@ self._helper = helper.rMakeHelper(buildConfig=self._rmakeCfg) + self.cvc = Cvc(self._cfg, self._ccfg, self._client, self._formatInput) + def build(self, troveSpecs): """ Build a list of troves. diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py new file mode 100644 --- /dev/null +++ b/updatebot/build/cvc.py @@ -0,0 +1,102 @@ +# +# Copyright (c) 2009 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +An abstraction layer around cvc cook. +""" + +import logging + +log = logging.getLogger('updatebot.build.cvc') + +from conary.build import cook + +from updatebot.lib import conarycallbacks +from updatebot.errors import LocalCookFailedError + +class Cvc(object): + """ + This is initially intended to only implement interfaces for cooking groups + locally, but could and should be extended to building other kinds of + troves. This would be useful for building packages that do not depend on + environment, such as factories and superclasses. Additionally implementing + a general abstraction around cvc for all cvc operations could be very handy. + @param cfg: updatebot config object + @type cfg: updatebot.config.UpdateBotConfiguration + @param ccfg: conary configuration object + @type ccfg: conary.conarycfg.ConaryConfiguration + @param client: conary client object + @type client: conary.conaryclient.ConaryClient + @param inputFormatter: method to format trove lists into approriate tuples. + @type inputFormatter: method + """ + + def __init__(self, cfg, ccfg, client, inputFormatter): + self._cfg = cfg + self._ccfg = ccfg + self._client = client + self._formatInput = inputFormatter + + def cook(self, troveSpecs): + """ + Cook a set of trove specs, currently limited to groups. + @params troveSpecs: list of name, version, and flavor tuples. + @type troveSpecs: [(name, version, flavor), ... ] + """ + + # TODO: Look at conary.build.cook.cookCommand for how to setup + # environment when building anything other than groups. + + troveSpecs = self._formatInput(troveSpecs) + + # make sure all troves are groups + assert not [ x for x in troveSpecs if not x[0].startswith('group-') ] + + # make sure that all groups are the same name and version. + assert len(set([ (x[0], x[1]) for x in troveSpecs])) == 1 + + # pulled from conary.cvc + groupCookOptions = cook.GroupCookOptions( + alwaysBumpCount = True, + errorOnFlavorChange = True, + shortenFlavors = self._ccfg.shortenGroupFlavors + ) + + # extract flavor set + flavors = set([ x[2] for x in troveSpecs ]) + item = (troveSpecs[0][0], troveSpecs[0][1], flavors) + + built = cook.cookItem( + self._client.repos, + self._ccfg, + item, + ignoreDeps=True, + logBuild=True, + callback=conarycallbacks.UpdateBotCookCallback(), + groupOptions=groupOptions + ) + + if built is None: + raise LocalCookFailedError(troveSpecs=troveSpecs) + + components, csFile = built + + if not components: + raise LocalCookFailedError(troveSpecs=troveSpecs) + + if csFile is None: + log.info('changeset committed to repository') + + res = { (troveSpecs[0][0], troveSpecs[0][1], None): set(components) } + return res diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -354,3 +354,19 @@ _params = [] _template = 'This platform has not yet been created.' + +class CvcError(UpdateBotError): + """ + Generic cvc related error. + """ + + _params = [] + _template = 'cvc failed' + +class LocalCookFailedError(CvcError): + """ + LocalCookFailedError, raised when cvc.cook fails. + """ + + _parms = ['troveSpecs', ] + _template = 'Failed while cooking %(troveSpecs)s' diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -103,7 +103,7 @@ """ groupTroves = ((self._sourceName, self._sourceVersion, None), ) - built = self._builder.build(groupTroves) + built = self._builder.cvc.cook(groupTroves) return built @checkout diff --git a/updatebot/lib/conarycallbacks.py b/updatebot/lib/conarycallbacks.py --- a/updatebot/lib/conarycallbacks.py +++ b/updatebot/lib/conarycallbacks.py @@ -20,6 +20,7 @@ import logging log = logging.getLogger('updatebot.lib.conarycallbacks') +from conary.build.cook import CookCallback from conary.conaryclient.callbacks import CloneCallback def callonce(func): @@ -29,16 +30,19 @@ return func(self, *args, **kwargs) return wrapper -class UpdateBotCloneCallback(CloneCallback): +class BaseCallback(object): def __init__(self, *args, **kwargs): - self._log = kwargs.pop('log', log) - CloneCallback.__init__(self, *args, **kwargs) - self._last = None def _message(self, message): self._log.info(message) +class UpdateBotCloneCallback(BaseCallback, CloneCallback): + def __init__(self, *args, **kwargs): + self._log = kwargs.pop('log', log) + BaseCallback.__init__(self, *args, **kwargs) + CloneCallback.__init__(self, *args, **kwargs) + @callonce def determiningCloneTroves(self, current=0, total=0): CloneCallback.determiningCloneTroves(self, current=0, total=0) @@ -82,3 +86,9 @@ @callonce def sendingChangset(self, got, need): self._message('uploading changeset') + +class UpdateBotCookCallback(BaseCallback, CookCallback): + def __init__(self, *args, **kwargs): + self._log = kwargs.pop('log', log) + BaseCallback.__init__(self, *args, **kwargs) + CookCallback.__init__(self, *args, **kwargs) From elliot at rpath.com Mon Mar 15 17:12:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:02 +0000 Subject: mirrorball: use a copy of the conarycfg since dbPath needs to be set for the system db Message-ID: <201003152112.o2FLC3IR011957@scc.eng.rpath.com> changeset: e9d344caef12 user: Elliot Peele date: Wed, 16 Dec 2009 18:28:42 -0500 use a copy of the conarycfg since dbPath needs to be set for the system db when validating policy diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -142,7 +142,7 @@ self._helper = helper.rMakeHelper(buildConfig=self._rmakeCfg) - self.cvc = Cvc(self._cfg, self._ccfg, self._client, self._formatInput) + self.cvc = Cvc(self._cfg, self._ccfg, self._formatInput) def build(self, troveSpecs): """ diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py --- a/updatebot/build/cvc.py +++ b/updatebot/build/cvc.py @@ -16,10 +16,12 @@ An abstraction layer around cvc cook. """ +import copy import logging log = logging.getLogger('updatebot.build.cvc') +from conary import conarycfg from conary.build import cook from updatebot.lib import conarycallbacks @@ -42,12 +44,14 @@ @type inputFormatter: method """ - def __init__(self, cfg, ccfg, client, inputFormatter): + def __init__(self, cfg, ccfg, inputFormatter): self._cfg = cfg - self._ccfg = ccfg - self._client = client + self._ccfg = copy.deepcopy(ccfg) self._formatInput = inputFormatter + # Restet dbPath to the default value for local cooking. + self._ccfg.dbPath = conarycfg.ConaryContext.dbPath + def cook(self, troveSpecs): """ Cook a set of trove specs, currently limited to groups. @@ -84,7 +88,7 @@ ignoreDeps=True, logBuild=True, callback=conarycallbacks.UpdateBotCookCallback(), - groupOptions=groupOptions + groupOptions=groupCookOptions, ) if built is None: From elliot at rpath.com Mon Mar 15 17:12:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:10 +0000 Subject: mirrorball: add a conary client instance Message-ID: <201003152112.o2FLCAaK011992@scc.eng.rpath.com> changeset: 76864e8757cf user: Elliot Peele date: Wed, 16 Dec 2009 18:55:50 -0500 add a conary client instance diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py --- a/updatebot/build/cvc.py +++ b/updatebot/build/cvc.py @@ -22,6 +22,7 @@ log = logging.getLogger('updatebot.build.cvc') from conary import conarycfg +from conary import conaryclient from conary.build import cook from updatebot.lib import conarycallbacks @@ -52,6 +53,8 @@ # Restet dbPath to the default value for local cooking. self._ccfg.dbPath = conarycfg.ConaryContext.dbPath + self._client = conaryclient.ConaryClient(self._ccfg) + def cook(self, troveSpecs): """ Cook a set of trove specs, currently limited to groups. From elliot at rpath.com Mon Mar 15 17:12:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:13 +0000 Subject: mirrorball: Do a better job of mapping binaries to sources when the source packages are Message-ID: <201003152112.o2FLCDcr012022@scc.eng.rpath.com> changeset: 8d7872ef4189 user: Elliot Peele date: Thu, 17 Dec 2009 16:58:41 -0500 Do a better job of mapping binaries to sources when the source packages are not available and the source and binary have different epochs. diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -370,3 +370,24 @@ _parms = ['troveSpecs', ] _template = 'Failed while cooking %(troveSpecs)s' + +class PackageSourceError(UpdateBotError): + """ + Generic error for all package source errors to decend from. + """ + + _parms = [] + _template = 'an error has occured in the package source' + +class CanNotFindSourceForBinariesError(PackageSourceError): + """ + CanNotFindSourceForBinariesError, raise when synthesizing sources and the + binary package has a source of a different name and a source of that name, + version, and release can not be found. + """ + + _parms = ['count', ] + _template = ('Could not find %(count) sources for matching binary ' + 'packages. This generally means that there is a binary package with a ' + 'source of a different name and a source can not be found with a ' + 'matching source name, version, and release.') diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -277,18 +277,17 @@ Create a source map from the binary map if no sources are available. """ - # Return if sources should be available in repos. - if not self._cfg.synthesizeSources: - return + def getSourcePackage(nevra, bins): + # unpack the nevra + n, e, v, r, a = nevra - # Create a fake source rpm object for each key in the rpmMap. - for (name, epoch, version, release, arch), bins in self._rpmMap.iteritems(): + # create a source package object based on the nevra srcPkg = self.PkgClass() - srcPkg.name = name - srcPkg.epoch = epoch - srcPkg.version = version - srcPkg.release = release - srcPkg.arch = arch + srcPkg.name = n + srcPkg.epoch = e + srcPkg.version = v + srcPkg.release = r + srcPkg.arch = a # grab the first binary package pkg = sorted(bins)[0] @@ -297,24 +296,58 @@ # of the file. The factory will take care of the rest. srcPkg.location = pkg.sourcerpm + return srcPkg + + # Return if sources should be available in repos. + if not self._cfg.synthesizeSources: + return + + deffer = set() + # Create a fake source rpm object for each key in the rpmMap. + for nevra, bins in self._rpmMap.iteritems(): + srcPkg = getSourcePackage(nevra, bins) + # Handle sub packages with different epochs that should be taken # care of with the epoch fuzzing that happens in finalize. This # should only happen with differently named packages. - if name not in [ x.name for x in bins ]: - # Find sources that match on all cases except epoch. - sources = [ x for x in self._srcMap.iterkeys() - if (name, version, release, arch) == (x[0], x[2], x[3], x[4]) ] - - # leave it up to fuzzing - if sources: continue + if nevra[0] not in [ x.name for x in bins ]: + deffer.add(nevra) + continue # add source to structures - if (name, epoch, version, release, arch) not in self._srcMap: + if nevra not in self._srcMap: log.warn('synthesizing source package %s' % srcPkg) self._procSrc(srcPkg) # Add location mappings for packages that may have once been # synthesized so that parsing old manifest files still works. elif srcPkg.location not in self.locationMap: - pkg = self._srcMap[(name, epoch, version, release, arch)] + pkg = self._srcMap[nevra] self.locationMap[srcPkg.location] = pkg + + broken = set() + # Make an attempt to sort out the binaries that have different names + # than the related sources. + for nevra in deffer: + bins = self._rpmMap[nevra] + srcPkg = getSourcePackage(nevra, bins) + + name, epoch, version, release, arch = nevra + # Find sources that match on all cases except epoch. + sources = [ x for x in self._srcMap.iterkeys() + if (name, version, release, arch) == (x[0], x[2], x[3], x[4]) ] + # leave it up to fuzzing + if sources: continue + + # If we get here this is a set of binary packages that have a + # different name than the source rpm. This is possible, but should + # be an extremely rare case. + log.warn('found binary without matching source name %s' + % list(bins)[0].name) + + broken.add((nevra, bins)) + + # Raise an exception if this ever happens. We can figure out the right + # thing to do then, purhaps on a case by case basis. + if broken: + raise CanNotFindSourceForBinariesError(count=len(broken)) From elliot at rpath.com Mon Mar 15 17:12:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:15 +0000 Subject: mirrorball: remove extra versions, this can be caused by having packages that change Message-ID: <201003152112.o2FLCFkH012050@scc.eng.rpath.com> changeset: 9a5532b5293d user: Elliot Peele date: Thu, 17 Dec 2009 18:17:51 -0500 remove extra versions, this can be caused by having packages that change flavor over time diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -624,6 +624,22 @@ flavors.add(section.buildFlavor) trvMap = self._repos.getTroveLeavesByLabel({None: {label: flavors}}) + + # Remove anything with multiple versions, this results from something + # changing flavor at any point. + for name, verDict in trvMap.iteritems(): + if len(verDict) == 1: + continue + versions = verDict.keys() + versions.sort() + latest = versions[-1] + versions.remove(latest) + for version in versions: + for flavor in verDict[version]: + log.warn('removing extra version of %s=%s[%s]' % + (name, version, flavor)) + del trvMap[name][version] + return trvMap def getLatestVersions(self): From elliot at rpath.com Mon Mar 15 17:12:21 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:21 +0000 Subject: mirrorball: more compat for source packages that move from being synthesized to not Message-ID: <201003152112.o2FLCM21012087@scc.eng.rpath.com> changeset: 713d9e8687d8 user: Elliot Peele date: Thu, 17 Dec 2009 22:28:36 -0500 more compat for source packages that move from being synthesized to not sythenssized diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -17,6 +17,7 @@ Module for interacting with packages in multiple yum repositories. """ +import os import logging import repomd @@ -137,6 +138,13 @@ self.locationMap[package.location] = package + # In case the a synthesized source ever turns into real source add the + # short name for backward compatibility. + if self._cfg.synthesizeSources: + baseLoc = os.path.basename(package.location) + if baseLoc not in self.locationMap: + self.locationMap[baseLoc] = package + self._srcPkgs.add(package) self._srcMap[(package.name, package.epoch, package.version, package.release, package.arch)] = package From elliot at rpath.com Mon Mar 15 17:12:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:27 +0000 Subject: mirrorball: new errors Message-ID: <201003152112.o2FLCRRi012120@scc.eng.rpath.com> changeset: 1726e82171ad user: Elliot Peele date: Sun, 20 Dec 2009 13:23:21 -0500 new errors diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -75,8 +75,16 @@ """ _params = ['jobId', 'why'] - _templates = 'rMake job %(jobId)s failed: %(why)s' + _template = 'rMake job %(jobId)s failed: %(why)s' +class JobNotCompleteError(UpdateBotError): + """ + JobNotCompleteError, raised when the build dispatcher thinks that the job + should be done, but it isn't. + """ + + _params = ['jobId', ] + _template = 'Build job not complete %(jobId)s' class UnhandledUpdateError(UpdateBotError): """ @@ -391,3 +399,21 @@ 'packages. This generally means that there is a binary package with a ' 'source of a different name and a source can not be found with a ' 'matching source name, version, and release.') + +class ErrataFilterError(UpdateBotError): + """ + Generic errata filter error. + """ + + _params = [] + _template = 'generic errata filter error' + +class UnableToMergeUpdatesError(ErrataFilterError): + """ + UnableToMergeUpdatesError, raised when errata buckets can not be merged + together. + """ + + _params = [ 'source', 'target', 'package', ] + _template = ('Can not merge %(source)s into %(target)s due to conflicting ' + 'package %(package)s') From elliot at rpath.com Mon Mar 15 17:12:30 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:30 +0000 Subject: mirrorball: disable major version detection for now Message-ID: <201003152112.o2FLCWdO012154@scc.eng.rpath.com> changeset: cff41140646c user: Elliot Peele date: Sun, 20 Dec 2009 13:25:00 -0500 disable major version detection for now diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -38,7 +38,8 @@ def __init__(self, cfg, errataSource): BotSuperClass.__init__(self, cfg) - self._errata = errata.ErrataFilter(self._pkgSource, errataSource) + self._errata = errata.ErrataFilter(self._cfg, self._pkgSource, + errataSource) self._groupmgr = groupmgr.GroupManager(self._cfg) def _addPackages(self, pkgMap): @@ -138,15 +139,17 @@ name += '_rolling' errataVersions.add(name) + # FIXME: Might want to re-enable this one day. # Get current set of source names and versions. - nvMap = self._updater.getSourceVersionMap() + #nvMap = self._updater.getSourceVersionMap() # Add in new names and versions that have just been built. - for n, v, f in pkgMap.iterkeys(): - n = n.split(':')[0] - nvMap[n] = v - pkgSet = set(nvMap.items()) + #for n, v, f in pkgMap.iterkeys(): + # n = n.split(':')[0] + # nvMap[n] = v + #pkgSet = set(nvMap.items()) # Get the major distro verisons from the group manager. - majorVersions = self._groupmgr.getVersions(pkgSet) + #majorVersions = self._groupmgr.getVersions(pkgSet) + #import epdb; epdb.st() # Store current updateId. self._groupmgr.setErrataState(updateId) @@ -156,14 +159,15 @@ # Build various group verisons. #expected = self._flattenSetDict(pkgMap) - versions = sorted(errataVersions | majorVersions, cmp=verCmp) + versions = sorted(errataVersions, cmp=verCmp) if not versions: - versions = set(['unknown.%s' % updateId, ]) + versions = ['unknown.%s' % updateId, ] for version in versions: log.info('setting version %s' % version) self._groupmgr.setVersion(version) grpTrvMap = self._groupmgr.build() + # FIXME: enable promotes at some point #log.info('promoting version %s' % version) #toPublish = self._flattenSetDict(grpTrvMap) #newTroves = self._updater.publish( From elliot at rpath.com Mon Mar 15 17:12:36 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:36 +0000 Subject: mirrorball: disable major version detection Message-ID: <201003152112.o2FLCaDD012187@scc.eng.rpath.com> changeset: 03ff098adc9c user: Elliot Peele date: Sun, 20 Dec 2009 13:25:16 -0500 disable major version detection diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -67,7 +67,7 @@ self._helper = GroupHelper(self._cfg) self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') - self._versionFactory = VersionFactory(cfg) + #self._versionFactory = VersionFactory(cfg) self._sourceName = self._cfg.topSourceGroup[0] self._sourceVersion = self._cfg.topSourceGroup[1] @@ -291,7 +291,8 @@ packages from the version factory. """ - return self._versionFactory.getVersions(pkgSet) + return set() + #return self._versionFactory.getVersions(pkgSet) class GroupHelper(ConaryHelper): From elliot at rpath.com Mon Mar 15 17:12:38 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:38 +0000 Subject: mirrorball: implement interface for doing splitting each source into its own job, but Message-ID: <201003152112.o2FLCdo1012215@scc.eng.rpath.com> changeset: f0cff5b8d4aa user: Elliot Peele date: Sun, 20 Dec 2009 13:26:43 -0500 implement interface for doing splitting each source into its own job, but waiting until all jobs are complete to commit diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -47,6 +47,7 @@ from updatebot.build.cvc import Cvc from updatebot.build.dispatcher import Dispatcher +from updatebot.build.dispatcher import NonCommittalDispatcher from updatebot.build.callbacks import StatusOnlyDisplay log = logging.getLogger('updatebot.build') @@ -160,15 +161,22 @@ ret = self._formatOutput(trvMap) return ret - def buildmany(self, troveSpecs): + def buildmany(self, troveSpecs, lateCommit=False): """ Build many troves in separate jobs. @param troveSpecs: list of trove specs @type troveSpecs: [(name, versionObj, flavorObj), ...] + @param lateCommit: if True, build all troves, then commit. (defaults to + False) + @type lateCommit: boolean @return troveMap: dictionary of troveSpecs to built troves """ - dispatcher = Dispatcher(self, 15) + workers = 15 + if not lateCommit: + dispatcher = Dispatcher(self, workers) + else: + dispatcher = NonCommittalDispatcher(self, workers) return dispatcher.buildmany(troveSpecs) def buildsplitarch(self, troveSpecs): diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -29,6 +29,8 @@ from updatebot.build.monitor import JobCommitter from updatebot.build.constants import JobStatus +from updatebot.errors import JobNotCompleteError + log = logging.getLogger('updatebot.build') class Dispatcher(object): @@ -176,6 +178,10 @@ results = {} for jobId, (trove, status, result) in self._jobs.iteritems(): + # don't return errors if we intentionaly didn't commit. + built = buildjob.JOB_STATE_BUILT + if status == built and built in self._completed and not result: + continue # log failed jobs if status == buildjob.JOB_STATE_FAILED or not result: log.info('[%s] failed job: %s' % (jobId, trove)) @@ -205,3 +211,47 @@ avail = util.getAvailableFileDescriptors(setMax=setMax) return math.floor(0.8 * avail) + + +class NonCommittalDispatcher(Dispatcher): + """ + A dispatcher class that is configured to not commit until all jobs are + complete. + """ + + # States where the job is considered complete. + _completed = ( + JobStatus.ERROR_MONITOR_FAILURE, + JobStatus.ERROR_COMMITTER_FAILURE, + buildjob.JOB_STATE_FAILED, + buildjob.JOB_STATE_BUILT, + ) + + def __init__(self, builder, maxSlots): + Dispatcher.__init__(self, builder, maxSlots) + + # Disable commits by removing all commit slots. + self._maxCommitSlots = 0 + self._commitSlots = 0 + + def buildmany(self, troveSpecs): + """ + Build all packages in seperate jobs, then commit. + """ + + results, self._failures = Dispatcher.buildmany(self, troveSpecs) + + # Make sure there are no failures. + assert not self._failures + + for jobId, (trove, status, result) in self._jobs.iteritems(): + # Make sure all jobs are built. + if status != buildjob.JOB_STATE_BUILT: + raise JobNotCompleteError(jobId=jobId) + + # If we get here, all jobs have built successfully and are ready to be + # committed. + jobIds = self._jobs.keys() + res = self._builder.commit(jobIds) + + return res, self._failures From elliot at rpath.com Mon Mar 15 17:12:41 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:41 +0000 Subject: mirrorball: always use buildmany with late commit if building in ordered mode Message-ID: <201003152112.o2FLCfkZ012245@scc.eng.rpath.com> changeset: fb92eeb43ac6 user: Elliot Peele date: Sun, 20 Dec 2009 13:27:19 -0500 always use buildmany with late commit if building in ordered mode diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -202,10 +202,16 @@ # sources that have been updated, but not built. buildTroves = set([ x[0] for x in toAdvise ]) + # If importing specific packages, they might require each other so + # always use buildmany, but wait to commit. + if updatePkgs: + trvMap, failed = self._builder.buildmany(buildTroves, + lateCommit=True) + # Switch to splitarch if a build is larger than maxBuildSize. This # number is kinda arbitrary. Builds tend to break when architectures # are combind if the build is significantly large - if len(buildTroves) < self._cfg.maxBuildSize: + elif len(buildTroves) < self._cfg.maxBuildSize: trvMap = self._builder.build(buildTroves) else: trvMap = self._builder.buildsplitarch(buildTroves) From elliot at rpath.com Mon Mar 15 17:12:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:43 +0000 Subject: mirrorball: support merging updates and reordering updates Message-ID: <201003152112.o2FLChaL012276@scc.eng.rpath.com> changeset: 0dc186f2b9f6 user: Elliot Peele date: Sun, 20 Dec 2009 13:27:57 -0500 support merging updates and reordering updates diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -21,8 +21,9 @@ from conary.lib import cfg from conary import versions from conary.conarycfg import CfgFlavor, CfgLabel -from conary.lib.cfgtypes import CfgString, CfgList, CfgRegExp, CfgBool, CfgDict, CfgInt from conary.lib.cfgtypes import ParseError +from conary.lib.cfgtypes import CfgInt, CfgQuotedLineList +from conary.lib.cfgtypes import CfgString, CfgList, CfgRegExp, CfgBool, CfgDict from rmake.build.buildcfg import CfgTroveSpec @@ -218,6 +219,21 @@ # a magic number, but at least it is configurable? maxBuildSize = (CfgInt, 10) + # List of errata timestamps to merge together. This is used when one errata + # leaves the platform in a non dependency closed state and a later update + # should solve the dependency problem. All updates are folded into the first + # bucket listed. + mergeUpdates = (CfgList(CfgQuotedLineList(CfgInt)), []) + + # Errata timestamp pairs for rescheduling when updates are applied. The + # first element is the current timestamp of the update. The second element + # is the new timestamp. You may need to use this option if it appears that + # the upstream provider has somehow managed to release updates out of order + # and has caused dependency closure problems. Note that you will need to + # mark remove anything that has been committed past the destination + # timestamp to get mirrorball to go back and apply this update. + reorderUpdates = (CfgList(CfgQuotedLineList(CfgInt)), []) + class UpdateBotConfig(cfg.SectionedConfigFile): """ diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -36,11 +36,15 @@ Filter data from a given errataSource in chronological order. """ - def __init__(self, pkgSource, errataSource): + def __init__(self, cfg, pkgSource, errataSource): + self._cfg = cfg self._pkgSource = pkgSource self._errata = errataSource + # timestamp: srcPkg Set self._order = {} + + # timestamp: advisory info self._advMap = {} @loadErrata @@ -119,6 +123,48 @@ src = self._pkgSource.binPkgMap[pkg] self._order[bucketId].add(src) + # fold together updates to preserve dep closure. + for mergeList in self._cfg.mergeUpdates: + target = mergeList[0] + # merge remaining updates into target. + for source in mergeList[1:]: + log.info('merging errata bucket %s -> %s' % (source, target)) + updateSet = self._order.pop(source) + oldNames = set([ x.name for x in self._order[target]]) + newNames = set([ x.name for x in updateSet ]) + # Check for overlapping updates. If there is overlap, these + # things can't be merged since all versions need to be + # represented in the repository. + if oldNames & newNames: + raise UnableToMergeUpdatesError( + source=source, target=target, + package=', '.join(oldNames & newNames) + ) + self._order[target].update(updateSet) + + # merge advisory detail. + if source in self._advMap: + advInfo = self._advMap.pop(source) + if target not in self._advMap: + self._advMap[target] = set() + self._advMap[target].update(advInfo) + + # reschedule any updates that may have been released out of order. + for source, dest in self._cfg.reorderUpdates: + # Probably don't want to move an update into an already + # existing bucket. + assert dest not in self._order + + log.info('rescheduling %s -> %s' % (source, dest)) + + # remove old version + bucket = self._order.pop(source) + adv = self._advMap.pop(source) + + # move to new version + self._order[dest] = bucket + self._advMap[dest] = adv + def _getNevra(self, pkg): """ Get the NEVRA of a package object and do any transformation required. From elliot at rpath.com Mon Mar 15 17:12:45 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:45 +0000 Subject: mirrorball: log when updates go backwards, may not always raise an exception Message-ID: <201003152112.o2FLCjTS012305@scc.eng.rpath.com> changeset: 80bf5236d762 user: Elliot Peele date: Sun, 20 Dec 2009 13:28:26 -0500 log when updates go backwards, may not always raise an exception diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -223,6 +223,8 @@ # make sure new package is actually newer if util.packagevercmp(srpm, srcPkg) == -1: + log.warn('version goes backwards %s -> %s' % + (srpm.getNevra(), srcPkg.getNevra())) raise UpdateGoesBackwardsError(why=(srcPkg, srpm)) # make sure we aren't trying to remove a package From elliot at rpath.com Mon Mar 15 17:12:47 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:47 +0000 Subject: mirrorball: split out getSourceVersions from getSourceVersionMap Message-ID: <201003152112.o2FLClij012333@scc.eng.rpath.com> changeset: 93743b9c3e5c user: Elliot Peele date: Mon, 21 Dec 2009 02:02:10 -0500 split out getSourceVersions from getSourceVersionMap diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -127,11 +127,11 @@ log.info('found %s protentially updatable troves' % len(troves)) return troves - def getSourceVersionMap(self): + def getSourceVersions(self): """ - Query the repository for a list of the latest source names and versions - that have binary versions. - @return {sourceName: sourceVersion} + Query the repository for a list of latest source trove specs and + matching binaries. + @return {srcTroveSpec: set(binTrovSpec, ...)} """ # Get the latest versions from repository @@ -150,6 +150,17 @@ # Get the sources for all binary packages. sourceVersions = self._conaryhelper.getSourceVersions(troveSpecs) + return sourceVersions + + def getSourceVersionMap(self): + """ + Query the repository for a list of the latest source names and versions + that have binary versions. + @return {sourceName: sourceVersion} + """ + + sourceVersions = self.getSourceVersions() + # Convert to sourceName: version map. return dict([ (x.split(':')[0], y) for x, y, z in sourceVersions.iterkeys() From elliot at rpath.com Mon Mar 15 17:12:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:49 +0000 Subject: mirrorball: add script for diffing the contents of a label against the latest binary group Message-ID: <201003152112.o2FLCngK012362@scc.eng.rpath.com> changeset: 5da8b06e5705 user: Elliot Peele date: Mon, 21 Dec 2009 02:14:24 -0500 add script for diffing the contents of a label against the latest binary group diff --git a/scripts/order_update.py b/scripts/difflabel copy from scripts/order_update.py copy to scripts/difflabel --- a/scripts/order_update.py +++ b/scripts/difflabel @@ -32,10 +32,10 @@ from conary.lib import util sys.excepthook = util.genExcepthook() -import rhnmirror +from conary import checkin from updatebot import config -from updatebot import ordered +from updatebot.bot import Bot from updatebot import log as logSetup logSetup.addRootLogger() @@ -54,13 +54,35 @@ cfg = config.UpdateBotConfig() cfg.read(confDir + '/updatebotrc') -mcfg = rhnmirror.MirrorConfig() -mcfg.read(confDir + '/erratarc') +bot = Bot(cfg) +updater = bot._updater +helper = updater._conaryhelper -errata = rhnmirror.Errata(mcfg) -errata.fetch() +labelPkgs = updater.getSourceVersions() +groupPkgs = updater._conaryhelper.getSourceTroves(cfg.topGroup) -bot = ordered.Bot(cfg, errata) -pkgMap = bot.update() +lks = set([ x for x in labelPkgs.keys() + if not updater._fltrPkg(x[0].split(':')[0]) ]) +gks = set([ x for x in groupPkgs.keys() + if not updater._fltrPkg(x[0].split(':')[0]) ]) + +newPkgs = lks.difference(gks) + +toRemove = set() +for srcTrv in newPkgs: + toRemove.add(srcTrv) + toRemove.update(labelPkgs[srcTrv]) + +ccfg = helper._ccfg +repos = helper._repos + +# This will prompt for confirmation. +for rspec in sorted(toRemove): + n, v, f = rspec + spec = '%s=%s' % (n, v) + if f is not None: + spec += '[%s]' % f + + checkin.markRemoved(ccfg, repos, spec) import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:12:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:53 +0000 Subject: mirrorball: add difflabel doc string Message-ID: <201003152112.o2FLCrse012394@scc.eng.rpath.com> changeset: c435d65244f9 user: Elliot Peele date: Mon, 21 Dec 2009 02:15:27 -0500 add difflabel doc string diff --git a/scripts/difflabel b/scripts/difflabel --- a/scripts/difflabel +++ b/scripts/difflabel @@ -13,6 +13,11 @@ # full details. # +""" +Script for diffing the contents of a label against the latest binary +group version. +""" + import os import sys import logging From elliot at rpath.com Mon Mar 15 17:12:56 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:56 +0000 Subject: mirrorball: handle case of reordering update that does not have an associated advisory Message-ID: <201003152112.o2FLCviY012424@scc.eng.rpath.com> changeset: 19c668674401 user: Elliot Peele date: Thu, 24 Dec 2009 01:57:14 -0500 handle case of reordering update that does not have an associated advisory diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -159,11 +159,13 @@ # remove old version bucket = self._order.pop(source) - adv = self._advMap.pop(source) + # There will not be an entry for sources that do not have + # advisories, default to None. + adv = self._advMap.pop(source, None) # move to new version self._order[dest] = bucket - self._advMap[dest] = adv + if adv: self._advMap[dest] = adv def _getNevra(self, pkg): """ From elliot at rpath.com Mon Mar 15 17:12:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:12:59 +0000 Subject: mirrorball: replace python rpmvercmp with implemenation from librpm.ao Message-ID: <201003152113.o2FLCxtG012450@scc.eng.rpath.com> changeset: 6951eca100b0 user: Erik Troan date: Thu, 24 Dec 2009 08:53:19 -0500 replace python rpmvercmp with implemenation from librpm.ao diff --git a/rpmutils/vercmp.py b/rpmutils/vercmp.py --- a/rpmutils/vercmp.py +++ b/rpmutils/vercmp.py @@ -16,65 +16,7 @@ Module that implements rpm version comparison. """ -def _rpmversplit(s): - """ - Split version strings. - """ +import ctypes +rpmlib = ctypes.cdll.LoadLibrary('librpm.so') - l = [] - isNumericHunk = s[0].isdigit() - - i = 1 - start = 0 - - - while i < len(s): - if not s[i].isalnum(): - l.append(s[start:i]) - start = i + 1 - elif isNumericHunk != s[i].isdigit(): - l.append(s[start:i]) - start = i - - i += 1 - - l.append(s[start:i]) - - # filter out empty strings - return [ x for x in l if x ] - -def rpmvercmp(ver1string, ver2string): - """ - Compare rpm version strings. - """ - - # R0911 - Too many return statements - # pylint: disable-msg=R0911 - - ver1list = _rpmversplit(ver1string) - ver2list = _rpmversplit(ver2string) - - while ver1list or ver2list: - if not ver1list: - return -1 - elif not ver2list: - return 1 - - v1 = ver1list.pop(0) - v2 = ver2list.pop(0) - - if v1.isdigit() and v2.isdigit(): - v1 = int(v1) - v2 = int(v2) - elif v1.isdigit() and not v2.isdigit(): - # numbers are newer than letters - return 1 - elif not v1.isdigit() and v2.isdigit(): - return -1 - - if v1 < v2: - return -1 - elif v1 > v2: - return 1 - - return 0 +rpmvercmp = rpmlib.rpmvercmp From elliot at rpath.com Mon Mar 15 17:13:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:02 +0000 Subject: mirrorball: branch merge Message-ID: <201003152113.o2FLD292012481@scc.eng.rpath.com> changeset: 428e7557fda7 user: Elliot Peele date: Thu, 24 Dec 2009 09:15:04 -0500 branch merge diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -159,11 +159,13 @@ # remove old version bucket = self._order.pop(source) - adv = self._advMap.pop(source) + # There will not be an entry for sources that do not have + # advisories, default to None. + adv = self._advMap.pop(source, None) # move to new version self._order[dest] = bucket - self._advMap[dest] = adv + if adv: self._advMap[dest] = adv def _getNevra(self, pkg): """ From elliot at rpath.com Mon Mar 15 17:13:04 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:04 +0000 Subject: mirrorball: stringify unicode Message-ID: <201003152113.o2FLD4gb012504@scc.eng.rpath.com> changeset: 6c1758988617 user: Elliot Peele date: Thu, 24 Dec 2009 10:07:48 -0500 stringify unicode diff --git a/rpmutils/vercmp.py b/rpmutils/vercmp.py --- a/rpmutils/vercmp.py +++ b/rpmutils/vercmp.py @@ -19,4 +19,8 @@ import ctypes rpmlib = ctypes.cdll.LoadLibrary('librpm.so') -rpmvercmp = rpmlib.rpmvercmp +def rpmvercmp(a, b): + # rpmlib.rpmvercmp does not handle unicode strings, so convert + a = str(a) + b = str(b) + return rpmlib.rpmvercmp(a, b) From elliot at rpath.com Mon Mar 15 17:13:07 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:07 +0000 Subject: mirrorball: Added allowRemovePackages to deal with deprecations. Message-ID: <201003152113.o2FLD79V012541@scc.eng.rpath.com> changeset: cc10659f0474 user: Elliot Peele date: Sun, 27 Dec 2009 18:26:31 -0500 Added allowRemovePackages to deal with deprecations. diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -68,6 +68,34 @@ raise ParseError, e +class CfgIntDict(CfgDict): + """ + Config class to represent dictionaries keyed by integers rather than + strings. + """ + + def updateFromString(self, val, str): + # update the dict value -- don't just overwrite it, it might be + # that the dict value is a list, so we call updateFromString + strs = str.split(None, 1) + if len(strs) == 1: + dkey, dvalue = strs[0], '' + else: + (dkey, dvalue) = strs + + dkey = CfgInt().parseString(dkey) + + if dkey in val: + val[dkey] = self.valueType.updateFromString(val[dkey], dvalue) + else: + val[dkey] = self.parseValueString(dkey, dvalue) + return val + + def toStrings(self, value, displayOptions): + value = dict([ (str(x), y) for x, y in value.iteritems() ]) + return CfgDict.toStrings(self, value, displayOptions) + + class UpdateBotConfigSection(cfg.ConfigSection): """ Config class for updatebot. @@ -234,6 +262,9 @@ # timestamp to get mirrorball to go back and apply this update. reorderUpdates = (CfgList(CfgQuotedLineList(CfgInt)), []) + # Dictionary of bucketIds and packages that are expected to be removed. + updateRemovesPackages = (CfgIntDict(CfgList(CfgString)), {}) + class UpdateBotConfig(cfg.SectionedConfigFile): """ diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -154,6 +154,14 @@ # Store current updateId. self._groupmgr.setErrataState(updateId) + # Remove any packages that are scheduled for removal. + if updateId in self._cfg.updateRemovesPackages: + pkgs = self._cfg.updateRemovesPackages[updateId] + log.info('removing the following packages from the managed ' + 'group: %s' % ', '.join(pkgs)) + for pkg in pkgs: + self._groupmgr.remove(pkg) + # Make sure built troves are part of group. self._addPackages(pkgMap) From elliot at rpath.com Mon Mar 15 17:13:11 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:11 +0000 Subject: mirrorball: imported patch rhelordercleanup Message-ID: <201003152113.o2FLDBLT012571@scc.eng.rpath.com> changeset: a81b4ba8337e user: Elliot Peele date: Sun, 27 Dec 2009 18:28:28 -0500 imported patch rhelordercleanup diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py --- a/scripts/rhelorder.py +++ b/scripts/rhelorder.py @@ -2,6 +2,7 @@ import os import sys +import time import tempfile sys.path.insert(0, os.environ['HOME'] + '/hg/conary') @@ -49,4 +50,11 @@ bot._pkgSource.load() bot._errata._orderErrata() +order = bot._errata._order +advMap = bot._errata._advMap +sorder = sorted(order) + +def tconv(tstamp): + return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp)) + import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:13:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:15 +0000 Subject: mirrorball: always write versions to group xml defs Message-ID: <201003152113.o2FLDFRC012604@scc.eng.rpath.com> changeset: 37ee0256d259 user: Elliot Peele date: Wed, 30 Dec 2009 23:30:24 -0500 always write versions to group xml defs diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -148,15 +148,15 @@ flavor = flavors[0] # noarch package, add unconditionally if flavor == plain: - self.add(name) + self.add(name, version=version) # x86, add with use=x86 elif flavor.satisfies(x86): - self.add(name, flavor=flavor, use='x86') + self.add(name, version=version, flavor=flavor, use='x86') # x86_64, add with use=x86_64 elif flavor.satisfies(x86_64): - self.add(name, flavor=flavor, use='x86_64') + self.add(name, version=version, flavor=flavor, use='x86_64') else: raise UnsupportedTroveFlavorError(name=name, flavor=flavor) @@ -183,7 +183,7 @@ assert len([ x for x, y in flvCount.iteritems() if y != 1 ]) == 1 # In this case just add the package unconditionally - self.add(name) + self.add(name, version=version) return @@ -213,9 +213,9 @@ # with use=x86_64 for flavor in flavors: if flavor.satisfies(x86): - self.add(name, flavor=flavor, use='x86') + self.add(name, version=version, flavor=flavor, use='x86') elif flavor.satisfies(x86_64): - self.add(name, flavor=flavor, use='x86_64') + self.add(name, version=version, flavor=flavor, use='x86_64') else: raise UnsupportedTroveFlavorError(name=name, flavor=flavor) @@ -250,9 +250,11 @@ for flavor in flavors: if flavor.satisfies(x86): - self.add(name, flavor=flavor, use='x86') + self.add(name, version=version, + flavor=flavor, use='x86') elif flavor.satisfies(x86_64): - self.add(name, flavor=flavor, use='x86_64') + self.add(name, version=version, + flavor=flavor, use='x86_64') else: raise UnsupportedTroveFlavorError(name=name, flavor=flavor) diff --git a/updatebot/lib/xobjects.py b/updatebot/lib/xobjects.py --- a/updatebot/lib/xobjects.py +++ b/updatebot/lib/xobjects.py @@ -183,7 +183,6 @@ use=None, source=None): self.name = name - self.version = version self.source = source if byDefault in (True, False): @@ -196,6 +195,11 @@ else: self.use = use + if isinstance(version, conary.versions.Version): + self.version = version.freeze() + else: + self.version = version + if isinstance(flavor, conary.deps.deps.Flavor): self.flavor = flavor.freeze() else: From elliot at rpath.com Mon Mar 15 17:13:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:18 +0000 Subject: mirrorball: increase build slots to 20 Message-ID: <201003152113.o2FLDJWe012634@scc.eng.rpath.com> changeset: 887794d4a97b user: Elliot Peele date: Wed, 30 Dec 2009 23:32:58 -0500 increase build slots to 20 diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -172,7 +172,7 @@ @return troveMap: dictionary of troveSpecs to built troves """ - workers = 15 + workers = 20 if not lateCommit: dispatcher = Dispatcher(self, workers) else: From elliot at rpath.com Mon Mar 15 17:13:22 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:22 +0000 Subject: mirrorball: copy package versions from managed package group to other managed groups Message-ID: <201003152113.o2FLDMp0012661@scc.eng.rpath.com> changeset: 5c9a24e893c3 user: Elliot Peele date: Fri, 01 Jan 2010 16:50:57 -0500 copy package versions from managed package group to other managed groups diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -90,12 +90,37 @@ Commit current changes to the group. """ + # sync versions from the package group to the other managed groups. + self._copyVersions() + + # write out the model data self._helper.setModel(self._sourceName, self._groups) + + # commit to the repository self._helper.commit(self._sourceName, commitMessage='automated commit') self._checkedout = False save = _commit + def _copyVersions(self): + """ + Copy versions from the packages group to the other managed groups. + """ + + pkgs = [ (x[1].name, x[1]) for x in + self._groups[self._pkgGroupName].iteritems() ] + + for group in self._groups.itervalues(): + # skip over package group since it is the version source. + if group.groupName == self._pkgGroupName: + continue + + # for all other groups iterate over contents and set versions to + # match package group. + for k, pkg in group.iteritems(): + if pkg.name in pkgs: + pkg.version = pkgs[pkg.name].version + @commit def build(self): """ From elliot at rpath.com Mon Mar 15 17:13:26 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:26 +0000 Subject: mirrorball: add ability to copy/deepcopy a package source object Message-ID: <201003152113.o2FLDRQh012695@scc.eng.rpath.com> changeset: 76c7f2e37111 user: Elliot Peele date: Fri, 01 Jan 2010 16:52:15 -0500 add ability to copy/deepcopy a package source object diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py --- a/updatebot/pkgsource/common.py +++ b/updatebot/pkgsource/common.py @@ -16,6 +16,7 @@ Common module between all pkgSource implementations. """ +import copy import logging log = logging.getLogger('updatebot.pkgsource') @@ -49,6 +50,28 @@ # {binName: [binPkg, ... ] } self.binNameMap = dict() + def __copy__(self): + log.info('copying pkgsource') + cls = self.__class__ + obj = cls(self._cfg) + obj.locationMap = copy.copy(self.locationMap) + obj.srcPkgMap = copy.copy(self.srcPkgMap) + obj.binPkgMap = copy.copy(self.binPkgMap) + obj.srcNameMap = copy.copy(self.srcNameMap) + obj.binNameMap = copy.copy(self.binNameMap) + return obj + + def __deepcopy__(self, memo): + log.info('deepcopying pkgsource') + cls = self.__class__ + obj = cls(self._cfg) + obj.locationMap = copy.deepcopy(self.locationMap, memo) + obj.srcPkgMap = copy.deepcopy(self.srcPkgMap, memo) + obj.binPkgMap = copy.deepcopy(self.binPkgMap, memo) + obj.srcNameMap = copy.deepcopy(self.srcNameMap, memo) + obj.binNameMap = copy.deepcopy(self.binNameMap, memo) + return obj + def getClients(self): """ Get instances of repository clients. From elliot at rpath.com Mon Mar 15 17:13:29 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:29 +0000 Subject: mirrorball: make sure to log a warning when updates remove packages in case the exception Message-ID: <201003152113.o2FLDTE5012725@scc.eng.rpath.com> changeset: 938437b5eace user: Elliot Peele date: Fri, 01 Jan 2010 16:52:50 -0500 make sure to log a warning when updates remove packages in case the exception in handled at a higher level diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -254,6 +254,8 @@ # equal. if (rpmvercmp(pkg.epoch, srpm.epoch) != 0 or rpmvercmp(pkg.version, srpm.version) != 0): + log.warn('update removes package (%s) %s -> %s' + % (pkg.name, srpm.getNevra(), srcPkg.getNevra())) raise UpdateRemovesPackageError(why='all rpms in the ' 'manifest should have the same version, trying ' 'to add %s' % (pkg, )) From elliot at rpath.com Mon Mar 15 17:13:33 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:33 +0000 Subject: mirrorball: fix bugs in version copying and raise an error when managed packages have been Message-ID: <201003152113.o2FLDXUR012754@scc.eng.rpath.com> changeset: e5cb91b58e2c user: Elliot Peele date: Fri, 01 Jan 2010 17:14:50 -0500 fix bugs in version copying and raise an error when managed packages have been removed that are mentioned in other groups diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -339,6 +339,18 @@ _params = ['name', ] _template = 'I do not know what to do with this package %(name)s.' +class UnknownPackageFoundInManagedGroupError(GroupManagerError): + """ + UnknownPackageFoundInManagedGroupError, raised when a package is dicovered + in one of the managed non package groups that no longer exists in the + package group. + """ + + _params = ['what', ] + _template = ('The following package is not longer managed as part of the ' + 'version group %(what)s, you may need to remove this package from any ' + 'other static group definitions.') + class ImportError(UpdateBotError): """ General purpose error for all import related issues. diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -107,8 +107,8 @@ Copy versions from the packages group to the other managed groups. """ - pkgs = [ (x[1].name, x[1]) for x in - self._groups[self._pkgGroupName].iteritems() ] + pkgs = dict([ (x[1].name, x[1]) for x in + self._groups[self._pkgGroupName].iteritems() ]) for group in self._groups.itervalues(): # skip over package group since it is the version source. @@ -120,6 +120,8 @@ for k, pkg in group.iteritems(): if pkg.name in pkgs: pkg.version = pkgs[pkg.name].version + else: + raise UnknownPackageFoundInManagedGroupError(what=pkg.name) @commit def build(self): From elliot at rpath.com Mon Mar 15 17:13:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:37 +0000 Subject: mirrorball: add missing import Message-ID: <201003152113.o2FLDcaX012786@scc.eng.rpath.com> changeset: 64cd691139b2 user: Elliot Peele date: Fri, 01 Jan 2010 17:15:24 -0500 add missing import diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -37,6 +37,7 @@ from updatebot.errors import UnknownBuildContextError from updatebot.errors import UnsupportedTroveFlavorError from updatebot.errors import UnhandledPackageAdditionError +from updatebot.errors import UnknownPackageFoundInManagedGroupError log = logging.getLogger('updatebot.groupmgr') From elliot at rpath.com Mon Mar 15 17:13:40 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:40 +0000 Subject: mirrorball: add an assert to ensure versions are always present when adding packages to Message-ID: <201003152113.o2FLDfRe012817@scc.eng.rpath.com> changeset: d32451884b48 user: Elliot Peele date: Fri, 01 Jan 2010 17:35:23 -0500 add an assert to ensure versions are always present when adding packages to managed groups diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -166,6 +166,9 @@ @type flavors: [conary.deps.deps.Flavor, ...] """ + # Now that versions are actually used for something make sure they + # are always present. + assert version assert len(flavors) plain = deps.parseFlavor('') From elliot at rpath.com Mon Mar 15 17:13:44 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:44 +0000 Subject: mirrorball: expose ordered bot Message-ID: <201003152113.o2FLDi4c012846@scc.eng.rpath.com> changeset: e4b073fd334f user: Elliot Peele date: Tue, 05 Jan 2010 15:25:32 -0500 expose ordered bot diff --git a/updatebot/__init__.py b/updatebot/__init__.py --- a/updatebot/__init__.py +++ b/updatebot/__init__.py @@ -18,4 +18,5 @@ """ from updatebot.bot import Bot +from updatebot.ordered import Bot as OrderedBot from updatebot.config import UpdateBotConfig From elliot at rpath.com Mon Mar 15 17:13:51 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:51 +0000 Subject: mirrorball: expose findTroves interface Message-ID: <201003152113.o2FLDp9t012880@scc.eng.rpath.com> changeset: d19360c53ff5 user: Elliot Peele date: Tue, 05 Jan 2010 15:28:49 -0500 expose findTroves interface add support for checking out specific versions of a source diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -92,6 +92,14 @@ return self._ccfg + def findTroves(self, troveList, *args, **kwargs): + """ + Mapped to conaryclient.repos.findTroves. Will always search buildLabel. + """ + + return self._repos.findTroves(self._ccfg.buildLabel, troveList, + *args, **kwargs) + def getSourceTroves(self, group): """ Find all of the source troves included in group. If group is None use @@ -207,9 +215,7 @@ sources = self._repos.getTroveInfo(tiSourceName, troveSpecs) for i, (n, v, f) in enumerate(troveSpecs): src = (sources[i](), v.getSourceVersion(), None) - if src not in srcTrvs: - srcTrvs[src] = set() - srcTrvs[src].add((n, v, f)) + srcTrvs.setdefault(src, set()).add((n, v, f)) return srcTrvs @@ -413,7 +419,7 @@ # make sure version file has been added to package self._addFile(recipeDir, 'version') - def commit(self, pkgname, commitMessage=''): + def commit(self, pkgname, version=None, commitMessage=''): """ Commit the cached checkout of a source component. @param pkgname name of the package @@ -421,10 +427,13 @@ @param commitMessage: optional argument for setting the commit message to use when committing to the repository. @type commitMessage: string + @param version optional source version to checkout. + @type version conary.versions.Version @return version of the source commit. """ - if pkgname not in self._checkoutCache: + pkgkey = (pkgname, version) + if pkgkey not in self._checkoutCache: raise NoCheckoutFoundError(pkgname=pkgname) # Setup flavor objects @@ -432,7 +441,7 @@ error=False) # Commit to repository. - recipeDir = self._checkoutCache[pkgname] + recipeDir = self._checkoutCache[pkgkey] self._commit(recipeDir, commitMessage) # Get new version of the source trove. @@ -440,25 +449,29 @@ assert version is not None return version - def _edit(self, pkgname): + def _edit(self, pkgname, version=None): """ Checkout/Create source checkout. @param pkgname name of the package @type pkgname string + @param version optional source version to checkout. + @type version conary.versions.Version @return path to checkout """ - if pkgname in self._checkoutCache: - return self._checkoutCache[pkgname] + pkgkey = (pkgname, version) + if pkgkey in self._checkoutCache: + return self._checkoutCache[pkgkey] # Figure out if we should create or update. if not self.getLatestSourceVersion(pkgname): + assert version is None recipeDir = self._newpkg(pkgname) else: - recipeDir = self._checkout(pkgname) + recipeDir = self._checkout(pkgname, version=version) - self._checkoutCache[pkgname] = recipeDir - self._checkoutCache[recipeDir] = pkgname + self._checkoutCache[pkgkey] = recipeDir + self._checkoutCache[recipeDir] = pkgkey return recipeDir @@ -472,18 +485,24 @@ return tempfile.mkdtemp(prefix='%s-' % pkgname, dir=self._cacheDir) - def _checkout(self, pkgname): + def _checkout(self, pkgname, version=None): """ Checkout a source component from the repository. @param pkgname: name of the package to checkout @type pkgname: string + @param version optional source version to checkout. + @type version conary.versions.Version @return checkout directory """ - log.info('checking out %s' % pkgname) + troveSpec = pkgname + if version: + troveSpec += '=%s' % version + + log.info('checking out %s' % troveSpec) recipeDir = self._getRecipeDir(pkgname) - checkin.checkout(self._repos, self._ccfg, recipeDir, [pkgname, ]) + checkin.checkout(self._repos, self._ccfg, recipeDir, [troveSpec, ]) return recipeDir From elliot at rpath.com Mon Mar 15 17:13:55 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:55 +0000 Subject: mirrorball: add a cache for fetching source versions Message-ID: <201003152113.o2FLDtYA012910@scc.eng.rpath.com> changeset: e5ed06704230 user: Elliot Peele date: Tue, 05 Jan 2010 17:28:25 -0500 add a cache for fetching source versions diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -84,6 +84,10 @@ self._cacheDir = tempfile.mkdtemp( prefix='conaryhelper-%s-' % cfg.platformName) + # caches source names and versions for binaries past into getSourceVersions. + # binTroveSpec: sourceTroveSpec + self._sourceVersionCache = {} + def getConaryConfig(self): """ Get a conary config instance. @@ -194,6 +198,8 @@ withFileContents=False, recurse=False) + # Iterate over both strong and weak refs because msw said it was a + # good idea. topTrove = self._getTrove(cs, name, version, flavor) troves = topTrove.iterTroveList(weakRefs=True, strongRefs=True) @@ -208,14 +214,21 @@ @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ...]} """ - # Iterate over both strong and weak refs because msw said it was a - # good idea. + # check the cache for any source we have already retrieved from the + # repository. + cached = set(x for x in troveSpecs if x in self._sourceVersionCache) + uncached = sorted(set(troveSpecs).difference(cached)) + srcTrvs = {} tiSourceName = trove._TROVEINFO_TAG_SOURCENAME - sources = self._repos.getTroveInfo(tiSourceName, troveSpecs) - for i, (n, v, f) in enumerate(troveSpecs): + sources = self._repos.getTroveInfo(tiSourceName, uncached) + for i, (n, v, f) in enumerate(uncached): src = (sources[i](), v.getSourceVersion(), None) srcTrvs.setdefault(src, set()).add((n, v, f)) + self._sourceVersionCache[(n, v, f)] = src + + for spec in cached: + srcTrvs.setdefault(self._sourceVersionCache[spec], set()).add(spec) return srcTrvs From elliot at rpath.com Mon Mar 15 17:13:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:13:58 +0000 Subject: mirrorball: properly subclass rpm entry elements Message-ID: <201003152113.o2FLDxUT012941@scc.eng.rpath.com> changeset: 215ab42856bd user: Elliot Peele date: Tue, 05 Jan 2010 23:29:12 -0500 properly subclass rpm entry elements diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -176,7 +176,7 @@ return ver -class _RpmRequires(xmllib.BaseNode): +class _RpmEntry(xmllib.BaseNode): """ Parse any element that contains rpm:entry or suse:entry elements. """ @@ -217,49 +217,55 @@ raise UnknownElementError(child) -class _RpmRecommends(_RpmRequires): +class _RpmRequires(_RpmEntry): + """ + Parse rpm:requires children. + """ + + +class _RpmRecommends(_RpmEntry): """ Parse rpm:recommends children. """ -class _RpmProvides(_RpmRequires): +class _RpmProvides(_RpmEntry): """ Parse rpm:provides children. """ -class _RpmObsoletes(_RpmRequires): +class _RpmObsoletes(_RpmEntry): """ Parse rpm:obsoletes children. """ -class _RpmConflicts(_RpmRequires): +class _RpmConflicts(_RpmEntry): """ Parse rpm:conflicts children. """ -class _RpmEnhances(_RpmRequires): +class _RpmEnhances(_RpmEntry): """ Parse rpm:enhances children. """ -class _RpmSupplements(_RpmRequires): +class _RpmSupplements(_RpmEntry): """ Parse rpm:supplements children. """ -class _RpmSuggests(_RpmRequires): +class _RpmSuggests(_RpmEntry): """ Parse rpm:suggests children. """ -class _SuseFreshens(_RpmRequires): +class _SuseFreshens(_RpmEntry): """ Parse suse:freshens children. """ From elliot at rpath.com Mon Mar 15 17:14:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:02 +0000 Subject: mirrorball: add option to get specific versions and from sources. Message-ID: <201003152114.o2FLE3S8012974@scc.eng.rpath.com> changeset: 9648f8ec54ec user: Elliot Peele date: Thu, 07 Jan 2010 11:00:30 -0500 add option to get specific versions and from sources. diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -385,7 +385,7 @@ # add file to the source compoent self._addFile(recipeDir, 'buildrequires') - def getVersion(self, pkgname): + def getVersion(self, pkgname, version=None): """ Get the version of the specified package if this package has a version file in the source component, otherwise return None. @@ -395,7 +395,7 @@ log.info('getting version info for %s' % pkgname) - recipeDir = self._edit(pkgname) + recipeDir = self._edit(pkgname, version=version) versionFileName = util.join(recipeDir, 'version') if not os.path.exists(versionFileName): From elliot at rpath.com Mon Mar 15 17:14:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:10 +0000 Subject: mirrorball: 1. add validation for group contents to catch common issues Message-ID: <201003152114.o2FLEAON013009@scc.eng.rpath.com> changeset: e624721b8756 user: Elliot Peele date: Thu, 07 Jan 2010 11:02:23 -0500 1. add validation for group contents to catch common issues 2. add support for manging specific source versions, this makes it easier to fix bugs in old source models. diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -19,7 +19,9 @@ import os import copy import logging +import itertools +from conary import versions from conary.deps import deps from updatebot.lib import util @@ -33,10 +35,13 @@ from updatebot.lib.xobjects import XPackageData from updatebot.lib.xobjects import XPackageItem +from updatebot.errors import OldVersionsFoundError from updatebot.errors import FlavorCountMismatchError from updatebot.errors import UnknownBuildContextError +from updatebot.errors import GroupValidationFailedError from updatebot.errors import UnsupportedTroveFlavorError from updatebot.errors import UnhandledPackageAdditionError +from updatebot.errors import NameVersionConflictsFoundError from updatebot.errors import UnknownPackageFoundInManagedGroupError log = logging.getLogger('updatebot.groupmgr') @@ -71,7 +76,7 @@ #self._versionFactory = VersionFactory(cfg) self._sourceName = self._cfg.topSourceGroup[0] - self._sourceVersion = self._cfg.topSourceGroup[1] + self._sourceVersion = None self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName @@ -83,47 +88,62 @@ Get current group state from the repository. """ - self._groups = self._helper.getModel(self._sourceName) + self._groups = self._helper.getModel(self._sourceName, + version=self._sourceVersion) self._checkedout = True - def _commit(self): + def _commit(self, copyToLatest=False): """ Commit current changes to the group. """ + if self._sourceVersion and not copyToLatest: + log.error('refusing to commit out of date source') + raise NotCommittingOutOfDateSourceError() + + # Copy forward data when we are fixing up old group versions so that + # this is the latest source. + if copyToLatest: + log.info('copying information to latest version') + # Get data from the old versoin + version = self._helper.getVersion(self._sourceName, version=self._sourceVersion) + errataState = self._helper.getErrataState(self._sourceName, version=self._sourceVersion) + groups = self._groups + + log.info('version: %s' % version) + log.info('errataState: %s' % errataState) + + # Set version to None to get the latest source. + self._sourceVersion = None + + # Checkout latest source. + self._checkout() + + # Set back to old data + self.setVersion(version) + self.setErrataState(errataState) + self._groups = groups + # sync versions from the package group to the other managed groups. self._copyVersions() + # validate group contents. + self._validateGroups() + # write out the model data self._helper.setModel(self._sourceName, self._groups) # commit to the repository - self._helper.commit(self._sourceName, commitMessage='automated commit') + version = self._helper.commit(self._sourceName, + version=self._sourceVersion, + commitMessage='automated commit') + if self._sourceVersion: + self._sourceVersion = version self._checkedout = False + return version save = _commit - def _copyVersions(self): - """ - Copy versions from the packages group to the other managed groups. - """ - - pkgs = dict([ (x[1].name, x[1]) for x in - self._groups[self._pkgGroupName].iteritems() ]) - - for group in self._groups.itervalues(): - # skip over package group since it is the version source. - if group.groupName == self._pkgGroupName: - continue - - # for all other groups iterate over contents and set versions to - # match package group. - for k, pkg in group.iteritems(): - if pkg.name in pkgs: - pkg.version = pkgs[pkg.name].version - else: - raise UnknownPackageFoundInManagedGroupError(what=pkg.name) - @commit def build(self): """ @@ -327,6 +347,183 @@ return set() #return self._versionFactory.getVersions(pkgSet) + def _copyVersions(self): + """ + Copy versions from the packages group to the other managed groups. + """ + + pkgs = dict([ (x[1].name, x[1]) for x in + self._groups[self._pkgGroupName].iteritems() ]) + + for group in self._groups.itervalues(): + # skip over package group since it is the version source. + if group.groupName == self._pkgGroupName: + continue + + # for all other groups iterate over contents and set versions to + # match package group. + for k, pkg in group.iteritems(): + if pkg.name in pkgs: + pkg.version = pkgs[pkg.name].version + else: + raise UnknownPackageFoundInManagedGroupError(what=pkg.name) + + def _validateGroups(self): + """ + Validate the contents of the package group to ensure sanity: + 1. Check for packages that have the same source name, but + different versions. + 2. Check that the version in the group is the latest source/build + of that version. + """ + + errors = [] + for name, group in self._groups.iteritems(): + log.info('checking consistentcy of %s' % name) + try: + self._checkNameVersionConflict(group) + except NameVersionConflictsFoundError, e: + errors.append((group, e)) + + try: + self._checkLatestVersion(group) + except OldVersionsFoundError, e: + errors.append((group, e)) + + if errors: + raise GroupValidationFailedError(errors=errors) + + def _checkNameVersionConflict(self, group): + """ + Check for packages taht have the same source name, but different + versions. + """ + + # get names and versions + troves = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + version = str(versions.ThawVersion(pkgData.version).asString()) + + flavor = None + # FIXME: At some point we might want to add proper flavor handling, + # note that group flavor handling is different than what + # findTroves normally does. + #if pkgData.flavor: + # flavor = deps.ThawFlavor(str(pkgData.flavor)) + + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = set([ x for x in + itertools.chain(*self._helper.findTroves(troves).itervalues()) ]) + + # get sources for each name version pair + sources = self._helper.getSourceVersions(foundTroves) + + seen = {} + for (n, v, f), pkgSet in sources.iteritems(): + binVer = list(pkgSet)[1][1] + seen.setdefault(n, set()).add(binVer) + + binPkgs = {} + conflicts = {} + for name, vers in seen.iteritems(): + if len(vers) > 1: + log.error('found multiple versions of %s' % name) + for binVer in vers: + srcVer = binVer.getSourceVersion() + nvf = (name, srcVer, None) + conflicts.setdefault(name, []).append(srcVer) + binPkgs[nvf] = sources[nvf] + + if conflicts: + raise NameVersionConflictsFoundError(groupName=group.groupName, + conflicts=conflicts, + binPkgs=binPkgs) + + def _checkLatestVersion(self, group): + """ + Check to make sure each specific conary version is the latest source + and build count of the upstream version. + """ + + # get names and versions + troves = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + version = versions.ThawVersion(pkgData.version) + # get upstream version + revision = version.trailingRevision() + upstreamVersion = revision.getVersion() + + # FIXME: This should probably be a fully formed version + # as above. + version = upstreamVersion + + flavor = None + # FIXME: At some point we might want to add proper flavor handling, + # note that group flavor handling is different than what + # findTroves normally does. + #if pkgData.flavor: + # flavor = deps.ThawFlavor(str(pkgData.flavor)) + + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = dict([ (x[0], y) for x, y in + self._helper.findTroves(troves).iteritems() ]) + + pkgs = {} + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + version = None + if pkgData.version: + version = versions.ThawVersion(pkgData.version) + flavor = None + if pkgData.flavor: + flavor = deps.ThawFlavor(str(pkgData.flavor)) + + pkgs.setdefault(name, []).append((name, version, flavor)) + + assert len(pkgs) == len(foundTroves) + + errors = {} + for name, found in foundTroves.iteritems(): + assert name in pkgs + current = pkgs[name] + + if len(current) > len(found): + log.warn('found more packages in the model than in the ' + 'repository, assuming that multiversion policy will ' + 'catch this.') + continue + + assert len(current) == 1 or len(found) == len(current) + + foundError = False + for i, (n, v, f) in enumerate(found): + if len(current) == 1: + i = 0 + cn, cv, cf = current[i] + assert n == cn + + if v != cv: + foundError = True + + if foundError: + log.error('found old version for %s' % name) + errors[name] = (current, found) + + if errors: + raise OldVersionsFoundError(pkgNames=errors.keys(), errors=errors) + class GroupHelper(ConaryHelper): """ @@ -346,14 +543,14 @@ # autoLoadRecipes here anyway. self._ccfg.autoLoadRecipes = [] - def getModel(self, pkgName): + def getModel(self, pkgName, version=None): """ Get a thawed data representation of the group xml data from the repository. """ log.info('loading model for %s' % pkgName) - recipeDir = self._edit(pkgName) + recipeDir = self._edit(pkgName, version=version) groupFileName = util.join(recipeDir, 'groups.xml') # load group model @@ -384,8 +581,10 @@ (name, byDefault, depCheck) ) - # override anything from the repo - groups[name] = contentsModel + # override anything from the repo, unless retriveing a + # specific version. + if version is None: + groups[name] = contentsModel return groups @@ -412,7 +611,7 @@ groupModel.freeze(groupFileName) self._addFile(recipeDir, 'groups.xml') - def getErrataState(self, pkgname): + def getErrataState(self, pkgname, version=None): """ Get the contents of the errata state file from the specified package, if file does not exist, return None. @@ -420,7 +619,7 @@ log.info('getting errata state information from %s' % pkgname) - recipeDir = self._edit(pkgname) + recipeDir = self._edit(pkgname, version=version) stateFileName = util.join(recipeDir, 'erratastate') if not os.path.exists(stateFileName): From elliot at rpath.com Mon Mar 15 17:14:16 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:16 +0000 Subject: mirrorball: add script for finding and fixing group brokenness Message-ID: <201003152114.o2FLEGje013041@scc.eng.rpath.com> changeset: 35d5f4a66231 user: Elliot Peele date: Thu, 07 Jan 2010 13:34:04 -0500 add script for finding and fixing group brokenness diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py new file mode 100755 --- /dev/null +++ b/scripts/grpchecker.py @@ -0,0 +1,152 @@ +#!/usr/bin/python +# +# Copyright (c) 2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# + +""" +Script for finding and fixing group issues. +""" + +import os +import sys + +sys.path.insert(0, os.environ['HOME'] + '/hg/conary') +sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') + +from conary.lib import util +sys.excepthook = util.genExcepthook() + +mbdir = os.path.abspath('../') +sys.path.insert(0, mbdir) + +confDir = os.path.join(mbdir, 'config', sys.argv[1]) + +from updatebot import log +from updatebot import groupmgr +from updatebot import conaryhelper +from updatebot import UpdateBotConfig + +from updatebot.errors import OldVersionsFoundError +from updatebot.errors import GroupValidationFailedError +from updatebot.errors import NameVersionConflictsFoundError + +import time +import logging + +slog = log.addRootLogger() +cfg = UpdateBotConfig() +cfg.read(os.path.join(confDir, 'updatebotrc')) + +mgr = groupmgr.GroupManager(cfg) +helper = conaryhelper.ConaryHelper(cfg) + +def handleVersionConflicts(group, error): + conflicts = error.conflicts + binPkgs = error.binPkgs + + toAdd = {} + toRemove = set() + for source, svers in conflicts.iteritems(): + assert len(svers) == 2 + svers.sort() + old = binPkgs[(source, svers[0], None)] + new = binPkgs[(source, svers[1], None)] + + toRemove.update(set([ x[0] for x in old ])) + + for n, v, f, in new: + if n in toRemove: + toAdd.setdefault((n, v), set()).add(f) + + return toRemove, toAdd + +def handleVersionErrors(group, error): + toAdd = {} + toRemove = set() + for pkgName, (modelContents, repoContents) in error.errors.iteritems(): + toRemove.update(set([ x[0] for x in modelContents ])) + + for n, v, f in repoContents: + if n in toRemove: + toAdd.setdefault((n, v), set()).add(f) + + return toRemove, toAdd + +def checkVersion(ver): + mgr._sourceVersion = ver + mgr._checkout() + mgr._copyVersions() + + changes = [] + + try: + mgr._validateGroups() + except GroupValidationFailedError, e: + for group, error in e.errors: + if isinstance(error, NameVersionConflictsFoundError): + changes.append(handleVersionConflicts(group, error)) + elif isinstance(error, OldVersionsFoundError): + changes.append(handleVersionErrors(group, error)) + else: + raise error + + return ver, changes + +troves = helper.findTroves((cfg.topSourceGroup, ), getLeaves=False).values()[0] + +nv = [] +for n, v, f in sorted(troves): + nsv = (n, v.getSourceVersion()) + if nsv not in nv: + nv.append(nsv) + +toUpdate = [] +for n, v in nv: + ver, changed = checkVersion(v) + if not changed: + continue + toUpdate.append((ver, changed)) + +slog.critical('Before going any further, be aware that this will only rebuild ' + 'the groups that need to change, not any other existing groups, if you ' + 'want to maintain order on the devel label you will need to write that ' + 'code.') +assert False + + +jobIds = [] +for ver, changed in toUpdate: + mgr._sourceVersion = ver + mgr._checkout() + for toRemove, toAdd in changed: + for pkg in toRemove: + slog.info('removing %s' % pkg) + mgr.remove(pkg) + for (n, v), flvs in toAdd.iteritems(): + slog.info('adding %s=%s' % (n, v)) + mgr.addPackage(n, v, flvs) + mgr._copyVersions() + mgr._validateGroups() + version = mgr.save(copyToLatest=True) + jobId = mgr._builder.start(((mgr._sourceName, version, None), )) + jobIds.append(jobId) + +for jobId in jobIds: + mgr._builder.watch(jobId) + +import epdb; epdb.st() + +for jobId in jobIds: + mgr._builder.commit(jobId) + +import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:14:22 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:22 +0000 Subject: mirrorball: add script for generating xml model from package list Message-ID: <201003152114.o2FLENf1013073@scc.eng.rpath.com> changeset: 92d47b82de9c user: Elliot Peele date: Thu, 07 Jan 2010 13:35:24 -0500 add script for generating xml model from package list diff --git a/scripts/grouptoxml.py b/scripts/grouptoxml.py new file mode 100755 --- /dev/null +++ b/scripts/grouptoxml.py @@ -0,0 +1,404 @@ +#!/usr/bin/python +# +# Copyright (c) 2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# + +""" +Script to used to generate xml model for standard rhel groups. +""" + +rhel4corePackages = ( + ('kernel', 'kernel.smp,!kernel.largesmp,!kernel.hugemem,!xen,!domU,!dom0'), + 'acl', + 'ash', + 'attr', + 'audit', + 'audit-libs', + 'authconfig', + 'basesystem', + 'bash', + 'beecrypt', + 'bzip2', + 'bzip2-libs', + 'checkpolicy', + 'chkconfig', + 'coreutils', + 'cpio', + 'cracklib', + 'cracklib-dicts', + 'crontabs', + 'cyrus-sasl', + 'cyrus-sasl-md5', + 'db4', + 'dbus', + 'dbus-glib', + 'device-mapper', + 'dhclient', + 'diffutils', + 'dmraid', + 'e2fsprogs', + 'ed', + 'elfutils-libelf', + 'ethtool', + 'expat', + 'file', + 'filesystem', + 'findutils', + 'fontconfig', + 'freetype', + 'gawk', + 'gdbm', + 'glib2', + 'glibc', + 'glibc-common', + 'gmp', + 'grep', + 'grub', + 'gzip', + 'hal', + 'hdparm', + 'hesiod', + 'hotplug', + 'hwdata', + 'indexhtml', + 'info', + 'initscripts', + 'iproute', + 'iptables', + 'iputils', + 'kbd', + 'krb5-libs', + 'kudzu', + 'less', + 'libacl', + 'libattr', + 'libcap', + 'libgcc', + 'libgcrypt', + 'libgpg-error', + 'libselinux', + 'libsepol', + 'libstdc++', + 'libtermcap', + 'libuser', + 'libxml2', + 'libxml2-python', + 'libxslt', + 'libxslt-python', + 'logrotate', + 'lvm2', + 'MAKEDEV', + 'mdadm', + 'mingetty', + 'mkinitrd', + 'mktemp', + 'module-init-tools', + 'ncurses', + 'net-tools', + 'newt', + 'nscd', + 'openldap', + 'openssl', + 'pam', + 'parted', + 'passwd', + 'pciutils', + 'pcre', + 'perl', + 'perl-Filter', + 'policycoreutils', + 'popt', + 'prelink', + 'procmail', + 'procps', + 'psmisc', + 'python', + 'pyxf86config', + 'readline', + 'redhat-logos', + 'redhat-release', + 'rhpl', + 'rootfiles', + 'rpm', + 'rpmdb-redhat', + 'rpm-libs', + 'sed', + 'selinux-doc', + 'selinux-policy-targeted', + 'sendmail', + 'setools', + 'setserial', + 'setup', + 'shadow-utils', + 'slang', + 'sysklogd', + 'system-config-mouse', + 'SysVinit', + 'tar', + 'tcp_wrappers', + 'termcap', + 'tmpwatch', + 'tzdata', + 'udev', + 'usbutils', + 'usermode', + 'util-linux', + 'vim-minimal', + 'vixie-cron', + 'wireless-tools', + 'xorg-x11-libs', + 'xorg-x11-Mesa-libGL', + 'zlib', +) + + +rhel5corePackages = ( + ('kernel', 'kernel.smp,!kernel.debug,!kernel.pae,!xen,!domU,!dom0'), + 'acl', + 'alchemist', + 'atk', + 'attr', + 'audit', + 'audit-libs', + 'audit-libs-python', + 'authconfig', + 'basesystem', + 'bash', + 'beecrypt', + 'bzip2', + 'bzip2-libs', + 'cairo', + 'checkpolicy', + 'chkconfig', + 'coreutils', + 'cpio', + 'cracklib', + 'cracklib-dicts', + 'crontabs', + 'cryptsetup-luks', + 'cups-libs', + 'cyrus-sasl', + 'cyrus-sasl-lib', + 'cyrus-sasl-md5', + 'db4', + 'dbus', + 'dbus-glib', + 'Deployment_Guide-en-US', + 'device-mapper', + 'device-mapper-multipath', + 'dhclient', + 'diffutils', + 'dmidecode', + 'dmraid', + 'e2fsprogs', + 'e2fsprogs-libs', + 'ed', + 'elfutils-libelf', + 'ethtool', + 'expat', + 'file', + 'filesystem', + 'findutils', + 'fontconfig', + 'freetype', + 'gawk', + 'gdbm', + 'glib2', + 'glibc', + 'glibc-common', + 'gmp', + 'gnutls', + 'grep', + 'grub', + 'gtk2', + 'gzip', + 'hal', + 'hdparm', + 'hesiod', + 'hicolor-icon-theme', + 'hwdata', + 'info', + 'initscripts', + 'iproute', + 'iptables', + 'iputils', + 'kbd', + 'kernel-headers', + 'keyutils-libs', + 'kpartx', + 'krb5-libs', + 'kudzu', + 'less', + 'libacl', + 'libattr', + 'libcap', + 'libgcc', + 'libgcrypt', + 'libgpg-error', + 'libhugetlbfs', + 'libhugetlbfs-lib', + 'libjpeg', + 'libpng', + 'libselinux', + 'libselinux-python', + 'libsemanage', + 'libsepol', + 'libstdc++', + 'libsysfs', + 'libtermcap', + 'libtiff', + 'libusb', + 'libuser', + 'libvolume_id', + 'libX11', + 'libXau', + 'libXcursor', + 'libXdmcp', + 'libXext', + 'libXfixes', + 'libXft', + 'libXi', + 'libXinerama', + 'libxml2', + 'libxml2-python', + 'libXrandr', + 'libXrender', + 'libxslt', + 'libxslt-python', + 'logrotate', + 'lvm2', + 'MAKEDEV', + 'mcstrans', + 'mdadm', + 'mingetty', + 'mkinitrd', + 'mktemp', + 'module-init-tools', + 'nash', + 'ncurses', + 'net-tools', + 'newt', + 'nscd', + 'openldap', + 'openssl', + 'pam', + 'pango', + 'parted', + 'passwd', + 'pciutils', + 'pcre', + 'pm-utils', + 'policycoreutils', + 'popt', + 'prelink', + 'procmail', + 'procps', + 'psmisc', + 'pycairo', + 'pygobject2', + 'pygtk2', + 'python', + 'python-numeric', + 'pyxf86config', + 'readline', + 'redhat-logos', + 'redhat-release', + 'redhat-release-notes', + 'rhpl', + 'rootfiles', + 'rpm', + 'rpm-libs', + 'sed', + 'selinux-policy', + 'selinux-policy-targeted', + 'sendmail', + 'setools', + 'setserial', + 'setup', + 'shadow-utils', + 'slang', + 'sqlite', + 'sysfsutils', + 'sysklogd', + 'SysVinit', + 'tar', + 'tcl', + 'tcp_wrappers', + 'termcap', + 'tmpwatch', + 'tzdata', + 'udev', + 'usbutils', + 'usermode', + 'util-linux', + 'vim-minimal', + 'vixie-cron', + 'wireless-tools', + 'xorg-x11-filesystem', + 'zlib', +) + +import os +import sys +import time +import logging + +sys.path.insert(0, os.environ['HOME'] + '/hg/conary') +sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py') +sys.path.insert(0, os.environ['HOME'] + '/hg/rbuilder-trunk/rpath-xmllib') + +from conary.lib import util +sys.excepthook = util.genExcepthook() + +from conary.deps import deps + +mbdir = os.path.abspath('../') +sys.path.insert(0, mbdir) + +from updatebot import log +from updatebot import groupmgr + +slog = log.addRootLogger() + +def toxml(pkgList, toFile): + groupName = 'group-rhel-standard' + byDefault = True + depCheck = True + + contents = groupmgr.GroupContentsModel(groupName, + byDefault=byDefault, + depCheck=depCheck) + + for pkg in pkgList: + slog.info('adding %s' % (pkg, )) + if type(pkg) == tuple: + pkg, flavor = pkg + contents.add(pkg, flavor=deps.parseFlavor(flavor)) + else: + contents.add(pkg) + + contents.freeze(toFile) + + +if __name__ == '__main__': + platforms = { + 'rhel4': rhel4corePackages, + 'rhel5': rhel5corePackages, + } + + platform = sys.argv[1] + assert platform in platforms + + toFile = sys.argv[2] + + pkgs = platforms[platform] + toxml(pkgs, toFile) From elliot at rpath.com Mon Mar 15 17:14:28 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:28 +0000 Subject: mirrorball: sanity check the order Message-ID: <201003152114.o2FLES6s013109@scc.eng.rpath.com> changeset: f743f3d4e8ae user: Elliot Peele date: Thu, 07 Jan 2010 13:35:55 -0500 sanity check the order diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py --- a/scripts/rhelorder.py +++ b/scripts/rhelorder.py @@ -16,7 +16,7 @@ mbdir = os.path.abspath('../') sys.path.insert(0, mbdir) -confDir = os.path.join(mbdir, 'config', 'rhel5') +confDir = os.path.join(mbdir, 'config', sys.argv[1]) from updatebot import log from updatebot.ordered import Bot @@ -48,6 +48,7 @@ bot = Bot(cfg, errata) bot._pkgSource.load() + bot._errata._orderErrata() order = bot._errata._order @@ -57,4 +58,6 @@ def tconv(tstamp): return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp)) +bot._errata.sanityCheckOrder() + import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:14:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:35 +0000 Subject: mirrorball: add copyright info and validate group model Message-ID: <201003152114.o2FLEbuv013143@scc.eng.rpath.com> changeset: 1c9c5ae953ed user: Elliot Peele date: Thu, 07 Jan 2010 13:37:55 -0500 add copyright info and validate group model diff --git a/scripts/gengroup.py b/scripts/gengroup.py --- a/scripts/gengroup.py +++ b/scripts/gengroup.py @@ -1,4 +1,20 @@ #!/usr/bin/python +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# + +""" +Script for generating group model from current state of the repository. +""" import os import sys @@ -49,26 +65,27 @@ slog.error('unhandled flavor found %s' % k1) raise RuntimeError -import epdb; epdb.st() +#import epdb; epdb.st() for name, vf in troves.iteritems(): if ':' in name or bot._updater._fltrPkg(name): continue versions = vf.keys() + assert len(versions) == 1 versions.sort() version = versions[-1] flavors = vf[version] - if name == 'kernel': - import epdb; epdb.st() + mgr.addPackage(name, version, flavors) - mgr.addPackage(name, version, flavors) - -import epdb; epdb.st() +#import epdb; epdb.st() mgr.setVersion('0') mgr.setErrataState('0') +mgr._copyVersions() +mgr._validateGroups() +mgr._helper.setModel(mgr._sourceName, mgr._groups) #mgr._commit() #mgr.build() From elliot at rpath.com Mon Mar 15 17:14:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:39 +0000 Subject: mirrorball: adapt to new mirrorball dirs Message-ID: <201003152114.o2FLEdAv013173@scc.eng.rpath.com> changeset: fa9681ebcd49 user: Elliot Peele date: Thu, 07 Jan 2010 13:39:36 -0500 adapt to new mirrorball dirs diff --git a/scripts/checkoutall b/scripts/checkoutall --- a/scripts/checkoutall +++ b/scripts/checkoutall @@ -10,7 +10,8 @@ import os import sys -sys.path.insert(0, os.environ['HOME'] + '/hg/mirrorball') +mirrorballDir = os.path.abspath('../') +sys.path.insert(0, mirrorballDir) from conary.lib import util sys.excepthook = util.genExcepthook() @@ -20,9 +21,10 @@ log.addRootLogger() cfg = config.UpdateBotConfig() -cfg.read(os.environ['HOME'] + '/hg/mirrorball/config/%s/updatebotrc' % sys.argv[1]) +cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1]) helper = conaryhelper.ConaryHelper(cfg) -pkgs = [ x.split(':')[0] for x in helper.getLatestVersions() if x.endswith(':source') ] +pkgs = [ x.split(':')[0] for x in helper.getLatestVersions() + if x.endswith(':source') ] checkin.checkout(helper._repos, helper._ccfg, None, pkgs) From elliot at rpath.com Mon Mar 15 17:14:45 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:45 +0000 Subject: mirrorball: add mapping of rpm obsoletes Message-ID: <201003152114.o2FLEjxE013209@scc.eng.rpath.com> changeset: e5b5f8212203 user: Elliot Peele date: Thu, 07 Jan 2010 13:51:44 -0500 add mapping of rpm obsoletes diff --git a/updatebot/pkgsource/common.py b/updatebot/pkgsource/common.py --- a/updatebot/pkgsource/common.py +++ b/updatebot/pkgsource/common.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -50,6 +50,9 @@ # {binName: [binPkg, ... ] } self.binNameMap = dict() + # {binPkg: set(binNames) } + self.obsoletesMap = dict() + def __copy__(self): log.info('copying pkgsource') cls = self.__class__ diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2006,2008-2009 rPath, Inc. +# Copyright (c) 2006,2008-2010 rPath, Inc. # # # This program is distributed under the terms of the Common Public License, @@ -18,6 +18,7 @@ """ import os +import itertools import logging import repomd @@ -170,6 +171,25 @@ self._rpmMap[rpmMapKey] = set() self._rpmMap[rpmMapKey].add(package) + # the normal case of "obsoletes foo < version" (or with + # "requires foo", though that normally also follows the + # "< version" pattern) is "keep in sync" which we do already. + # The point of this is what redirects are used for in + # native conary packages, not for what groups are used for. + obsoleteNames = set( + x.name for x in itertools.chain(*[ + y.getChildren('rpm:entry') for y in package.format + if isinstance(y, repomd.packagexml._RpmObsoletes)]) + if not x.version) + if obsoleteNames: + requiresNames = set( + x.name for x in itertools.chain(*[ + y.getChildren('rpm:entry') for y in package.format + if isinstance(y, repomd.packagexml._RpmRequires)])) + obsoleteNames -= requiresNames + if obsoleteNames: + self.obsoletesMap[package] = obsoleteNames + if package.name not in self.binNameMap: self.binNameMap[package.name] = set() self.binNameMap[package.name].add(package) From elliot at rpath.com Mon Mar 15 17:14:48 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:48 +0000 Subject: mirrorball: 1. remove all flavors of a package before adding Message-ID: <201003152114.o2FLEn0D013240@scc.eng.rpath.com> changeset: cb990c1da0dd user: Elliot Peele date: Fri, 08 Jan 2010 16:26:01 -0500 1. remove all flavors of a package before adding 2. validate package removes to make sure we remove all packages stated in the config file diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -42,6 +42,7 @@ from updatebot.errors import UnsupportedTroveFlavorError from updatebot.errors import UnhandledPackageAdditionError from updatebot.errors import NameVersionConflictsFoundError +from updatebot.errors import ExpectedRemovalValidationFailedError from updatebot.errors import UnknownPackageFoundInManagedGroupError log = logging.getLogger('updatebot.groupmgr') @@ -191,6 +192,10 @@ assert version assert len(flavors) + # Remove all versions and flavors of this name before adding this + # package. This avoids flavor change issues by replacing all flavors. + self.remove(name) + plain = deps.parseFlavor('') x86 = deps.parseFlavor('is: x86') x86_64 = deps.parseFlavor('is: x86_64') @@ -375,6 +380,8 @@ different versions. 2. Check that the version in the group is the latest source/build of that version. + 3. Check that package removals specified in the config file have + occured. """ errors = [] @@ -390,6 +397,11 @@ except OldVersionsFoundError, e: errors.append((group, e)) + try: + self._checkRemovals(group) + except ExpectedRemovalValidationFailedError, e: + errors.append((group, e)) + if errors: raise GroupValidationFailedError(errors=errors) @@ -524,6 +536,30 @@ if errors: raise OldVersionsFoundError(pkgNames=errors.keys(), errors=errors) + def _checkRemovals(self, group): + """ + Check to make sure that all configured package removals have happened. + """ + + updateId = self.getErrataState() + + # get package removals from the config object. + removePackages = self._cfg.updateRemovesPackages.get(updateId, []) + removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) + + # take the union + removals = set(removePackages) | set(removeObsoleted) + + errors = [] + # Make sure these packages are not in the group model. + for pkgKey, pkgData in group.iteritems(): + if pkgData.name in removals: + errors.append(pkgData.name) + + if errors: + raise ExpectedRemovalValidationFailedError(updateId=updateId, + pkgNames=errors) + class GroupHelper(ConaryHelper): """ From elliot at rpath.com Mon Mar 15 17:14:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:14:58 +0000 Subject: mirrorball: new exceptions to support lots of new validation Message-ID: <201003152114.o2FLEw32013278@scc.eng.rpath.com> changeset: dd2ce8068373 user: Elliot Peele date: Fri, 08 Jan 2010 16:26:52 -0500 new exceptions to support lots of new validation diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -119,6 +119,20 @@ UpdateRemovesPackageError, raised when the bot tries to remove an rpm from the manifest. """ + _params = ['pkgList', 'pkgNames', + 'oldspkg', 'newspkg', + 'oldNevra', 'newNevra'] + _template = '%(pkgList)s removed going from %(oldNevra)s to %(newNevra)s' + +class UpdateReusesPackageError(UnhandledUpdateError): + """ + UpdateReusesPackageError, raised when the bot tries to use an old + package with a newer srpm. + """ + _params = ['pkgList', 'pkgNames', + 'oldspkg', 'newspkg', + 'oldNevra', 'newNevra'] + _template = '%(pkgList)s reused going from %(oldNevra)s to %(newNevra)s' class GroupNotFound(UnhandledUpdateError): """ @@ -289,10 +303,28 @@ """ _params = ['broken', ] - _tempalte = ('Found missing information when parsing errata source. This ' + _template = ('Found missing information when parsing errata source. This ' 'normally means that the errat source is corrupt or incorect. ' '%(broken)s') +class AdvisoryPackageMissingFromBucketError(ErrataError): + """ + AdvisoryPackageMissingFromBucketError, raised when moving advisories between + buckets and the expected packages can not be found. + """ + + _params = ['nevra', ] + _template = 'Failed to find %(nevra)s in source update bucket.' + +class PackageNotFoundInBucketError(ErrataError): + """ + PackageNotFoundInBucketError, raised when moving sources between buckets and + the specified nevra can not be found in the source bucket. + """ + + _params = ['nevra', 'bucketId', ] + _template = 'Can not find %(nevra)s in %(bucketId)s' + class GroupManagerError(UpdateBotError): """ GroupManagerError, generic error for group manager related errors. @@ -351,6 +383,54 @@ 'version group %(what)s, you may need to remove this package from any ' 'other static group definitions.') +class NameVersionConflictsFoundError(GroupManagerError): + """ + NameVersionConflictsFoundError, raised when multiple versions of the same + source are referenced by binaries in a managed group. + """ + + _params = ['conflicts', 'groupName', 'binPkgs', ] + _template = ('Multiple versions of the following sources are referenced ' + 'in %(groupName)s: %(conflicts)s') + +class OldVersionsFoundError(GroupManagerError): + """ + OldVersionsFoundError, raised when the latest source/build version of a + package is not in a group. This happens when an old package has been + rebuilt and has a new build count or source, but the same upstream version. + """ + + _params = ['pkgNames', 'errors', ] + _template = 'Newer versions of $(pkgNames)s were found.' + +class GroupValidationFailedError(GroupManagerError): + """ + GroupValidationFailedError, raised when errors are discovered in validate + the group model. Contains a reference to all errors discovered. + """ + + _params = ['errors', ] + _template = 'Errors were discovered with the group model.' + +class NotCommittingOutOfDateSourceError(GroupManagerError): + """ + NotCommittingOutOfDateSourceError, raised when trying to commit a group + model that is not the latest version of the source component. + """ + + _params = [] + _template = 'Can not commit out of date source.' + +class ExpectedRemovalValidationFailedError(GroupManagerError): + """ + ExpectedRemovalValidationFailedError, raised when package remove validation + fails. + """ + + _params = ['updateId', 'pkgNames', ] + _template = ('The following package names were not properly removed from ' + 'the group model at updateId %(updateId)s: %(pkgNames)s') + class ImportError(UpdateBotError): """ General purpose error for all import related issues. @@ -429,3 +509,21 @@ _params = [ 'source', 'target', 'package', ] _template = ('Can not merge %(source)s into %(target)s due to conflicting ' 'package %(package)s') + +class ConfigurationError(UpdateBotError): + """ + Generic exception class for configuration related errors. + """ + + _params = [] + _template = 'An error has been discovered with your configuration.' + +class UnknownRemoveSourceError(ConfigurationError): + """ + UnknownRemoveSourceError, raised when an unknown source nevra is specified + in the removeSource directive in the updatebotrc. + """ + + _params = ['nevra', ] + _template = ('The following source nevra, mentioned in a removeSource line ' + 'in your config, was not found: %(nevra)s') From elliot at rpath.com Mon Mar 15 17:15:07 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:07 +0000 Subject: mirrorball: comment cleanup Message-ID: <201003152115.o2FLF89o013311@scc.eng.rpath.com> changeset: c74124b33849 user: Elliot Peele date: Fri, 08 Jan 2010 16:27:39 -0500 comment cleanup diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -171,11 +171,13 @@ self._rpmMap[rpmMapKey] = set() self._rpmMap[rpmMapKey].add(package) - # the normal case of "obsoletes foo < version" (or with + # The normal case of "obsoletes foo < version" (or with # "requires foo", though that normally also follows the - # "< version" pattern) is "keep in sync" which we do already. - # The point of this is what redirects are used for in - # native conary packages, not for what groups are used for. + # "< version" pattern) is "keep in sync", which we do + # already through groups. + # The point of explicit obsoletes handling is what redirects + # are used for in native conary packages, not for what groups + # are used for. obsoleteNames = set( x.name for x in itertools.chain(*[ y.getChildren('rpm:entry') for y in package.format From elliot at rpath.com Mon Mar 15 17:15:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:10 +0000 Subject: mirrorball: new config options to support sanity checking exceptions Message-ID: <201003152115.o2FLFBLE013338@scc.eng.rpath.com> changeset: 80f7ac6e60af user: Elliot Peele date: Fri, 08 Jan 2010 16:29:38 -0500 new config options to support sanity checking exceptions diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -68,6 +68,60 @@ raise ParseError, e +class CfgAdvisoryOrder(CfgString): + """ + Class for parsing advisor order config. + """ + + def parseString(self, val): + splt = val.split() + assert len(splt) == 3 + fromId, toId, advisory = splt + fromId = int(fromId) + toId = int(toId) + return fromId, toId, advisory + + +class CfgSourceOrder(CfgString): + """ + Class for parsing source order config. + """ + + def parseString(self, val): + splt = val.split() + assert len(splt) == 7 + fromId, toId = splt[:2] + fromId = int(fromId) + toId = int(toId) + nevra = tuple(splt[2:]) + return fromId, toId, nevra + + +class CfgNevra(CfgString): + """ + Class for parsing nevras + """ + + def parseString(self, val): + splt = tuple(val.split()) + assert len(splt) == 5 + return splt + + +class CfgObsoletes(CfgString): + """ + Class for parsing obsolete mappings: + + """ + + def parseString(self, val): + splt = val.split() + assert len(splt) == 10 + obsoleter = tuple(splt[0:5]) + obsoleted = tuple(splt[5:10]) + return obsoleter, obsoleted + + class CfgIntDict(CfgDict): """ Config class to represent dictionaries keyed by integers rather than @@ -262,9 +316,50 @@ # timestamp to get mirrorball to go back and apply this update. reorderUpdates = (CfgList(CfgQuotedLineList(CfgInt)), []) + # fromUpdateId toUpdateId advisory + # Sometimes advisories are released out of order, but it is inconvienent to + # move the entire update bucket. + reorderAdvisory = (CfgList(CfgAdvisoryOrder), []) + + # fromUpdateId toUpdateId sourceNevra + # Sometimes multiple versions of the same package are released as part of a + # single advisory. This does not fit the conary model of doing things, so we + # have to reschedule one or more of these sources to another time so that + # they end up in a binary group and get updated in the correct order. + reorderSource = (CfgList(CfgSourceOrder), []) + + # reuse old revisions as used in SLES, where if on a rebuild with the + # same version but different revision a subpackage does not change + # content, the new build is not used + reuseOldRevisions = (CfgBool, False) + + # updateId binaryPackageName # Dictionary of bucketIds and packages that are expected to be removed. updateRemovesPackages = (CfgIntDict(CfgList(CfgString)), {}) + # updateId sourceNevra + # As of updateId, remove source package specified by sourceNevra + # from the package model + removeSource = (CfgIntDict(CfgList(CfgNevra)), {}) + + # updateId sourceNevra + # As of updateId, the specified src is fully obsoleted, but + # should be retained in groups + keepObsoleteSource = (CfgIntDict(CfgList(CfgNevra)), {}) + + # Some obsoletes are merely conceptual preferences, and should not + # turn into removals. + # We would prefer CfgSet(CfgTuple(CfgNevra, CfgNevra), set()) + # but CfgSet and CfgTuple do not exist at this point; + # maybe we can add them later. + # keepObsolete + keepObsolete = (CfgList(CfgObsoletes), []) + + # updateId packageName [packageName ...] + # remove obsoleted packages when other subpackages of the same + # srpm are not obsoleted, so we cannot use removeSource + removeObsoleted = (CfgIntDict(CfgList(CfgString)), {}) + class UpdateBotConfig(cfg.SectionedConfigFile): """ From elliot at rpath.com Mon Mar 15 17:15:16 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:16 +0000 Subject: mirrorball: 1. allow packages to be removed when specified in the config Message-ID: <201003152115.o2FLFHXe013379@scc.eng.rpath.com> changeset: 81a1bf5ff6f8 user: Elliot Peele date: Fri, 08 Jan 2010 16:33:58 -0500 1. allow packages to be removed when specified in the config 2. validate ordering 3. handle obsoletes 4. add support for * merging update buckets * reordering update buckets * reordering individual advisories * reordering individual sources diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -140,13 +140,16 @@ return trvMap, failed - def update(self, force=None, updatePkgs=None): + def update(self, force=None, updatePkgs=None, expectedRemovals=None): """ Update the conary repository from the yum repositories. @param force: list of packages to update without exception @type force: list(pkgName, pkgName, ...) @param updatePkgs: set of source package objects to update @type updatePkgs: iterable of source package objects + @param expectedRemovals: set of packages that are expected to be + removed. + @type expectedRemovals: set of package names """ if force is not None: @@ -164,7 +167,9 @@ self._pkgSource.load() # Get troves to update and send advisories. - toAdvise, toUpdate = self._updater.getUpdates(updateTroves=updateTroves) + toAdvise, toUpdate = self._updater.getUpdates( + updateTroves=updateTroves, + expectedRemovals=expectedRemovals) # If forcing an update, make sure that all packages are listed in # toAdvise and toUpdate as needed. diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2009 rPath, Inc. +# Copyright (c) 2009-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -16,11 +16,22 @@ Module for ordering errata. """ +import os +import copy import time import logging +from updatebot import update +from updatebot import conaryhelper from updatebot.errors import ErrataPackageNotFoundError from updatebot.errors import ErrataSourceDataMissingError +from updatebot.errors import PackageNotFoundInBucketError +from updatebot.errors import AdvisoryPackageMissingFromBucketError + +# update errors +from updatebot.errors import UpdateGoesBackwardsError +from updatebot.errors import UpdateRemovesPackageError +from updatebot.errors import UpdateReusesPackageError log = logging.getLogger('updatebot.errata') @@ -47,6 +58,12 @@ # timestamp: advisory info self._advMap = {} + # advisory: nevras + self._advPkgMap = {} + + # nevra: advisories + self._advPkgRevMap = {} + @loadErrata def getInitialPackages(self): """ @@ -78,6 +95,29 @@ return '%s (no detail found)' % bucketId @loadErrata + def getVersions(self, bucketId): + """ + Get a set of group versions that should be built for the given bucketId. + @param bucketId: identifier for a given update slice + @type bucketId: integer (unix time) + """ + + versions = set() + for advisory in sellf.getUpdateDetail(bucketId): + versions.add(self._errata.getGroupVersion(advisory['name'])) + return versions + + def getBucketVersion(self, bucketId): + """ + Convert a bucketId to a conary version. + @param bucketId: identifier for a given update slice + @type bucketId: integer (unix time) + """ + + version = time.strftime('%Y.%m.%d_%H%M.%S', time.gmtime(bucketId)) + return version + + @loadErrata def iterByIssueDate(self, current=None): """ Yield sets of srcPkgs by errata release date. @@ -91,6 +131,294 @@ continue yield stamp, self._order[stamp] + @loadErrata + def sanityCheckOrder(self): + """ + Validate the update order for: + 1. package revisions going backwards + 2. packages being removed + 3. same package in bucket multiple times + 4. obsolete packages still included in groups + Raise an exception if sanity checks are not satisfied. + """ + + log.info('sanity checking ordering') + + def tconv(stamp): + return time.strftime('%m-%d-%Y %H:%M:%S', time.gmtime(stamp)) + + def rhnUrl(errataId): + errataId = errataId.replace(':', '-') + return 'http://rhn.redhat.com/errata/%s.html' % errataId + + def rhnUrls(errataSet): + return ' '.join(rhnUrl(x) for x in errataSet) + + # duplicate updater and pkgsource so as to not change state. + pkgSource = copy.copy(self._pkgSource) + updater = update.Updater(self._cfg, pkgSource) + updater._conaryhelper = _ConaryHelperShim(self._cfg) + + # build a mapping of srpm to bucketId for debuging purposes + srpmToBucketId = {} + for bucketId, srpms in self._order.iteritems(): + for srpm in srpms: + srpmToBucketId[srpm] = bucketId + + # build a mapping of srpm to advisory for debuging purposes + srpmToAdvisory = {} + for advisory, srpms in self._advPkgMap.iteritems(): + for srpm in srpms: + srpmToAdvisory.setdefault(srpm, set()).add(advisory) + + # convert keepObsolete config into set of edges + keepObsolete = set(self._cfg.keepObsolete) + + errors = {} + # Make sure there no buckets that contain the same srpm name twice. + for updateId, srpms in sorted(self._order.iteritems()): + seen = {} + dups = {} + for srpm in srpms: + if srpm.name in seen: + thisdup = dups.setdefault(srpm.name, set()) + thisdup.add(seen[srpm.name]) + thisdup.add(srpm) + else: + seen[srpm.name] = srpm + continue + if dups: + log.warn('found duplicates in %s' % updateId) + errors.setdefault(updateId, []).append(('duplicates', dups)) + + # Play though entire update history to check for iregularities. + current = {} + removals = self._cfg.updateRemovesPackages + currentlyRemovedBinaryNevras = set() + foundObsoleteEdges = set() + foundObsoleteSrcs = set() + for updateId in sorted(self._order.keys()): + log.info('validating %s' % updateId) + expectedRemovals = removals.get(updateId, None) + explicitSourceRemovals = self._cfg.removeSource.get(updateId, set()) + explicitBinaryRemovals = self._cfg.removeObsoleted.get(updateId, set()) + for srpm in self._order[updateId]: + nvf = (srpm.name, None, None) + + # validate updates + try: + assert updater._sanitizeTrove(nvf, srpm, + expectedRemovals=expectedRemovals) + except (UpdateGoesBackwardsError, + UpdateRemovesPackageError, + UpdateReusesPackageError), e: + errors.setdefault(updateId, []).append(e) + + # apply update to checkout + if srpm.getNevra() in explicitSourceRemovals: + log.error('Removing %s in %s would cause it never to be promoted' % + (str(' '.join(srpm.getNevra())), updateId)) + current[srpm.name] = srpm + assert not updater.update(nvf, srpm) + + # all package names obsoleted by packages in the current set + obsoleteNames = set() + obsoleteBinaries = set() + obsoleteSources = set() + obsoletingPkgMap = {} + pkgNames = set() + pkgNameMap = {} + srpmNameMap = {} + # Create maps for processing obsoletes + for srpm in sorted(current.itervalues()): + if srpm.getNevra() in explicitSourceRemovals: + current.pop(srpm.name, None) + continue + for pkg in sorted(self._pkgSource.srcPkgMap[srpm]): + if pkg.arch == 'src': + continue + pkgNames.add(pkg.name) + pkgNameMap[pkg.name] = pkg + if pkg in self._pkgSource.obsoletesMap: + pkgObsoleteNames = self._pkgSource.obsoletesMap[pkg] + for obsoleteName in pkgObsoleteNames: + obsoletingPkgMap[obsoleteName] = pkg + obsoleteNames.add(obsoleteName) + + for obsoleteName in explicitBinaryRemovals: + # these nevra-nevra edges are already handled in config, + # do not report them in the wrong bucket + obsoletingPkg = obsoletingPkgMap[obsoleteName] + obsoletedPkg = pkgNameMap[obsoleteName] + obsoleteEdge = (obsoletingPkg, obsoletedPkg) + foundObsoleteEdges.add(obsoleteEdge) + + # coalesce obsoleted packages by src package, filtering + # by explicit configs + obsoletePkgMap = {} + for obsoleteName in obsoleteNames: + if obsoleteName in pkgNames: + obsoletingPkg = obsoletingPkgMap[obsoleteName] + obsoletedPkg = pkgNameMap[obsoleteName] + obsoleteNevraEdge = (obsoletingPkg.getNevra(), + obsoletedPkg.getNevra()) + + if obsoleteNevraEdge in keepObsolete: + # We have configured to keep this "obsolete" package + continue + + obsoleteEdge = (obsoletingPkg, obsoletedPkg) + if obsoleteEdge in foundObsoleteEdges: + # report each obsoleting relationship only once + continue + foundObsoleteEdges.add(obsoleteEdge) + + obsoleteSrcPkg = self._pkgSource.binPkgMap[obsoletedPkg] + obsoletePkgMap.setdefault(obsoleteSrcPkg, + set()).add(obsoleteEdge) + + # report sets of obsoleted packages inappropriately included + for srcPkg, obsoleteEdgeSet in sorted(obsoletePkgMap.iteritems()): + # determine whether bins or srcs need removal + pkgsBySrc = self._pkgSource.srcPkgMap[srcPkg] + binPkgs = tuple(sorted(set(x for x in + pkgsBySrc if x.arch != 'src'))) + unremovedBinPkgs = tuple(sorted(set(x for x in + pkgsBySrc if x.arch != 'src' + and x.name not in obsoleteNames))) + + if unremovedBinPkgs: + obsoleteBinaries.add( + (tuple(sorted(obsoleteEdgeSet)), + srcPkg, + binPkgs, + unremovedBinPkgs)) + else: + # choose whether to include or exclude pkg sets by sources + if srcPkg not in foundObsoleteSrcs: + obsoletingSrcPkgs = tuple(sorted(set( + self._pkgSource.binPkgMap[x] + for x, y in obsoleteEdgeSet))) + # we exclude any source only once, not per bucket + obsoleteSources.add( + (obsoletedPkg.name, + obsoletedPkg, + obsoletingSrcPkgs, + srcPkg, + binPkgs)) + foundObsoleteSrcs.add(srcPkg) + + + if obsoleteBinaries: + log.warn('found obsolete binary packages in %s' % updateId) + errors.setdefault(updateId, []).append(('obsoleteBinaries', + obsoleteBinaries)) + if obsoleteSources: + log.warn('found obsolete source packages in %s' % updateId) + errors.setdefault(updateId, []).append(('obsoleteSources', + obsoleteSources)) + + # Report errors. + for updateId, error in sorted(errors.iteritems()): + for e in error: + if isinstance(e, UpdateGoesBackwardsError): + one, two = e.why + oneNevra = str(' '.join(one.getNevra())) + twoNevra = str(' '.join(two.getNevra())) + log.warn('%s %s -revertsTo-> %s' % (updateId, + oneNevra, twoNevra)) + log.info('%s -revertsTo-> %s' % ( + rhnUrls(srpmToAdvisory[one]), + rhnUrls(srpmToAdvisory[two]))) + log.info('%s %s (%d) -> %s (%d)' % (updateId, + tconv(srpmToBucketId[one]), srpmToBucketId[one], + tconv(srpmToBucketId[two]), srpmToBucketId[two])) + log.info('? reorderSource %s otherId<%s %s' % ( + updateId, srpmToBucketId[one], twoNevra)) + for errataId in srpmToAdvisory[two]: + log.info('? reorderAdvisory %s otherId<%s %s' % ( + updateId, srpmToBucketId[one], errataId)) + elif isinstance(e, UpdateReusesPackageError): + # Note that updateObsoletesPackages not yet implemented... + log.warn('%s %s reused in %s; check for obsoletes?' % ( + updateId, e.pkgNames, e.newspkg)) + for name in sorted(set(p.name for p in e.pkgList)): + log.info('? updateObsoletesPackages %s %s' % ( + updateId, name)) + elif isinstance(e, UpdateRemovesPackageError): + log.warn('%s %s removed in %s' % ( + updateId, e.pkgNames, e.newspkg)) + for name in sorted(set(p.name for p in e.pkgList)): + log.info('? updateRemovesPackages %s %s' % ( + updateId, name)) + elif isinstance(e, tuple): + if e[0] == 'duplicates': + sortedOrder = sorted(self._order) + previousId = sortedOrder[sortedOrder.index(updateId)-1] + for dupName, dupSet in e[1].iteritems(): + dupList = sorted(dupSet) + log.warn('%s contains duplicate %s %s' %(updateId, + dupName, dupList)) + for srcPkg in dupList[:-1]: + srcNevra = str(' '.join(srcPkg.getNevra())) + log.info('%s : %s' % ( + srcNevra, rhnUrls(srpmToAdvisory[srcPkg]))) + log.info('? reorderSource %s earlierId>%s %s' % + (updateId, previousId, srcNevra)) + for errataId in srpmToAdvisory[srcPkg]: + log.info( + '? reorderAdvisory %s earlierId>%s %r' + % (updateId, previousId, errataId)) + + elif e[0] == 'obsoleteBinaries': + for (obsoleteEdgeList, srcPkg, binPkgs, + unremovedBinPkgs) in e[1]: + obsoletingNevra = str(' '.join(obsoletingPkg.getNevra())) + obsoletedNevra = str(' '.join(obsoletedPkg.getNevra())) + srcNevra = srcPkg.getNevra() + srcNevraStr = str(' '.join(srcNevra)) + unremovedStr = str(' '.join(sorted( + repr(x) for x in unremovedBinPkgs))) + obsoleteNames = set() + for obsoleteEdge in obsoleteEdgeList: + obsoletingPkg, obsoletedPkg = obsoleteEdge + obsoletingNevra = str(' '.join(obsoletingPkg.getNevra())) + obsoletedNevra = str(' '.join(obsoletedPkg.getNevra())) + obsoleteName = obsoletedPkg.name + obsoleteNames.add(obsoleteName) + log.warn('%s %s obsoletes %s (%s)' % ( + updateId, obsoletingPkg, + obsoletedPkg, obsoleteName)) + log.info('? keepObsolete %s %s' % + (obsoletingNevra, obsoletedNevra)) + log.info('? removeObsoleted %s %s' % (updateId, + obsoleteName)) + log.warn('Not "removeSource %s"; that would remove non-obsoleted %s' % + (srcNevraStr, unremovedStr)) + + elif e[0] == 'obsoleteSources': + for (obsoleteName, obsoletedPkg, obsoletingSrcPkgs, + srcPkg, binPkgs) in e[1]: + srcNevra = srcPkg.getNevra() + srcNevraStr = str(' '.join(srcNevra)) + obsoletingSrcPkgNames = str(' '.join(sorted(set( + x.name for x in obsoletingSrcPkgs)))) + pkgList = str(' '.join(repr(x) for x in binPkgs)) + log.warn('%s %s obsolete(s) %s (%s)' % ( + updateId, obsoletingSrcPkgNames, + obsoletedPkg, obsoleteName)) + log.info('? removeSource %s %s # %s' % ( + updateId, srcNevraStr, + obsoletingSrcPkgNames)) + log.info(' will remove the following: %s' % pkgList) + + + # Fail if there are any errors. + assert not errors + + import epdb; epdb.st() + + def _orderErrata(self): """ Order errata by timestamp. @@ -123,49 +451,157 @@ src = self._pkgSource.binPkgMap[pkg] self._order[bucketId].add(src) + ## + # Start order munging here + ## + + # Make sure we don't drop any updates + totalPkgs = sum([ len(x) for x in self._order.itervalues() ]) + pkgs = set() + for pkgSet in self._order.itervalues(): + pkgs.update(pkgSet) + assert len(pkgs) == totalPkgs + # fold together updates to preserve dep closure. for mergeList in self._cfg.mergeUpdates: - target = mergeList[0] - # merge remaining updates into target. - for source in mergeList[1:]: - log.info('merging errata bucket %s -> %s' % (source, target)) - updateSet = self._order.pop(source) - oldNames = set([ x.name for x in self._order[target]]) - newNames = set([ x.name for x in updateSet ]) - # Check for overlapping updates. If there is overlap, these - # things can't be merged since all versions need to be - # represented in the repository. - if oldNames & newNames: - raise UnableToMergeUpdatesError( - source=source, target=target, - package=', '.join(oldNames & newNames) - ) - self._order[target].update(updateSet) - - # merge advisory detail. - if source in self._advMap: - advInfo = self._advMap.pop(source) - if target not in self._advMap: - self._advMap[target] = set() - self._advMap[target].update(advInfo) + self._mergeUpdates(mergeList) # reschedule any updates that may have been released out of order. for source, dest in self._cfg.reorderUpdates: - # Probably don't want to move an update into an already - # existing bucket. - assert dest not in self._order + self._reorderUpdates(source, dest) - log.info('rescheduling %s -> %s' % (source, dest)) + # reschedule any individual advisories. + for source, dest, advisory in self._cfg.reorderAdvisory: + self._reorderAdvisory(source, dest, advisory) - # remove old version - bucket = self._order.pop(source) - # There will not be an entry for sources that do not have - # advisories, default to None. - adv = self._advMap.pop(source, None) + # reschedule individual packages + for source, dest, nevra in self._cfg.reorderSource: + self._reorderSource(source, dest, nevra) - # move to new version - self._order[dest] = bucket - if adv: self._advMap[dest] = adv + # Make sure we don't drop any updates + totalPkgs2 = sum([ len(x) for x in self._order.itervalues() ]) + pkgs = set() + for pkgSet in self._order.itervalues(): + pkgs.update(pkgSet) + assert len(pkgs) == totalPkgs2 + assert totalPkgs2 == totalPkgs + + def _mergeUpdates(self, mergeList): + """ + Merge a list of updates into one bucket. + """ + + target = mergeList[0] + # merge remaining updates into target. + for source in mergeList[1:]: + log.info('merging errata bucket %s -> %s' % (source, target)) + updateSet = self._order.pop(source) + oldNames = set([ x.name for x in self._order[target]]) + newNames = set([ x.name for x in updateSet ]) + # Check for overlapping updates. If there is overlap, these + # things can't be merged since all versions need to be + # represented in the repository. + if oldNames & newNames: + raise UnableToMergeUpdatesError( + source=source, target=target, + package=', '.join(oldNames & newNames) + ) + self._order[target].update(updateSet) + + # merge advisory detail. + if source in self._advMap: + advInfo = self._advMap.pop(source) + if target not in self._advMap: + self._advMap[target] = set() + self._advMap[target].update(advInfo) + + def _reorderUpdates(self, source, dest): + """ + Reschedule an update from one timestamp to another. + """ + + # Probably don't want to move an update into an already + # existing bucket. + assert dest not in self._order + + log.info('rescheduling %s -> %s' % (source, dest)) + + # remove old version + bucket = self._order.pop(source) + # There will not be an entry for sources that do not have + # advisories, default to None. + adv = self._advMap.pop(source, None) + + # move to new version + self._order[dest] = bucket + if adv: self._advMap[dest] = adv + + + def _reorderAdvisory(self, source, dest, advisory): + """ + Reschedule a single advisory. + """ + + # Warn when moving advisory into existing bucket. + if dest in self._order: + log.warn('inserting %s into pre-existing bucket' % advisory) + + log.info('rescheduling %s %s -> %s' % (advisory, source, dest)) + + # Find the srpms that apply to this advisory + srpms = self._advPkgMap[advisory] + + # Remove them from the source bucket Id, while making sure they are + # all in the source bucketId. + bucketNevras = dict([ (x.getNevra(), x) + for x in self._order[source] ]) + for srpm in srpms: + nevra = srpm.getNevra() + if nevra not in bucketNevras: + raise AdvisoryPackageMissingFromBucketError(nevra=nevra) + self._order[source].remove(srpm) + + # Make sure that each package that we are moving is only + # mentioned in one advisory. + assert len(self._advPkgRevMap[srpm]) == 1 + + # Move packages to destination bucket Id. + self._order.setdefault(dest, set()).add(srpm) + + # Remove the advisory information for the source bucket Id. + for advInfo in self._advMap[source]: + name = dict(advInfo)['name'] + if name == advisory: + self._advMap[source].remove(advInfo) + self._advMap.setdefault(dest, set()).add(advInfo) + break + + + def _reorderSource(self, source, dest, nevra): + """ + Reschedule an individual srpm to another bucket. + """ + + # Warn when moving srpm into existing bucket. + if dest in self._order: + log.warn('inserting %s into pre-existing bucket' % '-'.join(nevra)) + + log.info('rescheduling %s %s -> %s' % (nevra, source, dest)) + + # Remove specified source nevra from the source bucket + bucketNevras = dict([ (x.getNevra(), x) + for x in self._order[source] ]) + if nevra not in bucketNevras: + raise PackageNotFoundInBucketError(nevra=nevra, bucketId=source) + srpm = bucketNevras[nevra] + self._order[source].remove(srpm) + + # Remove all references to this srpm being part of an advisory + for advisory in self._advPkgRevMap.pop(srpm, set()): + self._advPkgMap[advisory].remove(srpm) + + # Move srpm to destination bucket + self._order.setdefault(dest, set()).add(srpm) def _getNevra(self, pkg): """ @@ -187,6 +623,10 @@ Sort packages by errata release timestamp. """ + # get mapping of nevra to source pkg object + sources = dict( ((x.name, x.epoch, x.version, x.release, x.arch), y) + for x, y in self._pkgSource.binPkgMap.iteritems() ) + # get mapping of nevra to pkg obj nevras = dict(((x.name, x.epoch, x.version, x.release, x.arch), x) for x in self._pkgSource.binPkgMap.keys() @@ -219,6 +659,10 @@ if not indexedChannels & channels: continue + # add package to advisory package map + self._advPkgMap.setdefault(e.advisory, set()).add(sources[nevra]) + self._advPkgRevMap.setdefault(sources[nevra], set()).add(e.advisory) + # move nevra to errata buckets if nevra in nevras: binPkg = nevras.pop(nevra) @@ -274,3 +718,63 @@ buckets[0] = golden return buckets, other + + +class _ConaryHelperShim(conaryhelper.ConaryHelper): + """ + Shim class that doesn't actually change the repository. + """ + + def __init__(self, cfg): + conaryhelper.ConaryHelper.__init__(self, cfg) + self._client.repos = None + self._repos = None + conaryhelper.checkin = None + + def getLatestSourceVersion(self, pkgname): + """ + Stub for latest version. + """ + + return False + + def _not_implemented(self, *args, **kwargs): + """ + Stub for methods that this class does not implemented. + """ + raise NotImplementedError + + setTroveMetadata = _not_implemented + mirror = _not_implemented + promote = _not_implemented + getSourceTroves = _not_implemented + getSourceVersions = _not_implemented + + def _checkout(self, pkgname): + """ + Checkout stub. + """ + + #log.info('checking out %s' % pkgname) + + recipeDir = self._getRecipeDir(pkgname) + return recipeDir + + _newpkg = _checkout + + @staticmethod + def _addFile(pkgDir, fileName): + """ + addFile stub. + """ + + #log.info('adding file %s' % fileName) + + _removeFile = _addFile + + def _commit(self, pkgDir, commitMessage): + """ + commit stub. + """ + + log.info('committing %s' % os.path.basename(pkgDir)) diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2009 rPath, Inc. +# Copyright (c) 2009-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -23,6 +23,7 @@ from updatebot import groupmgr from updatebot.bot import Bot as BotSuperClass +from updatebot.errors import UnknownRemoveSourceError from updatebot.errors import PlatformNotImportedError from updatebot.errors import PlatformAlreadyImportedError @@ -71,7 +72,7 @@ """ # Make sure this platform has not already been imported. - if self._groupmgr.getErrataState(): + if self._groupmgr.getErrataState() is not None: raise PlatformAlreadyImportedError self._pkgSource.load() @@ -119,25 +120,34 @@ # Load package source. self._pkgSource.load() + # Sanity check errata ordering. + self._errata.sanityCheckOrder() + updateSet = {} for updateId, updates in self._errata.iterByIssueDate(current=current): start = time.time() detail = self._errata.getUpdateDetailMessage(updateId) log.info('attempting to apply %s' % detail) + # remove packages from config + removePackages = self._cfg.updateRemovesPackages.get(updateId, []) + removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) + + # take the union of the two lists to get a unique list of packages + # to remove. + expectedRemovals = set(removePackages) | set(removeObsoleted) + # Update package set. - pkgMap = self._update(*args, updatePkgs=updates, **kwargs) + pkgMap = self._update(*args, updatePkgs=updates, + expectedRemovals=expectedRemovals, **kwargs) + # FIXME: we might actually want to do this one day # Find errata group versions. + #errataVersions = self._errata.getVersions(updateId) errataVersions = set() - for advisory in self._errata.getUpdateDetail(updateId): - # advisory names are in the form RHSA-2009:1234 - # transform to a conary version. - name = advisory['name'] - name = name.replace('-', '_') - name = name.replace(':', '.') - name += '_rolling' - errataVersions.add(name) + + # Add timestamp version. + errataVersions.add(self._errata.getBucketVersion(updateId)) # FIXME: Might want to re-enable this one day. # Get current set of source names and versions. @@ -155,14 +165,38 @@ self._groupmgr.setErrataState(updateId) # Remove any packages that are scheduled for removal. - if updateId in self._cfg.updateRemovesPackages: - pkgs = self._cfg.updateRemovesPackages[updateId] + # NOTE: This should always be done before adding packages so that + # any packages that move between sources will be removed and + # then readded. + if expectedRemovals: log.info('removing the following packages from the managed ' - 'group: %s' % ', '.join(pkgs)) - for pkg in pkgs: + 'group: %s' % ', '.join(expectedRemovals)) + for pkg in expectedRemovals: self._groupmgr.remove(pkg) - # Make sure built troves are part of group. + # Handle the case of entire source being obsoleted, this causes all + # binaries from that source to be removed from the group model. + if updateId in self._cfg.removeSource: + # get nevras from the config + nevras = self._cfg.removeSource[updateId] + + # get a map of source nevra to binary package list. + nevraMap = dict((x.getNevra(), y) for x, y in + self._pkgSource.srcPkgMap.iteritems() + if x.getNevra() in nevras) + + for nevra in nevras: + # if for some reason the nevra from the config is not in + # the pkgSource, raise an error. + if nevra not in nevraMap: + raise UnknownRemoveSourceError(nevra=nevra) + + # remove all binary names from the group. + binNames = set([ x.name for x in nevraMap[nevra] ]) + for name in binNames: + self._groupmgr.remove(name) + + # Make sure built troves are part of the group. self._addPackages(pkgMap) # Build various group verisons. diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -42,12 +42,15 @@ self._conaryhelper = conaryhelper.ConaryHelper(self._cfg) - def getUpdates(self, updateTroves=None): + def getUpdates(self, updateTroves=None, expectedRemovals=None): """ Find all packages that need updates and/or advisories from a top level binary group. @param updateTroves: set of troves to update @type updateTroves: iterable + @param expectedRemovals: set of package names that are expected to be + removed. + @type expectedRemovals: set of package names @return list of packages to send advisories for and list of packages to update """ @@ -64,7 +67,8 @@ for nvf, srpm in updateTroves: # Will raise exception if any errors are found, halting execution. - if self._sanitizeTrove(nvf, srpm): + if self._sanitizeTrove(nvf, srpm, + expectedRemovals=expectedRemovals): toUpdate.append((nvf, srpm)) toAdvise.append((nvf, srpm)) @@ -179,7 +183,7 @@ srpms.sort(util.packagevercmp) return srpms[-1] - def _sanitizeTrove(self, nvf, srpm): + def _sanitizeTrove(self, nvf, srpm, expectedRemovals=None): """ Verifies the package update to make sure it looks correct and is a case that the bot knows how to handle. @@ -193,6 +197,9 @@ @type nvf: (name, versionObj, flavorObj) @param srpm: src pacakge object @type srpm: repomd.packagexml._Package + @param expectedRemovals: set of package names that are expected to be + removed. + @type expectedRemovals: set of package names @return needsUpdate boolean @raises UpdateGoesBackwardsError @raises UpdateRemovesPackageError @@ -201,6 +208,8 @@ needsUpdate = False newNames = [ (x.name, x.arch) for x in self._pkgSource.srcPkgMap[srpm] ] metadata = None + removedPackages = set() + reusedPackages = set() try: manifest = self._conaryhelper.getManifest(nvf[0]) @@ -235,7 +244,7 @@ # make sure new package is actually newer if util.packagevercmp(srpm, srcPkg) == -1: log.warn('version goes backwards %s -> %s' % - (srpm.getNevra(), srcPkg.getNevra())) + (srcPkg.getNevra(), srpm.getNevra())) raise UpdateGoesBackwardsError(why=(srcPkg, srpm)) # make sure we aren't trying to remove a package @@ -251,17 +260,47 @@ if x.arch == binPkg.arch ][0] # Raise an exception if the versions of the packages aren't - # equal. + # equal or the discovered package comes from a different source. if (rpmvercmp(pkg.epoch, srpm.epoch) != 0 or - rpmvercmp(pkg.version, srpm.version) != 0): + rpmvercmp(pkg.version, srpm.version) != 0 or + # in the suse case we have to ignore release + (self._cfg.reuseOldRevisions or + rpmvercmp(pkg.release, srpm.release) != 0) or + # binary does not come from the same source as it used to + self._pkgSource.binPkgMap[pkg].name != srpm.name): log.warn('update removes package (%s) %s -> %s' % (pkg.name, srpm.getNevra(), srcPkg.getNevra())) - raise UpdateRemovesPackageError(why='all rpms in the ' - 'manifest should have the same version, trying ' - 'to add %s' % (pkg, )) - log.warn('using old version of package %s' % (pkg, )) - self._pkgSource.srcPkgMap[srpm].add(pkg) + #import epdb; epdb.st() + + # allow some packages to be removed. + if expectedRemovals and pkg.name in expectedRemovals: + log.info('package removal (%s) handled in configuration' + % pkg.name) + continue + + removedPackages.add(pkg) + + if not removedPackages: + reusedPackages.add(pkg) + #log.warn('using old version of package %s' % (pkg, )) + #self._pkgSource.srcPkgMap[srpm].add(pkg) + + if removedPackages: + pkgList=sorted(removedPackages) + raise UpdateRemovesPackageError(pkgList=pkgList, + pkgNames=' '.join([str(x) for x in pkgList]), + newspkg=srpm, oldspkg=srcPkg, + oldNevra=str(' '.join(srcPkg.getNevra())), + newNevra=str(' '.join(srpm.getNevra()))) + + if reusedPackages: + pkgList=sorted(reusedPackages) + raise UpdateReusesPackageError(pkgList=pkgList, + pkgNames=' '.join([str(x) for x in pkgList]), + newspkg=srpm, oldspkg=srcPkg, + oldNevra=str(' '.join(srcPkg.getNevra())), + newNevra=str(' '.join(srpm.getNevra()))) return needsUpdate From elliot at rpath.com Mon Mar 15 17:15:20 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:20 +0000 Subject: mirrorball: 1. split out binaries that are not moved into a different package into the Message-ID: <201003152115.o2FLFL19013410@scc.eng.rpath.com> changeset: d3406d1746f3 user: Elliot Peele date: Sat, 09 Jan 2010 02:47:44 -0500 1. split out binaries that are not moved into a different package into the updateReplacesPackages config item. 2. add sanity checking for packages that should have been removed to the group manager. diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -337,6 +337,11 @@ # Dictionary of bucketIds and packages that are expected to be removed. updateRemovesPackages = (CfgIntDict(CfgList(CfgString)), {}) + # updateId binaryPackageName + # Dictionary of bucketIds and packages that are expected to be moved + # between sources. + updateReplacesPackages = (CfgIntDict(CfgList(CfgString)), {}) + # updateId sourceNevra # As of updateId, remove source package specified by sourceNevra # from the package model diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -194,12 +194,14 @@ # Play though entire update history to check for iregularities. current = {} removals = self._cfg.updateRemovesPackages + replaces = self._cfg.updateReplacesPackages currentlyRemovedBinaryNevras = set() foundObsoleteEdges = set() foundObsoleteSrcs = set() for updateId in sorted(self._order.keys()): log.info('validating %s' % updateId) - expectedRemovals = removals.get(updateId, None) + expectedRemovals = removals.get(updateId, []) + expectedReplaces = replaces.get(updateId, []) explicitSourceRemovals = self._cfg.removeSource.get(updateId, set()) explicitBinaryRemovals = self._cfg.removeObsoleted.get(updateId, set()) for srpm in self._order[updateId]: @@ -208,7 +210,7 @@ # validate updates try: assert updater._sanitizeTrove(nvf, srpm, - expectedRemovals=expectedRemovals) + expectedRemovals=expectedRemovals + expectedReplaces) except (UpdateGoesBackwardsError, UpdateRemovesPackageError, UpdateReusesPackageError), e: @@ -245,6 +247,9 @@ obsoletingPkgMap[obsoleteName] = pkg obsoleteNames.add(obsoleteName) + # packages that really moved from one source to another + removedShouldBeReplaced = set(expectedRemovals) & pkgNames + for obsoleteName in explicitBinaryRemovals: # these nevra-nevra edges are already handled in config, # do not report them in the wrong bucket @@ -317,6 +322,10 @@ log.warn('found obsolete source packages in %s' % updateId) errors.setdefault(updateId, []).append(('obsoleteSources', obsoleteSources)) + if removedShouldBeReplaced: + log.warn('found removals for replacements in %s' % updateId) + errors.setdefault(updateId, []).append(('removedShouldBeReplaced', + removedShouldBeReplaced)) # Report errors. for updateId, error in sorted(errors.iteritems()): @@ -411,14 +420,17 @@ updateId, srcNevraStr, obsoletingSrcPkgNames)) log.info(' will remove the following: %s' % pkgList) + elif e[0] == 'removedShouldBeReplaced': + for pkgName in e[1]: + log.warn('%s removed package %s should be replaced' % ( + updateId, pkgName)) + log.info('? updateReplacesPackages %s %s' % ( + updateId, pkgName)) # Fail if there are any errors. assert not errors - import epdb; epdb.st() - - def _orderErrata(self): """ Order errata by timestamp. @@ -727,9 +739,8 @@ def __init__(self, cfg): conaryhelper.ConaryHelper.__init__(self, cfg) - self._client.repos = None + self._client = None self._repos = None - conaryhelper.checkin = None def getLatestSourceVersion(self, pkgname): """ diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -341,7 +341,8 @@ Get the errata state info. """ - return self._helper.getErrataState(self._sourceName) + return self._helper.getErrataState(self._sourceName, + version=self._sourceVersion) def getVersions(self, pkgSet): """ @@ -546,9 +547,38 @@ # get package removals from the config object. removePackages = self._cfg.updateRemovesPackages.get(updateId, []) removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) + removeSource = [ x[0] for x in + self._cfg.removeSource.get(updateId, []) ] + + # get names and versions + troves = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + version = str(versions.ThawVersion(pkgData.version).asString()) + + flavor = None + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = set([ x for x in + itertools.chain(*self._helper.findTroves(troves).itervalues()) ]) + + # get sources for each name version pair + sources = self._helper.getSourceVersions(foundTroves) + + # collapse to sourceName: [ binNames, ] dictionary + sourceNameMap = dict([ (x[0].split(':')[0], [ z[0] for z in y ]) + for x, y in sources.iteritems() ]) + + binRemovals = set(itertools.chain(*[ sourceNameMap[x] + for x in removeSource + if x in sourceNameMap ])) # take the union - removals = set(removePackages) | set(removeObsoleted) + removals = set(removePackages) | set(removeObsoleted) | binRemovals errors = [] # Make sure these packages are not in the group model. @@ -557,6 +587,7 @@ errors.append(pkgData.name) if errors: + log.info('found packages that should be removed %s' % errors) raise ExpectedRemovalValidationFailedError(updateId=updateId, pkgNames=errors) diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -132,10 +132,13 @@ # remove packages from config removePackages = self._cfg.updateRemovesPackages.get(updateId, []) removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) + removeReplaced = self._cfg.updateReplacesPackages.get(updateId, []) # take the union of the two lists to get a unique list of packages # to remove. - expectedRemovals = set(removePackages) | set(removeObsoleted) + expectedRemovals = (set(removePackages) | + set(removeObsoleted) | + set(removeReplaced)) # Update package set. pkgMap = self._update(*args, updatePkgs=updates, From elliot at rpath.com Mon Mar 15 17:15:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:24 +0000 Subject: mirrorball: don't try to remove packages that aren't in the model Message-ID: <201003152115.o2FLFOgl013439@scc.eng.rpath.com> changeset: 35989aab2088 user: Elliot Peele date: Sat, 09 Jan 2010 16:47:27 -0500 don't try to remove packages that aren't in the model diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -176,6 +176,14 @@ return self._groups[self._pkgGroupName].remove(name) + @checkout + def hasPackage(self, name): + """ + Check if a given package name is in the group. + """ + + return name in self._groups[self._pkgGroupName] + def addPackage(self, name, version, flavors): """ Add a package to the model. @@ -194,7 +202,8 @@ # Remove all versions and flavors of this name before adding this # package. This avoids flavor change issues by replacing all flavors. - self.remove(name) + if self.hasPackage(name): + self.remove(name) plain = deps.parseFlavor('') x86 = deps.parseFlavor('is: x86') @@ -801,6 +810,13 @@ self._removeItem(name) + def __contains__(self, name): + """ + Check if element name is in the model. + """ + + return name in self._nameMap + class GroupModel(AbstractModel): """ From elliot at rpath.com Mon Mar 15 17:15:29 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:29 +0000 Subject: mirrorball: handle removing packages Message-ID: <201003152115.o2FLFT98013472@scc.eng.rpath.com> changeset: bf3887335ba1 user: Elliot Peele date: Sat, 09 Jan 2010 16:47:45 -0500 handle removing packages diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py --- a/scripts/grpchecker.py +++ b/scripts/grpchecker.py @@ -26,11 +26,18 @@ from conary.lib import util sys.excepthook = util.genExcepthook() +from conary import versions +from conary.conaryclient import cmdline + mbdir = os.path.abspath('../') sys.path.insert(0, mbdir) confDir = os.path.join(mbdir, 'config', sys.argv[1]) +sourceVersion = None +if len(sys.argv) == 3: + sourceVersion = cmdline.parseTroveSpec(sys.argv[2])[1] + from updatebot import log from updatebot import groupmgr from updatebot import conaryhelper @@ -39,6 +46,7 @@ from updatebot.errors import OldVersionsFoundError from updatebot.errors import GroupValidationFailedError from updatebot.errors import NameVersionConflictsFoundError +from updatebot.errors import ExpectedRemovalValidationFailedError import time import logging @@ -82,6 +90,12 @@ return toRemove, toAdd +def handleRemovalErrors(group, error): + toAdd = {} + toRemove = set(error.pkgNames) + + return toRemove, toAdd + def checkVersion(ver): mgr._sourceVersion = ver mgr._checkout() @@ -97,6 +111,8 @@ changes.append(handleVersionConflicts(group, error)) elif isinstance(error, OldVersionsFoundError): changes.append(handleVersionErrors(group, error)) + elif isinstance(error, ExpectedRemovalValidationFailedError): + changes.append(handleRemovalErrors(group, error)) else: raise error @@ -104,37 +120,72 @@ troves = helper.findTroves((cfg.topSourceGroup, ), getLeaves=False).values()[0] +troveSpec = (cfg.topSourceGroup[0], sourceVersion, None) +sourceVersion = helper.findTroves((troveSpec, )).values()[0][0][1] + nv = [] +start = False for n, v, f in sorted(troves): + if sourceVersion and v == sourceVersion: + start = True + if not start: + continue nsv = (n, v.getSourceVersion()) if nsv not in nv: nv.append(nsv) +if not start: + raise RuntimeError + toUpdate = [] for n, v in nv: ver, changed = checkVersion(v) - if not changed: + # Make sure to only rebuild groups that have changed + # and every group after. + if not changed and not toUpdate: continue toUpdate.append((ver, changed)) -slog.critical('Before going any further, be aware that this will only rebuild ' - 'the groups that need to change, not any other existing groups, if you ' - 'want to maintain order on the devel label you will need to write that ' - 'code.') -assert False - +#slog.critical('Before going any further, be aware that this will only rebuild ' +# 'the groups that need to change, not any other existing groups, if you ' +# 'want to maintain order on the devel label you will need to write that ' +# 'code.') +#assert False jobIds = [] +removed = {} for ver, changed in toUpdate: mgr._sourceVersion = ver mgr._checkout() + + slog.info('updating %s' % ver) + + # Find all names and versions in the group model + nv = dict([ (y.name, versions.ThawVersion(y.version)) + for x, y in mgr._groups[mgr._pkgGroupName].iteritems() ]) + + # Set of packages that should be removed + removals = set([ x for x, y in removed.iteritems() if nv[x] == y ]) + + # Remove old removals + for pkg in removals: + slog.info('removing %s' % pkg) + mgr.remove(pkg) + for toRemove, toAdd in changed: + # Handle removes for pkg in toRemove: slog.info('removing %s' % pkg) mgr.remove(pkg) + + # cache removals + removed[pkg] = nv[pkg] + + # Handle adds for (n, v), flvs in toAdd.iteritems(): slog.info('adding %s=%s' % (n, v)) mgr.addPackage(n, v, flvs) + mgr._copyVersions() mgr._validateGroups() version = mgr.save(copyToLatest=True) @@ -144,7 +195,7 @@ for jobId in jobIds: mgr._builder.watch(jobId) -import epdb; epdb.st() +#import epdb; epdb.st() for jobId in jobIds: mgr._builder.commit(jobId) From elliot at rpath.com Mon Mar 15 17:15:32 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:32 +0000 Subject: mirrorball: log when the sanity checking is complete Message-ID: <201003152115.o2FLFWjn013501@scc.eng.rpath.com> changeset: 9410d665fd78 user: Elliot Peele date: Sun, 10 Jan 2010 14:51:27 -0500 log when the sanity checking is complete diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -431,6 +431,8 @@ # Fail if there are any errors. assert not errors + log.info('order sanity checking complete') + def _orderErrata(self): """ Order errata by timestamp. From elliot at rpath.com Mon Mar 15 17:15:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:35 +0000 Subject: mirrorball: expose a findTrove interface as well Message-ID: <201003152115.o2FLFZ9Y013533@scc.eng.rpath.com> changeset: b67ac54daa0f user: Elliot Peele date: Sun, 10 Jan 2010 14:52:17 -0500 expose a findTrove interface as well diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -96,6 +96,14 @@ return self._ccfg + def findTrove(self, nvf, *args, **kwargs): + """ + Mapped to conaryclient.repos.findTrove. Will always search buildLabel. + """ + + return self._repos.findTrove(self._ccfg.buildLabel, nvf, + *args, **kwargs) + def findTroves(self, troveList, *args, **kwargs): """ Mapped to conaryclient.repos.findTroves. Will always search buildLabel. From elliot at rpath.com Mon Mar 15 17:15:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:37 +0000 Subject: mirrorball: Handle the case where the group failed to build in the last run, the errors Message-ID: <201003152115.o2FLFcov013560@scc.eng.rpath.com> changeset: d4062c92d68f user: Elliot Peele date: Sun, 10 Jan 2010 14:53:16 -0500 Handle the case where the group failed to build in the last run, the errors have been fixed, and now we are restarting the build and just want to build the group rather than building all of the packages in the current state diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -145,6 +145,30 @@ save = _commit + def hasBinaryVersion(self): + """ + Check if there is a binary version for the current source version. + """ + + # Get a mapping of all source version to binary versions for all + # existing binary versions. + srcVersions = dict([ (x[1].getSourceVersion(), x[1]) + for x in self._helper.findTrove( + (self._sourceName, None, None), + getLeaves=False + ) + ]) + + # Get the version of the specified source, usually the latest + # source version. + srcVersion = self._helper.findTrove(('%s:source' % self._sourceName, + self._sourceVersion, None))[0][1] + + # Check to see if the latest source version is in the map of + # binary versions. + return srcVersion in srcVersions + + @checkout @commit def build(self): """ diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -117,6 +117,14 @@ if current is None: raise PlatformNotImportedError + # Check to see if there is a binary version if the current group. + # This handles restarts where the group failed to build, but we don't + # want to rebuild all of the packages again. + if not self._groupmgr.hasBinaryVersion(): + # grpmgr.build will make sure to refresh the group model and sync + # up the standard group contents before building. + self._groupmgr.build() + # Load package source. self._pkgSource.load() From elliot at rpath.com Mon Mar 15 17:15:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:39 +0000 Subject: mirrorball: add option for always grouping packages into buckets Message-ID: <201003152115.o2FLFdnL013589@scc.eng.rpath.com> changeset: 355e5725ed53 user: Elliot Peele date: Mon, 11 Jan 2010 12:04:32 -0500 add option for always grouping packages into buckets diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -257,6 +257,40 @@ ret = self._formatOutput(trvMap) return ret + def orderJobs(self, troveSpecs): + """ + Create a sorted list of troveSpecs, grouped according to the config. + @param troveSpecs: list of source name, source version, and flavor. + @type troveSpecs: iterable of three tuples + """ + + order = [] + + bucketMap = {} + for grouping in self._cfg.combinePackages: + for pkg in grouping: + bucketMap[pkg] = grouping + + buckets = {} + for nvf in sorted(troveSpecs): + # Package has already been added. + if nvf[0] in buckets: + continue + + # Process any combined packages + elif nvf[0] in bucketMap: + order.append([]) + idx = len(order) - 1 + for n in bucketMap[nvf[0]]: + buckets[n] = idx + order[idx].append(n) + + # Normal packages + else: + order.append(nvf) + + return order + def setCommitFailed(self, jobId, reason=None): """ Sets the job as failed in rmake. @@ -277,6 +311,10 @@ @return list((name, version, flavor, context), ...) """ + # Make sure troveSpecs is an iterable of three tuples. + if not isinsatance(troveSpecs[0], (list, set, tuple)): + troveSpecs = [ troveSpecs, ] + # Build all troves in defined contexts. troves = [] for name, version, flavor in troveSpecs: diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -82,8 +82,8 @@ if not troveSpecs: return {} - troves = list(troveSpecs) - troves.reverse() + # Sort troves into buckets. + troves = self._builder.orderJobs(troveSpecs) while troves or not self._jobDone(): # Only create more jobs once the last batch has been started. diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py --- a/updatebot/build/monitor.py +++ b/updatebot/build/monitor.py @@ -44,7 +44,7 @@ Start the specified build and report jobId. """ - jobId = self.builder.start((self.trove, )) + jobId = self.builder.start(self.trove) self.status.put((MessageTypes.DATA, (jobId, self.trove))) diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -270,6 +270,10 @@ # Save all binary changesets to disk before committing them. saveChangeSets = (CfgBool, False) + # Always build this list of package names in one job rather than splitting + # them up in the case that you are using a builder that splits by default. + combinePackages = (CfgList(CfgQuotedLineList(CfgString)), []) + # email information for sending advisories emailFromName = CfgString emailFrom = CfgString From elliot at rpath.com Mon Mar 15 17:15:41 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:41 +0000 Subject: mirrorball: check length first Message-ID: <201003152115.o2FLFfxo013616@scc.eng.rpath.com> changeset: 221dd5c67ceb user: Elliot Peele date: Mon, 11 Jan 2010 13:35:15 -0500 check length first diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -312,7 +312,9 @@ """ # Make sure troveSpecs is an iterable of three tuples. - if not isinsatance(troveSpecs[0], (list, set, tuple)): + if (len(troveSpecs) == 3 and + not isinstance(troveSpec[0], (list, set, tuple))): + # Assume that (n,v,f) was passed in troveSpecs = [ troveSpecs, ] # Build all troves in defined contexts. From elliot at rpath.com Mon Mar 15 17:15:42 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:42 +0000 Subject: mirrorball: fix typo Message-ID: <201003152115.o2FLFhTF013644@scc.eng.rpath.com> changeset: 94a5292974b4 user: Elliot Peele date: Mon, 11 Jan 2010 14:36:18 -0500 fix typo diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -313,7 +313,7 @@ # Make sure troveSpecs is an iterable of three tuples. if (len(troveSpecs) == 3 and - not isinstance(troveSpec[0], (list, set, tuple))): + not isinstance(troveSpecs[0], (list, set, tuple))): # Assume that (n,v,f) was passed in troveSpecs = [ troveSpecs, ] From elliot at rpath.com Mon Mar 15 17:15:44 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:44 +0000 Subject: mirrorball: only print downloads once Message-ID: <201003152115.o2FLFi4f013675@scc.eng.rpath.com> changeset: 322fe141de0a user: Elliot Peele date: Mon, 11 Jan 2010 17:48:13 -0500 only print downloads once diff --git a/updatebot/lib/conarycallbacks.py b/updatebot/lib/conarycallbacks.py --- a/updatebot/lib/conarycallbacks.py +++ b/updatebot/lib/conarycallbacks.py @@ -87,8 +87,21 @@ def sendingChangset(self, got, need): self._message('uploading changeset') + @callonce + def downloadingChangeSet(self, got, need): + self._message('Downloading changeset') + + class UpdateBotCookCallback(BaseCallback, CookCallback): def __init__(self, *args, **kwargs): self._log = kwargs.pop('log', log) BaseCallback.__init__(self, *args, **kwargs) CookCallback.__init__(self, *args, **kwargs) + + @callonce + def sendingChangset(self, got, need): + self._message('Committing changeset') + + @callonce + def downloadingChangeSet(self, got, need): + self._message('Downloading changeset') From elliot at rpath.com Mon Mar 15 17:15:46 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:46 +0000 Subject: mirrorball: handle removeObsoletes Message-ID: <201003152115.o2FLFkXH013704@scc.eng.rpath.com> changeset: bbf22f32b7e3 user: Elliot Peele date: Mon, 11 Jan 2010 20:55:35 -0500 handle removeObsoletes diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -193,12 +193,13 @@ self._groups[self._pkgGroupName].add(*args, **kwargs) @checkout - def remove(self, name): + def remove(self, name, missingOk=False): """ Remove a given trove from the package group contents. """ - return self._groups[self._pkgGroupName].remove(name) + return self._groups[self._pkgGroupName].remove(name, + missingOk=missingOk) @checkout def hasPackage(self, name): @@ -776,12 +777,16 @@ self._nameMap[item.name] = set() self._nameMap[item.name].add(item.key) - def _removeItem(self, name): + def _removeItem(self, name, missingOk=False): """ Remove an item from the appropriate structures. """ - keys = self._nameMap.pop(name) + if missingOk: + keys = self._nameMap.pop(name, []) + else: + keys = self._nameMap.pop(name) + for key in keys: self._data.pop(key) @@ -827,12 +832,12 @@ obj = self.elementClass(*args, **kwargs) self._addItem(obj) - def remove(self, name): + def remove(self, name, missingOk=False): """ Remove data element. """ - self._removeItem(name) + self._removeItem(name, missingOk=missingOk) def __contains__(self, name): """ diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -142,11 +142,18 @@ removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) removeReplaced = self._cfg.updateReplacesPackages.get(updateId, []) - # take the union of the two lists to get a unique list of packages + # take the union of the three lists to get a unique list of packages # to remove. expectedRemovals = (set(removePackages) | set(removeObsoleted) | set(removeReplaced)) + # The following packages are expected to exist and must be removed + # (removeObsoleted may be mentioned for buckets where the package + # is not in the model, in order to support later adding the ability + # for a package to re-appear if an RPM obsoletes entry disappears.) + requiredRemovals = (set(removePackages) | + set(removeReplaced)) + # Update package set. pkgMap = self._update(*args, updatePkgs=updates, @@ -179,11 +186,16 @@ # NOTE: This should always be done before adding packages so that # any packages that move between sources will be removed and # then readded. - if expectedRemovals: + if requiredRemovals: log.info('removing the following packages from the managed ' - 'group: %s' % ', '.join(expectedRemovals)) - for pkg in expectedRemovals: + 'group: %s' % ', '.join(requiredRemovals)) + for pkg in requiredRemovals: self._groupmgr.remove(pkg) + if removeObsoleted: + log.info('removing any of obsoleted packages from the managed ' + 'group: %s' % ', '.join(removeObsoleted)) + for pkg in removeObsoleted: + self._groupmgr.remove(pkg, missingOK = True) # Handle the case of entire source being obsoleted, this causes all # binaries from that source to be removed from the group model. From elliot at rpath.com Mon Mar 15 17:15:47 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:47 +0000 Subject: mirrorball: 1. allow bucket merging to result in package overlap Message-ID: <201003152115.o2FLFl8r013725@scc.eng.rpath.com> changeset: 279c154ea574 user: Elliot Peele date: Mon, 11 Jan 2010 21:00:09 -0500 1. allow bucket merging to result in package overlap 2. allow reorderAdvisory to reorder the same package provided by multiple advisories. diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -516,10 +516,7 @@ # things can't be merged since all versions need to be # represented in the repository. if oldNames & newNames: - raise UnableToMergeUpdatesError( - source=source, target=target, - package=', '.join(oldNames & newNames) - ) + log.warn('merge causes package overlap') self._order[target].update(updateSet) # merge advisory detail. @@ -569,15 +566,30 @@ # all in the source bucketId. bucketNevras = dict([ (x.getNevra(), x) for x in self._order[source] ]) + for srpm in srpms: + # Make sure to only move packages if they haven't already + # been moved. + if (srpm not in self._order[source] and + dest in self._order and + srpm in self._order[dest]): + continue + nevra = srpm.getNevra() + if nevra not in bucketNevras: raise AdvisoryPackageMissingFromBucketError(nevra=nevra) self._order[source].remove(srpm) # Make sure that each package that we are moving is only # mentioned in one advisory. - assert len(self._advPkgRevMap[srpm]) == 1 + advisories = self._advPkgRevMap[srpm] + if len(advisories) > 1: + advisories = advisories.difference(set(advisory)) + # Make sure all advisories this srpm is mentioned in are also + # scheduled to be moved to the same bucket. + for adv in advisories: + assert (source, dest, adv) in self._cfg.reorderAdvisory # Move packages to destination bucket Id. self._order.setdefault(dest, set()).add(srpm) From elliot at rpath.com Mon Mar 15 17:15:48 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:48 +0000 Subject: mirrorball: check source versions Message-ID: <201003152115.o2FLFnkJ013758@scc.eng.rpath.com> changeset: e692903f6706 user: Elliot Peele date: Mon, 11 Jan 2010 21:00:31 -0500 check source versions diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py --- a/scripts/grpchecker.py +++ b/scripts/grpchecker.py @@ -118,9 +118,10 @@ return ver, changes -troves = helper.findTroves((cfg.topSourceGroup, ), getLeaves=False).values()[0] +srcName, srcVersion, srcFlavor = cfg.topSourceGroup +troves = helper.findTrove(('%s:source' % srcName, srcVersion, srcFlavor), getLeaves=False) -troveSpec = (cfg.topSourceGroup[0], sourceVersion, None) +troveSpec = ('%s:source' % cfg.topSourceGroup[0], sourceVersion, None) sourceVersion = helper.findTroves((troveSpec, )).values()[0][0][1] nv = [] From elliot at rpath.com Mon Mar 15 17:15:50 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:50 +0000 Subject: mirrorball: fix typo Message-ID: <201003152115.o2FLFpMW013790@scc.eng.rpath.com> changeset: 1906f9d3007e user: Elliot Peele date: Tue, 12 Jan 2010 01:40:25 -0500 fix typo diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -195,7 +195,7 @@ log.info('removing any of obsoleted packages from the managed ' 'group: %s' % ', '.join(removeObsoleted)) for pkg in removeObsoleted: - self._groupmgr.remove(pkg, missingOK = True) + self._groupmgr.remove(pkg, missingOk=True) # Handle the case of entire source being obsoleted, this causes all # binaries from that source to be removed from the group model. From elliot at rpath.com Mon Mar 15 17:15:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:52 +0000 Subject: mirrorball: fix ordering issues Message-ID: <201003152115.o2FLFq5o013810@scc.eng.rpath.com> changeset: 337812df7961 user: Elliot Peele date: Tue, 12 Jan 2010 01:40:51 -0500 fix ordering issues diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -269,23 +269,21 @@ bucketMap = {} for grouping in self._cfg.combinePackages: for pkg in grouping: - bucketMap[pkg] = grouping + bucketMap[pkg] = self._cfg.combinePackages.index(grouping) buckets = {} for nvf in sorted(troveSpecs): - # Package has already been added. - if nvf[0] in buckets: - continue + name = nvf[0] + if name in bucketMap: + if name not in buckets: + order.append([]) + idx = len(order) - 1 - # Process any combined packages - elif nvf[0] in bucketMap: - order.append([]) - idx = len(order) - 1 - for n in bucketMap[nvf[0]]: - buckets[n] = idx - order[idx].append(n) + for pkg in self._cfg.combinePackages[bucketMap[name]]: + buckets[name] = idx - # Normal packages + idx = buckets[name] + order[idx].append(nvf) else: order.append(nvf) From elliot at rpath.com Mon Mar 15 17:15:54 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:54 +0000 Subject: mirrorball: don't try to hash a non hashable object Message-ID: <201003152115.o2FLFsMv013842@scc.eng.rpath.com> changeset: e3e672d6fae1 user: Elliot Peele date: Tue, 12 Jan 2010 01:41:08 -0500 don't try to hash a non hashable object diff --git a/updatebot/build/common.py b/updatebot/build/common.py --- a/updatebot/build/common.py +++ b/updatebot/build/common.py @@ -81,6 +81,12 @@ Add a job to the worker pool. """ + # Make sure job is hashable. + if isinstance(job, list): + job = tuple(job) + elif isinstance(job, set): + job = frozenset(job) + if job in self._workers: log.critical('job already being monitored: %s' % (job, )) import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:15:57 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:57 +0000 Subject: mirrorball: make sure we don't try to update an empty bucket Message-ID: <201003152115.o2FLFvIT013892@scc.eng.rpath.com> changeset: 183165aef0ee user: Elliot Peele date: Wed, 13 Jan 2010 10:20:00 -0500 make sure we don't try to update an empty bucket diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -57,6 +57,8 @@ log.info('searching for packages to update') + assert updateTroves is None or len(updateTroves) + toAdvise = [] toUpdate = [] From elliot at rpath.com Mon Mar 15 17:15:56 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:56 +0000 Subject: mirrorball: handle empty buckets Message-ID: <201003152115.o2FLFu9j013874@scc.eng.rpath.com> changeset: 60a0895b3d1c user: Elliot Peele date: Wed, 13 Jan 2010 10:19:43 -0500 handle empty buckets diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -204,6 +204,8 @@ expectedReplaces = replaces.get(updateId, []) explicitSourceRemovals = self._cfg.removeSource.get(updateId, set()) explicitBinaryRemovals = self._cfg.removeObsoleted.get(updateId, set()) + + assert len(self._order[updateId]) for srpm in self._order[updateId]: nvf = (srpm.name, None, None) @@ -580,6 +582,8 @@ if nevra not in bucketNevras: raise AdvisoryPackageMissingFromBucketError(nevra=nevra) self._order[source].remove(srpm) + if not len(self._order[source]): + del self._order[source] # Make sure that each package that we are moving is only # mentioned in one advisory. @@ -599,6 +603,8 @@ name = dict(advInfo)['name'] if name == advisory: self._advMap[source].remove(advInfo) + if not len(self._advMap[source]): + del self._advMap[source] self._advMap.setdefault(dest, set()).add(advInfo) break @@ -621,10 +627,14 @@ raise PackageNotFoundInBucketError(nevra=nevra, bucketId=source) srpm = bucketNevras[nevra] self._order[source].remove(srpm) + if not len(self._order[source]): + del self._order[source] # Remove all references to this srpm being part of an advisory for advisory in self._advPkgRevMap.pop(srpm, set()): self._advPkgMap[advisory].remove(srpm) + if not len(self._advPkgMap[advisory]): + del self._advPkgMap[advisory] # Move srpm to destination bucket self._order.setdefault(dest, set()).add(srpm) From elliot at rpath.com Mon Mar 15 17:15:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:15:59 +0000 Subject: mirrorball: add getBinaryVersions method for retrieving binary versions of a source Message-ID: <201003152115.o2FLFxNQ013918@scc.eng.rpath.com> changeset: be6ee62803f9 user: Elliot Peele date: Wed, 13 Jan 2010 19:59:54 -0500 add getBinaryVersions method for retrieving binary versions of a source version diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -40,6 +40,7 @@ from updatebot.errors import PromoteFailedError from updatebot.errors import PromoteMismatchError from updatebot.errors import MirrorFailedError +from updatebot.errors import BinariesNotFoundForSourceVersion from updatebot.lib.conarycallbacks import UpdateBotCloneCallback @@ -213,19 +214,19 @@ return self.getSourceVersions(list(troves)) - def getSourceVersions(self, troveSpecs): + def getSourceVersions(self, binTroveSpecs): """ Given a list of trove specs, query the repository for all of the related source versions. - @param troveSpecs: list of troves to query for. - @type troveSpecs: [(name, versionObj, flavObj), ... ] + @param binTroveSpecs: list of troves to query for. + @type binTroveSpecs: [(name, versionObj, flavObj), ... ] @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ...]} """ # check the cache for any source we have already retrieved from the # repository. - cached = set(x for x in troveSpecs if x in self._sourceVersionCache) - uncached = sorted(set(troveSpecs).difference(cached)) + cached = set(x for x in binTroveSpecs if x in self._sourceVersionCache) + uncached = sorted(set(binTroveSpecs).difference(cached)) srcTrvs = {} tiSourceName = trove._TROVEINFO_TAG_SOURCENAME @@ -240,6 +241,58 @@ return srcTrvs + def getBinaryVersions(self, srcTroveSpecs): + """ + Given a list of source trove specs, find the latest versions of all + binaries generated from these sources. + @param srcTroveSpecs: list of source troves. + @type srcTroveSpecs: [(name, versionObj, None), ... ] + @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ... ]} + """ + + # get all binary trove specs for the build label + binTrvMap = self._repos.getTroveVersionsByLabel( + {None: {self._ccfg.buildLabel: None}}) + binTrvSpecs = set() + for n, vermap in binTrvMap.iteritems(): + # filter out sources + if n.endswith(':source'): + continue + for v, flvs in vermap.iteritems(): + for f in flvs: + binTrvSpecs.add((n, v, f)) + + # get a map of source trove specs to binary trove specs + srcVerMap = self.getSourceVersions(binTrvSpecs) + + srcMap = {} + for srcTrv in srcTroveSpecs: + if srcTrv not in srcVerMap: + log.error('can not find requested source trove in repository') + raise BinariesNotFoundForSourceVersion(srcName=srvTrv[0], + srcVersion=srcTrv[1]) + + # Move binaries into a more convienient data structure. + binTrvMap = {} + binTrvs = srcVerMap[srcTrv] + for binTrv in binTrvs: + binTrvMap.setdefault(binTrv[1], set()).add(binTrv) + + # find the latest version. + latest = None + for binVer in binTrvMap: + if latest is None: + latest = binVer + continue + if latest < binVer: + latest = binVer + + # Store latest binary versions in source version map + assert srcTrv not in srcMap + srcMap[srcTrv] = binTrvMap[latest] + + return srcMap + @staticmethod def _getTrove(cs, name, version, flavor): """ @@ -740,7 +793,7 @@ # Build the label map. labelMap = {fromLabel: targetLabel} for label in sourceLabels: - assert(label is not None) + assert label is not None labelMap[label] = targetLabel # mix in the extra troves diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -167,6 +167,17 @@ _params = ['pkgname'] _template = 'No checked out version of %(pkgname)s was found.' +class BinariesNotFoundForSourceVersion(UnhandledUpdateError): + """ + BinariesNotFoundForSourceVersion, raised when querying by source name and + version for all binaries build from the given source version. Normally means + that an incorrect source name and/or version was provided or the source was + never built. + """ + + _params = ['srcName', 'srcVersion', ] + _template = 'Can not find binaries for %(srcName)s=%(srcVersion)s' + class PromoteFailedError(UnhandledUpdateError): """ PromoteFailedError, raised when the bot fails to promote the binary group From elliot at rpath.com Mon Mar 15 17:16:01 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:01 +0000 Subject: mirrorball: add support for saving and restoring the package map when there are errors Message-ID: <201003152116.o2FLG1AW013959@scc.eng.rpath.com> changeset: cdbe95e4a637 user: Elliot Peele date: Wed, 13 Jan 2010 20:00:25 -0500 add support for saving and restoring the package map when there are errors diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -17,7 +17,12 @@ """ import time +import pickle import logging +import tempfile + +from conary import versions +from conary.deps import deps from updatebot import errata from updatebot import groupmgr @@ -66,6 +71,46 @@ flavors = list(vf[version]) self._groupmgr.addPackage(name, version, flavors) + def _savePackages(self, pkgMap, fn=None): + """ + Save the package map to a file. + """ + + if fn is None: + fn = tempfile.mktemp() + + log.info('saving package map to file: %s' % fn) + + # freeze contents + frzPkgs = dict([ ((x[0], x[1].freeze(), x[2]), + set([ (z[0], z[1].freeze(), z[2].freeze()) + for z in y])) + for x, y in pkgMap.iteritems() ]) + + # pickle frozen contents + pickle.dump(frzPkgs, open(fn, 'w')) + + def _restorePackages(self, fn): + """ + Restore the frozen form of the package map. + """ + + log.info('restoring package map from file: %s' % fn) + + thawVersion = versions.ThawVersion + thawFlavor = deps.ThawFlavor + + # load pickle + frzPkgs = pickle.load(open(fn)) + + # thaw versions and flavors + pkgMap = dict([ ((x[0], thawVersion(x[1]), thawFlavor(x[2])), + set([ (z[0], thawVersion(z[1]), thawFlavor(z[2])) + for z in y ])) + for x, y in frzPkgs.iteritems() ]) + + return pkgMap + def create(self, *args, **kwargs): """ Handle initial import case. @@ -100,6 +145,9 @@ Handle update case. """ + # Load specific kwargs + restoreFile = kwargs.pop('restoreFile', None) + # FIXME: this should probably be provided by the errata object. # Method for sorting versions. def verCmp(a, b): @@ -155,9 +203,18 @@ set(removeReplaced)) + # If recovering from a failure, restore the pkgMap from disk. + if restoreFile: + pkgMap = self._restorePackages(restoreFile) + restoreFile = None + # Update package set. - pkgMap = self._update(*args, updatePkgs=updates, - expectedRemovals=expectedRemovals, **kwargs) + else: + pkgMap = self._update(*args, updatePkgs=updates, + expectedRemovals=expectedRemovals, **kwargs) + + # Save package map in case we run into trouble later. + self._savePackages(pkgMap) # FIXME: we might actually want to do this one day # Find errata group versions. From elliot at rpath.com Mon Mar 15 17:16:03 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:03 +0000 Subject: mirrorball: add support for restoreFile if specified Message-ID: <201003152116.o2FLG3AC013988@scc.eng.rpath.com> changeset: 3b651e6209f5 user: Elliot Peele date: Wed, 13 Jan 2010 20:00:50 -0500 add support for restoreFile if specified diff --git a/scripts/order_update.py b/scripts/order_update.py --- a/scripts/order_update.py +++ b/scripts/order_update.py @@ -49,6 +49,10 @@ if platform not in os.listdir(mirrorballDir + '/config'): usage() +restoreFile=None +if len(sys.argv) > 2: + restoreFile = sys.argv[2] + confDir = mirrorballDir + '/config/' + platform cfg = config.UpdateBotConfig() @@ -61,6 +65,6 @@ errata.fetch() bot = ordered.Bot(cfg, errata) -pkgMap = bot.update() +pkgMap = bot.update(restoreFile=restoreFile) import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:16:04 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:04 +0000 Subject: mirrorball: initial stab at a script that uses ordered data to promote groups Message-ID: <201003152116.o2FLG5Oh014015@scc.eng.rpath.com> changeset: e0b66b42ec5d user: Elliot Peele date: Thu, 14 Jan 2010 00:41:14 -0500 initial stab at a script that uses ordered data to promote groups diff --git a/scripts/grpchecker.py b/scripts/order_promote.py copy from scripts/grpchecker.py copy to scripts/order_promote.py --- a/scripts/grpchecker.py +++ b/scripts/order_promote.py @@ -13,11 +13,12 @@ # """ -Script for finding and fixing group issues. +Script for promoting groups in the correct order. """ import os import sys +import itertools sys.path.insert(0, os.environ['HOME'] + '/hg/conary') sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py') @@ -34,171 +35,111 @@ confDir = os.path.join(mbdir, 'config', sys.argv[1]) -sourceVersion = None -if len(sys.argv) == 3: - sourceVersion = cmdline.parseTroveSpec(sys.argv[2])[1] +import rhnmirror from updatebot import log -from updatebot import groupmgr from updatebot import conaryhelper +from updatebot import OrderedBot from updatebot import UpdateBotConfig -from updatebot.errors import OldVersionsFoundError -from updatebot.errors import GroupValidationFailedError -from updatebot.errors import NameVersionConflictsFoundError -from updatebot.errors import ExpectedRemovalValidationFailedError - -import time -import logging - slog = log.addRootLogger() cfg = UpdateBotConfig() cfg.read(os.path.join(confDir, 'updatebotrc')) -mgr = groupmgr.GroupManager(cfg) +mcfg = rhnmirror.MirrorConfig() +mcfg.read(confDir + '/erratarc') + +errata = rhnmirror.Errata(mcfg) +errata.fetch() + +bot = OrderedBot(cfg, errata) +bot._pkgSource.load() + helper = conaryhelper.ConaryHelper(cfg) -def handleVersionConflicts(group, error): - conflicts = error.conflicts - binPkgs = error.binPkgs +def getUpstreamVersionMap(groupvers): + # Turn groupvers into something we can work with. + grpVerMap = {} + for n, v, f in groupvers: + upver = v.trailingRevision().getVersion() + grpVerMap.setdefault(upver, dict()).setdefault(v, set()).add((n, v, f)) - toAdd = {} - toRemove = set() - for source, svers in conflicts.iteritems(): - assert len(svers) == 2 - svers.sort() - old = binPkgs[(source, svers[0], None)] - new = binPkgs[(source, svers[1], None)] + # Find the latest conary versions of all upstream versions + latestMap = {} + for upver, vers in grpVerMap.iteritems(): + latest = None + for v, nvfs in vers.iteritems(): + if latest is None: + latest = v + continue + if latest < v: + latest = v - toRemove.update(set([ x[0] for x in old ])) + # Store the latest versions that we need to promote + assert upver not in latestMap + latestMap[upver] = vers[latest] + return latestMap - for n, v, f, in new: - if n in toRemove: - toAdd.setdefault((n, v), set()).add(f) +# Get all of the binary versions of the top level group +slog.info('querying repository for all group versions') +groupvers = helper.findTrove(cfg.topGroup, getLeaves=False) +latestMap = getUpstreamVersionMap(groupvers) - return toRemove, toAdd +# Get all target versions +slog.info('querying target label for all group versions') +targetGroupvers = helper.findTrove((cfg.topGroup[0], cfg.targetLabel, None), getLeaves=False) +targetLatest = getUpstreamVersionMap(targetGroupvers) -def handleVersionErrors(group, error): - toAdd = {} - toRemove = set() - for pkgName, (modelContents, repoContents) in error.errors.iteritems(): - toRemove.update(set([ x[0] for x in modelContents ])) +# Get all updates after the first bucket. +missing = False +for updateId, bucket in bot._errata.iterByIssueDate(current=1): + upver = bot._errata.getBucketVersion(updateId) - for n, v, f in repoContents: - if n in toRemove: - toAdd.setdefault((n, v), set()).add(f) + if upver in targetLatest: + slog.info('%s found on target label, skipping' % upver) + continue - return toRemove, toAdd + slog.info('starting promote of %s' % upver) -def handleRemovalErrors(group, error): - toAdd = {} - toRemove = set(error.pkgNames) + # make sure version has been imported + if upver not in latestMap: + missing = upver + continue - return toRemove, toAdd + # If we find a missing version and then find a version in the + # repository report an error. + if missing: + slog.critical('found missing version %s' % missing) + raise RuntimeError -def checkVersion(ver): - mgr._sourceVersion = ver - mgr._checkout() - mgr._copyVersions() + # Get conary versions to promote + topGroups = latestMap[upver] + toPromote = [] + for n, v, f in topGroups: + toPromote.append((n, v, f)) + toPromote.append(('group-rhel-packages', v, f)) + toPromote.append(('group-rhel-standard', v, f)) - changes = [] + # Make sure we have the expected number of flavors + if len(topGroups) != len(cfg.groupFlavors): + slog.error('did not find expected number of flavors') + raise RuntimeError - try: - mgr._validateGroups() - except GroupValidationFailedError, e: - for group, error in e.errors: - if isinstance(error, NameVersionConflictsFoundError): - changes.append(handleVersionConflicts(group, error)) - elif isinstance(error, OldVersionsFoundError): - changes.append(handleVersionErrors(group, error)) - elif isinstance(error, ExpectedRemovalValidationFailedError): - changes.append(handleRemovalErrors(group, error)) - else: - raise error + # Find expected promote packages. + csrcTrvs = [ ('%s:source' % x.name, x.getConaryVersion(), None) + for x in bucket ] + srcTrvs = helper.findTroves(csrcTrvs) - return ver, changes + # Map source versions to binary versions. + slog.info('retrieving binary trove map') + srcTrvMap = helper.getBinaryVersions([ (x, y, None) for x, y, z in + itertools.chain(*srcTrvs.itervalues()) ]) -srcName, srcVersion, srcFlavor = cfg.topSourceGroup -troves = helper.findTrove(('%s:source' % srcName, srcVersion, srcFlavor), getLeaves=False) + # These are the binary trove specs that we expect to be promoted. + expected = [ x for x in itertools.chain(*srcTrvMap.itervalues()) ] -troveSpec = ('%s:source' % cfg.topSourceGroup[0], sourceVersion, None) -sourceVersion = helper.findTroves((troveSpec, )).values()[0][0][1] - -nv = [] -start = False -for n, v, f in sorted(troves): - if sourceVersion and v == sourceVersion: - start = True - if not start: - continue - nsv = (n, v.getSourceVersion()) - if nsv not in nv: - nv.append(nsv) - -if not start: - raise RuntimeError - -toUpdate = [] -for n, v in nv: - ver, changed = checkVersion(v) - # Make sure to only rebuild groups that have changed - # and every group after. - if not changed and not toUpdate: - continue - toUpdate.append((ver, changed)) - -#slog.critical('Before going any further, be aware that this will only rebuild ' -# 'the groups that need to change, not any other existing groups, if you ' -# 'want to maintain order on the devel label you will need to write that ' -# 'code.') -#assert False - -jobIds = [] -removed = {} -for ver, changed in toUpdate: - mgr._sourceVersion = ver - mgr._checkout() - - slog.info('updating %s' % ver) - - # Find all names and versions in the group model - nv = dict([ (y.name, versions.ThawVersion(y.version)) - for x, y in mgr._groups[mgr._pkgGroupName].iteritems() ]) - - # Set of packages that should be removed - removals = set([ x for x, y in removed.iteritems() if nv[x] == y ]) - - # Remove old removals - for pkg in removals: - slog.info('removing %s' % pkg) - mgr.remove(pkg) - - for toRemove, toAdd in changed: - # Handle removes - for pkg in toRemove: - slog.info('removing %s' % pkg) - mgr.remove(pkg) - - # cache removals - removed[pkg] = nv[pkg] - - # Handle adds - for (n, v), flvs in toAdd.iteritems(): - slog.info('adding %s=%s' % (n, v)) - mgr.addPackage(n, v, flvs) - - mgr._copyVersions() - mgr._validateGroups() - version = mgr.save(copyToLatest=True) - jobId = mgr._builder.start(((mgr._sourceName, version, None), )) - jobIds.append(jobId) - -for jobId in jobIds: - mgr._builder.watch(jobId) - -#import epdb; epdb.st() - -for jobId in jobIds: - mgr._builder.commit(jobId) + # Create and validate promote changeset + packageList = helper.promote(toPromote, expected, cfg.sourceLabel, + cfg.targetLabel, commit=True) import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:16:06 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:06 +0000 Subject: mirrorball: move log line to avoid confusion Message-ID: <201003152116.o2FLG6Zj014042@scc.eng.rpath.com> changeset: 9266fae33531 user: Elliot Peele date: Mon, 18 Jan 2010 10:41:13 -0500 move log line to avoid confusion diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -99,13 +99,13 @@ slog.info('%s found on target label, skipping' % upver) continue - slog.info('starting promote of %s' % upver) - # make sure version has been imported if upver not in latestMap: missing = upver continue + slog.info('starting promote of %s' % upver) + # If we find a missing version and then find a version in the # repository report an error. if missing: From elliot at rpath.com Mon Mar 15 17:16:08 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:08 +0000 Subject: mirrorball: add breakpoint Message-ID: <201003152116.o2FLG8ob014074@scc.eng.rpath.com> changeset: 5acb3845b7ce user: Elliot Peele date: Mon, 18 Jan 2010 10:41:24 -0500 add breakpoint diff --git a/scripts/grpchecker.py b/scripts/grpchecker.py --- a/scripts/grpchecker.py +++ b/scripts/grpchecker.py @@ -153,6 +153,8 @@ # 'code.') #assert False +import epdb; epdb.st() + jobIds = [] removed = {} for ver, changed in toUpdate: From elliot at rpath.com Mon Mar 15 17:16:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:10 +0000 Subject: mirrorball: add ability to specify extra troves that are expected to be promoted Message-ID: <201003152116.o2FLGBKL014103@scc.eng.rpath.com> changeset: de9b3953b71b user: Elliot Peele date: Mon, 18 Jan 2010 18:20:32 -0500 add ability to specify extra troves that are expected to be promoted diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -138,8 +138,12 @@ # These are the binary trove specs that we expect to be promoted. expected = [ x for x in itertools.chain(*srcTrvMap.itervalues()) ] + # Get list of extra troves from the config + extra = cfg.expectedExtraPromoteTroves.get(updateId, []) + # Create and validate promote changeset packageList = helper.promote(toPromote, expected, cfg.sourceLabel, - cfg.targetLabel, commit=True) + cfg.targetLabel, commit=True, + expectedExtraPromoteTroves=extra) import epdb; epdb.st() diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -757,7 +757,7 @@ def promote(self, trvLst, expected, sourceLabels, targetLabel, checkPackageList=True, extraPromoteTroves=None, - commit=True): + expectedExtraPromoteTroves=None, commit=True): """ Promote a group and its contents to a target label. @param trvLst: list of troves to publish @@ -775,6 +775,12 @@ @param extraPromoteTroves: troves to promote in addition to the troves that have been built. @type extraPromoteTroves: list of trove specs. + @param expectedExtraPromoteTroves: list of trove nvfs that are expected + to be promoted, but are only filtered + by name, rather than version and + flavor. + @type expectedExtraPromoteTroves: list of name, version, flavor tuples + where version and flavor may be None. @param commit: commit the promote changeset or just return it. @type commit: boolean """ @@ -787,6 +793,10 @@ if extraPromoteTroves is None: extraPromoteTroves = [] + # make expectedExtraPromoteTroves a list if it is not specified. + if expectedExtraPromoteTroves is None: + expectedExtraPromoteTroves = [] + # Get the label that the group is on. fromLabel = trvLst[0][1].trailingLabel() @@ -836,7 +846,8 @@ grpTrvs = set([ (x[0], x[2]) for x in trvLst if not x[0].endswith(':source') ]) grpDiff = set([ x[0] for x in trvDiff.difference(grpTrvs) ]) - extraTroves = set([ x[0] for x in extraPromoteTroves ]) + extraTroves = set([ x[0] for x in extraPromoteTroves | + expectedExtraPromoteTroves ]) if checkPackageList and grpDiff.difference(extraTroves): raise PromoteMismatchError(expected=oldPkgs, actual=newPkgs) diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -210,6 +210,11 @@ # Packages other than the topGroup that need to be promoted. extraPromoteTroves = (CfgList(CfgTroveSpec), []) + # Extra packages that are expected to be in the promote result set at a + # given bucketId. These are normally packages that for some reason or + # another, usually deps, we had to rebuild. + extraExpectedPromoteTroves = (CfgIntDict(CfgList(CfgTroveSpec)), {}) + # Packages to import package = (CfgList(CfgString), []) From elliot at rpath.com Mon Mar 15 17:16:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:13 +0000 Subject: mirrorball: make consistent with config Message-ID: <201003152116.o2FLGDdo014131@scc.eng.rpath.com> changeset: dea39a87dca1 user: Elliot Peele date: Mon, 18 Jan 2010 19:56:17 -0500 make consistent with config diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -139,11 +139,11 @@ expected = [ x for x in itertools.chain(*srcTrvMap.itervalues()) ] # Get list of extra troves from the config - extra = cfg.expectedExtraPromoteTroves.get(updateId, []) + extra = cfg.extraExpectedPromoteTroves.get(updateId, []) # Create and validate promote changeset packageList = helper.promote(toPromote, expected, cfg.sourceLabel, cfg.targetLabel, commit=True, - expectedExtraPromoteTroves=extra) + extraExpectedPromoteTroves=extra) import epdb; epdb.st() diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -757,7 +757,7 @@ def promote(self, trvLst, expected, sourceLabels, targetLabel, checkPackageList=True, extraPromoteTroves=None, - expectedExtraPromoteTroves=None, commit=True): + extraExpectedPromoteTroves=None, commit=True): """ Promote a group and its contents to a target label. @param trvLst: list of troves to publish @@ -775,11 +775,11 @@ @param extraPromoteTroves: troves to promote in addition to the troves that have been built. @type extraPromoteTroves: list of trove specs. - @param expectedExtraPromoteTroves: list of trove nvfs that are expected + @param extraExpectedPromoteTroves: list of trove nvfs that are expected to be promoted, but are only filtered by name, rather than version and flavor. - @type expectedExtraPromoteTroves: list of name, version, flavor tuples + @type extraExpectedPromoteTroves: list of name, version, flavor tuples where version and flavor may be None. @param commit: commit the promote changeset or just return it. @type commit: boolean @@ -793,9 +793,9 @@ if extraPromoteTroves is None: extraPromoteTroves = [] - # make expectedExtraPromoteTroves a list if it is not specified. - if expectedExtraPromoteTroves is None: - expectedExtraPromoteTroves = [] + # make extraExpectedPromoteTroves a list if it is not specified. + if extraExpectedPromoteTroves is None: + extraExpectedPromoteTroves = [] # Get the label that the group is on. fromLabel = trvLst[0][1].trailingLabel() @@ -847,7 +847,7 @@ if not x[0].endswith(':source') ]) grpDiff = set([ x[0] for x in trvDiff.difference(grpTrvs) ]) extraTroves = set([ x[0] for x in extraPromoteTroves | - expectedExtraPromoteTroves ]) + extraExpectedPromoteTroves ]) if checkPackageList and grpDiff.difference(extraTroves): raise PromoteMismatchError(expected=oldPkgs, actual=newPkgs) From elliot at rpath.com Mon Mar 15 17:16:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:15 +0000 Subject: mirrorball: always turn extras lists into sets Message-ID: <201003152116.o2FLGFSH014163@scc.eng.rpath.com> changeset: a8406438aa8a user: Elliot Peele date: Mon, 18 Jan 2010 20:51:13 -0500 always turn extras lists into sets diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -789,13 +789,17 @@ log.info('starting promote') log.info('creating changeset') - # make extraPromoteTroves a list if it was not specified. + # make extraPromoteTroves a set if it was not specified. if extraPromoteTroves is None: - extraPromoteTroves = [] + extraPromoteTroves = set() + else: + extraPromoteTroves = set(extraPromoteTroves) # make extraExpectedPromoteTroves a list if it is not specified. if extraExpectedPromoteTroves is None: - extraExpectedPromoteTroves = [] + extraExpectedPromoteTroves = set() + else: + extraExpectedPromoteTroves = set(extraExpectedPromoteTroves) # Get the label that the group is on. fromLabel = trvLst[0][1].trailingLabel() From elliot at rpath.com Mon Mar 15 17:16:17 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:17 +0000 Subject: mirrorball: make getManifest and getMetadata version aware Message-ID: <201003152116.o2FLGHho014190@scc.eng.rpath.com> changeset: 7301bc095251 user: Elliot Peele date: Wed, 20 Jan 2010 10:52:37 -0500 make getManifest and getMetadata version aware diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -313,17 +313,19 @@ trv = trove.Trove(troveCs, skipIntegrityChecks=True) return trv - def getManifest(self, pkgname): + def getManifest(self, pkgname, version=None): """ Get the contents of the manifest file from the source component for a given package. @param pkgname: name of the package to retrieve @type pkgname: string + @param version optional source version to checkout. + @type version conary.versions.Version @return manifest for pkgname """ log.info('retrieving manifest for %s' % pkgname) - recipeDir = self._edit(pkgname) + recipeDir = self._edit(pkgname, version=version) manifestFileName = util.join(recipeDir, 'manifest') if not os.path.exists(manifestFileName): @@ -355,16 +357,18 @@ # Make sure manifest file has been added. self._addFile(recipeDir, 'manifest') - def getMetadata(self, pkgname): + def getMetadata(self, pkgname, version=None): """ Get the metadata.xml file from a source componet named pkgname. @param pkgname name of the package to checkout @type pkgname string @return list like object + @param version optional source version to checkout. + @type version conary.versions.Version """ log.info('retrieving metadata for %s' % pkgname) - recipeDir = self._edit(pkgname) + recipeDir = self._edit(pkgname, version=version) metadataFileName = util.join(recipeDir, 'metadata.xml') if not os.path.exists(metadataFileName): @@ -452,6 +456,8 @@ file in the source component, otherwise return None. @param pkgname: name of hte package to edit @type pkgname: string + @param version optional source version to checkout. + @type version conary.versions.Version """ log.info('getting version info for %s' % pkgname) From elliot at rpath.com Mon Mar 15 17:16:19 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:19 +0000 Subject: mirrorball: add method for checking for updated errata that have already been imported Message-ID: <201003152116.o2FLGJ51014219@scc.eng.rpath.com> changeset: fd5bc2d78956 user: Elliot Peele date: Wed, 20 Jan 2010 11:43:49 -0500 add method for checking for updated errata that have already been imported into the conary repository diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -118,6 +118,38 @@ return version @loadErrata + def getModifiedErrata(self, current): + """ + Get all updates that were issued before current, but have been modified + after current. + @param current: the current state, start iterating after this state has + been reached. + @type current: int + """ + + # Get modified errata from the model + modified = self._errata.getModifiedErrata(current) + + # Map this errata to srpms + modMap = {} + for e in modified: + advisory = e.advisory + last_modified = e.last_modified_date + issue_date = e.issue_date + pkgs = self._advPkgMap[advisory] + + assert advisory not in modMap + + modMap[advisory] = { + 'advisory': advisory, + 'last_modified': last_modified, + 'issue_date': issue_date, + 'srpms': pkgs, + } + + return modMap + + @loadErrata def iterByIssueDate(self, current=None): """ Yield sets of srcPkgs by errata release date. From elliot at rpath.com Mon Mar 15 17:16:21 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:21 +0000 Subject: mirrorball: add method for validating already imported manifests Message-ID: <201003152116.o2FLGLl1014246@scc.eng.rpath.com> changeset: c620a192fea5 user: Elliot Peele date: Wed, 20 Jan 2010 11:44:14 -0500 add method for validating already imported manifests diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -178,6 +178,21 @@ _params = ['srcName', 'srcVersion', ] _template = 'Can not find binaries for %(srcName)s=%(srcVersion)s' +class RepositoryPackageSourceInconsistencyError(UnhandledUpdateError): + """ + RepositoryPackageSourceInconsistencyError, raised when the manifest for a + given source component does not match the state of the package source. This + should only be triggered when the upstream provider of updates has gone back + and changed the package set for an update that has already been imported + into the conary repository. + """ + + _params = ['nvf', 'srpm', ] + _template = ('An inconsistency has been discovered between the conary ' + 'repository contents and the upstream package source for %(srpm)s. ' + 'This is normally due to upstream modifying the package set for an ' + 'update that has already been imported into the conary repository.') + class PromoteFailedError(UnhandledUpdateError): """ PromoteFailedError, raised when the bot fails to promote the binary group diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -214,7 +214,7 @@ reusedPackages = set() try: - manifest = self._conaryhelper.getManifest(nvf[0]) + manifest = self._conaryhelper.getManifest(nvf[0], version=nvf[1]) except NoManifestFoundError, e: # Create packages that do not have manifests. # TODO: might want to make this a config option? @@ -230,7 +230,8 @@ srcPkg = self._pkgSource.binPkgMap[binPkg] else: if metadata is None: - pkgs = self._getMetadataFromConaryRepository(nvf[0]) + pkgs = self._getMetadataFromConaryRepository(nvf[0], + verison=nvf[1]) metadata = util.Metadata(pkgs) if metadata: binPkg = metadata.locationMap[line] @@ -273,8 +274,6 @@ log.warn('update removes package (%s) %s -> %s' % (pkg.name, srpm.getNevra(), srcPkg.getNevra())) - #import epdb; epdb.st() - # allow some packages to be removed. if expectedRemovals and pkg.name in expectedRemovals: log.info('package removal (%s) handled in configuration' @@ -306,6 +305,27 @@ return needsUpdate + def sanityCheckSource(self, srpm): + """ + Look up the matching source version in the conary repository and verify + that the manifest matches the package list in the package source. + @param srpm: src pacakge object + @type srpm: repomd.packagexml._Package + """ + + nvflst = self._conaryhelper.findTrove(('%s:source' % srpm.name, + srpm.getConaryVersion(), None)) + + assert len(nvflst) == 1 + n, v, f = nvflst[0] + nvf = (n.split(':')[0], v, None) + + needsUpdate = self._sanitizeTrove(nvf, srpm) + + # If anything has chnaged raise an error. + if needsUpdate: + raise RepositoryPackageSourceInconsistencyError(nvf=nvf, srpm=srpm) + @staticmethod def _getLatestOfAvailableArches(pkgLst): """ @@ -542,15 +562,17 @@ return self._pkgSource.srcPkgMap[srcPkg] - def _getMetadataFromConaryRepository(self, pkgName): + def _getMetadataFromConaryRepository(self, pkgName, version=None): """ Get the metadata from the repository and generate required mappings. @param pkgName: source package name @type pkgName: string + @param version optional source version to checkout. + @type version conary.versions.Version @return dictionary of infomation that looks like a pkgsource. """ - return self._conaryhelper.getMetadata(pkgName) + return self._conaryhelper.getMetadata(pkgName, version=version) def _getBuildRequiresFromPkgSource(self, srcPkg): """ From elliot at rpath.com Mon Mar 15 17:16:22 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:22 +0000 Subject: mirrorball: check for errata that have been updating upstream, but have already been Message-ID: <201003152116.o2FLGNga014278@scc.eng.rpath.com> changeset: 7bf5d62f3127 user: Elliot Peele date: Wed, 20 Jan 2010 11:45:28 -0500 check for errata that have been updating upstream, but have already been imported into the conary repository, this will raise an exception if the existing manifest fails to validating against the package source. diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -179,6 +179,20 @@ # Sanity check errata ordering. self._errata.sanityCheckOrder() + # Check for updated errata that may require some manual changes to the + # repository. These are errata that were issued before the current + # errata state, but have been modified in the upstream errata source. + changed = self._errata.getModifiedErrata(current) + # Iterate through changed and verify the current conary repository + # contents against any changes. + if changed: + log.info('found modified updates, validating repository state') + for advisory, advInfo in changed.iteritems(): + log.info('validating %s' % advisory) + for srpm in advInfo['srpms']: + log.info(srpm) + self._updater.sanityCheckSource(srpm) + updateSet = {} for updateId, updates in self._errata.iterByIssueDate(current=current): start = time.time() From elliot at rpath.com Mon Mar 15 17:16:24 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:24 +0000 Subject: mirrorball: add comment explaining that sanity checking function will raise an exception Message-ID: <201003152116.o2FLGO1K014305@scc.eng.rpath.com> changeset: 27fa068a6d6e user: Elliot Peele date: Wed, 20 Jan 2010 14:10:39 -0500 add comment explaining that sanity checking function will raise an exception when insanity is reached diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -191,6 +191,8 @@ log.info('validating %s' % advisory) for srpm in advInfo['srpms']: log.info(srpm) + # This will raise an exception if any inconsistencies are + # detected. self._updater.sanityCheckSource(srpm) updateSet = {} From elliot at rpath.com Mon Mar 15 17:16:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:25 +0000 Subject: mirrorball: refresh latest cache everytime through Message-ID: <201003152116.o2FLGP8Q014334@scc.eng.rpath.com> changeset: c18c795f8553 user: Elliot Peele date: Wed, 20 Jan 2010 14:37:45 -0500 refresh latest cache everytime through diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -80,10 +80,14 @@ latestMap[upver] = vers[latest] return latestMap -# Get all of the binary versions of the top level group -slog.info('querying repository for all group versions') -groupvers = helper.findTrove(cfg.topGroup, getLeaves=False) -latestMap = getUpstreamVersionMap(groupvers) +def updateLatestMap(): + # Get all of the binary versions of the top level group + slog.info('querying repository for all group versions') + groupvers = helper.findTrove(cfg.topGroup, getLeaves=False) + latestMap = getUpstreamVersionMap(groupvers) + return latestMap + +latestMap = updateLatestMap() # Get all target versions slog.info('querying target label for all group versions') @@ -146,4 +150,7 @@ cfg.targetLabel, commit=True, extraExpectedPromoteTroves=extra) + # Update latest map for the next loop + latestMap = updateLatestMap() + import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:16:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:27 +0000 Subject: mirrorball: handle partial errata Message-ID: <201003152116.o2FLGRko014361@scc.eng.rpath.com> changeset: 4ac10202ad5c user: Elliot Peele date: Mon, 01 Feb 2010 16:14:08 -0500 handle partial errata diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -485,19 +485,34 @@ srcMap[src].append(pkg) # insert bins by buildstamp + extras = {} for src, bins in srcMap.iteritems(): + # Pull out any package sets that look like they are incomplete. + if len(bins) != len(self._pkgSource.srcPkgMap[src]) - 1: + extras[src] = bins + continue + buildstamp = int(sorted(bins)[0].buildTimestamp) if buildstamp not in buckets: buckets[buildstamp] = [] buckets[buildstamp].extend(bins) # get sources to build + srpmToBucketId = {} for bucketId in sorted(buckets.keys()): bucket = buckets[bucketId] self._order[bucketId] = set() for pkg in bucket: src = self._pkgSource.binPkgMap[pkg] self._order[bucketId].add(src) + srpmToBucketId[src] = bucketId + + # Make sure extra packages are already included in the order. + for src, bins in extras.iteritems(): + assert src in srpmToBucketId + assert src in self._order[srpmToBucketId[src]] + for bin in bins: + assert bin in self._pkgSource.srcPkgMap[src] ## # Start order munging here @@ -508,6 +523,11 @@ pkgs = set() for pkgSet in self._order.itervalues(): pkgs.update(pkgSet) + + # This assert validates that no one srpm is mentioned in more than + # one bucket. This can happen when a partial set of packages was + # released and the code tried to fill in the other packages by build + # time. assert len(pkgs) == totalPkgs # fold together updates to preserve dep closure. From elliot at rpath.com Mon Mar 15 17:16:29 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:29 +0000 Subject: mirrorball: a bit of a start at some high-level conceptual README docs Message-ID: <201003152116.o2FLGTAY014393@scc.eng.rpath.com> changeset: e480d637e0f5 user: Michael K. Johnson date: Wed, 03 Feb 2010 15:20:12 -0500 a bit of a start at some high-level conceptual README docs diff --git a/README b/README --- a/README +++ b/README @@ -1,9 +1,79 @@ -Date: 2008-09-02 -Author: Elliot Peele +Mirrorball README + +================ += Introduction = +================ + +Mirrorball is a tool for importing external projects with two main +facets: + o updatebot: a maintenance tool for automatically processing updates + o scripts for initial import and for resolving maintenance exceptions + +Mirrorball is a toolkit, not a product. When mirrorball finds an +exceptional case, it drops you into a debugger. This is not a bug +in mirrorball; it is an opportunity to use the data structures in +mirrorball to understand what caused the exception and determine +the best way to resolve it. Automating (or partially automating) +exception handling is good; adding debugging output with potential +solutions is good, but the debugger is part of the toolkit. It is +common when using the scripts for initial import and resolving +exceptions during maintenance to add breakpoints to the scripts +and verify what they are going to do in the debugger, and then +let the scripts continue and do the work. + +To successfully use mirrororball, you need to understand its theory +of operation. Mirrorball synchronizes sources of package data, +including packages and advisories, with associated Conary repositories. +It uses Conary and rMake as the main underlying tools to drive the +process of converting upstream packages into content in a Conary +repository. + +Mirrorball has two main way of representing upstream package state +in a Conary repository. In the first ("latest"), if the latest +versions of upstream packages differ from what is in the Conary +repository, mirrorball represents the latest state of the upstream +packages in the Conary repository. In the second ("ordered"), +mirrorball represents a view of the entire history of the upstream +package source in the Conary repository. + +Using the "latest" model is easiest; it can often be done without +writing new code in mirrorball. It requires a source of packages +that it recognizes (for example, a yum repository of RPMs), a +configuration that tells it what to do with the packages, and +a conary repository to put the packages into. (It also requires +a source of packages to use for building, and the bootstrap +process of making this source available is an advanced topic.) + +Using the "ordered" model is more demanding. In order to represent +all the packages in the historical record, it needs a source of +data for the ordering. One potential source of data is errata +advisories with dates attached to them somehow. In order to use +such advisories, code has to be written to import critical data +into mirrorball's "errata" representation. Then the import process +has to create a complete ordering representing the entire history, +validate that history, warn about conditions in the history that +require manual adjustment, and then apply all history past the +current state of the repository to the repository. + +In both models, there are multiple configuration files that need +to be written: essentially, one for each tool used. These will +include conary configuration (e.g. flavors, superclass packages, +installLabelPath, buildLabel), rmake configuration (e.g. rmakeUrl, +resolveTroves, defaultBuildReqs), and elements of mirrorball per +set (e.g. errata sources and updatebot configuration) + +In both models, there are bootstrap requirements: Conary packages +representing sufficient binaries to build with in a repository, +and all necessary Conary superclass and factory packages in the +repository. Mirrorball does not automatically create these things; +creating the prerequisites for the platform is the responsibility +of the packager. + This document is meant as a guide to get people up and running with a working updatebot configuration and some basic usage information to get -people started. +people started. It is currently insufficient, so after reading this +document, join #conary on irc.freenode.net and ask for help. * Setting up updatebot to work in your environment. * How to do: From elliot at rpath.com Mon Mar 15 17:16:30 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:30 +0000 Subject: mirrorball: branch merge Message-ID: <201003152116.o2FLGU7W014420@scc.eng.rpath.com> changeset: 54eb1de4d603 user: Elliot Peele date: Wed, 03 Feb 2010 15:38:42 -0500 branch merge diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -485,19 +485,34 @@ srcMap[src].append(pkg) # insert bins by buildstamp + extras = {} for src, bins in srcMap.iteritems(): + # Pull out any package sets that look like they are incomplete. + if len(bins) != len(self._pkgSource.srcPkgMap[src]) - 1: + extras[src] = bins + continue + buildstamp = int(sorted(bins)[0].buildTimestamp) if buildstamp not in buckets: buckets[buildstamp] = [] buckets[buildstamp].extend(bins) # get sources to build + srpmToBucketId = {} for bucketId in sorted(buckets.keys()): bucket = buckets[bucketId] self._order[bucketId] = set() for pkg in bucket: src = self._pkgSource.binPkgMap[pkg] self._order[bucketId].add(src) + srpmToBucketId[src] = bucketId + + # Make sure extra packages are already included in the order. + for src, bins in extras.iteritems(): + assert src in srpmToBucketId + assert src in self._order[srpmToBucketId[src]] + for bin in bins: + assert bin in self._pkgSource.srcPkgMap[src] ## # Start order munging here @@ -508,6 +523,11 @@ pkgs = set() for pkgSet in self._order.itervalues(): pkgs.update(pkgSet) + + # This assert validates that no one srpm is mentioned in more than + # one bucket. This can happen when a partial set of packages was + # released and the code tried to fill in the other packages by build + # time. assert len(pkgs) == totalPkgs # fold together updates to preserve dep closure. From elliot at rpath.com Mon Mar 15 17:16:32 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:32 +0000 Subject: mirrorball: fix a couple of typos Message-ID: <201003152116.o2FLGWcl014450@scc.eng.rpath.com> changeset: 6aaa8e220793 user: Elliot Peele date: Wed, 03 Feb 2010 15:52:17 -0500 fix a couple of typos diff --git a/README b/README --- a/README +++ b/README @@ -28,7 +28,7 @@ process of converting upstream packages into content in a Conary repository. -Mirrorball has two main way of representing upstream package state +Mirrorball has two main ways of representing upstream package state in a Conary repository. In the first ("latest"), if the latest versions of upstream packages differ from what is in the Conary repository, mirrorball represents the latest state of the upstream @@ -60,7 +60,7 @@ include conary configuration (e.g. flavors, superclass packages, installLabelPath, buildLabel), rmake configuration (e.g. rmakeUrl, resolveTroves, defaultBuildReqs), and elements of mirrorball per -set (e.g. errata sources and updatebot configuration) +say (e.g. errata sources and updatebot configuration) In both models, there are bootstrap requirements: Conary packages representing sufficient binaries to build with in a repository, From elliot at rpath.com Mon Mar 15 17:16:34 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:34 +0000 Subject: mirrorball: add support for parent platforms (PFM-565) Message-ID: <201003152116.o2FLGYsx014477@scc.eng.rpath.com> changeset: 8cc261c6a78e user: Elliot Peele date: Tue, 09 Feb 2010 17:59:14 -0500 add support for parent platforms (PFM-565) diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -109,15 +109,16 @@ # Import sources into repository. - toBuild, fail = self._updater.create(toPackage, - buildAll=rebuild, - recreate=bool(recreate), - toCreate=toCreate) + toBuild, parentPkgMap, fail = self._updater.create( + toPackage, + buildAll=rebuild, + recreate=bool(recreate), + toCreate=toCreate) log.info('failed to create %s packages' % len(fail)) log.info('found %s packages to build' % len(toBuild)) - trvMap = [] + trvMap = {} failed = () if len(toBuild): if not rebuild or (rebuild and toCreate): @@ -138,6 +139,9 @@ log.info('elapsed time %s' % (time.time() - start, )) + # Add any platform packages to the trove map. + trvMap.update(parentPkgMap) + return trvMap, failed def update(self, force=None, updatePkgs=None, expectedRemovals=None): @@ -198,10 +202,17 @@ self._advisor.check(toAdvise) # Update source + parentPackages = set() for nvf, srcPkg in toUpdate: toAdvise.remove((nvf, srcPkg)) newVersion = self._updater.update(nvf, srcPkg) - toAdvise.append(((nvf[0], newVersion, nvf[2]), srcPkg)) + if self._updater.isPlatformTrove(newVersion): + toAdvise.append(((nvf[0], newVersion, nvf[2]), srcPkg)) + else: + parentPackages.add((nvf[0], newVersion, nvf[2])) + + parentPkgMap = self._updater.getBinaryVersions(parentPackages, + labels=self._platformSearchPath) # Make sure to build everything in the toAdvise list, there may be # sources that have been updated, but not built. @@ -244,4 +255,7 @@ % (len(toUpdate), len(toAdvise))) log.info('elapsed time %s' % (time.time() - start, )) + # Add any platform packages to the trove map. + trvMap.update(parentPkgMap) + return trvMap diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -30,6 +30,7 @@ from conary import conaryclient from conary.lib import log as clog from conary.conaryclient import mirror +from conary import errors as conaryerrors from updatebot.lib import util from updatebot.lib import xobjects @@ -97,21 +98,47 @@ return self._ccfg - def findTrove(self, nvf, *args, **kwargs): + def findTrove(self, nvf, labels=None, *args, **kwargs): """ Mapped to conaryclient.repos.findTrove. Will always search buildLabel. """ - return self._repos.findTrove(self._ccfg.buildLabel, nvf, - *args, **kwargs) + if not labels: + labels = self._ccfg.buildLabel - def findTroves(self, troveList, *args, **kwargs): + try: + return self._repos.findTrove(labels, nvf, *args, **kwargs) + except conaryerrors.TroveNotFound, e: + return [] + + def findTroves(self, troveList, labels=None, *args, **kwargs): """ Mapped to conaryclient.repos.findTroves. Will always search buildLabel. """ - return self._repos.findTroves(self._ccfg.buildLabel, troveList, - *args, **kwargs) + if not labels: + labels = self._ccfg.buildLabel + + try: + return self._repos.findTroves(labels, troveList, *args, **kwargs) + except conaryerrors.TroveNotFound, e: + return {} + + def isOnBuildLabel(self, version): + """ + Check if version is on the build label. + @param version: conary version object + @type version: conary.versions.Version + @return True if version is on the buildLabel. + @rtype boolean + """ + + if hasattr(version, 'label'): + label = version.label() + else: + label = version.trailingLabel() + + return self._ccfg.buildLabel == label def getSourceTroves(self, group): """ @@ -241,18 +268,26 @@ return srcTrvs - def getBinaryVersions(self, srcTroveSpecs): + def getBinaryVersions(self, srcTroveSpecs, labels=None): """ Given a list of source trove specs, find the latest versions of all binaries generated from these sources. @param srcTroveSpecs: list of source troves. @type srcTroveSpecs: [(name, versionObj, None), ... ] + @param labels: list of labels to search, defaults to the buildLabel + @type labels: list(conary.versions.Label, ...) @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ... ]} """ - # get all binary trove specs for the build label - binTrvMap = self._repos.getTroveVersionsByLabel( - {None: {self._ccfg.buildLabel: None}}) + # default to the build label if not other labels were specified + if not labels: + labels = [ self._ccfg.buildLabel, ] + + # get all binary trove specs for the specified labels + req = {None: dict([ (x, None) for x in labels ])} + binTrvMap = self._repos.getTroveVersionsByLabel(req) + + # build a list of the binary troves on the labels binTrvSpecs = set() for n, vermap in binTrvMap.iteritems(): # filter out sources @@ -544,7 +579,8 @@ return self._checkoutCache[pkgkey] # Figure out if we should create or update. - if not self.getLatestSourceVersion(pkgname): + if (not self.getLatestSourceVersion(pkgname) and + (version is None or self.isOnBuildLabel(version))): assert version is None recipeDir = self._newpkg(pkgname) else: diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -197,6 +197,9 @@ # The top level source group. topSourceGroup = CfgTroveSpec + # Path to search for packages to be included in the platform. + platformSearchPath = (CfgQuotedLineList(CfgLabel), []) + # Group contents info. groupContents = (CfgDict(CfgDict(CfgString)), {}) @@ -401,7 +404,8 @@ # Find configPath. ret = cfg.SectionedConfigFile.read(self, *args, **kwargs) - if not self.configPath.startswith(os.sep): + if (not self.configPath.startswith(os.sep) and + args[0].endswith('updatebotrc')): # configPath is relative dirname = os.path.dirname(args[0]) self.configPath = os.path.normpath(os.path.join(dirname, diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -225,6 +225,8 @@ # Play though entire update history to check for iregularities. current = {} + childPackages = [] + parentPackages = [] removals = self._cfg.updateRemovesPackages replaces = self._cfg.updateReplacesPackages currentlyRemovedBinaryNevras = set() @@ -255,7 +257,13 @@ log.error('Removing %s in %s would cause it never to be promoted' % (str(' '.join(srpm.getNevra())), updateId)) current[srpm.name] = srpm - assert not updater.update(nvf, srpm) + version = updater.update(nvf, srpm) + assert (not version or + not updater.isPlatformTrove((None, version, None))) + if version: + parentPackages.append(((nvf, srpm), version)) + else: + childPackages.append(((nvf, srpm), None)) # all package names obsoleted by packages in the current set obsoleteNames = set() @@ -467,6 +475,8 @@ log.info('order sanity checking complete') + return childPackages, parentPackages + def _orderErrata(self): """ Order errata by timestamp. @@ -816,7 +826,6 @@ def __init__(self, cfg): conaryhelper.ConaryHelper.__init__(self, cfg) self._client = None - self._repos = None def getLatestSourceVersion(self, pkgname): """ @@ -837,12 +846,14 @@ getSourceTroves = _not_implemented getSourceVersions = _not_implemented - def _checkout(self, pkgname): + def _checkout(self, pkgname, version=None): """ Checkout stub. """ - #log.info('checking out %s' % pkgname) + if version and not self.isOnBuildLabel(version): + return conaryhelper.ConaryHelper._checkout(self, pkgname, + version=version) recipeDir = self._getRecipeDir(pkgname) return recipeDir @@ -855,8 +866,6 @@ addFile stub. """ - #log.info('adding file %s' % fileName) - _removeFile = _addFile def _commit(self, pkgDir, commitMessage): diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -193,6 +193,17 @@ 'This is normally due to upstream modifying the package set for an ' 'update that has already been imported into the conary repository.') +class ParentPlatformManifestInconsistencyError(UnhandledUpdateError): + """ + ParentPlatformManifestInconsistencyError, raised when the manifest contents + that would be generated by the platform package source differs from the + upstream platform manifest. + """ + + _params = ['srcPkg', 'manifest', 'parentManifest', ] + _template = ('An inconsistency has been dicovered between the platform ' + 'package source and the parent platform manifest for %(srcPkg)s') + class PromoteFailedError(UnhandledUpdateError): """ PromoteFailedError, raised when the bot fails to promote the binary group diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -28,6 +28,7 @@ from updatebot.errors import OldVersionNotFoundError from updatebot.errors import UpdateGoesBackwardsError from updatebot.errors import UpdateRemovesPackageError +from updatebot.errors import ParentPlatformManifestInconsistencyError log = logging.getLogger('updatebot.update') @@ -173,6 +174,30 @@ if not self._fltrPkg(x.split(':')[0]) ]) + def getBinaryVersions(self, srcTrvSpecs, labels=None): + """ + Find the latest version of all binaries built from the specified + sources. + @param srcTroveSpecs: list of source troves. + @type srcTroveSpecs: [(name, versionObj, None), ... ] + @param labels: list of labels to search, defaults to the buildLabel + @type labels: list(conary.versions.Label, ...) + @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ... ]} + """ + + # Short circuit trove caching if trove list is empty. + if not srcTrvSpecs: + return {} + + # Make sure all sources end in :source + req = [] + for n, v, f in srcTrvSpecs: + if not n.endswith(':source'): + n = '%s:source' % n + req.append((n, v, f)) + + return self._conaryhelper.getBinaryVersions(req, labels=labels) + def _getLatestSource(self, name): """ Get the latest src package for a given package name. @@ -418,6 +443,7 @@ # Update all of the unique sources. fail = set() toBuild = set() + parentPackages = set() for pkg in sorted(toCreate): try: # Only import packages that haven't been imported before @@ -427,7 +453,10 @@ version = self.update((pkg.name, None, None), pkg) if not verCache.get(pkg.name) or buildAll or recreate: - toBuild.add((pkg.name, version, None)) + if self._isPlatformTrove(version): + toBuild.add((pkg.name, version, None)) + else: + parentPackages.add((pkg.name, version, None)) else: log.info('not building %s' % pkg.name) except Exception, e: @@ -440,7 +469,10 @@ for x in pkgs if not self._fltrPkg(x) ] ) - return toBuild, fail + parentPkgMap = self.getBinaryVersions(parentPackages, + labels=self._cfg.platformSearchPath) + + return toBuild, parentPkgMap, fail def _getExistingPackageNames(self): """ @@ -514,6 +546,13 @@ @return version of the updated source trove """ + # Try to use package from a parent platform if the manifests match, + # unless there is already a version on the platform label. + parentVersion = self._getUpstreamPackageVersion(nvf, srcPkg) + if parentVersion: + log.info('using version from parent platform %s' % parentVersion) + return parentVersion + manifest = self._getManifestFromPkgSource(srcPkg) self._conaryhelper.setManifest(nvf[0], manifest) @@ -533,6 +572,61 @@ commitMessage=self._cfg.commitMessage) return newVersion + def _getUpstreamPackageVersion(self, nvf, srcPkg): + """ + Check if a package is maintained as part of an upstream platform. + @param nvf: name, version, flavor tuple of source trove + @type nvf: tuple(name, versionObj, flavorObj) + @param srcPkg: package object for source rpm + @type srcPkg: repomd.packagexml._Package + """ + + # If there is no parent platform search path definied, don't + # bother looking. + if not self._cfg.platformSearchPath: + return None + + srcName = '%s:source' % nvf[0] + + # Check if this package is maintained as part of the current platform. + hasName = self._conaryhelper.findTrove((srcName, None, None)) + + # This is an existing source on the child label + if hasName: + return None + + # Search for source upstream + srcSpec = (srcName, srcPkg.getConaryVersion(), None) + srcTrvs = self._conaryhelper.findTrove(srcSpec, + labels=self._cfg.platformSearchPath) + + # This is a new package on the child label + if not srcTrvs: + return None + + assert len(srcTrvs) == 1 + srcVersion = srcTrvs[0][1] + + manifest = self._getManifestFromPkgSource(srcPkg) + parentManifest = self._conaryhelper.getManifest(nvf[0], + version=srcVersion) + + # FIXME: This assumes that if the rpm filenames are the same the rpm + # contents are the same. + + # Take the basename of all paths in the manifest since the same rpm will + # be in different repositories for each platform. + baseManifest = sorted([ os.path.basename(x) for x in manifest ]) + parentBaseManifest = sorted([ os.path.basename(x) + for x in parentManifest ]) + + if baseManifest != parentBaseManifest: + log.error('found matching parent trove, but manifests differ') + raise ParentPlatformManifestInconsistencyError(srcPkg=srcPkg, + manifest=manifest, parentManifest=parentManifest) + + return srcVersion + def _getManifestFromPkgSource(self, srcPkg): """ Get the contents of the a manifest file from the pkgSource object. @@ -665,3 +759,14 @@ desc=srcPkg.description, shortDesc=srcPkg.summary, ) + + def isPlatformTrove(self, nvf): + """ + Check if the version is on the platform label. + @param nvf: name, version, and flavor of a trove + @type nvf: (str, versionObj, flavorObj) + @return True if the version part is on the build label + @rtype boolean + """ + + return self._conaryhelper.isOnBuildLabel(nvf[1]) From elliot at rpath.com Mon Mar 15 17:16:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:35 +0000 Subject: mirrorball: store return value for later evaluation Message-ID: <201003152116.o2FLGZnY014504@scc.eng.rpath.com> changeset: 5603f4f99839 user: Elliot Peele date: Tue, 09 Feb 2010 18:02:04 -0500 store return value for later evaluation diff --git a/scripts/rhelorder.py b/scripts/rhelorder.py --- a/scripts/rhelorder.py +++ b/scripts/rhelorder.py @@ -58,6 +58,6 @@ def tconv(tstamp): return time.strftime('%m-%d-%Y %H:%M:%S', time.localtime(tstamp)) -bot._errata.sanityCheckOrder() +childPackages, parentPackages = bot._errata.sanityCheckOrder() import epdb; epdb.st() From elliot at rpath.com Mon Mar 15 17:16:36 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:36 +0000 Subject: mirrorball: fix typo Message-ID: <201003152116.o2FLGa7P014531@scc.eng.rpath.com> changeset: 3fb89aa971c2 user: Elliot Peele date: Wed, 10 Feb 2010 11:48:52 -0500 fix typo diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -453,7 +453,7 @@ version = self.update((pkg.name, None, None), pkg) if not verCache.get(pkg.name) or buildAll or recreate: - if self._isPlatformTrove(version): + if self.isPlatformTrove(version): toBuild.add((pkg.name, version, None)) else: parentPackages.add((pkg.name, version, None)) From elliot at rpath.com Mon Mar 15 17:16:38 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:38 +0000 Subject: mirrorball: use only the version Message-ID: <201003152116.o2FLGcT2014558@scc.eng.rpath.com> changeset: 4cb8c814ac27 user: Elliot Peele date: Wed, 10 Feb 2010 13:05:21 -0500 use only the version diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -259,7 +259,7 @@ current[srpm.name] = srpm version = updater.update(nvf, srpm) assert (not version or - not updater.isPlatformTrove((None, version, None))) + not updater.isPlatformTrove(version)) if version: parentPackages.append(((nvf, srpm), version)) else: diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -760,13 +760,13 @@ shortDesc=srcPkg.summary, ) - def isPlatformTrove(self, nvf): + def isPlatformTrove(self, version): """ Check if the version is on the platform label. - @param nvf: name, version, and flavor of a trove - @type nvf: (str, versionObj, flavorObj) + @param version: version of a trove + @type version: versionObj @return True if the version part is on the build label @rtype boolean """ - return self._conaryhelper.isOnBuildLabel(nvf[1]) + return self._conaryhelper.isOnBuildLabel(version) From elliot at rpath.com Mon Mar 15 17:16:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:39 +0000 Subject: mirrorball: ensure pkg group instance exists before trying to access it Message-ID: <201003152116.o2FLGdKq014590@scc.eng.rpath.com> changeset: 56870b92e975 user: Elliot Peele date: Wed, 10 Feb 2010 18:03:38 -0500 ensure pkg group instance exists before trying to access it diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -198,6 +198,9 @@ Remove a given trove from the package group contents. """ + if self._pkgGroupName not in self._groups: + return + return self._groups[self._pkgGroupName].remove(name, missingOk=missingOk) @@ -207,7 +210,8 @@ Check if a given package name is in the group. """ - return name in self._groups[self._pkgGroupName] + return (self._pkgGroupName in self._groups and + name in self._groups[self._pkgGroupName]) def addPackage(self, name, version, flavors): """ From elliot at rpath.com Mon Mar 15 17:16:47 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:47 +0000 Subject: mirrorball: add handling for parent platform labels Message-ID: <201003152116.o2FLGlPT014626@scc.eng.rpath.com> changeset: 19206bba0ee2 user: Elliot Peele date: Thu, 11 Feb 2010 11:10:54 -0500 add handling for parent platform labels diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -452,12 +452,15 @@ # get names and versions troves = set() + labels = set() for pkgKey, pkgData in group.iteritems(): name = str(pkgData.name) version = None if pkgData.version: - version = str(versions.ThawVersion(pkgData.version).asString()) + versionObj = versions.ThawVersion(pkgData.version) + labels.add(versionObj.branch().label()) + version = str(versionObj.asString()) flavor = None # FIXME: At some point we might want to add proper flavor handling, @@ -470,7 +473,8 @@ # Get flavors and such. foundTroves = set([ x for x in - itertools.chain(*self._helper.findTroves(troves).itervalues()) ]) + itertools.chain(*self._helper.findTroves(troves, + labels=labels).itervalues()) ]) # get sources for each name version pair sources = self._helper.getSourceVersions(foundTroves) @@ -504,12 +508,14 @@ # get names and versions troves = set() + labels = set() for pkgKey, pkgData in group.iteritems(): name = str(pkgData.name) version = None if pkgData.version: version = versions.ThawVersion(pkgData.version) + labels.add(version.branch().label()) # get upstream version revision = version.trailingRevision() upstreamVersion = revision.getVersion() @@ -529,7 +535,7 @@ # Get flavors and such. foundTroves = dict([ (x[0], y) for x, y in - self._helper.findTroves(troves).iteritems() ]) + self._helper.findTroves(troves, labels=labels).iteritems() ]) pkgs = {} for pkgKey, pkgData in group.iteritems(): @@ -590,19 +596,23 @@ # get names and versions troves = set() + labels = set() for pkgKey, pkgData in group.iteritems(): name = str(pkgData.name) version = None if pkgData.version: - version = str(versions.ThawVersion(pkgData.version).asString()) + versionObj = versions.ThawVersion(pkgData.version) + labels.add(versionObj.branch().label()) + version = str(versionObj.asString()) flavor = None troves.add((name, version, flavor)) # Get flavors and such. foundTroves = set([ x for x in - itertools.chain(*self._helper.findTroves(troves).itervalues()) ]) + itertools.chain(*self._helper.findTroves(troves, + labels=labels).itervalues()) ]) # get sources for each name version pair sources = self._helper.getSourceVersions(foundTroves) From elliot at rpath.com Mon Mar 15 17:16:48 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:48 +0000 Subject: mirrorball: add cache for getTroveVersionsByLabel results to avoid expensive call Message-ID: <201003152116.o2FLGmqs014657@scc.eng.rpath.com> changeset: e23bc38c1105 user: Elliot Peele date: Thu, 11 Feb 2010 18:44:06 -0500 add cache for getTroveVersionsByLabel results to avoid expensive call diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -86,10 +86,17 @@ self._cacheDir = tempfile.mkdtemp( prefix='conaryhelper-%s-' % cfg.platformName) - # caches source names and versions for binaries past into getSourceVersions. + # caches source names and versions for binaries past into + # getSourceVersions. # binTroveSpec: sourceTroveSpec self._sourceVersionCache = {} + # Keep a cache of all binary versions that have been looked up in + # getBinaryVersions to avoid lots of expensive getTroveVersionsByLabel + # calls. + # frzenset(labels): binTroveNVFSet + self._binaryVerisonCache = {} + def getConaryConfig(self): """ Get a conary config instance. @@ -283,19 +290,32 @@ if not labels: labels = [ self._ccfg.buildLabel, ] - # get all binary trove specs for the specified labels - req = {None: dict([ (x, None) for x in labels ])} - binTrvMap = self._repos.getTroveVersionsByLabel(req) + # Needs to be a frozenset so that it is hashable. + labels = frozenset(labels) - # build a list of the binary troves on the labels - binTrvSpecs = set() - for n, vermap in binTrvMap.iteritems(): - # filter out sources - if n.endswith(':source'): - continue - for v, flvs in vermap.iteritems(): - for f in flvs: - binTrvSpecs.add((n, v, f)) + # FIXME: If this is ever used in a long running process the cache will + # need to be refreshed and/or expired. + + # Check the cache before going on. + if labels in self._binaryVersionCache: + binTrvSpecs = self._binaryVersionCache[labels] + else: + # get all binary trove specs for the specified labels + req = {None: dict([ (x, None) for x in labels ])} + binTrvMap = self._repos.getTroveVersionsByLabel(req) + + # build a list of the binary troves on the labels + binTrvSpecs = set() + for n, vermap in binTrvMap.iteritems(): + # filter out sources + if n.endswith(':source'): + continue + for v, flvs in vermap.iteritems(): + for f in flvs: + binTrvSpecs.add((n, v, f)) + + # Populate cache. + self._binaryVersionCache[labels] = binTrvSpecs # get a map of source trove specs to binary trove specs srcVerMap = self.getSourceVersions(binTrvSpecs) From elliot at rpath.com Mon Mar 15 17:16:50 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:50 +0000 Subject: mirrorball: add logging and fix typo Message-ID: <201003152116.o2FLGooH014685@scc.eng.rpath.com> changeset: 5a21a0eed9c3 user: Elliot Peele date: Thu, 11 Feb 2010 18:44:27 -0500 add logging and fix typo diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -211,8 +211,9 @@ else: parentPackages.add((nvf[0], newVersion, nvf[2])) + log.info('looking up binary versions of all parent platform packages') parentPkgMap = self._updater.getBinaryVersions(parentPackages, - labels=self._platformSearchPath) + labels=self._cfg.platformSearchPath) # Make sure to build everything in the toAdvise list, there may be # sources that have been updated, but not built. From elliot at rpath.com Mon Mar 15 17:16:51 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:51 +0000 Subject: mirrorball: don't try to build if no troves are listed Message-ID: <201003152116.o2FLGpmL014713@scc.eng.rpath.com> changeset: efd121799bbc user: Elliot Peele date: Thu, 11 Feb 2010 18:44:47 -0500 don't try to build if no troves are listed diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -153,6 +153,9 @@ @return troveMap: dictionary of troveSpecs to built troves """ + if not troveSpecs: + return {} + troves = self._formatInput(troveSpecs) jobId = self._startJob(troves) self._monitorJob(jobId, retry=2) @@ -188,6 +191,9 @@ @return troveMap: dictionary of troveSpecs to built troves """ + if not troveSpecs: + return {} + # Split troves by context. jobs = {} for trv in self._formatInput(troveSpecs): From elliot at rpath.com Mon Mar 15 17:16:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:53 +0000 Subject: mirrorball: make sure to return the correct number of items Message-ID: <201003152116.o2FLGroH014740@scc.eng.rpath.com> changeset: 86651af32a48 user: Elliot Peele date: Thu, 11 Feb 2010 18:45:06 -0500 make sure to return the correct number of items diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py --- a/updatebot/build/dispatcher.py +++ b/updatebot/build/dispatcher.py @@ -80,7 +80,7 @@ # Must have at least one trove to build, otherwise will end up in # an infinite loop. if not troveSpecs: - return {} + return {}, self._failures # Sort troves into buckets. troves = self._builder.orderJobs(troveSpecs) @@ -239,6 +239,11 @@ Build all packages in seperate jobs, then commit. """ + # Must have at least one trove to build, otherwise will end up in + # an infinite loop. + if not troveSpecs: + return {}, self._failures + results, self._failures = Dispatcher.buildmany(self, troveSpecs) # Make sure there are no failures. From elliot at rpath.com Mon Mar 15 17:16:54 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:54 +0000 Subject: mirrorball: 1. assume that manifest does not need to be checked on parent platform troves. Message-ID: <201003152116.o2FLGsL3014768@scc.eng.rpath.com> changeset: 717e75f70c2d user: Elliot Peele date: Thu, 11 Feb 2010 18:46:56 -0500 1. assume that manifest does not need to be checked on parent platform troves. 2. lookup pre built troves when creating a platform now that there is a mechanism for handling prebuilt troves diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -256,7 +256,7 @@ else: if metadata is None: pkgs = self._getMetadataFromConaryRepository(nvf[0], - verison=nvf[1]) + version=nvf[1]) metadata = util.Metadata(pkgs) if metadata: binPkg = metadata.locationMap[line] @@ -338,8 +338,18 @@ @type srpm: repomd.packagexml._Package """ - nvflst = self._conaryhelper.findTrove(('%s:source' % srpm.name, - srpm.getConaryVersion(), None)) + srcQuery = ('%s:source' % srpm.name, srpm.getConaryVersion(), None) + nvflst = self._conaryhelper.findTrove(srcQuery) + + # If this package was not found on the platform label, check if there + # are parent platforms involved. + if not nvflst and self._cfg.platformSearchPath: + nvflst = self._conaryhelper.findTrove(srcQuery, + labels=self._cfg.platformSearchPath) + + # Trust that the parent platform has sanity checked this source. + if nvflst: + return None assert len(nvflst) == 1 n, v, f = nvflst[0] @@ -433,7 +443,7 @@ # Skip all components, including sources if len(n.split(':')) > 1: continue - # If binary is in the group and the verison on the label is the + # If binary is in the group and the version on the label is the # same it needs to be updated. if n in nv and v == nv[n][0] and nv[n][1] in toCreateMap: create.add(toCreateMap[nv[n][1]]) @@ -443,6 +453,7 @@ # Update all of the unique sources. fail = set() toBuild = set() + preBuiltPackages = set() parentPackages = set() for pkg in sorted(toCreate): try: @@ -459,7 +470,9 @@ parentPackages.add((pkg.name, version, None)) else: log.info('not building %s' % pkg.name) + preBuiltPackages.add((pkg.name, version, None)) except Exception, e: + raise log.error('failed to import %s: %s' % (pkg, e)) fail.add((pkg, e)) @@ -469,10 +482,28 @@ for x in pkgs if not self._fltrPkg(x) ] ) + # Find all of the binaries that match the upstream platform sources. + log.info('looking up binary versions of all parent platform packages') parentPkgMap = self.getBinaryVersions(parentPackages, labels=self._cfg.platformSearchPath) - return toBuild, parentPkgMap, fail + # Find all of the binaries that match the pre-built sources. + log.info('looking up binary version information for all prebuilt ' + 'packages') + preBuiltPackageMap = self.getBinaryVersions(preBuiltPackages) + + # Combine the two package maps by name where pre built packages + # override parent packages. + parentNames = dict([ (x[0], x) for x in parentPkgMap ]) + preBuiltNames = dict([ (x[0], x) for x in preBuiltPackageMap ]) + + parentNames.update(preBuiltNames) + parentPkgMap.update(preBuiltPackageMap) + + pkgMap = dict([ (parentNames[x], parentPkgMap[parentNames[x]]) + for x in parentNames ]) + + return toBuild, pkgMap, fail def _getExistingPackageNames(self): """ From elliot at rpath.com Mon Mar 15 17:16:56 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:56 +0000 Subject: mirrorball: be more informative when checking manifests Message-ID: <201003152116.o2FLGvv0014799@scc.eng.rpath.com> changeset: 60d7c33f3f97 user: Elliot Peele date: Tue, 16 Feb 2010 14:34:22 -0500 be more informative when checking manifests diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -190,7 +190,7 @@ for advisory, advInfo in changed.iteritems(): log.info('validating %s' % advisory) for srpm in advInfo['srpms']: - log.info(srpm) + log.info('checking %s' % srpm.name) # This will raise an exception if any inconsistencies are # detected. self._updater.sanityCheckSource(srpm) From elliot at rpath.com Mon Mar 15 17:16:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:58 +0000 Subject: mirrorball: Don't try to thaw None Message-ID: <201003152116.o2FLGwYk014826@scc.eng.rpath.com> changeset: 4a4db0aaff78 user: Elliot Peele date: Thu, 18 Feb 2010 15:30:42 -0500 Don't try to thaw None diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -98,7 +98,12 @@ log.info('restoring package map from file: %s' % fn) thawVersion = versions.ThawVersion - thawFlavor = deps.ThawFlavor + + def thawFlavor(flv): + if flv is None: + return flv + else: + return deps.ThawFlavor(flv) # load pickle frzPkgs = pickle.load(open(fn)) From elliot at rpath.com Mon Mar 15 17:16:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:16:59 +0000 Subject: mirrorball: add multi package checkout, replacing conary's checkin.checkout Message-ID: <201003152116.o2FLGx6U014855@scc.eng.rpath.com> changeset: 2087a8adb42f user: Elliot Peele date: Thu, 18 Feb 2010 15:50:53 -0500 add multi package checkout, replacing conary's checkin.checkout use our findTroves for findTrove fix typos diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -21,9 +21,11 @@ import time import logging import tempfile +import itertools import conary from conary import trove +from conary import state from conary import checkin from conary.build import use from conary import conarycfg @@ -95,7 +97,7 @@ # getBinaryVersions to avoid lots of expensive getTroveVersionsByLabel # calls. # frzenset(labels): binTroveNVFSet - self._binaryVerisonCache = {} + self._binaryVersionCache = {} def getConaryConfig(self): """ @@ -105,17 +107,17 @@ return self._ccfg - def findTrove(self, nvf, labels=None, *args, **kwargs): + def findTrove(self, nvf, *args, **kwargs): """ Mapped to conaryclient.repos.findTrove. Will always search buildLabel. """ - if not labels: - labels = self._ccfg.buildLabel + trvs = self.findTroves([ nvf, ], *args, **kwargs) + assert len(trvs) in (0, 1) - try: - return self._repos.findTrove(labels, nvf, *args, **kwargs) - except conaryerrors.TroveNotFound, e: + if trvs: + return trvs.values()[0] + else: return [] def findTroves(self, troveList, labels=None, *args, **kwargs): @@ -637,8 +639,9 @@ log.info('checking out %s' % troveSpec) - recipeDir = self._getRecipeDir(pkgname) - checkin.checkout(self._repos, self._ccfg, recipeDir, [troveSpec, ]) + req = (pkgname, version, None) + coMap = self._multiCheckout([ req, ]) + recipeDir = coMap[req] return recipeDir @@ -982,3 +985,131 @@ metadata = [ (x, mi) for x in trvSpecs ] self._repos.addMetadataItems(metadata) + + def _multiCheckout(self, trvSpecs): + """ + Checkout several sources at once and return a map of source trove spec + to checkout directory. + @param trvSpecs: list of sources to checkout. + @type trvSpecs: list((name, version, None), ...) + @return map of source requests to checkout directories + @rtype dict((name, version, None)=/path/to/checkout) + """ + + # NOTE: This code is strongly based off of conary.checkin._checkout, + # which should possibly be refactored to allow something like + # this. + + # Make sure everything is :source. + req = set() + reqMap = {} + for n, v, f in trvSpecs: + nvf = (n, v, f) + if not n.endswith(':source'): + n += ':source' + req.add((n, v, f)) + reqMap[(n, v, f)] = nvf + + # Find all requested versions. + trvMap = self._repos.findTroves(self._ccfg.buildLabel, req) + + # Build rev map for later lookups. + revTrvMap = dict([ (y[0], x) for x, y in trvMap.iteritems() ]) + + # Get the list of results from the findTroves query. + trvList = [ x for x in itertools.chain(*trvMap.itervalues()) ] + + # Build a changeset request. + csJob = [ (x[0], (None, None), (x[1], x[2]), True) for x in trvList ] + + callback = checkin.CheckinCallback( + trustThreshold=self._ccfg.trustThreshold) + + # Request a changeset with all of the sources except for files that are + # autosourced. + cs = self._repos.createChangeSet(csJob, + excludeAutoSource=True, + callback=callback) + checkin.verifyAbsoluteChangesetSignatures(cs, callback) + + pathMap = {} + checkoutMap = {} + sourceStateMap = {} + conaryStateTargets = {} + + # Prepare to unpack sources from the changeset. + for nvf in trvList: + troveCs = cs.getNewTroveVersion(*nvf) + trv = trove.Trove(troveCs) + + # Create target directory + targetDir = self._getRecipeDir(nvf[0]) + checkoutMap[reqMap[revTrvMap[nvf]]] = targetDir + + # Store source state + sourceState = state.SourceState(nvf[0], nvf[1], nvf[1].branch()) + sourceStateMap[trv.getNameVersionFlavor()] = sourceState + + # Store conary state + conaryState = state.ConaryState(self._ccfg.context, sourceState) + conaryStateTargets[targetDir] = conaryState + + # Store factory info + if trv.getFactory(): + sourceState.setFactory(trv.getFactory()) + + # Extract file info from changeset + for (pathId, path, fileId, version) in troveCs.getNewFileList(): + pathMap[(nvf, path)] = (targetDir, pathId, fileId, version) + + # Explode changeset contents. + checkin.CheckoutExploder(cs, pathMap, sourceStateMap) + + # Write out CONARY state files. + for targetDir, conaryState in conaryStateTargets.iteritems(): + conaryState.write(targetDir + '/CONARY') + + return checkoutMap + + def cacheSources(self, label, latest=True): + """ + Checkout all sources on a label and add them to the checkout cache. This + is significantly more efficient than requesting one sources at a time + when we know ahead of time that we will fetching all or almost all + sources. + @param label: label to search for sources. + @type label: conary.versions.Label + @param latest: optional, if True check only latest versions rather + than all versions. + @type latest: boolean + @return map of source nvf to checkout directories + @rtype dict((name, version, None)=/path/to/checkout) + """ + + log.info('caching %s sources for %s' + % (latest and 'latest' or 'all', label)) + + # Find correct query function + if latest: + query = self._repos.getTroveLeavesByLabel + else: + query = self._repos.getTroveVersionsByLabel + + trvMap = query({None: {label: None}}) + + srcSet = set() + for n, verDict in trvMap.iteritems(): + # skip anything that isn't a source + if not n.endswith(':source'): + continue + for v, flvs in verDict.iteritems(): + srcSet.add((n, v, None)) + + coMap = self._multiCheckout(srcSet) + + # Update the checkout cache. + self._checkoutCache.update(coMap) + self._checkoutCache.update(dict([ (y, x) + for x, y in coMap.iteritems() ])) + + return coMap From elliot at rpath.com Mon Mar 15 17:17:01 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:01 +0000 Subject: mirrorball: prefetch platform label source checkouts in one repository call Message-ID: <201003152117.o2FLH1du014882@scc.eng.rpath.com> changeset: 643fc35b6ef8 user: Elliot Peele date: Thu, 18 Feb 2010 15:51:24 -0500 prefetch platform label source checkouts in one repository call diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -191,6 +191,11 @@ updater = update.Updater(self._cfg, pkgSource) updater._conaryhelper = _ConaryHelperShim(self._cfg) + if self._cfg.platformSearchPath: + log.info('prefetching sources for parent platform labels') + for label in self._cfg.platformSearchPath: + updater._conaryhelper.cacheSources(label, latest=False) + # build a mapping of srpm to bucketId for debuging purposes srpmToBucketId = {} for bucketId, srpms in self._order.iteritems(): From elliot at rpath.com Mon Mar 15 17:17:02 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:02 +0000 Subject: mirrorball: add findtroves cache for prefetching all results for a label Message-ID: <201003152117.o2FLH2jN014914@scc.eng.rpath.com> changeset: 420c80878a42 user: Elliot Peele date: Mon, 22 Feb 2010 17:49:38 -0500 add findtroves cache for prefetching all results for a label diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -831,6 +831,61 @@ def __init__(self, cfg): conaryhelper.ConaryHelper.__init__(self, cfg) self._client = None + self._findTrovesCache = {} + + @staticmethod + def _getCacheKey(nvf): + n, v, f = nvf + if v and hasattr(v, 'asString'): + v = v.asString() + return (n, v) + + def populateFindTrovesCache(self, labels): + """ + Pre populate the find troves cache with all versions from all labels + listed. + """ + + req = { None: dict((x, None) for x in labels) } + trvs = self._repos.getTroveVersionsByLabel(req) + + for n, vd in trvs.iteritems(): + for v, fs in vd.iteritems(): + key = self._getCacheKey((n, v, None)) + self._findTrovesCache[key] = [ (n, v, f) for f in fs ] + + def findTroves(self, troveList, labels=None, *args, **kwargs): + """ + Aggresivly cache all findTroves queries as they are not likely to change + while sanity checking. + """ + + if not labels: + return [] + + # Find any requests that are already cached. + cached = set([ x for x in troveList + if self._getCacheKey(x) in self._findTrovesCache ]) + + # Filter out cached requests. + needed = set(troveList) - cached + + # Query for new requets. + if needed: + trvs = conaryhelper.ConaryHelper.findTroves(self, needed, + labels=None, *args, **kwargs) + else: + trvs = {} + + # Cache new requests. + self._findTrovesCache.update(dict([ (self._getCacheKey(x), y) + for x, y in trvs.iteritems() ])) + + # Pull results out of the cache. + res = dict([ (x, self._findTrovesCache[self._getCacheKey(x)]) + for x in troveList ]) + + return res def getLatestSourceVersion(self, pkgname): """ From elliot at rpath.com Mon Mar 15 17:17:04 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:04 +0000 Subject: mirrorball: adapt errata processing to new capsule indexer interface Message-ID: <201003152117.o2FLH4I0014942@scc.eng.rpath.com> changeset: 2859070196b7 user: Elliot Peele date: Wed, 24 Feb 2010 15:23:23 -0500 adapt errata processing to new capsule indexer interface add config option for ignoring errata errors diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -377,6 +377,10 @@ # srpm are not obsoleted, so we cannot use removeSource removeObsoleted = (CfgIntDict(CfgList(CfgString)), {}) + # List of broken errata that have been researched and should be ignored + # when reporting errors. + brokenErrata = (CfgList(CfgString), []) + class UpdateBotConfig(cfg.SectionedConfigFile): """ diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -712,7 +712,7 @@ """ # convert nevra to yum compatible nevra - nevra = list(pkg.getNevra()) + nevra = list(pkg.nevra.getNevra()) if nevra[1] is None: nevra[1] = '0' if type(nevra[1]) == int: @@ -750,21 +750,19 @@ allocated = [] bucketId = None #log.info('processing %s' % e.advisory) - for pkg in e.packages: - nevra = self._getNevra(pkg) - # ignore arches we don't know about. - if nevra[4] not in arches: - continue + # Get unique list of nevras for which we have packages indexed and + # are of a supported arch. + errataNevras = set([ self._getNevra(x) for x in e.nevraChannels + if x.channel.label in indexedChannels and + x.nevra.arch in arches ]) - # filter out channels we don't have indexed - channels = set([ x.label for x in pkg.channels ]) - if not indexedChannels & channels: - continue - + for nevra in errataNevras: # add package to advisory package map - self._advPkgMap.setdefault(e.advisory, set()).add(sources[nevra]) - self._advPkgRevMap.setdefault(sources[nevra], set()).add(e.advisory) + self._advPkgMap.setdefault(e.advisory, + set()).add(sources[nevra]) + self._advPkgRevMap.setdefault(sources[nevra], + set()).add(e.advisory) # move nevra to errata buckets if nevra in nevras: @@ -784,8 +782,12 @@ # already be in an existing bucket (bucketId != None), if there # aren't the errata store is probably broken. if not bucket and bucketId is None: - broken.append(e.advisory) - log.critical('broken advisory: %s' % e.advisory) + if e.advisory in self._cfg.brokenErrata: + msg = log.warn + else: + broken.append(e.advisory) + msg = log.critical + msg('broken advisory: %s' % e.advisory) continue if bucketId is None: From elliot at rpath.com Mon Mar 15 17:17:06 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:06 +0000 Subject: mirrorball: fix findtroves caching bugs Message-ID: <201003152117.o2FLH6Dw014970@scc.eng.rpath.com> changeset: 5fc203a28eb4 user: Elliot Peele date: Fri, 26 Feb 2010 11:58:21 -0500 fix findtroves caching bugs add support for handling broken errata that do not have packages associated with them diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -379,7 +379,7 @@ # List of broken errata that have been researched and should be ignored # when reporting errors. - brokenErrata = (CfgList(CfgString), []) + brokenErrata = (CfgDict(CfgList(CfgNevra)), {}) class UpdateBotConfig(cfg.SectionedConfigFile): diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -195,6 +195,10 @@ log.info('prefetching sources for parent platform labels') for label in self._cfg.platformSearchPath: updater._conaryhelper.cacheSources(label, latest=False) + log.info('prefetching findTroves information for parent platform ' + 'labels') + updater._conaryhelper.populateFindTrovesCache( + self._cfg.platformSearchPath) # build a mapping of srpm to bucketId for debuging purposes srpmToBucketId = {} @@ -501,16 +505,38 @@ # insert bins by buildstamp extras = {} + + # Build a reverse map of broken errata so that we can match packages + # and advisories + nevraAdvMap = {} + for adv, nevras in self._cfg.brokenErrata.iteritems(): + for nevra in nevras: + assert nevra not in nevraAdvMap + nevraAdvMap[nevra] = adv + + # Build reverse map of advisory to bucketId. + advRevMap = {} + for bucketId, advInfoList in self._advMap.iteritems(): + for advInfo in advInfoList: + advDict = dict(advInfo) + assert advDict['name'] not in advRevMap + advRevMap[advDict['name']] = bucketId + for src, bins in srcMap.iteritems(): # Pull out any package sets that look like they are incomplete. if len(bins) != len(self._pkgSource.srcPkgMap[src]) - 1: extras[src] = bins continue + if src.getNevra() in nevraAdvMap: + advisory = nevraAdvMap[src.getNevra()] + bucketId = advRevMap[advisory] + log.info('inserting %s for advisory %s into bucket %s' % (src, advisory, bucketId)) + buckets.setdefault(bucketId, set()).add(src) + continue + buildstamp = int(sorted(bins)[0].buildTimestamp) - if buildstamp not in buckets: - buckets[buildstamp] = [] - buckets[buildstamp].extend(bins) + buckets.setdefault(buildstamp, set()).update(set(bins)) # get sources to build srpmToBucketId = {} @@ -788,7 +814,6 @@ broken.append(e.advisory) msg = log.critical msg('broken advisory: %s' % e.advisory) - continue if bucketId is None: bucketId = int(time.mktime(time.strptime(e.issue_date, @@ -838,8 +863,8 @@ @staticmethod def _getCacheKey(nvf): n, v, f = nvf - if v and hasattr(v, 'asString'): - v = v.asString() + if v and hasattr(v, 'trailingRevision'): + v = v.trailingRevision().getVersion() return (n, v) def populateFindTrovesCache(self, labels): @@ -874,17 +899,20 @@ # Query for new requets. if needed: + #log.critical('CACHE MISS') + #log.critical('request: %s' % troveList) trvs = conaryhelper.ConaryHelper.findTroves(self, needed, - labels=None, *args, **kwargs) + labels=labels, *args, **kwargs) else: + #log.info('CACHE HIT') trvs = {} # Cache new requests. self._findTrovesCache.update(dict([ (self._getCacheKey(x), y) - for x, y in trvs.iteritems() ])) + for x, y in trvs.iteritems() ])) # Pull results out of the cache. - res = dict([ (x, self._findTrovesCache[self._getCacheKey(x)]) + res = dict([ (x, self._findTrovesCache.get(self._getCacheKey(x), [])) for x in troveList ]) return res From elliot at rpath.com Mon Mar 15 17:17:07 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:07 +0000 Subject: mirrorball: only use checksum for checking if something is equal, not for sorting Message-ID: <201003152117.o2FLH7dA014997@scc.eng.rpath.com> changeset: 295a6d6cee7d user: Elliot Peele date: Fri, 26 Feb 2010 11:58:54 -0500 only use checksum for checking if something is equal, not for sorting diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -155,6 +155,11 @@ if pkgcmp != 0: return pkgcmp + if (self.checksum and other.checksum and + self.checksumType == other.checksumType and + self.checksum == other.checksum): + return 0 + return cmp(self.location, other.location) def getNevra(self): From elliot at rpath.com Mon Mar 15 17:17:08 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:08 +0000 Subject: mirrorball: fix formatting Message-ID: <201003152117.o2FLH8bm015025@scc.eng.rpath.com> changeset: 8ea8f1493271 user: Elliot Peele date: Fri, 26 Feb 2010 13:27:44 -0500 fix formatting diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -531,7 +531,8 @@ if src.getNevra() in nevraAdvMap: advisory = nevraAdvMap[src.getNevra()] bucketId = advRevMap[advisory] - log.info('inserting %s for advisory %s into bucket %s' % (src, advisory, bucketId)) + log.info('inserting %s for advisory %s into bucket %s' + % (src, advisory, bucketId)) buckets.setdefault(bucketId, set()).add(src) continue From elliot at rpath.com Mon Mar 15 17:17:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:10 +0000 Subject: mirrorball: allow for requests for errata state of different versions Message-ID: <201003152117.o2FLHAMN015056@scc.eng.rpath.com> changeset: afd30006a802 user: Elliot Peele date: Fri, 26 Feb 2010 13:30:57 -0500 allow for requests for errata state of different versions diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -374,13 +374,16 @@ self._helper.setErrataState(self._sourceName, state) @checkout - def getErrataState(self): + def getErrataState(self, version=None): """ Get the errata state info. """ + if version is None: + version = self._sourceVersion + return self._helper.getErrataState(self._sourceName, - version=self._sourceVersion) + version=version) def getVersions(self, pkgSet): """ From elliot at rpath.com Mon Mar 15 17:17:11 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:11 +0000 Subject: mirrorball: dump cached errata state when done Message-ID: <201003152117.o2FLHC1G015081@scc.eng.rpath.com> changeset: 742c68619f33 user: Elliot Peele date: Sun, 28 Feb 2010 22:38:26 -0500 dump cached errata state when done diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -848,6 +848,9 @@ buckets[0] = golden + # Dump cached errata results once we are done with them. + self._errata.cleanup() + return buckets, other From elliot at rpath.com Mon Mar 15 17:17:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:13 +0000 Subject: mirrorball: add script for syncing centos vault mirror Message-ID: <201003152117.o2FLHD34015112@scc.eng.rpath.com> changeset: 160abbe4086d user: Elliot Peele date: Mon, 01 Mar 2010 16:37:29 -0500 add script for syncing centos vault mirror diff --git a/scripts/sync-centos.sh b/scripts/sync-centos-vault.sh copy from scripts/sync-centos.sh copy to scripts/sync-centos-vault.sh --- a/scripts/sync-centos.sh +++ b/scripts/sync-centos-vault.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,8 +13,8 @@ # full details. # -SOURCE=rsync://mirrors.us.kernel.org/CentOS-nodvd -DEST=/l/CentOS/ +SOURCE=rsync://archive.kernel.org/centos-vault +DEST=/l/CentOS-vault/ date rsync -arv --progress --bwlimit=700 --exclude 2* --exclude 3* $SOURCE $DEST From elliot at rpath.com Mon Mar 15 17:17:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:15 +0000 Subject: mirrorball: add config options for parent platform group information and using old Message-ID: <201003152117.o2FLHFQf015137@scc.eng.rpath.com> changeset: 65fcd071c2df user: Elliot Peele date: Wed, 03 Mar 2010 11:36:03 -0500 add config options for parent platform group information and using old versions of parent troves diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -197,6 +197,9 @@ # The top level source group. topSourceGroup = CfgTroveSpec + # Parent top level source group + topParentSourceGroup = CfgTroveSpec + # Path to search for packages to be included in the platform. platformSearchPath = (CfgQuotedLineList(CfgLabel), []) @@ -381,6 +384,11 @@ # when reporting errors. brokenErrata = (CfgDict(CfgList(CfgNevra)), {}) + # Dictionary of updateId to list of trove specs. When the bucketId has been + # reached, update to the version specified in the trovespec rather than the + # latest that matches the current rpm version. + useOldVersion = (CfgIntDict(CfgList(CfgTroveSpec)), {}) + class UpdateBotConfig(cfg.SectionedConfigFile): """ From elliot at rpath.com Mon Mar 15 17:17:17 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:17 +0000 Subject: mirrorball: 1. provide option to not always pull the latest versions when querying for Message-ID: <201003152117.o2FLHHhi015171@scc.eng.rpath.com> changeset: 7be079982d08 user: Elliot Peele date: Wed, 03 Mar 2010 11:38:46 -0500 1. provide option to not always pull the latest versions when querying for binaries 2. add method for retreiving a mapping of source trove spec to list of related binary trove specs from a single binary trove spec diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -277,7 +277,7 @@ return srcTrvs - def getBinaryVersions(self, srcTroveSpecs, labels=None): + def getBinaryVersions(self, srcTroveSpecs, labels=None, latest=True): """ Given a list of source trove specs, find the latest versions of all binaries generated from these sources. @@ -285,6 +285,8 @@ @type srcTroveSpecs: [(name, versionObj, None), ... ] @param labels: list of labels to search, defaults to the buildLabel @type labels: list(conary.versions.Label, ...) + @param latest: get only the latest binaries. + @type latest: boolean @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ... ]} """ @@ -326,7 +328,7 @@ for srcTrv in srcTroveSpecs: if srcTrv not in srcVerMap: log.error('can not find requested source trove in repository') - raise BinariesNotFoundForSourceVersion(srcName=srvTrv[0], + raise BinariesNotFoundForSourceVersion(srcName=srcTrv[0], srcVersion=srcTrv[1]) # Move binaries into a more convienient data structure. @@ -335,21 +337,45 @@ for binTrv in binTrvs: binTrvMap.setdefault(binTrv[1], set()).add(binTrv) - # find the latest version. - latest = None - for binVer in binTrvMap: - if latest is None: - latest = binVer - continue - if latest < binVer: - latest = binVer + if latest: + # find the latest version. + latestVer = None + for binVer in binTrvMap: + if latestVer is None: + latestVer = binVer + continue + if latestVer < binVer: + latestVer = binVer - # Store latest binary versions in source version map - assert srcTrv not in srcMap - srcMap[srcTrv] = binTrvMap[latest] + # Store latest binary versions in source version map + assert srcTrv not in srcMap + srcMap[srcTrv] = binTrvMap[latestVer] + else: + srcMap[srcTrv] = [ x for x in + itertools.chain(*binTrvMap.values()) ] return srcMap + def getSourceVersionMapFromBinaryVersion(self, (n, v, f), labels=None, + latest=False): + """ + Find a mapping of source to binaries, given a single binary name, + version, and flavor. + @param nvf: binary name, version, and flavor + @type nvf: tuple(name, versionObj, falvorObj) + @param labels: list of labels to search, defaults to buildLabel + @type labels: list(conary.versions.Label, ...) + @param latest: check for only the latest versions or not + @type latest: boolean + @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ...]} + """ + + trvs = self.findTrove((n, v, f), labels=labels) + srcVersions = self.getSourceVersions(trvs) + srcSpecs = srcVersions.keys() + srcMap = self.getBinaryVersions(srcSpecs, labels=labels, latest=latest) + return srcMap + @staticmethod def _getTrove(cs, name, version, flavor): """ From elliot at rpath.com Mon Mar 15 17:17:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:18 +0000 Subject: mirrorball: map helper method for retrieiving source map from a single binary Message-ID: <201003152117.o2FLHJwE015200@scc.eng.rpath.com> changeset: 97bdc3204fd9 user: Elliot Peele date: Wed, 03 Mar 2010 11:39:21 -0500 map helper method for retrieiving source map from a single binary diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -198,6 +198,23 @@ return self._conaryhelper.getBinaryVersions(req, labels=labels) + def getSourceVersionMapFromBinaryVersion(self, (n, v, f), labels=None, + latest=False): + """ + Find a mapping of source to binaries, given a single binary name, + version, and flavor. + @param nvf: binary name, version, and flavor + @type nvf: tuple(name, versionObj, falvorObj) + @param labels: list of labels to search, defaults to buildLabel + @type labels: list(conary.versions.Label, ...) + @param latest: check for only the latest versions or not + @type latest: boolean + @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ...]} + """ + + return self._conaryhelper.getSourceVersionMapFromBinaryVersion( + (n, v, f), labels=labels, latest=latest) + def _getLatestSource(self, name): """ Get the latest src package for a given package name. From elliot at rpath.com Mon Mar 15 17:17:20 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:20 +0000 Subject: mirrorball: 1. add support for querying parent group information Message-ID: <201003152117.o2FLHKLw015227@scc.eng.rpath.com> changeset: bd19722a50a3 user: Elliot Peele date: Wed, 03 Mar 2010 11:40:11 -0500 1. add support for querying parent group information 2. add exceptions to sanity checking when old versions are used diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -57,6 +57,9 @@ def commit(func): def wrapper(self, *args, **kwargs): + if self._readonly: + return + if self._checkedout: self._commit() @@ -69,15 +72,29 @@ Manage group of all packages for a platform. """ - def __init__(self, cfg): + def __init__(self, cfg, parentGroup=False): self._cfg = cfg self._helper = GroupHelper(self._cfg) self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') #self._versionFactory = VersionFactory(cfg) - self._sourceName = self._cfg.topSourceGroup[0] - self._sourceVersion = None + if parentGroup: + topGroup = list(self._cfg.topParentSourceGroup) + topGroup[0] = '%s:source' % topGroup[0] + trvs = self._helper.findTrove(tuple(topGroup), + labels=self._cfg.platformSearchPath) + + assert len(trvs) + + self._sourceName = topGroup[0] + self._sourceVersion = trvs[0][1] + + self._readonly = True + else: + self._sourceName = self._cfg.topSourceGroup[0] + self._sourceVersion = None + self._readonly = False self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName @@ -107,8 +124,10 @@ if copyToLatest: log.info('copying information to latest version') # Get data from the old versoin - version = self._helper.getVersion(self._sourceName, version=self._sourceVersion) - errataState = self._helper.getErrataState(self._sourceName, version=self._sourceVersion) + version = self._helper.getVersion(self._sourceName, + version=self._sourceVersion) + errataState = self._helper.getErrataState(self._sourceName, + version=self._sourceVersion) groups = self._groups log.info('version: %s' % version) @@ -554,6 +573,16 @@ assert len(pkgs) == len(foundTroves) + # Get all old versions so that we can make sure any version conflicts + # were introduced by old version handling. + oldVersions = set() + for nvfLst in self._cfg.useOldVersion.itervalues(): + for nvf in nvfLst: + srcMap = self._helper.getSourceVersionMapFromBinaryVersion(nvf, + labels=self._cfg.platformSearchPath, latest=False) + oldVersions |= set(itertools.chain(*srcMap.itervalues())) + + errors = {} for name, found in foundTroves.iteritems(): assert name in pkgs @@ -575,6 +604,10 @@ assert n == cn if v != cv: + if (n, v, f) in oldVersions: + log.info('found %s=%s[%s] in oldVersions exceptions' + % (n, v, f)) + continue foundError = True if foundError: From elliot at rpath.com Mon Mar 15 17:17:21 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:21 +0000 Subject: mirrorball: 1. add support for fencing updates on parent platform Message-ID: <201003152117.o2FLHM6a015254@scc.eng.rpath.com> changeset: 5a98da154763 user: Elliot Peele date: Wed, 03 Mar 2010 11:43:35 -0500 1. add support for fencing updates on parent platform 2. add handling for usong old versions of packages from a parent platform diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -48,6 +48,10 @@ errataSource) self._groupmgr = groupmgr.GroupManager(self._cfg) + if self._cfg.platformSearchPath: + self._parentGroup = groupmgr.GroupManager(self._cfg, + parentGroup=True) + def _addPackages(self, pkgMap): """ Add pkgMap to group. @@ -206,6 +210,18 @@ detail = self._errata.getUpdateDetailMessage(updateId) log.info('attempting to apply %s' % detail) + # If on a derived platform and the current updateId is greater than + # the parent updateId, stop applying updates. + if self._cfg.platformSearchPath: + # FIXME: This means that if there is an update the the child + # platform that is not included in the parent platform, + # we will not apply the update until there is a later + # update to the parent platform. + parentState = self._parentGroup.getErrataState() + if parentState < updateId: + log.info('reached end of parent platform update stream') + continue + # remove packages from config removePackages = self._cfg.updateRemovesPackages.get(updateId, []) removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) @@ -234,6 +250,30 @@ pkgMap = self._update(*args, updatePkgs=updates, expectedRemovals=expectedRemovals, **kwargs) + # When deriving from an upstream platform sometimes we don't want + # the latest versions. + oldVersions = self._cfg.useOldVersion.get(updateId, None) + if self._cfg.platformSearchPath and oldVersions: + for nvf in oldVersions: + # Lookup all source and binaries that match this binary. + srcMap = self._updater.getSourceVersionMapFromBinaryVersion( + nvf, labels=self._cfg.platformSearchPath, latest=False) + + # Make sure there is only one + assert len(srcMap) == 1 + + # Filter out any versions that don't match the version we + # are looking for. + curVerMap = dict((x, [ z for z in y + if z[1].asString() == nvf[1] ]) + for x, y in srcMap.iteritems()) + + # Make sure the version we are looking for is in the list + assert curVerMap and curVerMap.values()[0] + + # Update the package map with the new versions. + pkgMap.update(curVerMap) + # Save package map in case we run into trouble later. self._savePackages(pkgMap) From elliot at rpath.com Mon Mar 15 17:17:23 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:23 +0000 Subject: mirrorball: add module for forking a process and watching for a result Message-ID: <201003152117.o2FLHNSn015286@scc.eng.rpath.com> changeset: 13a98b49efd9 user: Elliot Peele date: Thu, 04 Mar 2010 16:58:19 -0500 add module for forking a process and watching for a result diff --git a/updatebot/lib/watchdog.py b/updatebot/lib/watchdog.py new file mode 100644 --- /dev/null +++ b/updatebot/lib/watchdog.py @@ -0,0 +1,73 @@ +# +# Copyright (c) 2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Watch and wait for processes to fail and then restart them. +""" + +import os +import logging + +log = logging.getLogger('updatebot.lib.watchdog') + +def runOnce(func, funcArgs): + """ + Fork and run a function. + """ + + pid = os.fork() + if not pid: + log.info('started %s(pid %s)' % (func.__name__, os.getpid())) + args, kwargs = funcArgs + rc = func(*args, **kwargs) + os._exit(rc) + else: + return pid + +def watchOnce(func, funcArgs): + """ + Fork and wait for a function to complete. + """ + + pid = runOnce(func, funcArgs) + + log.info('waiting for %s' % pid) + pid, status = os.waitpid(pid, 0) + if os.WIFEXITED(status): + rc = os.WEXITSTATUS(status) + return rc + +def watch(func, funcArgs): + """ + Run a forked function in a loop. + """ + + while True: + log.info('looping %s' % func.__name__) + rc = watchOnce(func, funcArgs) + if not rc: + log.info('completed function loop') + break + + return rc + +def forknwatch(func): + """ + Decorator to run a function in a forked process in a loop. + """ + + def wrapper(*args, **kwargs): + funcArgs = (args, kwargs) + return watch(func, funcArgs) + return wrapper From elliot at rpath.com Mon Mar 15 17:17:25 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:25 +0000 Subject: mirrorball: use __slots__ properly Message-ID: <201003152117.o2FLHPno015315@scc.eng.rpath.com> changeset: f40b582d25b8 user: Elliot Peele date: Mon, 08 Mar 2010 15:00:43 -0500 use __slots__ properly diff --git a/repomd/packagexml.py b/repomd/packagexml.py --- a/repomd/packagexml.py +++ b/repomd/packagexml.py @@ -181,25 +181,19 @@ return ver -class _RpmEntry(xmllib.BaseNode): +class _RpmEntry(SlotNode): """ Parse any element that contains rpm:entry or suse:entry elements. """ + __slots__ = () + def addChild(self, child): """ Parse rpm:entry and suse:entry nodes. """ if child.getName() in ('rpm:entry', 'suse:entry'): - child.kind = None - child.name = None - child.epoch = None - child.version = None - child.release = None - child.flags = None - child.pre = None - for attr, value in child.iterAttributes(): if attr == 'kind': child.kind = value @@ -217,64 +211,91 @@ child.pre = value else: raise UnknownAttributeError(child, attr) - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) else: raise UnknownElementError(child) +class _RpmEntries(SlotNode): + """ + Class to represent all rpm:entry and suse:entry types. + """ + + __slots__ = ('kind', 'name', 'epoch', 'version', 'release', 'flags', + 'pre', ) + + class _RpmRequires(_RpmEntry): """ Parse rpm:requires children. """ + __slots__ = () + class _RpmRecommends(_RpmEntry): """ Parse rpm:recommends children. """ + __slots__ = () + class _RpmProvides(_RpmEntry): """ Parse rpm:provides children. """ + __slots__ = () + class _RpmObsoletes(_RpmEntry): """ Parse rpm:obsoletes children. """ + __slots__ = () + class _RpmConflicts(_RpmEntry): """ Parse rpm:conflicts children. """ + __slots__ = () + class _RpmEnhances(_RpmEntry): """ Parse rpm:enhances children. """ + __slots__ = () + class _RpmSupplements(_RpmEntry): """ Parse rpm:supplements children. """ + __slots__ = () + class _RpmSuggests(_RpmEntry): """ Parse rpm:suggests children. """ + __slots__ = () + class _SuseFreshens(_RpmEntry): """ Parse suse:freshens children. """ + __slots__ = () + class PackageXmlMixIn(object): """ @@ -293,6 +314,10 @@ self._databinder.registerType(xmllib.StringNode, name='summary') self._databinder.registerType(xmllib.StringNode, name='description') self._databinder.registerType(xmllib.StringNode, name='url') + self._databinder.registerType(_RpmEntries, name='entry', + namespace='rpm') + self._databinder.registerType(_RpmEntries, name='entry', + namespace='suse') self._databinder.registerType(_RpmRequires, name='requires', namespace='rpm') self._databinder.registerType(_RpmRecommends, name='recommends', diff --git a/repomd/patchesxml.py b/repomd/patchesxml.py --- a/repomd/patchesxml.py +++ b/repomd/patchesxml.py @@ -25,11 +25,13 @@ from repomd.xmlcommon import XmlFileParser, SlotNode from repomd.errors import UnknownElementError -class _Patches(xmllib.BaseNode): +class _Patches(SlotNode): """ Python representation of patches.xml from the repository metadata. """ + __slots__ = () + def addChild(self, child): """ Parse children of patches element. @@ -42,7 +44,7 @@ child.id = child.getAttribute('id') child._parser = PatchXml(None, child.location) child.parseChildren = child._parser.parse - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) else: raise UnknownElementError(child) @@ -59,6 +61,7 @@ """ Parser for patch element of patches.xml. """ + __slots__ = ('id', 'checksum', 'checksumType', 'location') # All attributes are defined in __init__ by iterating over __slots__, diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -115,11 +115,13 @@ self.description)) -class _Atoms(xmllib.BaseNode): +class _Atoms(SlotNode): """ Parser for the atoms element of a path-*.xml file. """ + __slots__ = () + def addChild(self, child): """ Parse children of atoms element. @@ -128,7 +130,7 @@ n = child.getName() if n == 'package': child.type = child.getAttribute('type') - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) elif n == 'message': pass elif n == 'script': diff --git a/repomd/primaryxml.py b/repomd/primaryxml.py --- a/repomd/primaryxml.py +++ b/repomd/primaryxml.py @@ -18,17 +18,17 @@ __all__ = ('PrimaryXml', ) -from rpath_xmllib import api1 as xmllib - -from repomd.xmlcommon import XmlFileParser from repomd.packagexml import PackageXmlMixIn from repomd.errors import UnknownElementError +from repomd.xmlcommon import XmlFileParser, SlotNode -class _Metadata(xmllib.BaseNode): +class _Metadata(SlotNode): """ Python representation of primary.xml.gz from the repository metadata. """ + __slots__ = () + def addChild(self, child): """ Parse children of metadata element. @@ -36,7 +36,7 @@ if child.getName() == 'package': child.type = child.getAttribute('type') - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) else: raise UnknownElementError(child) diff --git a/repomd/repomdxml.py b/repomd/repomdxml.py --- a/repomd/repomdxml.py +++ b/repomd/repomdxml.py @@ -28,7 +28,7 @@ from repomd.xmlcommon import XmlFileParser, SlotNode from repomd.errors import UnknownElementError -class _RepoMd(xmllib.BaseNode): +class _RepoMd(SlotNode): """ Python representation of repomd.xml from the repository metadata. """ @@ -60,7 +60,7 @@ elif child.type == 'updateinfo': child._parser = UpdateInfoXml(None, child.location) child.parseChildren = child._parser.parse - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) else: raise UnknownElementError(child) @@ -90,7 +90,7 @@ """ __slots__ = ('location', 'checksum', 'checksumType', 'timestamp', 'openChecksum', 'openChecksumType', 'databaseVersion', - 'size', 'openSize') + 'size', 'openSize', 'type', '_parser', 'parseChildren', ) # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py --- a/repomd/updateinfoxml.py +++ b/repomd/updateinfoxml.py @@ -30,6 +30,8 @@ Represents updates in an updateinfo.xml. """ + __slots__ = () + def addChild(self, child): """ Set attributes on child nodes. @@ -54,7 +56,7 @@ else: raise UnknownAttributeError(child, attr) - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) def getUpdateInfo(self): """ @@ -108,6 +110,8 @@ Represent a list of references in updateinfo.xml. """ + __slots__ = () + def addChild(self, child): if child.getName() != 'reference': raise UnknownElementError(child) @@ -129,7 +133,7 @@ else: raise UnknownAttributeError(child, attr) - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) class _Collection(SlotNode): @@ -137,6 +141,8 @@ Represents a pkglist collection in updateinfo.xml. """ + __slots__ = () + def addChild(self, child): """ Update child attributes. @@ -164,7 +170,7 @@ else: raise UnknownAttributeError(child, attr) - xmllib.BaseNode.addChild(self, child) + SlotNode.addChild(self, child) class _UpdateInfoPackage(SlotNode, PackageCompare): diff --git a/repomd/xmlcommon.py b/repomd/xmlcommon.py --- a/repomd/xmlcommon.py +++ b/repomd/xmlcommon.py @@ -52,7 +52,7 @@ data = self._databinder.parseFile(fn) for child in data.iterChildren(): - if hasattr(child, '_parser'): + if hasattr(child, '_parser') and child._parser is not None: child._parser._repository = self._repository return data From elliot at rpath.com Mon Mar 15 17:17:26 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:26 +0000 Subject: mirrorball: cleanups Message-ID: <201003152117.o2FLHRDh015342@scc.eng.rpath.com> changeset: 1eaf12068f40 user: Elliot Peele date: Mon, 08 Mar 2010 18:28:05 -0500 cleanups diff --git a/updatebot/lib/watchdog.py b/updatebot/lib/watchdog.py --- a/updatebot/lib/watchdog.py +++ b/updatebot/lib/watchdog.py @@ -21,7 +21,7 @@ log = logging.getLogger('updatebot.lib.watchdog') -def runOnce(func, funcArgs): +def runOnce(func, *args, **kwargs): """ Fork and run a function. """ @@ -29,18 +29,17 @@ pid = os.fork() if not pid: log.info('started %s(pid %s)' % (func.__name__, os.getpid())) - args, kwargs = funcArgs rc = func(*args, **kwargs) os._exit(rc) else: return pid -def watchOnce(func, funcArgs): +def waitOnce(func, *args, **kwargs): """ Fork and wait for a function to complete. """ - pid = runOnce(func, funcArgs) + pid = runOnce(func, *args, **kwargs) log.info('waiting for %s' % pid) pid, status = os.waitpid(pid, 0) @@ -48,26 +47,34 @@ rc = os.WEXITSTATUS(status) return rc -def watch(func, funcArgs): +def loopwait(func, *args, **kwargs): """ Run a forked function in a loop. """ while True: log.info('looping %s' % func.__name__) - rc = watchOnce(func, funcArgs) + rc = waitOnce(func, *args, **kwargs) if not rc: log.info('completed function loop') break return rc -def forknwatch(func): +def forknloop(func): """ Decorator to run a function in a forked process in a loop. """ def wrapper(*args, **kwargs): - funcArgs = (args, kwargs) - return watch(func, funcArgs) + return loopwait(func, *args, **kwargs) return wrapper + +def forknwait(func): + """ + Decorator to run a function in a forked process and wait for it to complete. + """ + + def wrapper(*args, **kwargs): + return waitOnce(func, *args, **kwargs) + return wrapper From elliot at rpath.com Mon Mar 15 17:17:28 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:28 +0000 Subject: mirrorball: pull in convertPackageNameToClassName from conary.lib.util Message-ID: <201003152117.o2FLHSRF015370@scc.eng.rpath.com> changeset: ab39ea1e0767 user: Elliot Peele date: Tue, 09 Mar 2010 15:17:21 -0500 pull in convertPackageNameToClassName from conary.lib.util diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2008-2009 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -24,6 +24,7 @@ import signal import resource from conary.lib.util import rmtree +from conary.lib.util import convertPackageNameToClassName from rpmutils import rpmvercmp From elliot at rpath.com Mon Mar 15 17:17:29 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:29 +0000 Subject: mirrorball: 1. drop support for building multiple group versions with the same contents Message-ID: <201003152117.o2FLHTRd015399@scc.eng.rpath.com> changeset: e845658e951c user: Elliot Peele date: Tue, 09 Mar 2010 17:11:07 -0500 1. drop support for building multiple group versions with the same contents 2. order promote will be implemented as a separate method of the bot diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -157,18 +157,6 @@ # Load specific kwargs restoreFile = kwargs.pop('restoreFile', None) - # FIXME: this should probably be provided by the errata object. - # Method for sorting versions. - def verCmp(a, b): - if a.startswith('RH') and b.startswith('RH'): - return cmp(a.split('_')[1], b.split('_')[1]) - elif a.startswith('RH') and not b.startswith('RH'): - return 1 - elif not a.startswith('RH') and b.startswith('RH'): - return -1 - else: - return cmp(a, b) - # Get current timestamp current = self._groupmgr.getErrataState() if current is None: @@ -277,26 +265,6 @@ # Save package map in case we run into trouble later. self._savePackages(pkgMap) - # FIXME: we might actually want to do this one day - # Find errata group versions. - #errataVersions = self._errata.getVersions(updateId) - errataVersions = set() - - # Add timestamp version. - errataVersions.add(self._errata.getBucketVersion(updateId)) - - # FIXME: Might want to re-enable this one day. - # Get current set of source names and versions. - #nvMap = self._updater.getSourceVersionMap() - # Add in new names and versions that have just been built. - #for n, v, f in pkgMap.iterkeys(): - # n = n.split(':')[0] - # nvMap[n] = v - #pkgSet = set(nvMap.items()) - # Get the major distro verisons from the group manager. - #majorVersions = self._groupmgr.getVersions(pkgSet) - #import epdb; epdb.st() - # Store current updateId. self._groupmgr.setErrataState(updateId) @@ -340,27 +308,15 @@ # Make sure built troves are part of the group. self._addPackages(pkgMap) - # Build various group verisons. - #expected = self._flattenSetDict(pkgMap) - versions = sorted(errataVersions, cmp=verCmp) - if not versions: - versions = ['unknown.%s' % updateId, ] - for version in versions: - log.info('setting version %s' % version) - self._groupmgr.setVersion(version) - grpTrvMap = self._groupmgr.build() + # Get timestamp version. + version = self._errata.getBucketVersion(updateId) + if not version: + version = 'unknown.%s' % updateId - # FIXME: enable promotes at some point - #log.info('promoting version %s' % version) - #toPublish = self._flattenSetDict(grpTrvMap) - #newTroves = self._updater.publish( - # toPublish, - # expected, - # self._cfg.targetLabel - #) - - # After the first promote, packages should not be repromoted. - #expected = set() + # Build groups. + log.info('setting version %s' % version) + self._groupmgr.setVersion(version) + grpTrvMap = self._groupmgr.build() updateSet.update(pkgMap) From elliot at rpath.com Mon Mar 15 17:17:30 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:30 +0000 Subject: mirrorball: 1. add support for generating group recipes for new sources Message-ID: <201003152117.o2FLHUsB015428@scc.eng.rpath.com> changeset: aaa3e8b694fe user: Elliot Peele date: Tue, 09 Mar 2010 17:12:57 -0500 1. add support for generating group recipes for new sources 2. add class for managing single group contents diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -18,6 +18,7 @@ import os import copy +import time import logging import itertools @@ -47,6 +48,18 @@ log = logging.getLogger('updatebot.groupmgr') +GROUP_RECIPE = """\ +# +# Copyright (c) %(year)s rPath, Inc. +# This file is distributed under the terms of the MIT License. +# A copy is available at http://www.rpath.com/permanent/mit-license.html +# +class %%(className)s(FactoryRecipeClass): + \"\"\" + Groups require that a recipe exists. + \"\"\" +""" % {'year': time.gmtime().tm_year, } + def checkout(func): def wrapper(self, *args, **kwargs): if not self._checkedout: @@ -187,6 +200,14 @@ # binary versions. return srcVersion in srcVersions + @commit + def getBuildJob(self): + """ + Get the list of trove specs to submit to the build system. + """ + + return ((self._sourceName, self._sourceVersion, None), ) + @checkout @commit def build(self): @@ -194,7 +215,7 @@ Build all configured flavors of the group. """ - groupTroves = ((self._sourceName, self._sourceVersion, None), ) + groupTroves = self.getBuildJob() built = self._builder.cvc.cook(groupTroves) return built @@ -676,6 +697,18 @@ pkgNames=errors) +class SingleGroupManager(GroupManager): + """ + Class to manage a single group that contains only packages with no + subgroups. + """ + + def __init__(self, *args, **kwargs): + GroupManager.__init__(self, *args, **kwargs) + self._pkgGroupName = self._sourceName + self._helper._groupContents = {} + + class GroupHelper(ConaryHelper): """ Modified conary helper to deal with managing group sources. @@ -694,6 +727,24 @@ # autoLoadRecipes here anyway. self._ccfg.autoLoadRecipes = [] + def _newpkg(self, pkgName): + """ + Wrap newpkg to add a group recipe since group recipes are required. + """ + + recipeDir = ConaryHelper._newpkg(self, pkgName) + + recipe = '%s.recipe' % pkgName + recipeFile = os.path.join(recipeDir, recipe) + if not os.path.exists(recipeFile): + className = util.convertPackageNameToClassName(pkgName) + fh = open(recipeFile, 'w') + fh.write(GROUP_RECIPE % {'className': className}) + fh.close() + self._addFile(recipeDir, recipe) + + return recipeDir + def getModel(self, pkgName, version=None): """ Get a thawed data representation of the group xml data from the From elliot at rpath.com Mon Mar 15 17:17:32 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:32 +0000 Subject: mirrorball: add generic caching for trove info Message-ID: <201003152117.o2FLHWXX015457@scc.eng.rpath.com> changeset: 0f1823d230ed user: Elliot Peele date: Tue, 09 Mar 2010 18:15:38 -0500 add generic caching for trove info add caching for cloned from diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -99,6 +99,10 @@ # frzenset(labels): binTroveNVFSet self._binaryVersionCache = {} + # Cache cloned from information + # srcNVF: destNVF + self._clonedFromCache = {} + def getConaryConfig(self): """ Get a conary config instance. @@ -250,6 +254,51 @@ return self.getSourceVersions(list(troves)) + def _cacheTroveInfo(self, troveSpecs, cache, tiType, tiFunc=None): + """ + Retrieve a bit of trove info for a listed trove specs and cache the + results. + @param troveSpecs: list of destination nvfs + @type troveSpecs: iterable of (name, verObj, flvObj) tuples. + @param cache: cache of trove spec to trove info + @type cache: dict(trvSpec=trvInfo) + @param tiType: trove info field to cache on + @type tiType: conary.trove._TROVEINFO_TAG_* + @param tiFunc: optional function to format trove info into the desired + value. This will be passed two arguments, the trove info + and the trvSpec this applies to. + @type tiFunc: function + """ + + # filter out already cached results + cached = set(x for x in troveSpecs if x in cache) + uncached = sorted(set(troveSpecs).difference(cached)) + + tiMap = {} + tiLst = self._repos.getTroveInfo(tiType, uncached) + for i, nvf in enumerate(uncached): + ti = tiLst[i]() + if tiFunc: + ti = tiFunc(ti, nvf) + tiMap.setdefault(ti, set()).add(nvf) + cache[nvf] = ti + + for spec in cached: + tiMap.setdefault(cache[spec], set()).add(spec) + + return tiMap + + def getClonedFrom(self, troveSpecs): + """ + Get a mapping of cloned from trove info for a list of trove specs in the + form srcSpec: destSpec. + @param troveSpecs: list of destination nvfs + @type troveSpecs: iterable of (name, verObj, flvObj) tuples. + """ + + return self._cacheTroveInfo(troveSpecs, self._clonedFromCache, + trove._TROVEINFO_TAG_CLONEDFROM) + def getSourceVersions(self, binTroveSpecs): """ Given a list of trove specs, query the repository for all of the related @@ -259,23 +308,11 @@ @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ...]} """ - # check the cache for any source we have already retrieved from the - # repository. - cached = set(x for x in binTroveSpecs if x in self._sourceVersionCache) - uncached = sorted(set(binTroveSpecs).difference(cached)) + def tiFunc(ti, nvf): + return (ti, nvf[0].getSourceVersion(), None) - srcTrvs = {} - tiSourceName = trove._TROVEINFO_TAG_SOURCENAME - sources = self._repos.getTroveInfo(tiSourceName, uncached) - for i, (n, v, f) in enumerate(uncached): - src = (sources[i](), v.getSourceVersion(), None) - srcTrvs.setdefault(src, set()).add((n, v, f)) - self._sourceVersionCache[(n, v, f)] = src - - for spec in cached: - srcTrvs.setdefault(self._sourceVersionCache[spec], set()).add(spec) - - return srcTrvs + return self._cacheTroveInfo(binTroveSpecs, self._sourceVersionCache, + trove._TROVEINFO_TAG_SOURCENAME, tiFunc=tiFunc) def getBinaryVersions(self, srcTroveSpecs, labels=None, latest=True): """ From elliot at rpath.com Mon Mar 15 17:17:33 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:33 +0000 Subject: mirrorball: add support for getting group info from the target label Message-ID: <201003152117.o2FLHY4a015485@scc.eng.rpath.com> changeset: 2e74254d6fff user: Elliot Peele date: Tue, 09 Mar 2010 18:16:23 -0500 add support for getting group info from the target label diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py --- a/updatebot/groupmgr.py +++ b/updatebot/groupmgr.py @@ -85,14 +85,26 @@ Manage group of all packages for a platform. """ - def __init__(self, cfg, parentGroup=False): + def __init__(self, cfg, parentGroup=False, targetGroup=False): self._cfg = cfg self._helper = GroupHelper(self._cfg) self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') #self._versionFactory = VersionFactory(cfg) - if parentGroup: + assert not (parentGroup and targetGroup) + + if targetGroup: + srcName = '%s:source' % self._cfg.topSourceGroup[0] + trvs = self._helper.findTrove((srcName, None, None), + labels=(self._cfg.targetLabel, )) + + assert len(trvs) + + self._sourceName = srcName + self._sourceVersion = trvs[0][1] + self._readonly = True + elif parentGroup: topGroup = list(self._cfg.topParentSourceGroup) topGroup[0] = '%s:source' % topGroup[0] trvs = self._helper.findTrove(tuple(topGroup), From elliot at rpath.com Mon Mar 15 17:17:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:35 +0000 Subject: mirrorball: refactor group manager Message-ID: <201003152117.o2FLHZZ0015513@scc.eng.rpath.com> changeset: dcf60290922b user: Elliot Peele date: Wed, 10 Mar 2010 17:06:33 -0500 refactor group manager diff --git a/updatebot/groupmgr.py b/updatebot/groupmgr.py deleted file mode 100644 --- a/updatebot/groupmgr.py +++ /dev/null @@ -1,1046 +0,0 @@ -# -# Copyright (c) 2009 rPath, Inc. -# -# This program is distributed under the terms of the Common Public License, -# version 1.0. A copy of this license should have been distributed with this -# source file in a file called LICENSE. If it is not present, the license -# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. -# -# This program is distributed in the hope that it will be useful, but -# without any warranty; without even the implied warranty of merchantability -# or fitness for a particular purpose. See the Common Public License for -# full details. -# - -""" -Module for managing conary groups. -""" - -import os -import copy -import time -import logging -import itertools - -from conary import versions -from conary.deps import deps - -from updatebot.lib import util -from updatebot.build import Builder -from updatebot.pkgsource import RpmSource -from updatebot.conaryhelper import ConaryHelper -from updatebot.lib.xobjects import XGroup -from updatebot.lib.xobjects import XGroupDoc -from updatebot.lib.xobjects import XGroupList -from updatebot.lib.xobjects import XPackageDoc -from updatebot.lib.xobjects import XPackageData -from updatebot.lib.xobjects import XPackageItem - -from updatebot.errors import OldVersionsFoundError -from updatebot.errors import FlavorCountMismatchError -from updatebot.errors import UnknownBuildContextError -from updatebot.errors import GroupValidationFailedError -from updatebot.errors import UnsupportedTroveFlavorError -from updatebot.errors import UnhandledPackageAdditionError -from updatebot.errors import NameVersionConflictsFoundError -from updatebot.errors import ExpectedRemovalValidationFailedError -from updatebot.errors import UnknownPackageFoundInManagedGroupError - -log = logging.getLogger('updatebot.groupmgr') - -GROUP_RECIPE = """\ -# -# Copyright (c) %(year)s rPath, Inc. -# This file is distributed under the terms of the MIT License. -# A copy is available at http://www.rpath.com/permanent/mit-license.html -# -class %%(className)s(FactoryRecipeClass): - \"\"\" - Groups require that a recipe exists. - \"\"\" -""" % {'year': time.gmtime().tm_year, } - -def checkout(func): - def wrapper(self, *args, **kwargs): - if not self._checkedout: - self._checkout() - - return func(self, *args, **kwargs) - return wrapper - -def commit(func): - def wrapper(self, *args, **kwargs): - if self._readonly: - return - - if self._checkedout: - self._commit() - - return func(self, *args, **kwargs) - return wrapper - - -class GroupManager(object): - """ - Manage group of all packages for a platform. - """ - - def __init__(self, cfg, parentGroup=False, targetGroup=False): - self._cfg = cfg - self._helper = GroupHelper(self._cfg) - self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') - - #self._versionFactory = VersionFactory(cfg) - - assert not (parentGroup and targetGroup) - - if targetGroup: - srcName = '%s:source' % self._cfg.topSourceGroup[0] - trvs = self._helper.findTrove((srcName, None, None), - labels=(self._cfg.targetLabel, )) - - assert len(trvs) - - self._sourceName = srcName - self._sourceVersion = trvs[0][1] - self._readonly = True - elif parentGroup: - topGroup = list(self._cfg.topParentSourceGroup) - topGroup[0] = '%s:source' % topGroup[0] - trvs = self._helper.findTrove(tuple(topGroup), - labels=self._cfg.platformSearchPath) - - assert len(trvs) - - self._sourceName = topGroup[0] - self._sourceVersion = trvs[0][1] - - self._readonly = True - else: - self._sourceName = self._cfg.topSourceGroup[0] - self._sourceVersion = None - self._readonly = False - - self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName - - self._checkedout = False - self._groups = {} - - def _checkout(self): - """ - Get current group state from the repository. - """ - - self._groups = self._helper.getModel(self._sourceName, - version=self._sourceVersion) - self._checkedout = True - - def _commit(self, copyToLatest=False): - """ - Commit current changes to the group. - """ - - if self._sourceVersion and not copyToLatest: - log.error('refusing to commit out of date source') - raise NotCommittingOutOfDateSourceError() - - # Copy forward data when we are fixing up old group versions so that - # this is the latest source. - if copyToLatest: - log.info('copying information to latest version') - # Get data from the old versoin - version = self._helper.getVersion(self._sourceName, - version=self._sourceVersion) - errataState = self._helper.getErrataState(self._sourceName, - version=self._sourceVersion) - groups = self._groups - - log.info('version: %s' % version) - log.info('errataState: %s' % errataState) - - # Set version to None to get the latest source. - self._sourceVersion = None - - # Checkout latest source. - self._checkout() - - # Set back to old data - self.setVersion(version) - self.setErrataState(errataState) - self._groups = groups - - # sync versions from the package group to the other managed groups. - self._copyVersions() - - # validate group contents. - self._validateGroups() - - # write out the model data - self._helper.setModel(self._sourceName, self._groups) - - # commit to the repository - version = self._helper.commit(self._sourceName, - version=self._sourceVersion, - commitMessage='automated commit') - if self._sourceVersion: - self._sourceVersion = version - self._checkedout = False - return version - - save = _commit - - def hasBinaryVersion(self): - """ - Check if there is a binary version for the current source version. - """ - - # Get a mapping of all source version to binary versions for all - # existing binary versions. - srcVersions = dict([ (x[1].getSourceVersion(), x[1]) - for x in self._helper.findTrove( - (self._sourceName, None, None), - getLeaves=False - ) - ]) - - # Get the version of the specified source, usually the latest - # source version. - srcVersion = self._helper.findTrove(('%s:source' % self._sourceName, - self._sourceVersion, None))[0][1] - - # Check to see if the latest source version is in the map of - # binary versions. - return srcVersion in srcVersions - - @commit - def getBuildJob(self): - """ - Get the list of trove specs to submit to the build system. - """ - - return ((self._sourceName, self._sourceVersion, None), ) - - @checkout - @commit - def build(self): - """ - Build all configured flavors of the group. - """ - - groupTroves = self.getBuildJob() - built = self._builder.cvc.cook(groupTroves) - return built - - @checkout - def add(self, *args, **kwargs): - """ - Add a trove to the package group contents. - """ - - # create package group model if it does not exist. - if self._pkgGroupName not in self._groups: - model = GroupContentsModel(self._pkgGroupName) - self._groups[self._pkgGroupName] = model - - self._groups[self._pkgGroupName].add(*args, **kwargs) - - @checkout - def remove(self, name, missingOk=False): - """ - Remove a given trove from the package group contents. - """ - - if self._pkgGroupName not in self._groups: - return - - return self._groups[self._pkgGroupName].remove(name, - missingOk=missingOk) - - @checkout - def hasPackage(self, name): - """ - Check if a given package name is in the group. - """ - - return (self._pkgGroupName in self._groups and - name in self._groups[self._pkgGroupName]) - - def addPackage(self, name, version, flavors): - """ - Add a package to the model. - @param name: name of the package - @type name: str - @param version: conary version from string object - @type version: conary.versions.VersionFromString - @param flavors: list of flavors - @type flavors: [conary.deps.deps.Flavor, ...] - """ - - # Now that versions are actually used for something make sure they - # are always present. - assert version - assert len(flavors) - - # Remove all versions and flavors of this name before adding this - # package. This avoids flavor change issues by replacing all flavors. - if self.hasPackage(name): - self.remove(name) - - plain = deps.parseFlavor('') - x86 = deps.parseFlavor('is: x86') - x86_64 = deps.parseFlavor('is: x86_64') - - if len(flavors) == 1: - flavor = flavors[0] - # noarch package, add unconditionally - if flavor == plain: - self.add(name, version=version) - - # x86, add with use=x86 - elif flavor.satisfies(x86): - self.add(name, version=version, flavor=flavor, use='x86') - - # x86_64, add with use=x86_64 - elif flavor.satisfies(x86_64): - self.add(name, version=version, flavor=flavor, use='x86_64') - - else: - raise UnsupportedTroveFlavorError(name=name, flavor=flavor) - - return - - elif len(flavors) == 2: - # This is most likely a normal package with both x86 and x86_64, but - # lets make sure anyway. - flvCount = {x86: 0, x86_64: 0, plain: 0} - for flavor in flavors: - if flavor.satisfies(x86): - flvCount[x86] += 1 - elif flavor.satisfies(x86_64): - flvCount[x86_64] += 1 - elif flavor.freeze() == '': - flvCount[plain] += 1 - else: - raise UnsupportedTroveFlavorError(name=name, flavor=flavor) - - # make sure there is only one instance of x86 and once instance of - # x86_64 in the flavor list. - assert (flvCount[x86_64] > 0) ^ (flvCount[plain] > 0) - assert len([ x for x, y in flvCount.iteritems() if y != 1 ]) == 1 - - # In this case just add the package unconditionally - self.add(name, version=version) - - return - - # These are special cases. - else: - # The way I see it there are a few ways you could end up here. - # 1. this is a kernel - # 2. this is a kernel module - # 3. this is a special package like glibc or openssl where there - # are i386, i686, and x86_64 varients. - # 4. this is a package with package flags that I don't know - # about. - # Lets see if we know about this package and think it should have - # more than two flavors. - - - # Get source trove name. - log.info('retrieving trove info for %s' % name) - srcTroveMap = self._helper._getSourceTroves( - (name, version, flavors[0]) - ) - srcTroveName = srcTroveMap.keys()[0][0].split(':')[0] - - # handle kernels. - if srcTroveName == 'kernel' or util.isKernelModulePackage(name): - # add all x86ish flavors with use=x86 and all x86_64ish flavors - # with use=x86_64 - for flavor in flavors: - if flavor.satisfies(x86): - self.add(name, version=version, flavor=flavor, use='x86') - elif flavor.satisfies(x86_64): - self.add(name, version=version, flavor=flavor, use='x86_64') - else: - raise UnsupportedTroveFlavorError(name=name, - flavor=flavor) - - # maybe this is one of the special flavors we know about. - elif srcTroveName in self._cfg.packageFlavors: - # separate packages into x86 and x86_64 by context name - # TODO: If we were really smart we would load the conary - # contexts and see what buildFlavors they contained. - flavorCount = {'x86': 0, 'x86_64': 0} - for context, bldflv in self._cfg.packageFlavors[srcTroveName]: - if context in ('i386', 'i486', 'i586', 'i686', 'x86'): - flavorCount['x86'] += 1 - elif context in ('x86_64', ): - flavorCount['x86_64'] += 1 - else: - raise UnknownBuildContextError(name=name, - flavor=context) - - for flavor in flavors: - if flavor.satisfies(x86): - flavorCount['x86'] -= 1 - elif flavor.satisfies(x86_64): - flavorCount['x86_64'] -= 1 - else: - raise UnsupportedTroveFlavorError(name=name, - flavor=flavor) - - errors = [ x for x, y in flavorCount.iteritems() if y != 0 ] - if errors: - raise FlavorCountMismatchError(name=name) - - for flavor in flavors: - if flavor.satisfies(x86): - self.add(name, version=version, - flavor=flavor, use='x86') - elif flavor.satisfies(x86_64): - self.add(name, version=version, - flavor=flavor, use='x86_64') - else: - raise UnsupportedTroveFlavorError(name=name, - flavor=flavor) - return - - # Unknown state. - raise UnhandledPackageAdditionError(name=name) - - @checkout - def setVersion(self, version): - """ - Set the version of the managed group. - """ - - self._helper.setVersion(self._sourceName, version) - - @checkout - def setErrataState(self, state): - """ - Set errata state info for the managed platform. - """ - - self._helper.setErrataState(self._sourceName, state) - - @checkout - def getErrataState(self, version=None): - """ - Get the errata state info. - """ - - if version is None: - version = self._sourceVersion - - return self._helper.getErrataState(self._sourceName, - version=version) - - def getVersions(self, pkgSet): - """ - Get the set of versions that are represented by the given set of - packages from the version factory. - """ - - return set() - #return self._versionFactory.getVersions(pkgSet) - - def _copyVersions(self): - """ - Copy versions from the packages group to the other managed groups. - """ - - pkgs = dict([ (x[1].name, x[1]) for x in - self._groups[self._pkgGroupName].iteritems() ]) - - for group in self._groups.itervalues(): - # skip over package group since it is the version source. - if group.groupName == self._pkgGroupName: - continue - - # for all other groups iterate over contents and set versions to - # match package group. - for k, pkg in group.iteritems(): - if pkg.name in pkgs: - pkg.version = pkgs[pkg.name].version - else: - raise UnknownPackageFoundInManagedGroupError(what=pkg.name) - - def _validateGroups(self): - """ - Validate the contents of the package group to ensure sanity: - 1. Check for packages that have the same source name, but - different versions. - 2. Check that the version in the group is the latest source/build - of that version. - 3. Check that package removals specified in the config file have - occured. - """ - - errors = [] - for name, group in self._groups.iteritems(): - log.info('checking consistentcy of %s' % name) - try: - self._checkNameVersionConflict(group) - except NameVersionConflictsFoundError, e: - errors.append((group, e)) - - try: - self._checkLatestVersion(group) - except OldVersionsFoundError, e: - errors.append((group, e)) - - try: - self._checkRemovals(group) - except ExpectedRemovalValidationFailedError, e: - errors.append((group, e)) - - if errors: - raise GroupValidationFailedError(errors=errors) - - def _checkNameVersionConflict(self, group): - """ - Check for packages taht have the same source name, but different - versions. - """ - - # get names and versions - troves = set() - labels = set() - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - - version = None - if pkgData.version: - versionObj = versions.ThawVersion(pkgData.version) - labels.add(versionObj.branch().label()) - version = str(versionObj.asString()) - - flavor = None - # FIXME: At some point we might want to add proper flavor handling, - # note that group flavor handling is different than what - # findTroves normally does. - #if pkgData.flavor: - # flavor = deps.ThawFlavor(str(pkgData.flavor)) - - troves.add((name, version, flavor)) - - # Get flavors and such. - foundTroves = set([ x for x in - itertools.chain(*self._helper.findTroves(troves, - labels=labels).itervalues()) ]) - - # get sources for each name version pair - sources = self._helper.getSourceVersions(foundTroves) - - seen = {} - for (n, v, f), pkgSet in sources.iteritems(): - binVer = list(pkgSet)[1][1] - seen.setdefault(n, set()).add(binVer) - - binPkgs = {} - conflicts = {} - for name, vers in seen.iteritems(): - if len(vers) > 1: - log.error('found multiple versions of %s' % name) - for binVer in vers: - srcVer = binVer.getSourceVersion() - nvf = (name, srcVer, None) - conflicts.setdefault(name, []).append(srcVer) - binPkgs[nvf] = sources[nvf] - - if conflicts: - raise NameVersionConflictsFoundError(groupName=group.groupName, - conflicts=conflicts, - binPkgs=binPkgs) - - def _checkLatestVersion(self, group): - """ - Check to make sure each specific conary version is the latest source - and build count of the upstream version. - """ - - # get names and versions - troves = set() - labels = set() - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - - version = None - if pkgData.version: - version = versions.ThawVersion(pkgData.version) - labels.add(version.branch().label()) - # get upstream version - revision = version.trailingRevision() - upstreamVersion = revision.getVersion() - - # FIXME: This should probably be a fully formed version - # as above. - version = upstreamVersion - - flavor = None - # FIXME: At some point we might want to add proper flavor handling, - # note that group flavor handling is different than what - # findTroves normally does. - #if pkgData.flavor: - # flavor = deps.ThawFlavor(str(pkgData.flavor)) - - troves.add((name, version, flavor)) - - # Get flavors and such. - foundTroves = dict([ (x[0], y) for x, y in - self._helper.findTroves(troves, labels=labels).iteritems() ]) - - pkgs = {} - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - version = None - if pkgData.version: - version = versions.ThawVersion(pkgData.version) - flavor = None - if pkgData.flavor: - flavor = deps.ThawFlavor(str(pkgData.flavor)) - - pkgs.setdefault(name, []).append((name, version, flavor)) - - assert len(pkgs) == len(foundTroves) - - # Get all old versions so that we can make sure any version conflicts - # were introduced by old version handling. - oldVersions = set() - for nvfLst in self._cfg.useOldVersion.itervalues(): - for nvf in nvfLst: - srcMap = self._helper.getSourceVersionMapFromBinaryVersion(nvf, - labels=self._cfg.platformSearchPath, latest=False) - oldVersions |= set(itertools.chain(*srcMap.itervalues())) - - - errors = {} - for name, found in foundTroves.iteritems(): - assert name in pkgs - current = pkgs[name] - - if len(current) > len(found): - log.warn('found more packages in the model than in the ' - 'repository, assuming that multiversion policy will ' - 'catch this.') - continue - - assert len(current) == 1 or len(found) == len(current) - - foundError = False - for i, (n, v, f) in enumerate(found): - if len(current) == 1: - i = 0 - cn, cv, cf = current[i] - assert n == cn - - if v != cv: - if (n, v, f) in oldVersions: - log.info('found %s=%s[%s] in oldVersions exceptions' - % (n, v, f)) - continue - foundError = True - - if foundError: - log.error('found old version for %s' % name) - errors[name] = (current, found) - - if errors: - raise OldVersionsFoundError(pkgNames=errors.keys(), errors=errors) - - def _checkRemovals(self, group): - """ - Check to make sure that all configured package removals have happened. - """ - - updateId = self.getErrataState() - - # get package removals from the config object. - removePackages = self._cfg.updateRemovesPackages.get(updateId, []) - removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) - removeSource = [ x[0] for x in - self._cfg.removeSource.get(updateId, []) ] - - # get names and versions - troves = set() - labels = set() - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - - version = None - if pkgData.version: - versionObj = versions.ThawVersion(pkgData.version) - labels.add(versionObj.branch().label()) - version = str(versionObj.asString()) - - flavor = None - troves.add((name, version, flavor)) - - # Get flavors and such. - foundTroves = set([ x for x in - itertools.chain(*self._helper.findTroves(troves, - labels=labels).itervalues()) ]) - - # get sources for each name version pair - sources = self._helper.getSourceVersions(foundTroves) - - # collapse to sourceName: [ binNames, ] dictionary - sourceNameMap = dict([ (x[0].split(':')[0], [ z[0] for z in y ]) - for x, y in sources.iteritems() ]) - - binRemovals = set(itertools.chain(*[ sourceNameMap[x] - for x in removeSource - if x in sourceNameMap ])) - - # take the union - removals = set(removePackages) | set(removeObsoleted) | binRemovals - - errors = [] - # Make sure these packages are not in the group model. - for pkgKey, pkgData in group.iteritems(): - if pkgData.name in removals: - errors.append(pkgData.name) - - if errors: - log.info('found packages that should be removed %s' % errors) - raise ExpectedRemovalValidationFailedError(updateId=updateId, - pkgNames=errors) - - -class SingleGroupManager(GroupManager): - """ - Class to manage a single group that contains only packages with no - subgroups. - """ - - def __init__(self, *args, **kwargs): - GroupManager.__init__(self, *args, **kwargs) - self._pkgGroupName = self._sourceName - self._helper._groupContents = {} - - -class GroupHelper(ConaryHelper): - """ - Modified conary helper to deal with managing group sources. - """ - - def __init__(self, cfg): - ConaryHelper.__init__(self, cfg) - self._configDir = cfg.configPath - self._newPkgFactory = 'managed-group' - self._groupContents = cfg.groupContents - - # FIXME: autoLoadRecipes causes group versioning to go sideways - # The group super class in the repository has a version defined, which - # overrides the version from factory-version. This should probably be - # considered a bug in factory-managed-group, but we don't need - # autoLoadRecipes here anyway. - self._ccfg.autoLoadRecipes = [] - - def _newpkg(self, pkgName): - """ - Wrap newpkg to add a group recipe since group recipes are required. - """ - - recipeDir = ConaryHelper._newpkg(self, pkgName) - - recipe = '%s.recipe' % pkgName - recipeFile = os.path.join(recipeDir, recipe) - if not os.path.exists(recipeFile): - className = util.convertPackageNameToClassName(pkgName) - fh = open(recipeFile, 'w') - fh.write(GROUP_RECIPE % {'className': className}) - fh.close() - self._addFile(recipeDir, recipe) - - return recipeDir - - def getModel(self, pkgName, version=None): - """ - Get a thawed data representation of the group xml data from the - repository. - """ - - log.info('loading model for %s' % pkgName) - recipeDir = self._edit(pkgName, version=version) - groupFileName = util.join(recipeDir, 'groups.xml') - - # load group model - groups = {} - if os.path.exists(groupFileName): - model = GroupModel.thaw(groupFileName) - for name, groupObj in model.iteritems(): - contentFileName = util.join(recipeDir, groupObj.filename) - contentsModel = GroupContentsModel.thaw(contentFileName, - (name, groupObj.byDefault, groupObj.depCheck)) - contentsModel.fileName = groupObj.filename - groups[groupObj.name] = contentsModel - - # copy in any group data - for name, data in self._groupContents.iteritems(): - newGroups = [ x for x in groups.itervalues() - if x.groupName == name and - x.fileName == data['filename'] ] - - assert len(newGroups) in (0, 1) - - byDefault = data['byDefault'] == 'True' and True or False - depCheck = data['depCheck'] == 'True' and True or False - - # load model - contentsModel = GroupContentsModel.thaw( - util.join(self._configDir, data['filename']), - (name, byDefault, depCheck) - ) - - # override anything from the repo, unless retriveing a - # specific version. - if version is None: - groups[name] = contentsModel - - return groups - - def setModel(self, pkgName, groups): - """ - Freeze group model and save to the repository. - """ - - log.info('saving model for %s' % pkgName) - recipeDir = self._edit(pkgName) - groupFileName = util.join(recipeDir, 'groups.xml') - - groupModel = GroupModel() - for name, model in groups.iteritems(): - groupfn = util.join(recipeDir, model.fileName) - - model.freeze(groupfn) - groupModel.add(name=name, - filename=model.fileName, - byDefault=model.byDefault, - depCheck=model.depCheck) - self._addFile(recipeDir, model.fileName) - - groupModel.freeze(groupFileName) - self._addFile(recipeDir, 'groups.xml') - - def getErrataState(self, pkgname, version=None): - """ - Get the contents of the errata state file from the specified package, - if file does not exist, return None. - """ - - log.info('getting errata state information from %s' % pkgname) - - recipeDir = self._edit(pkgname, version=version) - stateFileName = util.join(recipeDir, 'erratastate') - - if not os.path.exists(stateFileName): - return None - - state = open(stateFileName).read().strip() - if state.isdigit(): - state = int(state) - return state - - def setErrataState(self, pkgname, state): - """ - Set the current errata state for the given package. - """ - - log.info('storing errata state information in %s' % pkgname) - - recipeDir = self._edit(pkgname) - stateFileName = util.join(recipeDir, 'erratastate') - - # write state info - statefh = open(stateFileName, 'w') - statefh.write(str(state)) - - # source files must end in a trailing newline - statefh.write('\n') - - statefh.close() - - # make sure state file is part of source trove - self._addFile(recipeDir, 'erratastate') - - -class AbstractModel(object): - """ - Base object for models. - """ - - docClass = None - dataClass = None - elementClass = None - - def __init__(self): - self._data = {} - self._nameMap = {} - - def _addItem(self, item): - """ - Add an item to the appropriate structures. - """ - - self._data[item.key] = item - if item.name not in self._nameMap: - self._nameMap[item.name] = set() - self._nameMap[item.name].add(item.key) - - def _removeItem(self, name, missingOk=False): - """ - Remove an item from the appropriate structures. - """ - - if missingOk: - keys = self._nameMap.pop(name, []) - else: - keys = self._nameMap.pop(name) - - for key in keys: - self._data.pop(key) - - @classmethod - def thaw(cls, xmlfn, args=None): - """ - Thaw the model from xml. - """ - - model = cls.docClass.fromfile(xmlfn) - obj = args and cls(*args) or cls() - for item in model.data.items: - obj._addItem(item) - return obj - - def freeze(self, toFile): - """ - Freeze the model to a given output file. - """ - - def _srtByKey(a, b): - return cmp(a.key, b.key) - - model = self.dataClass() - model.items = sorted(self._data.values(), cmp=_srtByKey) - - doc = self.docClass() - doc.data = model - doc.tofile(toFile) - - def iteritems(self): - """ - Iterate over the model data. - """ - - return self._data.iteritems() - - def add(self, *args, **kwargs): - """ - Add an data element. - """ - - obj = self.elementClass(*args, **kwargs) - self._addItem(obj) - - def remove(self, name, missingOk=False): - """ - Remove data element. - """ - - self._removeItem(name, missingOk=missingOk) - - def __contains__(self, name): - """ - Check if element name is in the model. - """ - - return name in self._nameMap - - -class GroupModel(AbstractModel): - """ - Model for representing group name and file name. - """ - - docClass = XGroupDoc - dataClass = XGroupList - elementClass = XGroup - - -class GroupContentsModel(AbstractModel): - """ - Model for representing group data. - """ - - docClass = XPackageDoc - dataClass = XPackageData - elementClass = XPackageItem - - def __init__(self, groupName, byDefault=True, depCheck=True): - AbstractModel.__init__(self) - self.groupName = groupName - self.byDefault = byDefault - self.depCheck = depCheck - - # figure out file name based on group name - name = ''.join([ x.capitalize() for x in self.groupName.split('-') ]) - self.fileName = name[0].lower() + name[1:] + '.xml' - - -class URLVersionSource(object): - """ - Class for handling the indexing and comparison of ISO contents. - """ - - def __init__(self, cfg, version, url): - self._cfg = cfg - self._version = version - self._url = url - - pkgSource = RpmSource(cfg, ) - pkgSource.loadFromUrl(url) - pkgSource.finalize() - pkgSource.setLoaded() - - self._sourceSet = set([ x for x in pkgSource.iterPackageSet() ]) - - def areWeHereYet(self, pkgSet): - """ - Figure out if the given package set is this version. - @param pkgSet: set of source names and source versions - @type pkgSet: set([(conarySourceName, conarySourceVersion)]) - @return boolean - """ - - # Make sure that all versions that are on the ISO are in the conary - # repository. It appears to be a common case that the ISO is missing - # content from RHN. - return not self._sourceSet.difference(pkgSet) - - -class VersionFactory(object): - """ - Class for generating group verisons from a variety of sources. - """ - - def __init__(self, cfg): - self._cfg = copy.deepcopy(cfg) - self._cfg.synthesizeSources = True - self._sources = {} - - for version, url in sorted(cfg.versionSources.iteritems()): - self._sources[version] = URLVersionSource(self._cfg, version, url) - - def getVersions(self, pkgSet): - """ - Given the available package sources find the current versions. - """ - - versions = [] - for version, source in sorted(self._sources.iteritems()): - if source.areWeHereYet(pkgSet): - versions.append(version) - return set(versions) diff --git a/updatebot/groupmgr/__init__.py b/updatebot/groupmgr/__init__.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/__init__.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# diff --git a/updatebot/groupmgr/errata.py b/updatebot/groupmgr/errata.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/errata.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for managing errata groups. +""" + +from updatebot.groupmgr.single import SingleGroupHelper +from updatebot.groupmgr.single import SingleGroupManager +from updatebot.groupmgr.single import SingleGroupManagerSet + +class ErrataGroupHelper(GroupHelper): + """ + Class for managing errata group source troves. + """ + + def __init__(self, *args, **kwargs): + GroupHelper.__init__(self, *args, **kwargs) + self._newPkgFactory = 'managed-errata-group' + + +class ErrataGroupManager(SingleGroupManager): + """ + Class for managing errata group model. + """ + + _helperClass = ErrataGroupHelper + + +class ErrataGroupManagerSet(SingleGroupManagerSet): + """ + Class for managing a set of errata groups. + """ + + _managerClass = ErrataGroupManager diff --git a/updatebot/groupmgr/helper.py b/updatebot/groupmgr/helper.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/helper.py @@ -0,0 +1,187 @@ +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Conary interface for group content management. +""" + +import os +import time +import logging + +from updatebot.lib import util +from updatebot.conaryhelper import ConaryHelper + +from updatebot.groupmgr.model import GroupModel +from updatebot.groupmgr.model import GroupContentsModel + +log = logging.getLogger('updatebot.groupmgr') + +GROUP_RECIPE = """\ +# +# Copyright (c) %(year)s rPath, Inc. +# This file is distributed under the terms of the MIT License. +# A copy is available at http://www.rpath.com/permanent/mit-license.html +# +class %%(className)s(FactoryRecipeClass): + \"\"\" + Groups require that a recipe exists. + \"\"\" +""" % {'year': time.gmtime().tm_year, } + + +class GroupHelper(ConaryHelper): + """ + Modified conary helper to deal with managing group sources. + """ + + def __init__(self, cfg): + ConaryHelper.__init__(self, cfg) + self._configDir = cfg.configPath + self._newPkgFactory = 'managed-group' + self._groupContents = cfg.groupContents + + # FIXME: autoLoadRecipes causes group versioning to go sideways + # The group super class in the repository has a version defined, which + # overrides the version from factory-version. This should probably be + # considered a bug in factory-managed-group, but we don't need + # autoLoadRecipes here anyway. + self._ccfg.autoLoadRecipes = [] + + def _newpkg(self, pkgName): + """ + Wrap newpkg to add a group recipe since group recipes are required. + """ + + recipeDir = ConaryHelper._newpkg(self, pkgName) + + recipe = '%s.recipe' % pkgName + recipeFile = os.path.join(recipeDir, recipe) + if not os.path.exists(recipeFile): + className = util.convertPackageNameToClassName(pkgName) + fh = open(recipeFile, 'w') + fh.write(GROUP_RECIPE % {'className': className}) + fh.close() + self._addFile(recipeDir, recipe) + + return recipeDir + + def getModel(self, pkgName, version=None): + """ + Get a thawed data representation of the group xml data from the + repository. + """ + + log.info('loading model for %s' % pkgName) + recipeDir = self._edit(pkgName, version=version) + groupFileName = util.join(recipeDir, 'groups.xml') + + # load group model + groups = {} + if os.path.exists(groupFileName): + model = GroupModel.thaw(groupFileName) + for name, groupObj in model.iteritems(): + contentFileName = util.join(recipeDir, groupObj.filename) + contentsModel = GroupContentsModel.thaw(contentFileName, + (name, groupObj.byDefault, groupObj.depCheck)) + contentsModel.fileName = groupObj.filename + groups[groupObj.name] = contentsModel + + # copy in any group data + for name, data in self._groupContents.iteritems(): + newGroups = [ x for x in groups.itervalues() + if x.groupName == name and + x.fileName == data['filename'] ] + + assert len(newGroups) in (0, 1) + + byDefault = data['byDefault'] == 'True' and True or False + depCheck = data['depCheck'] == 'True' and True or False + + # load model + contentsModel = GroupContentsModel.thaw( + util.join(self._configDir, data['filename']), + (name, byDefault, depCheck) + ) + + # override anything from the repo, unless retriveing a + # specific version. + if version is None: + groups[name] = contentsModel + + return groups + + def setModel(self, pkgName, groups): + """ + Freeze group model and save to the repository. + """ + + log.info('saving model for %s' % pkgName) + recipeDir = self._edit(pkgName) + groupFileName = util.join(recipeDir, 'groups.xml') + + groupModel = GroupModel() + for name, model in groups.iteritems(): + groupfn = util.join(recipeDir, model.fileName) + + model.freeze(groupfn) + groupModel.add(name=name, + filename=model.fileName, + byDefault=model.byDefault, + depCheck=model.depCheck) + self._addFile(recipeDir, model.fileName) + + groupModel.freeze(groupFileName) + self._addFile(recipeDir, 'groups.xml') + + def getErrataState(self, pkgname, version=None): + """ + Get the contents of the errata state file from the specified package, + if file does not exist, return None. + """ + + log.info('getting errata state information from %s' % pkgname) + + recipeDir = self._edit(pkgname, version=version) + stateFileName = util.join(recipeDir, 'erratastate') + + if not os.path.exists(stateFileName): + return None + + state = open(stateFileName).read().strip() + if state.isdigit(): + state = int(state) + return state + + def setErrataState(self, pkgname, state): + """ + Set the current errata state for the given package. + """ + + log.info('storing errata state information in %s' % pkgname) + + recipeDir = self._edit(pkgname) + stateFileName = util.join(recipeDir, 'erratastate') + + # write state info + statefh = open(stateFileName, 'w') + statefh.write(str(state)) + + # source files must end in a trailing newline + statefh.write('\n') + + statefh.close() + + # make sure state file is part of source trove + self._addFile(recipeDir, 'erratastate') diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/manager.py @@ -0,0 +1,692 @@ +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for managing conary groups. +""" + +import logging +import itertools + +from conary import versions +from conary.deps import deps + +from updatebot.lib import util +from updatebot.build import Builder +from updatebot.errors import OldVersionsFoundError +from updatebot.errors import FlavorCountMismatchError +from updatebot.errors import UnknownBuildContextError +from updatebot.errors import GroupValidationFailedError +from updatebot.errors import UnsupportedTroveFlavorError +from updatebot.errors import UnhandledPackageAdditionError +from updatebot.errors import NameVersionConflictsFoundError +from updatebot.errors import ExpectedRemovalValidationFailedError +from updatebot.errors import UnknownPackageFoundInManagedGroupError + +from updatebot.groupmgr.helper import GroupHelper + +log = logging.getLogger('updatebot.groupmgr') + +def checkout(func): + def wrapper(self, *args, **kwargs): + if not self._checkedout: + self._checkout() + + return func(self, *args, **kwargs) + return wrapper + +def commit(func): + def wrapper(self, *args, **kwargs): + if self._readonly: + return + + if self._checkedout: + self._commit() + + return func(self, *args, **kwargs) + return wrapper + + +class GroupManager(object): + """ + Manage group of all packages for a platform. + """ + + _helperClass = GroupHelper + + def __init__(self, cfg, parentGroup=False, targetGroup=False): + self._cfg = cfg + self._helper = self._helperClass(self._cfg) + self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') + + #self._versionFactory = VersionFactory(cfg) + + assert not (parentGroup and targetGroup) + + if targetGroup: + srcName = '%s:source' % self._cfg.topSourceGroup[0] + trvs = self._helper.findTrove((srcName, None, None), + labels=(self._cfg.targetLabel, )) + + assert len(trvs) + + self._sourceName = srcName + self._sourceVersion = trvs[0][1] + self._readonly = True + elif parentGroup: + topGroup = list(self._cfg.topParentSourceGroup) + topGroup[0] = '%s:source' % topGroup[0] + trvs = self._helper.findTrove(tuple(topGroup), + labels=self._cfg.platformSearchPath) + + assert len(trvs) + + self._sourceName = topGroup[0] + self._sourceVersion = trvs[0][1] + + self._readonly = True + else: + self._sourceName = self._cfg.topSourceGroup[0] + self._sourceVersion = None + self._readonly = False + + self._pkgGroupName = 'group-%s-packages' % self._cfg.platformName + + self._checkedout = False + self._groups = {} + + def _checkout(self): + """ + Get current group state from the repository. + """ + + self._groups = self._helper.getModel(self._sourceName, + version=self._sourceVersion) + self._checkedout = True + + def _commit(self, copyToLatest=False): + """ + Commit current changes to the group. + """ + + if self._sourceVersion and not copyToLatest: + log.error('refusing to commit out of date source') + raise NotCommittingOutOfDateSourceError() + + # Copy forward data when we are fixing up old group versions so that + # this is the latest source. + if copyToLatest: + log.info('copying information to latest version') + # Get data from the old versoin + version = self._helper.getVersion(self._sourceName, + version=self._sourceVersion) + errataState = self._helper.getErrataState(self._sourceName, + version=self._sourceVersion) + groups = self._groups + + log.info('version: %s' % version) + log.info('errataState: %s' % errataState) + + # Set version to None to get the latest source. + self._sourceVersion = None + + # Checkout latest source. + self._checkout() + + # Set back to old data + self.setVersion(version) + self.setErrataState(errataState) + self._groups = groups + + # sync versions from the package group to the other managed groups. + self._copyVersions() + + # validate group contents. + self._validateGroups() + + # write out the model data + self._helper.setModel(self._sourceName, self._groups) + + # commit to the repository + version = self._helper.commit(self._sourceName, + version=self._sourceVersion, + commitMessage='automated commit') + if self._sourceVersion: + self._sourceVersion = version + self._checkedout = False + return version + + save = _commit + + def hasBinaryVersion(self, version=None): + """ + Check if there is a binary version for the current source version. + """ + + if not version: + verison = self._sourceVersion + + # Get a mapping of all source version to binary versions for all + # existing binary versions. + srcVersions = dict([ (x[1].getSourceVersion(), x[1]) + for x in self._helper.findTrove( + (self._sourceName, None, None), + getLeaves=False + ) + ]) + + # Get the version of the specified source, usually the latest + # source version. + srcVersion = self._helper.findTrove(('%s:source' % self._sourceName, + version, None))[0][1] + + # Check to see if the latest source version is in the map of + # binary versions. + return srcVersion in srcVersions + + @commit + def getBuildJob(self): + """ + Get the list of trove specs to submit to the build system. + """ + + return ((self._sourceName, self._sourceVersion, None), ) + + @checkout + @commit + def build(self): + """ + Build all configured flavors of the group. + """ + + groupTroves = self.getBuildJob() + built = self._builder.cvc.cook(groupTroves) + return built + + @checkout + def add(self, *args, **kwargs): + """ + Add a trove to the package group contents. + """ + + # create package group model if it does not exist. + if self._pkgGroupName not in self._groups: + model = GroupContentsModel(self._pkgGroupName) + self._groups[self._pkgGroupName] = model + + self._groups[self._pkgGroupName].add(*args, **kwargs) + + @checkout + def remove(self, name, missingOk=False): + """ + Remove a given trove from the package group contents. + """ + + if self._pkgGroupName not in self._groups: + return + + return self._groups[self._pkgGroupName].remove(name, + missingOk=missingOk) + + @checkout + def hasPackage(self, name): + """ + Check if a given package name is in the group. + """ + + return (self._pkgGroupName in self._groups and + name in self._groups[self._pkgGroupName]) + + def addPackage(self, name, version, flavors): + """ + Add a package to the model. + @param name: name of the package + @type name: str + @param version: conary version from string object + @type version: conary.versions.VersionFromString + @param flavors: list of flavors + @type flavors: [conary.deps.deps.Flavor, ...] + """ + + # Now that versions are actually used for something make sure they + # are always present. + assert version + assert len(flavors) + + # Remove all versions and flavors of this name before adding this + # package. This avoids flavor change issues by replacing all flavors. + if self.hasPackage(name): + self.remove(name) + + plain = deps.parseFlavor('') + x86 = deps.parseFlavor('is: x86') + x86_64 = deps.parseFlavor('is: x86_64') + + if len(flavors) == 1: + flavor = flavors[0] + # noarch package, add unconditionally + if flavor == plain: + self.add(name, version=version) + + # x86, add with use=x86 + elif flavor.satisfies(x86): + self.add(name, version=version, flavor=flavor, use='x86') + + # x86_64, add with use=x86_64 + elif flavor.satisfies(x86_64): + self.add(name, version=version, flavor=flavor, use='x86_64') + + else: + raise UnsupportedTroveFlavorError(name=name, flavor=flavor) + + return + + elif len(flavors) == 2: + # This is most likely a normal package with both x86 and x86_64, but + # lets make sure anyway. + flvCount = {x86: 0, x86_64: 0, plain: 0} + for flavor in flavors: + if flavor.satisfies(x86): + flvCount[x86] += 1 + elif flavor.satisfies(x86_64): + flvCount[x86_64] += 1 + elif flavor.freeze() == '': + flvCount[plain] += 1 + else: + raise UnsupportedTroveFlavorError(name=name, flavor=flavor) + + # make sure there is only one instance of x86 and once instance of + # x86_64 in the flavor list. + assert (flvCount[x86_64] > 0) ^ (flvCount[plain] > 0) + assert len([ x for x, y in flvCount.iteritems() if y != 1 ]) == 1 + + # In this case just add the package unconditionally + self.add(name, version=version) + + return + + # These are special cases. + else: + # The way I see it there are a few ways you could end up here. + # 1. this is a kernel + # 2. this is a kernel module + # 3. this is a special package like glibc or openssl where there + # are i386, i686, and x86_64 varients. + # 4. this is a package with package flags that I don't know + # about. + # Lets see if we know about this package and think it should have + # more than two flavors. + + + # Get source trove name. + log.info('retrieving trove info for %s' % name) + srcTroveMap = self._helper._getSourceTroves( + (name, version, flavors[0]) + ) + srcTroveName = srcTroveMap.keys()[0][0].split(':')[0] + + # handle kernels. + if srcTroveName == 'kernel' or util.isKernelModulePackage(name): + # add all x86ish flavors with use=x86 and all x86_64ish flavors + # with use=x86_64 + for flavor in flavors: + if flavor.satisfies(x86): + self.add(name, version=version, flavor=flavor, use='x86') + elif flavor.satisfies(x86_64): + self.add(name, version=version, flavor=flavor, use='x86_64') + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + + # maybe this is one of the special flavors we know about. + elif srcTroveName in self._cfg.packageFlavors: + # separate packages into x86 and x86_64 by context name + # TODO: If we were really smart we would load the conary + # contexts and see what buildFlavors they contained. + flavorCount = {'x86': 0, 'x86_64': 0} + for context, bldflv in self._cfg.packageFlavors[srcTroveName]: + if context in ('i386', 'i486', 'i586', 'i686', 'x86'): + flavorCount['x86'] += 1 + elif context in ('x86_64', ): + flavorCount['x86_64'] += 1 + else: + raise UnknownBuildContextError(name=name, + flavor=context) + + for flavor in flavors: + if flavor.satisfies(x86): + flavorCount['x86'] -= 1 + elif flavor.satisfies(x86_64): + flavorCount['x86_64'] -= 1 + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + + errors = [ x for x, y in flavorCount.iteritems() if y != 0 ] + if errors: + raise FlavorCountMismatchError(name=name) + + for flavor in flavors: + if flavor.satisfies(x86): + self.add(name, version=version, + flavor=flavor, use='x86') + elif flavor.satisfies(x86_64): + self.add(name, version=version, + flavor=flavor, use='x86_64') + else: + raise UnsupportedTroveFlavorError(name=name, + flavor=flavor) + return + + # Unknown state. + raise UnhandledPackageAdditionError(name=name) + + @checkout + def setVersion(self, version): + """ + Set the version of the managed group. + """ + + self._helper.setVersion(self._sourceName, version) + + @checkout + def setErrataState(self, state): + """ + Set errata state info for the managed platform. + """ + + self._helper.setErrataState(self._sourceName, state) + + @checkout + def getErrataState(self, version=None): + """ + Get the errata state info. + """ + + if version is None: + version = self._sourceVersion + + return self._helper.getErrataState(self._sourceName, + version=version) + + def getVersions(self, pkgSet): + """ + Get the set of versions that are represented by the given set of + packages from the version factory. + """ + + return set() + #return self._versionFactory.getVersions(pkgSet) + + def _copyVersions(self): + """ + Copy versions from the packages group to the other managed groups. + """ + + pkgs = dict([ (x[1].name, x[1]) for x in + self._groups[self._pkgGroupName].iteritems() ]) + + for group in self._groups.itervalues(): + # skip over package group since it is the version source. + if group.groupName == self._pkgGroupName: + continue + + # for all other groups iterate over contents and set versions to + # match package group. + for k, pkg in group.iteritems(): + if pkg.name in pkgs: + pkg.version = pkgs[pkg.name].version + else: + raise UnknownPackageFoundInManagedGroupError(what=pkg.name) + + def _validateGroups(self): + """ + Validate the contents of the package group to ensure sanity: + 1. Check for packages that have the same source name, but + different versions. + 2. Check that the version in the group is the latest source/build + of that version. + 3. Check that package removals specified in the config file have + occured. + """ + + errors = [] + for name, group in self._groups.iteritems(): + log.info('checking consistentcy of %s' % name) + try: + self._checkNameVersionConflict(group) + except NameVersionConflictsFoundError, e: + errors.append((group, e)) + + try: + self._checkLatestVersion(group) + except OldVersionsFoundError, e: + errors.append((group, e)) + + try: + self._checkRemovals(group) + except ExpectedRemovalValidationFailedError, e: + errors.append((group, e)) + + if errors: + raise GroupValidationFailedError(errors=errors) + + def _checkNameVersionConflict(self, group): + """ + Check for packages taht have the same source name, but different + versions. + """ + + # get names and versions + troves = set() + labels = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + versionObj = versions.ThawVersion(pkgData.version) + labels.add(versionObj.branch().label()) + version = str(versionObj.asString()) + + flavor = None + # FIXME: At some point we might want to add proper flavor handling, + # note that group flavor handling is different than what + # findTroves normally does. + #if pkgData.flavor: + # flavor = deps.ThawFlavor(str(pkgData.flavor)) + + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = set([ x for x in + itertools.chain(*self._helper.findTroves(troves, + labels=labels).itervalues()) ]) + + # get sources for each name version pair + sources = self._helper.getSourceVersions(foundTroves) + + seen = {} + for (n, v, f), pkgSet in sources.iteritems(): + binVer = list(pkgSet)[1][1] + seen.setdefault(n, set()).add(binVer) + + binPkgs = {} + conflicts = {} + for name, vers in seen.iteritems(): + if len(vers) > 1: + log.error('found multiple versions of %s' % name) + for binVer in vers: + srcVer = binVer.getSourceVersion() + nvf = (name, srcVer, None) + conflicts.setdefault(name, []).append(srcVer) + binPkgs[nvf] = sources[nvf] + + if conflicts: + raise NameVersionConflictsFoundError(groupName=group.groupName, + conflicts=conflicts, + binPkgs=binPkgs) + + def _checkLatestVersion(self, group): + """ + Check to make sure each specific conary version is the latest source + and build count of the upstream version. + """ + + # get names and versions + troves = set() + labels = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + version = versions.ThawVersion(pkgData.version) + labels.add(version.branch().label()) + # get upstream version + revision = version.trailingRevision() + upstreamVersion = revision.getVersion() + + # FIXME: This should probably be a fully formed version + # as above. + version = upstreamVersion + + flavor = None + # FIXME: At some point we might want to add proper flavor handling, + # note that group flavor handling is different than what + # findTroves normally does. + #if pkgData.flavor: + # flavor = deps.ThawFlavor(str(pkgData.flavor)) + + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = dict([ (x[0], y) for x, y in + self._helper.findTroves(troves, labels=labels).iteritems() ]) + + pkgs = {} + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + version = None + if pkgData.version: + version = versions.ThawVersion(pkgData.version) + flavor = None + if pkgData.flavor: + flavor = deps.ThawFlavor(str(pkgData.flavor)) + + pkgs.setdefault(name, []).append((name, version, flavor)) + + assert len(pkgs) == len(foundTroves) + + # Get all old versions so that we can make sure any version conflicts + # were introduced by old version handling. + oldVersions = set() + for nvfLst in self._cfg.useOldVersion.itervalues(): + for nvf in nvfLst: + srcMap = self._helper.getSourceVersionMapFromBinaryVersion(nvf, + labels=self._cfg.platformSearchPath, latest=False) + oldVersions |= set(itertools.chain(*srcMap.itervalues())) + + + errors = {} + for name, found in foundTroves.iteritems(): + assert name in pkgs + current = pkgs[name] + + if len(current) > len(found): + log.warn('found more packages in the model than in the ' + 'repository, assuming that multiversion policy will ' + 'catch this.') + continue + + assert len(current) == 1 or len(found) == len(current) + + foundError = False + for i, (n, v, f) in enumerate(found): + if len(current) == 1: + i = 0 + cn, cv, cf = current[i] + assert n == cn + + if v != cv: + if (n, v, f) in oldVersions: + log.info('found %s=%s[%s] in oldVersions exceptions' + % (n, v, f)) + continue + foundError = True + + if foundError: + log.error('found old version for %s' % name) + errors[name] = (current, found) + + if errors: + raise OldVersionsFoundError(pkgNames=errors.keys(), errors=errors) + + def _checkRemovals(self, group): + """ + Check to make sure that all configured package removals have happened. + """ + + updateId = self.getErrataState() + + # get package removals from the config object. + removePackages = self._cfg.updateRemovesPackages.get(updateId, []) + removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) + removeSource = [ x[0] for x in + self._cfg.removeSource.get(updateId, []) ] + + # get names and versions + troves = set() + labels = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + versionObj = versions.ThawVersion(pkgData.version) + labels.add(versionObj.branch().label()) + version = str(versionObj.asString()) + + flavor = None + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = set([ x for x in + itertools.chain(*self._helper.findTroves(troves, + labels=labels).itervalues()) ]) + + # get sources for each name version pair + sources = self._helper.getSourceVersions(foundTroves) + + # collapse to sourceName: [ binNames, ] dictionary + sourceNameMap = dict([ (x[0].split(':')[0], [ z[0] for z in y ]) + for x, y in sources.iteritems() ]) + + binRemovals = set(itertools.chain(*[ sourceNameMap[x] + for x in removeSource + if x in sourceNameMap ])) + + # take the union + removals = set(removePackages) | set(removeObsoleted) | binRemovals + + errors = [] + # Make sure these packages are not in the group model. + for pkgKey, pkgData in group.iteritems(): + if pkgData.name in removals: + errors.append(pkgData.name) + + if errors: + log.info('found packages that should be removed %s' % errors) + raise ExpectedRemovalValidationFailedError(updateId=updateId, + pkgNames=errors) diff --git a/updatebot/groupmgr/model.py b/updatebot/groupmgr/model.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/model.py @@ -0,0 +1,146 @@ +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Model representation of groups. +""" + +from updatebot.lib.xobjects import XGroup +from updatebot.lib.xobjects import XGroupDoc +from updatebot.lib.xobjects import XGroupList +from updatebot.lib.xobjects import XPackageDoc +from updatebot.lib.xobjects import XPackageData +from updatebot.lib.xobjects import XPackageItem + +class AbstractModel(object): + """ + Base object for models. + """ + + docClass = None + dataClass = None + elementClass = None + + def __init__(self): + self._data = {} + self._nameMap = {} + + def _addItem(self, item): + """ + Add an item to the appropriate structures. + """ + + self._data[item.key] = item + if item.name not in self._nameMap: + self._nameMap[item.name] = set() + self._nameMap[item.name].add(item.key) + + def _removeItem(self, name, missingOk=False): + """ + Remove an item from the appropriate structures. + """ + + if missingOk: + keys = self._nameMap.pop(name, []) + else: + keys = self._nameMap.pop(name) + + for key in keys: + self._data.pop(key) + + @classmethod + def thaw(cls, xmlfn, args=None): + """ + Thaw the model from xml. + """ + + model = cls.docClass.fromfile(xmlfn) + obj = args and cls(*args) or cls() + for item in model.data.items: + obj._addItem(item) + return obj + + def freeze(self, toFile): + """ + Freeze the model to a given output file. + """ + + def _srtByKey(a, b): + return cmp(a.key, b.key) + + model = self.dataClass() + model.items = sorted(self._data.values(), cmp=_srtByKey) + + doc = self.docClass() + doc.data = model + doc.tofile(toFile) + + def iteritems(self): + """ + Iterate over the model data. + """ + + return self._data.iteritems() + + def add(self, *args, **kwargs): + """ + Add an data element. + """ + + obj = self.elementClass(*args, **kwargs) + self._addItem(obj) + + def remove(self, name, missingOk=False): + """ + Remove data element. + """ + + self._removeItem(name, missingOk=missingOk) + + def __contains__(self, name): + """ + Check if element name is in the model. + """ + + return name in self._nameMap + + +class GroupModel(AbstractModel): + """ + Model for representing group name and file name. + """ + + docClass = XGroupDoc + dataClass = XGroupList + elementClass = XGroup + + +class GroupContentsModel(AbstractModel): + """ + Model for representing group data. + """ + + docClass = XPackageDoc + dataClass = XPackageData + elementClass = XPackageItem + + def __init__(self, groupName, byDefault=True, depCheck=True): + AbstractModel.__init__(self) + self.groupName = groupName + self.byDefault = byDefault + self.depCheck = depCheck + + # figure out file name based on group name + name = ''.join([ x.capitalize() for x in self.groupName.split('-') ]) + self.fileName = name[0].lower() + name[1:] + '.xml' diff --git a/updatebot/groupmgr/single.py b/updatebot/groupmgr/single.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/single.py @@ -0,0 +1,89 @@ +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for managing groups that contain only packages with no subgroups. +""" + +from updatebot.groupmgr.helper import GroupHelper +from updatebot.groupmgr.manager import GroupManager + +class SingleGroupHelper(GroupHelper): + """ + Group helper for managing groups with only packages and no other groups. + """ + + def __init__(self, *args, **kwargs): + GroupHelper.__init__(self, *args, **kwargs) + self._groupContents = {} + + +class SingleGroupManager(GroupManager): + """ + Class to manage a single group that contains only packages with no + subgroups. + """ + + _helperClass = SingleGrouphelper + + def __init__(self, name, *args, **kwargs): + GroupManager.__init__(self, *args, **kwargs) + self._sourceName = name + self._pkgGroupName = self._sourceName + + +class SingleGroupManagerSet(object): + """ + Class for working with a set of group manager instances. + """ + + _managerClass = SingleGroupManager + + def __init__(self, cfg): + self._cfg = cfg + self._groups = {} + + def newGroup(self, name): + """ + Create a new group instance with the provided name. + """ + + assert name not in self._groups + group = self._managerClass(name, self._cfg) + self._groups[name] = group + return group + + def build(self): + """ + Build all groups in the set. + """ + + # Make sure there are groups defined + assert self._groups + + toBuild = set() + builder = None + for group in self._groups.itervalues(): + # Select the builder from the first instance to build all groups. + if not builder: + builder = group._builder + + # Get the build job for each group. + trvSpecs = group.getBuildJob() + assert len(trvSpecs) == 1 + toBuild.add(trvSpecs[0]) + + # Build all groups in separate jobs, waiting until all have completed + # to commit. + return builder.buildmany(toBuild, lateCommit=True) diff --git a/updatebot/groupmgr/versions.py b/updatebot/groupmgr/versions.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/versions.py @@ -0,0 +1,76 @@ +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for generating platform versions from some upstream source. +""" + +import copy + +from updatebot.pkgsource import RpmSource + +class URLVersionSource(object): + """ + Class for handling the indexing and comparison of ISO contents. + """ + + def __init__(self, cfg, version, url): + self._cfg = cfg + self._version = version + self._url = url + + pkgSource = RpmSource(cfg, ) + pkgSource.loadFromUrl(url) + pkgSource.finalize() + pkgSource.setLoaded() + + self._sourceSet = set([ x for x in pkgSource.iterPackageSet() ]) + + def areWeHereYet(self, pkgSet): + """ + Figure out if the given package set is this version. + @param pkgSet: set of source names and source versions + @type pkgSet: set([(conarySourceName, conarySourceVersion)]) + @return boolean + """ + + # Make sure that all versions that are on the ISO are in the conary + # repository. It appears to be a common case that the ISO is missing + # content from RHN. + return not self._sourceSet.difference(pkgSet) + + +class VersionFactory(object): + """ + Class for generating group verisons from a variety of sources. + """ + + def __init__(self, cfg): + self._cfg = copy.deepcopy(cfg) + self._cfg.synthesizeSources = True + self._sources = {} + + for version, url in sorted(cfg.versionSources.iteritems()): + self._sources[version] = URLVersionSource(self._cfg, version, url) + + def getVersions(self, pkgSet): + """ + Given the available package sources find the current versions. + """ + + versions = [] + for version, source in sorted(self._sources.iteritems()): + if source.areWeHereYet(pkgSet): + versions.append(version) + return set(versions) From elliot at rpath.com Mon Mar 15 17:17:36 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:36 +0000 Subject: mirrorball: expose public inteface Message-ID: <201003152117.o2FLHaeg015544@scc.eng.rpath.com> changeset: 5bb3dd4cabea user: Elliot Peele date: Wed, 10 Mar 2010 17:07:59 -0500 expose public inteface diff --git a/updatebot/groupmgr/__init__.py b/updatebot/groupmgr/__init__.py --- a/updatebot/groupmgr/__init__.py +++ b/updatebot/groupmgr/__init__.py @@ -11,3 +11,9 @@ # or fitness for a particular purpose. See the Common Public License for # full details. # + +from updatebot.groupmgr.manager import GroupManager +from updatebot.groupmgr.single import SingleGroupManager +from updatebot.groupmgr.single import SingleGroupManagerSet +from updatebot.groupmgr.errata import ErrataGroupManager +from updatebot.groupmgr.errata import ErrataGroupManagerSet From elliot at rpath.com Mon Mar 15 17:17:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:37 +0000 Subject: mirrorball: use correct super class Message-ID: <201003152117.o2FLHboI015571@scc.eng.rpath.com> changeset: 97b84c636e46 user: Elliot Peele date: Wed, 10 Mar 2010 17:38:24 -0500 use correct super class diff --git a/updatebot/groupmgr/errata.py b/updatebot/groupmgr/errata.py --- a/updatebot/groupmgr/errata.py +++ b/updatebot/groupmgr/errata.py @@ -20,7 +20,7 @@ from updatebot.groupmgr.single import SingleGroupManager from updatebot.groupmgr.single import SingleGroupManagerSet -class ErrataGroupHelper(GroupHelper): +class ErrataGroupHelper(SingleGroupHelper): """ Class for managing errata group source troves. """ From elliot at rpath.com Mon Mar 15 17:17:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:39 +0000 Subject: mirrorball: fix typo Message-ID: <201003152117.o2FLHdGf015600@scc.eng.rpath.com> changeset: 35fd19f6cc1b user: Elliot Peele date: Wed, 10 Mar 2010 17:38:32 -0500 fix typo diff --git a/updatebot/groupmgr/single.py b/updatebot/groupmgr/single.py --- a/updatebot/groupmgr/single.py +++ b/updatebot/groupmgr/single.py @@ -35,7 +35,7 @@ subgroups. """ - _helperClass = SingleGrouphelper + _helperClass = SingleGroupHelper def __init__(self, name, *args, **kwargs): GroupManager.__init__(self, *args, **kwargs) From elliot at rpath.com Mon Mar 15 17:17:42 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:42 +0000 Subject: mirrorball: properly name the group Message-ID: <201003152117.o2FLHgNf015655@scc.eng.rpath.com> changeset: a44acc414c61 user: Elliot Peele date: Fri, 12 Mar 2010 14:14:29 -0500 properly name the group diff --git a/updatebot/groupmgr/single.py b/updatebot/groupmgr/single.py --- a/updatebot/groupmgr/single.py +++ b/updatebot/groupmgr/single.py @@ -39,7 +39,7 @@ def __init__(self, name, *args, **kwargs): GroupManager.__init__(self, *args, **kwargs) - self._sourceName = name + self._sourceName = 'group-%s' % name self._pkgGroupName = self._sourceName From elliot at rpath.com Mon Mar 15 17:17:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:43 +0000 Subject: mirrorball: use correct class name Message-ID: <201003152117.o2FLHhT7015686@scc.eng.rpath.com> changeset: 344276c62f3e user: Elliot Peele date: Fri, 12 Mar 2010 14:14:40 -0500 use correct class name diff --git a/updatebot/groupmgr/errata.py b/updatebot/groupmgr/errata.py --- a/updatebot/groupmgr/errata.py +++ b/updatebot/groupmgr/errata.py @@ -26,7 +26,7 @@ """ def __init__(self, *args, **kwargs): - GroupHelper.__init__(self, *args, **kwargs) + SingleGroupHelper.__init__(self, *args, **kwargs) self._newPkgFactory = 'managed-errata-group' From elliot at rpath.com Mon Mar 15 17:17:40 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:40 +0000 Subject: mirrorball: move sanity checking into separate class Message-ID: <201003152117.o2FLHeOl015627@scc.eng.rpath.com> changeset: fa9465c20d8f user: Elliot Peele date: Wed, 10 Mar 2010 17:38:47 -0500 move sanity checking into separate class diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py --- a/updatebot/groupmgr/manager.py +++ b/updatebot/groupmgr/manager.py @@ -19,22 +19,20 @@ import logging import itertools -from conary import versions from conary.deps import deps from updatebot.lib import util from updatebot.build import Builder -from updatebot.errors import OldVersionsFoundError + from updatebot.errors import FlavorCountMismatchError from updatebot.errors import UnknownBuildContextError -from updatebot.errors import GroupValidationFailedError from updatebot.errors import UnsupportedTroveFlavorError from updatebot.errors import UnhandledPackageAdditionError -from updatebot.errors import NameVersionConflictsFoundError -from updatebot.errors import ExpectedRemovalValidationFailedError +from updatebot.errors import NotCommittingOutOfDateSourceError from updatebot.errors import UnknownPackageFoundInManagedGroupError from updatebot.groupmgr.helper import GroupHelper +from updatebot.groupmgr.sanity import GroupSanityChecker log = logging.getLogger('updatebot.groupmgr') @@ -64,20 +62,20 @@ """ _helperClass = GroupHelper + _sanityCheckerClass = GroupSanityChecker def __init__(self, cfg, parentGroup=False, targetGroup=False): self._cfg = cfg self._helper = self._helperClass(self._cfg) self._builder = Builder(self._cfg, rmakeCfgFn='rmakerc-groups') - - #self._versionFactory = VersionFactory(cfg) + self._sanity = self._sanityCheckerClass(self._cfg, self._helper) assert not (parentGroup and targetGroup) if targetGroup: srcName = '%s:source' % self._cfg.topSourceGroup[0] - trvs = self._helper.findTrove((srcName, None, None), - labels=(self._cfg.targetLabel, )) + trvs = self._helper.findTrove( + (srcName, self._cfg.targetLabel, None)) assert len(trvs) @@ -153,7 +151,7 @@ self._copyVersions() # validate group contents. - self._validateGroups() + self._sanity.check(self._groups, self.getErrataState()) # write out the model data self._helper.setModel(self._sourceName, self._groups) @@ -427,7 +425,6 @@ """ return set() - #return self._versionFactory.getVersions(pkgSet) def _copyVersions(self): """ @@ -449,244 +446,3 @@ pkg.version = pkgs[pkg.name].version else: raise UnknownPackageFoundInManagedGroupError(what=pkg.name) - - def _validateGroups(self): - """ - Validate the contents of the package group to ensure sanity: - 1. Check for packages that have the same source name, but - different versions. - 2. Check that the version in the group is the latest source/build - of that version. - 3. Check that package removals specified in the config file have - occured. - """ - - errors = [] - for name, group in self._groups.iteritems(): - log.info('checking consistentcy of %s' % name) - try: - self._checkNameVersionConflict(group) - except NameVersionConflictsFoundError, e: - errors.append((group, e)) - - try: - self._checkLatestVersion(group) - except OldVersionsFoundError, e: - errors.append((group, e)) - - try: - self._checkRemovals(group) - except ExpectedRemovalValidationFailedError, e: - errors.append((group, e)) - - if errors: - raise GroupValidationFailedError(errors=errors) - - def _checkNameVersionConflict(self, group): - """ - Check for packages taht have the same source name, but different - versions. - """ - - # get names and versions - troves = set() - labels = set() - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - - version = None - if pkgData.version: - versionObj = versions.ThawVersion(pkgData.version) - labels.add(versionObj.branch().label()) - version = str(versionObj.asString()) - - flavor = None - # FIXME: At some point we might want to add proper flavor handling, - # note that group flavor handling is different than what - # findTroves normally does. - #if pkgData.flavor: - # flavor = deps.ThawFlavor(str(pkgData.flavor)) - - troves.add((name, version, flavor)) - - # Get flavors and such. - foundTroves = set([ x for x in - itertools.chain(*self._helper.findTroves(troves, - labels=labels).itervalues()) ]) - - # get sources for each name version pair - sources = self._helper.getSourceVersions(foundTroves) - - seen = {} - for (n, v, f), pkgSet in sources.iteritems(): - binVer = list(pkgSet)[1][1] - seen.setdefault(n, set()).add(binVer) - - binPkgs = {} - conflicts = {} - for name, vers in seen.iteritems(): - if len(vers) > 1: - log.error('found multiple versions of %s' % name) - for binVer in vers: - srcVer = binVer.getSourceVersion() - nvf = (name, srcVer, None) - conflicts.setdefault(name, []).append(srcVer) - binPkgs[nvf] = sources[nvf] - - if conflicts: - raise NameVersionConflictsFoundError(groupName=group.groupName, - conflicts=conflicts, - binPkgs=binPkgs) - - def _checkLatestVersion(self, group): - """ - Check to make sure each specific conary version is the latest source - and build count of the upstream version. - """ - - # get names and versions - troves = set() - labels = set() - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - - version = None - if pkgData.version: - version = versions.ThawVersion(pkgData.version) - labels.add(version.branch().label()) - # get upstream version - revision = version.trailingRevision() - upstreamVersion = revision.getVersion() - - # FIXME: This should probably be a fully formed version - # as above. - version = upstreamVersion - - flavor = None - # FIXME: At some point we might want to add proper flavor handling, - # note that group flavor handling is different than what - # findTroves normally does. - #if pkgData.flavor: - # flavor = deps.ThawFlavor(str(pkgData.flavor)) - - troves.add((name, version, flavor)) - - # Get flavors and such. - foundTroves = dict([ (x[0], y) for x, y in - self._helper.findTroves(troves, labels=labels).iteritems() ]) - - pkgs = {} - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - version = None - if pkgData.version: - version = versions.ThawVersion(pkgData.version) - flavor = None - if pkgData.flavor: - flavor = deps.ThawFlavor(str(pkgData.flavor)) - - pkgs.setdefault(name, []).append((name, version, flavor)) - - assert len(pkgs) == len(foundTroves) - - # Get all old versions so that we can make sure any version conflicts - # were introduced by old version handling. - oldVersions = set() - for nvfLst in self._cfg.useOldVersion.itervalues(): - for nvf in nvfLst: - srcMap = self._helper.getSourceVersionMapFromBinaryVersion(nvf, - labels=self._cfg.platformSearchPath, latest=False) - oldVersions |= set(itertools.chain(*srcMap.itervalues())) - - - errors = {} - for name, found in foundTroves.iteritems(): - assert name in pkgs - current = pkgs[name] - - if len(current) > len(found): - log.warn('found more packages in the model than in the ' - 'repository, assuming that multiversion policy will ' - 'catch this.') - continue - - assert len(current) == 1 or len(found) == len(current) - - foundError = False - for i, (n, v, f) in enumerate(found): - if len(current) == 1: - i = 0 - cn, cv, cf = current[i] - assert n == cn - - if v != cv: - if (n, v, f) in oldVersions: - log.info('found %s=%s[%s] in oldVersions exceptions' - % (n, v, f)) - continue - foundError = True - - if foundError: - log.error('found old version for %s' % name) - errors[name] = (current, found) - - if errors: - raise OldVersionsFoundError(pkgNames=errors.keys(), errors=errors) - - def _checkRemovals(self, group): - """ - Check to make sure that all configured package removals have happened. - """ - - updateId = self.getErrataState() - - # get package removals from the config object. - removePackages = self._cfg.updateRemovesPackages.get(updateId, []) - removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) - removeSource = [ x[0] for x in - self._cfg.removeSource.get(updateId, []) ] - - # get names and versions - troves = set() - labels = set() - for pkgKey, pkgData in group.iteritems(): - name = str(pkgData.name) - - version = None - if pkgData.version: - versionObj = versions.ThawVersion(pkgData.version) - labels.add(versionObj.branch().label()) - version = str(versionObj.asString()) - - flavor = None - troves.add((name, version, flavor)) - - # Get flavors and such. - foundTroves = set([ x for x in - itertools.chain(*self._helper.findTroves(troves, - labels=labels).itervalues()) ]) - - # get sources for each name version pair - sources = self._helper.getSourceVersions(foundTroves) - - # collapse to sourceName: [ binNames, ] dictionary - sourceNameMap = dict([ (x[0].split(':')[0], [ z[0] for z in y ]) - for x, y in sources.iteritems() ]) - - binRemovals = set(itertools.chain(*[ sourceNameMap[x] - for x in removeSource - if x in sourceNameMap ])) - - # take the union - removals = set(removePackages) | set(removeObsoleted) | binRemovals - - errors = [] - # Make sure these packages are not in the group model. - for pkgKey, pkgData in group.iteritems(): - if pkgData.name in removals: - errors.append(pkgData.name) - - if errors: - log.info('found packages that should be removed %s' % errors) - raise ExpectedRemovalValidationFailedError(updateId=updateId, - pkgNames=errors) diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py new file mode 100644 --- /dev/null +++ b/updatebot/groupmgr/sanity.py @@ -0,0 +1,278 @@ +# +# Copyright (c) 2009-2010 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +""" +Module for sanity checking group contents model. +""" + +import logging +import itertools + +from conary import versions +from conary.deps import deps + +from updatebot.errors import OldVersionsFoundError +from updatebot.errors import GroupValidationFailedError +from updatebot.errors import NameVersionConflictsFoundError +from updatebot.errors import ExpectedRemovalValidationFailedError + +log = logging.getLogger('updatebot.groupmgr') + +class GroupSanityChecker(object): + """ + Class for checking group model sanity. + """ + + def __init__(self, cfg, helper): + self._cfg = cfg + self._helper = helper + + def check(self, groups, errataState): + """ + Validate the contents of the package group to ensure sanity: + 1. Check for packages that have the same source name, but + different versions. + 2. Check that the version in the group is the latest source/build + of that version. + 3. Check that package removals specified in the config file have + occured. + """ + + errors = [] + for name, group in groups.iteritems(): + log.info('checking consistentcy of %s' % name) + try: + self._checkNameVersionConflict(group) + except NameVersionConflictsFoundError, e: + errors.append((group, e)) + + try: + self._checkLatestVersion(group) + except OldVersionsFoundError, e: + errors.append((group, e)) + + try: + self._checkRemovals(group, errataState) + except ExpectedRemovalValidationFailedError, e: + errors.append((group, e)) + + if errors: + raise GroupValidationFailedError(errors=errors) + + def _checkNameVersionConflict(self, group): + """ + Check for packages taht have the same source name, but different + versions. + """ + + # get names and versions + troves = set() + labels = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + versionObj = versions.ThawVersion(pkgData.version) + labels.add(versionObj.branch().label()) + version = str(versionObj.asString()) + + flavor = None + # FIXME: At some point we might want to add proper flavor handling, + # note that group flavor handling is different than what + # findTroves normally does. + #if pkgData.flavor: + # flavor = deps.ThawFlavor(str(pkgData.flavor)) + + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = set([ x for x in + itertools.chain(*self._helper.findTroves(troves, + labels=labels).itervalues()) ]) + + # get sources for each name version pair + sources = self._helper.getSourceVersions(foundTroves) + + seen = {} + for (n, v, f), pkgSet in sources.iteritems(): + binVer = list(pkgSet)[1][1] + seen.setdefault(n, set()).add(binVer) + + binPkgs = {} + conflicts = {} + for name, vers in seen.iteritems(): + if len(vers) > 1: + log.error('found multiple versions of %s' % name) + for binVer in vers: + srcVer = binVer.getSourceVersion() + nvf = (name, srcVer, None) + conflicts.setdefault(name, []).append(srcVer) + binPkgs[nvf] = sources[nvf] + + if conflicts: + raise NameVersionConflictsFoundError(groupName=group.groupName, + conflicts=conflicts, + binPkgs=binPkgs) + + def _checkLatestVersion(self, group): + """ + Check to make sure each specific conary version is the latest source + and build count of the upstream version. + """ + + # get names and versions + troves = set() + labels = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + version = versions.ThawVersion(pkgData.version) + labels.add(version.branch().label()) + # get upstream version + revision = version.trailingRevision() + upstreamVersion = revision.getVersion() + + # FIXME: This should probably be a fully formed version + # as above. + version = upstreamVersion + + flavor = None + # FIXME: At some point we might want to add proper flavor handling, + # note that group flavor handling is different than what + # findTroves normally does. + #if pkgData.flavor: + # flavor = deps.ThawFlavor(str(pkgData.flavor)) + + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = dict([ (x[0], y) for x, y in + self._helper.findTroves(troves, labels=labels).iteritems() ]) + + pkgs = {} + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + version = None + if pkgData.version: + version = versions.ThawVersion(pkgData.version) + flavor = None + if pkgData.flavor: + flavor = deps.ThawFlavor(str(pkgData.flavor)) + + pkgs.setdefault(name, []).append((name, version, flavor)) + + assert len(pkgs) == len(foundTroves) + + # Get all old versions so that we can make sure any version conflicts + # were introduced by old version handling. + oldVersions = set() + for nvfLst in self._cfg.useOldVersion.itervalues(): + for nvf in nvfLst: + srcMap = self._helper.getSourceVersionMapFromBinaryVersion(nvf, + labels=self._cfg.platformSearchPath, latest=False) + oldVersions |= set(itertools.chain(*srcMap.itervalues())) + + + errors = {} + for name, found in foundTroves.iteritems(): + assert name in pkgs + current = pkgs[name] + + if len(current) > len(found): + log.warn('found more packages in the model than in the ' + 'repository, assuming that multiversion policy will ' + 'catch this.') + continue + + assert len(current) == 1 or len(found) == len(current) + + foundError = False + for i, (n, v, f) in enumerate(found): + if len(current) == 1: + i = 0 + cn, cv, cf = current[i] + assert n == cn + + if v != cv: + if (n, v, f) in oldVersions: + log.info('found %s=%s[%s] in oldVersions exceptions' + % (n, v, f)) + continue + foundError = True + + if foundError: + log.error('found old version for %s' % name) + errors[name] = (current, found) + + if errors: + raise OldVersionsFoundError(pkgNames=errors.keys(), errors=errors) + + def _checkRemovals(self, group, updateId): + """ + Check to make sure that all configured package removals have happened. + """ + + # get package removals from the config object. + removePackages = self._cfg.updateRemovesPackages.get(updateId, []) + removeObsoleted = self._cfg.removeObsoleted.get(updateId, []) + removeSource = [ x[0] for x in + self._cfg.removeSource.get(updateId, []) ] + + # get names and versions + troves = set() + labels = set() + for pkgKey, pkgData in group.iteritems(): + name = str(pkgData.name) + + version = None + if pkgData.version: + versionObj = versions.ThawVersion(pkgData.version) + labels.add(versionObj.branch().label()) + version = str(versionObj.asString()) + + flavor = None + troves.add((name, version, flavor)) + + # Get flavors and such. + foundTroves = set([ x for x in + itertools.chain(*self._helper.findTroves(troves, + labels=labels).itervalues()) ]) + + # get sources for each name version pair + sources = self._helper.getSourceVersions(foundTroves) + + # collapse to sourceName: [ binNames, ] dictionary + sourceNameMap = dict([ (x[0].split(':')[0], [ z[0] for z in y ]) + for x, y in sources.iteritems() ]) + + binRemovals = set(itertools.chain(*[ sourceNameMap[x] + for x in removeSource + if x in sourceNameMap ])) + + # take the union + removals = set(removePackages) | set(removeObsoleted) | binRemovals + + errors = [] + # Make sure these packages are not in the group model. + for pkgKey, pkgData in group.iteritems(): + if pkgData.name in removals: + errors.append(pkgData.name) + + if errors: + log.info('found packages that should be removed %s' % errors) + raise ExpectedRemovalValidationFailedError(updateId=updateId, + pkgNames=errors) From elliot at rpath.com Mon Mar 15 17:17:44 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:44 +0000 Subject: mirrorball: be more consistent about package names and search labels Message-ID: <201003152117.o2FLHjrr015713@scc.eng.rpath.com> changeset: 6fe149300440 user: Elliot Peele date: Fri, 12 Mar 2010 14:15:09 -0500 be more consistent about package names and search labels diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py --- a/updatebot/groupmgr/manager.py +++ b/updatebot/groupmgr/manager.py @@ -79,7 +79,7 @@ assert len(trvs) - self._sourceName = srcName + self._sourceName = self._cfg.topSourceGroup[0] self._sourceVersion = trvs[0][1] self._readonly = True elif parentGroup: @@ -90,7 +90,7 @@ assert len(trvs) - self._sourceName = topGroup[0] + self._sourceName = self._cfg.topParentSourceGroup self._sourceVersion = trvs[0][1] self._readonly = True @@ -175,19 +175,27 @@ if not version: verison = self._sourceVersion + # Search the label from the source version. + if self._sourceVersion: + labels = (self._sourceVersion.trailingLabel(), ) + else: + labels = (self._cfg.topSourceGroup[1], ) + # Get a mapping of all source version to binary versions for all # existing binary versions. srcVersions = dict([ (x[1].getSourceVersion(), x[1]) for x in self._helper.findTrove( (self._sourceName, None, None), - getLeaves=False + getLeaves=False, + labels=labels, ) ]) # Get the version of the specified source, usually the latest # source version. - srcVersion = self._helper.findTrove(('%s:source' % self._sourceName, - version, None))[0][1] + srcVersion = self._helper.findTrove( + ('%s:source' % self._sourceName, version, None), + labels=labels)[0][1] # Check to see if the latest source version is in the map of # binary versions. From elliot at rpath.com Mon Mar 15 17:17:46 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:46 +0000 Subject: mirrorball: simplify flatten dictionary values Message-ID: <201003152117.o2FLHko3015742@scc.eng.rpath.com> changeset: bc7c50301205 user: Elliot Peele date: Fri, 12 Mar 2010 14:15:56 -0500 simplify flatten dictionary values diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -18,6 +18,7 @@ import time import logging +import itertools from updatebot import build from updatebot import update @@ -49,15 +50,12 @@ def _flattenSetDict(setDict): """ Convert a dictionary with values of sets to a list. - @param setList: dictionary of sets - @type setList: [set(), set(), ...] + @param setDict: dictionary of sets + @type setDict: [set(), set(), ...] @return list of items that were in the sets """ - lst = [] - for trvSet in setDict.itervalues(): - lst.extend(list(trvSet)) - return lst + return [ x for x in itertools.chain(*setDict.itervalues()) ] def create(self, rebuild=False, recreate=None, toCreate=None): """ From elliot at rpath.com Mon Mar 15 17:17:47 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:47 +0000 Subject: mirrorball: add caching of cloned from for an entire label Message-ID: <201003152117.o2FLHlE8015769@scc.eng.rpath.com> changeset: dc55656c5626 user: Elliot Peele date: Mon, 15 Mar 2010 16:47:46 -0400 add caching of cloned from for an entire label diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -27,6 +27,7 @@ from conary import trove from conary import state from conary import checkin +from conary.deps import deps from conary.build import use from conary import conarycfg from conary import conaryclient @@ -102,6 +103,7 @@ # Cache cloned from information # srcNVF: destNVF self._clonedFromCache = {} + self._labelClonedFromCache = {} def getConaryConfig(self): """ @@ -274,8 +276,14 @@ cached = set(x for x in troveSpecs if x in cache) uncached = sorted(set(troveSpecs).difference(cached)) + req = [] + for n, v, f in uncached: + if n.endswith(':source'): + f = deps.parseFlavor('') + req.append((n, v, f)) + tiMap = {} - tiLst = self._repos.getTroveInfo(tiType, uncached) + tiLst = self._repos.getTroveInfo(tiType, req) for i, nvf in enumerate(uncached): ti = tiLst[i]() if tiFunc: @@ -296,8 +304,45 @@ @type troveSpecs: iterable of (name, verObj, flvObj) tuples. """ + def tiFunc(ti, nvf): + return (nvf[0], ti, nvf[2]) + return self._cacheTroveInfo(troveSpecs, self._clonedFromCache, - trove._TROVEINFO_TAG_CLONEDFROM) + trove._TROVEINFO_TAG_CLONEDFROM, tiFunc=tiFunc) + + def getClonedFromForLabel(self, label): + """ + Get a mapping of cloned from trove info for all versions on a label. + @param label: conary label + @type label: conary.versions.Label + """ + + if hasattr(label, 'label'): + label = label.label() + + if label in self._labelClonedFromCache: + return self._labelClonedFromCache[label] + + req = {None: {label: None}} + binTrvMap = self._repos.getTroveVersionsByLabel(req) + + binTrvs = set() + for n, vMap in binTrvMap.iteritems(): + for v, flvs in vMap.iteritems(): + if n.endswith(':source'): + binTrvs.add((n, v, None)) + else: + binTrvs.update(set((n, v, x) for x in flvs)) + + cfMap = self.getClonedFrom(binTrvs) + + ret = {} + for f, t in cfMap.iteritems(): + assert len(cfMap[f]) == 1 + ret[f] = list(t)[0] + + self._labelClonedFromCache[label] = ret + return ret def getSourceVersions(self, binTroveSpecs): """ @@ -309,12 +354,13 @@ """ def tiFunc(ti, nvf): - return (ti, nvf[0].getSourceVersion(), None) + return (ti, nvf[1].getSourceVersion(), None) return self._cacheTroveInfo(binTroveSpecs, self._sourceVersionCache, trove._TROVEINFO_TAG_SOURCENAME, tiFunc=tiFunc) - def getBinaryVersions(self, srcTroveSpecs, labels=None, latest=True): + def getBinaryVersions(self, srcTroveSpecs, labels=None, latest=True, + includeBuildLabel=False, missingOk=False): """ Given a list of source trove specs, find the latest versions of all binaries generated from these sources. @@ -324,6 +370,9 @@ @type labels: list(conary.versions.Label, ...) @param latest: get only the latest binaries. @type latest: boolean + @param includeBuildLabel: search the build label in addition to + specified labels. + @type includeBuildLabel: boolean @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ... ]} """ @@ -331,6 +380,10 @@ if not labels: labels = [ self._ccfg.buildLabel, ] + # Insert build label if it isn't already in the list + if includeBuildLabel and self._ccfg.buildLabel not in labels: + labels.insert(0, self._ccfg.buildLabel) + # Needs to be a frozenset so that it is hashable. labels = frozenset(labels) @@ -364,7 +417,12 @@ srcMap = {} for srcTrv in srcTroveSpecs: if srcTrv not in srcVerMap: - log.error('can not find requested source trove in repository') + msg = ('can not find binaries for requested source trove in ' + 'repository %s' % (srcTrv, )) + if missingOk: + log.warn(msg) + continue + log.error(msg) raise BinariesNotFoundForSourceVersion(srcName=srcTrv[0], srcVersion=srcTrv[1]) From elliot at rpath.com Mon Mar 15 17:17:48 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:48 +0000 Subject: mirrorball: add missing import Message-ID: <201003152117.o2FLHmGe015796@scc.eng.rpath.com> changeset: 113e75d0bb43 user: Elliot Peele date: Mon, 15 Mar 2010 16:48:07 -0400 add missing import make sure flavors is indexable diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py --- a/updatebot/groupmgr/manager.py +++ b/updatebot/groupmgr/manager.py @@ -33,6 +33,7 @@ from updatebot.groupmgr.helper import GroupHelper from updatebot.groupmgr.sanity import GroupSanityChecker +from updatebot.groupmgr.model import GroupContentsModel log = logging.getLogger('updatebot.groupmgr') @@ -269,6 +270,7 @@ # are always present. assert version assert len(flavors) + flavors = list(flavors) # Remove all versions and flavors of this name before adding this # package. This avoids flavor change issues by replacing all flavors. From elliot at rpath.com Mon Mar 15 17:17:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:49 +0000 Subject: mirrorball: add methods for accessing more data from the errata model Message-ID: <201003152117.o2FLHoSx015822@scc.eng.rpath.com> changeset: c2bf51c2eba1 user: Elliot Peele date: Mon, 15 Mar 2010 16:48:42 -0400 add methods for accessing more data from the errata model diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -95,6 +95,15 @@ return '%s (no detail found)' % bucketId @loadErrata + def getAdvisoryPackages(self, advisory): + """ + Give a valid advisory name, return a set of applicable source package + objects. + """ + + return self._advPkgMap.get(advisory, set()) + + @loadErrata def getVersions(self, bucketId): """ Get a set of group versions that should be built for the given bucketId. @@ -102,11 +111,24 @@ @type bucketId: integer (unix time) """ - versions = set() - for advisory in sellf.getUpdateDetail(bucketId): - versions.add(self._errata.getGroupVersion(advisory['name'])) + versions = dict() + for advInfo in self.getUpdateDetail(bucketId): + advisory = advInfo['name'] + versions[advisory] = self._errata.getGroupVersion(advisory) return versions + @loadErrata + def getNames(self, bucketId): + """ + Get a map of group names by advisory. + """ + + names = dict() + for advInfo in self.getUpdateDetail(bucketId): + advisory = advInfo['name'] + names[advisory] = self._errata.getGroupName(advisory) + return names + def getBucketVersion(self, bucketId): """ Convert a bucketId to a conary version. From elliot at rpath.com Mon Mar 15 17:17:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 15 Mar 2010 21:17:52 +0000 Subject: mirrorball: initial stab at errata groups Message-ID: <201003152117.o2FLHqIZ015856@scc.eng.rpath.com> changeset: 799cd233cd6a user: Elliot Peele date: Mon, 15 Mar 2010 16:49:28 -0400 initial stab at errata groups diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -492,6 +492,16 @@ _params = [] _template = 'This platform has not yet been created.' +class TargetVersionNotFoundError(ImportError): + """ + TargetVersionNotFoundError, raised when a group version that is expected to + be on the targetLabel can not be found. + """ + + _params = ['updateId', 'version'] + _template = ('Could not find %(version)s of the top level group on ' + 'targetLabel') + class CvcError(UpdateBotError): """ Generic cvc related error. diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -30,6 +30,7 @@ from updatebot.errors import UnknownRemoveSourceError from updatebot.errors import PlatformNotImportedError +from updatebot.errors import TargetVersionNotFoundError from updatebot.errors import PlatformAlreadyImportedError log = logging.getLogger('updatebot.ordered') @@ -327,3 +328,109 @@ log.info('published update %s in %s seconds' % (advTime, totalTime)) return updateSet + + def promote(self): + pass + + def createErrataGroups(self): + """ + Create groups for each advisory that only contain the versions of + packages that were included in that advisory. Once created, promote + to production branch. + """ + + # Get current timestamp + current = self._groupmgr.getErrataState() + if current is None: + raise PlatformNotImportedError + + # Load package source. + self._pkgSource.load() + + # Get latest errataState from the targetLabel so that we can fence group + # building based on the target label state. + targetGroup = groupmgr.GroupManager(self._cfg, targetGroup=True) + targetErrataState = targetGroup.getErrataState() + + failures = {} + for updateId, updates in self._errata.iterByIssueDate(current=0): + # Stop if the updateId is greater than the state of the latest group + # on the production label. + if updateId > targetErrataState: + log.info('current updateId (%s) is newer than target label ' + 'contents' % updateId) + break + + # Make sure the group representing the current updateId has been + # imorted and promoted to the production label. + version = self._errata.getBucketVersion(updateId) + if not targetGroup.hasBinaryVersion(version=version): + raise TargetVersionNotFoundError(version=version, + updateId=updateId) + + # Now that we know that the packages that are part of this update + # should be on the target label we can separate things into + # advisories. + mgr = groupmgr.ErrataGroupManagerSet(self._cfg) + groupNames = self._errata.getNames(updateId) + for advInfo in self._errata.getUpdateDetail(updateId): + advisory = advInfo['name'] + log.info('%s: processing' % advisory) + + srcPkgs = self._errata.getAdvisoryPackages(advisory) + assert srcPkgs + + grp = mgr.newGroup(groupNames[advisory]) + grp.setVersion(version) + grp.setErrataState(updateId) + + log.info('%s: finding built packages' % advisory) + binTrvs = set() + for srcPkg in srcPkgs: + binTrvSpecs = \ + self._updater.getBinaryVersionsFromSourcePackage(srcPkg) + + # FIXME: Do something here to figure out what version should + # have been promoted from binTrvSpecs. Note that all + # built versions of a package will end up in + # binTrvSpecs, even if they didn't make it into a + # group. + + targetSpecs, failed = self._updater.getTargetVersions(binTrvSpecs) + binTrvs.update(set(targetSpecs)) + failures.setdefault(advisory, list()).extend(failed) + + binPkgs = {} + for n, v, f in binTrvs: + binPkgs.setdefault(n.split(':')[0], dict()).setdefault(v, set()).add(f) + + for n, vMap in binPkgs.iteritems(): + assert len(vMap) == 1 + + # FIXME: If there are two versions of the package that have + # been promoted I would expect it to something like + # the case of setroubleshoot where we had to rebuild + # due to our deps changing. This should be mentioned + # in the config and provide an advisory to tie the + # new version to. + + for v, flvs in vMap.iteritems(): + log.info('%s: adding package %s=%s' % (advisory, n, v)) + for f in flvs: + log.info('%s: %s' % (advisory, f)) + grp.addPackage(n, v, flvs) + + log.info('%s: would be building groups here' % advisory) + #log.info('building groups') + #trvMap = mgr.build() + + log.info('%s: would be promoting groups here' % advisory) + #log.info('promoting groups') + # Setting expected to an empty tuple since we don't expect anything + # other than groups to be promoted. + #expected = tuple() + #toPromote = self._flatenSetDict(trvMap) + #promoted = self._updater.publish(toPromote, expected, + # self._cfg.targetLabel) + + import epdb; epdb.st() diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -18,6 +18,7 @@ import os import logging +import itertools from rpmutils import rpmvercmp @@ -215,6 +216,61 @@ return self._conaryhelper.getSourceVersionMapFromBinaryVersion( (n, v, f), labels=labels, latest=latest) + def getBinaryVersionsFromSourcePackage(self, srcPkg): + """ + Get a list of all packages in the conary repository that were built from + the given source package. + @param srcPkg: source package object + @type srcPkg: repomd.packagexml._Package + @return set of binary trove specs + @rtype set([(str, conary.versions.Version, conary.deps.deps.Flavor), ]) + """ + + # Find the conary source trove + srcName = '%s:source' % srcPkg.name + srcSpec = (srcName, srcPkg.getConaryVersion(), None) + srcTrvs = [ (x, y, None) for x, y, z in + self._conaryhelper.findTrove(srcSpec, getLeaves=False) ] + + # Get a mapping of binaries + srcMap = self._conaryhelper.getBinaryVersions(srcTrvs, + labels=self._cfg.platformSearchPath, + includeBuildLabel=True, + latest=False, missingOk=True) + + # Return binary versions + bins = [ x for x in itertools.chain(*srcMap.itervalues()) ] + assert bins + return bins + + def getTargetVersions(self, binTrvSpecs): + """ + Given a list of binary trove specs from the devel label return a list of + promoted trove versions. + """ + + cfMap = self._conaryhelper.getClonedFromForLabel(self._cfg.targetLabel) + failed = [] + targetSpecs = [] + for spec in binTrvSpecs: + if spec not in cfMap: + failed.append(spec) + else: + targetSpecs.append(cfMap[spec]) + + seen = [ (x, y.getSourceVersion()) + for x, y, z in binTrvSpecs + if (x, y, z) not in failed ] + + fail = [ (x, y, z) + for x, y, z in failed + if (x, y.getSourceVersion()) not in seen ] + + for spec in fail: + log.critical('%s=%s[%s] not found in cloned from map' % spec) + + return targetSpecs, fail + def _getLatestSource(self, name): """ Get the latest src package for a given package name. From elliot at rpath.com Wed Mar 17 11:29:12 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 15:29:12 +0000 Subject: mirrorball: cast to list since sets don't support indexing Message-ID: <201003171529.o2HFTCu6014241@scc.eng.rpath.com> changeset: 955dfbf49439 user: Elliot Peele date: Wed, 17 Mar 2010 11:29:09 -0400 cast to list since sets don't support indexing diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -317,7 +317,7 @@ # Make sure troveSpecs is an iterable of three tuples. if (len(troveSpecs) == 3 and - not isinstance(troveSpecs[0], (list, set, tuple))): + not isinstance(list(troveSpecs)[0], (list, set, tuple))): # Assume that (n,v,f) was passed in troveSpecs = [ troveSpecs, ] From elliot at rpath.com Wed Mar 17 14:02:34 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 18:02:34 +0000 Subject: mirrorball: add method for displaying a trove map Message-ID: <201003171802.o2HI2Y8L024213@scc.eng.rpath.com> changeset: 4bcbda75e824 user: Elliot Peele date: Wed, 17 Mar 2010 13:58:26 -0400 add method for displaying a trove map diff --git a/updatebot/display.py b/updatebot/display.py --- a/updatebot/display.py +++ b/updatebot/display.py @@ -33,3 +33,21 @@ ret.append(' ' * indent + '\'%s\',' % name) return '\n'.join(ret) + +def displayTroveMap(trvMap, indent=4): + """ + Format a mapping of source to binaries. + @param trvMap dictionary of troves from a build + @type trvMap: {srcTrvs: set((n, v, f), ..)} + @return formatted string + """ + + tab = ' ' * indent + + ret = [] + for src, bins in sorted(trvMap.iteritems()): + ret.append('%s=%s' % (src[0], src[1])) + for bin in sorted(bins): + ret.append('%s%s=%s[%s]' % (tab, bin[0], bin[1], bin[2])) + + return '\n'.join(ret) From elliot at rpath.com Wed Mar 17 14:02:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 18:02:35 +0000 Subject: mirrorball: move display module into the cmdline handling Message-ID: <201003171802.o2HI2Zro024238@scc.eng.rpath.com> changeset: 0a3cd616a690 user: Elliot Peele date: Wed, 17 Mar 2010 13:58:44 -0400 move display module into the cmdline handling diff --git a/updatebot/display.py b/updatebot/cmdline/display.py rename from updatebot/display.py rename to updatebot/cmdline/display.py From elliot at rpath.com Wed Mar 17 14:02:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 18:02:37 +0000 Subject: mirrorball: bot.update now returns the map of built troves, rather than a return code. Message-ID: <201003171802.o2HI2bI0024273@scc.eng.rpath.com> changeset: d37df6ede381 user: Elliot Peele date: Wed, 17 Mar 2010 14:02:30 -0400 bot.update now returns the map of built troves, rather than a return code. diff --git a/scripts/auto_update b/scripts/auto_update --- a/scripts/auto_update +++ b/scripts/auto_update @@ -14,7 +14,7 @@ # import os -from updatebot import bot, config +from updatebot import bot, config, display def validatePlatform(platform, configDir): validPlatforms = os.listdir(configDir) @@ -39,7 +39,13 @@ cfg = config.UpdateBotConfig() cfg.read(os.path.join(configDir, platform, 'updatebotrc')) obj = bot.Bot(cfg) - return obj.update() + trvMap = obj.update() + + if trvMap: + print 'Updated the following troves:' + print display.displayTroveMap(trvMap) + + return 0 if __name__ == '__main__': import sys From elliot at rpath.com Wed Mar 17 14:39:31 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 18:39:31 +0000 Subject: mirrorball: import the correct module Message-ID: <201003171839.o2HIdVpQ024892@scc.eng.rpath.com> changeset: 31165be78af6 user: Elliot Peele date: Wed, 17 Mar 2010 14:39:29 -0400 import the correct module diff --git a/scripts/auto_update b/scripts/auto_update --- a/scripts/auto_update +++ b/scripts/auto_update @@ -14,7 +14,8 @@ # import os -from updatebot import bot, config, display +from updatebot import bot, config +from updatebot.cmdline import display def validatePlatform(platform, configDir): validPlatforms = os.listdir(configDir) From elliot at rpath.com Wed Mar 17 15:26:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 19:26:18 +0000 Subject: mirrorball: add missing elements Message-ID: <201003171926.o2HJQID8025506@scc.eng.rpath.com> changeset: e79d326b9d7f user: Elliot Peele date: Wed, 17 Mar 2010 15:26:15 -0400 add missing elements diff --git a/repomd/patchesxml.py b/repomd/patchesxml.py --- a/repomd/patchesxml.py +++ b/repomd/patchesxml.py @@ -62,7 +62,8 @@ Parser for patch element of patches.xml. """ - __slots__ = ('id', 'checksum', 'checksumType', 'location') + __slots__ = ('id', 'checksum', 'checksumType', 'location', '_parser', + 'parseChildren') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -35,7 +35,8 @@ __slots__ = ('name', 'summary', 'description', 'version', 'release', 'requires', 'recommends', 'rebootNeeded', 'licenseToConfirm', 'packageManager', 'category', - 'packages', 'provides', 'supplements', 'conflicts',) + 'packages', 'provides', 'supplements', 'conflicts', + 'obsoletes') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. From elliot at rpath.com Wed Mar 17 16:54:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 20:54:35 +0000 Subject: mirrorball: add slot node for reference Message-ID: <201003172054.o2HKsZT4029949@scc.eng.rpath.com> changeset: 088000cd6a03 user: Elliot Peele date: Wed, 17 Mar 2010 16:54:33 -0400 add slot node for reference diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py --- a/repomd/updateinfoxml.py +++ b/repomd/updateinfoxml.py @@ -136,6 +136,14 @@ SlotNode.addChild(self, child) +class _Reference(SlotNode): + """ + Prepesent a single reference. + """ + + __slots__ = ('href', 'id', 'title', 'type') + + class _Collection(SlotNode): """ Represents a pkglist collection in updateinfo.xml. @@ -218,6 +226,7 @@ self._databinder.registerType(_Updates, name='updates') self._databinder.registerType(_Update, name='update') self._databinder.registerType(_References, name='references') + self._databinder.registerType(_Reference, name='reference') self._databinder.registerType(_Collection, name='collection') self._databinder.registerType(_UpdateInfoPackage, name='package') From elliot at rpath.com Wed Mar 17 17:27:14 2010 From: elliot at rpath.com (Elliot Peele) Date: Wed, 17 Mar 2010 21:27:14 +0000 Subject: mirrorball: add more required attributes Message-ID: <201003172127.o2HLREHI030378@scc.eng.rpath.com> changeset: 8a2060d29ea4 user: Elliot Peele date: Wed, 17 Mar 2010 17:27:12 -0400 add more required attributes diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py --- a/repomd/updateinfoxml.py +++ b/repomd/updateinfoxml.py @@ -72,7 +72,8 @@ """ __slots__ = ('status', 'emailfrom', 'type', 'id', 'title', 'release', - 'issued', 'references', 'description', 'pkglist', 'packages') + 'issued', 'references', 'description', 'pkglist', 'packages', + 'summary', 'packages') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. From elliot at rpath.com Thu Mar 18 16:26:34 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:34 +0000 Subject: mirrorball: add named arg for including the build label in the search path when other Message-ID: <201003182026.o2IKQYfd026475@scc.eng.rpath.com> changeset: 12e8450db18b user: Elliot Peele date: Tue, 16 Mar 2010 15:32:28 -0400 add named arg for including the build label in the search path when other labels are specified diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -452,7 +452,7 @@ return srcMap def getSourceVersionMapFromBinaryVersion(self, (n, v, f), labels=None, - latest=False): + latest=False, includeBuildLabel=False): """ Find a mapping of source to binaries, given a single binary name, version, and flavor. @@ -462,13 +462,16 @@ @type labels: list(conary.versions.Label, ...) @param latest: check for only the latest versions or not @type latest: boolean + @param includeBuildLabel: search the build label in addition to + specified labels. @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ...]} """ trvs = self.findTrove((n, v, f), labels=labels) srcVersions = self.getSourceVersions(trvs) srcSpecs = srcVersions.keys() - srcMap = self.getBinaryVersions(srcSpecs, labels=labels, latest=latest) + srcMap = self.getBinaryVersions(srcSpecs, labels=labels, latest=latest, + includeBuildLabel=includeBuildLabel) return srcMap @staticmethod From elliot at rpath.com Thu Mar 18 16:26:35 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:35 +0000 Subject: mirrorball: return results in the same form as the rmake builder Message-ID: <201003182026.o2IKQZqj026502@scc.eng.rpath.com> changeset: a5fcca1f7f22 user: Elliot Peele date: Wed, 17 Mar 2010 11:41:33 -0400 return results in the same form as the rmake builder diff --git a/updatebot/build/cvc.py b/updatebot/build/cvc.py --- a/updatebot/build/cvc.py +++ b/updatebot/build/cvc.py @@ -21,6 +21,7 @@ log = logging.getLogger('updatebot.build.cvc') +from conary import versions from conary import conarycfg from conary import conaryclient from conary.build import cook @@ -105,5 +106,10 @@ if csFile is None: log.info('changeset committed to repository') - res = { (troveSpecs[0][0], troveSpecs[0][1], None): set(components) } + # Convert version strings to version objects to match the output from + # rMake builds. + results = set((x, versions.VersionFromString(y), z) + for x, y, z in components) + + res = { (troveSpecs[0][0], troveSpecs[0][1], None): results } return res From elliot at rpath.com Thu Mar 18 16:26:36 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:36 +0000 Subject: mirrorball: fix typo Message-ID: <201003182026.o2IKQakN026533@scc.eng.rpath.com> changeset: aa36b3bd0e28 user: Elliot Peele date: Wed, 17 Mar 2010 11:42:27 -0400 fix typo diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -416,7 +416,7 @@ """ _params = ['what', ] - _template = ('The following package is not longer managed as part of the ' + _template = ('The following package is no longer managed as part of the ' 'version group %(what)s, you may need to remove this package from any ' 'other static group definitions.') From elliot at rpath.com Thu Mar 18 16:26:37 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:37 +0000 Subject: mirrorball: override pkgname -> classname to handle names with dots Message-ID: <201003182026.o2IKQb8S026560@scc.eng.rpath.com> changeset: 96d76b575c03 user: Elliot Peele date: Wed, 17 Mar 2010 11:43:10 -0400 override pkgname -> classname to handle names with dots diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -24,7 +24,7 @@ import signal import resource from conary.lib.util import rmtree -from conary.lib.util import convertPackageNameToClassName +from conary.lib.util import convertPackageNameToClassName as _pkgNameToClassName from rpmutils import rpmvercmp @@ -180,3 +180,7 @@ epdb.serve() signal.signal(signal.SIGUSR1, handler) + +def convertPackageNameToClassName(pkgName): + name = _pkgNameToClassName(pkgName) + return name.replace('.', '_') From elliot at rpath.com Thu Mar 18 16:26:38 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:38 +0000 Subject: mirrorball: switch to using a common cache object to avoid hammering on the repositories Message-ID: <201003182026.o2IKQcsD026589@scc.eng.rpath.com> changeset: 9aecbf526e52 user: Elliot Peele date: Thu, 18 Mar 2010 16:05:37 -0400 switch to using a common cache object to avoid hammering on the repositories diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -50,35 +50,76 @@ log = logging.getLogger('updatebot.conaryhelper') +class ConaryHelperSharedCache(object): + def __init__(self): + self.clear() + + def clear(self): + # caches source names and versions for binaries past into + # getSourceVersions. + # binTroveSpec: sourceTroveSpec + self.sourceVersionCache = {} + + # Keep a cache of all binary versions that have been looked up in + # getBinaryVersions to avoid lots of expensive getTroveVersionsByLabel + # calls. + # frzenset(labels): binTroveNVFSet + self.binaryVersionCache = {} + + # Cache cloned from information + # srcNVF: destNVF + self.clonedFromCache = {} + self.labelClonedFromCache = {} + + self.conaryConfigCache = {} + self.sharedTmpDir = None + + class ConaryHelper(object): """ Wrapper object for conary api. """ + _cache = ConaryHelperSharedCache() + def __init__(self, cfg): self._groupFlavorCount = len(cfg.groupFlavors) - self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) - self._ccfg.read(util.join(cfg.configPath, 'conaryrc')) - self._ccfg.dbPath = ':memory:' - # Have to initialize flavors to commit to the repository. - self._ccfg.initializeFlavors() + if not self._cache.sharedTmpDir: + self._cache.sharedTmpDir = tempfile.mkdtemp( + prefix='conaryhelper-tmpdir-') - # FIXME: CNY-3256 - use unique tmp directory for lookaside until - # this issue is fixed. - self._ccfg.lookaside = tempfile.mkdtemp( - prefix='%s-lookaside-' % cfg.platformName) - log.info('using lookaside %s' % self._ccfg.lookaside) + conaryCfgFile = util.join(cfg.configPath, 'conaryrc') + if conaryCfgFile in self._cache.conaryConfigCache: + self._ccfg = self._cache.conaryConfigCache[conaryCfgFile] + else: + self._ccfg = conarycfg.ConaryConfiguration(readConfigFiles=False) + self._ccfg.read(conaryCfgFile) + self._ccfg.dbPath = ':memory:' + # Have to initialize flavors to commit to the repository. + self._ccfg.initializeFlavors() - mirrorDir = util.join(cfg.configPath, 'mirrors') - if os.path.exists(mirrorDir): - self._ccfg.mirrorDirs.insert(0, mirrorDir) + # FIXME: CNY-3256 - use unique tmp directory for lookaside until + # this issue is fixed. + self._ccfg.lookaside = tempfile.mkdtemp( + dir=self._cache.sharedTmpDir, + prefix='%s-lookaside-' % cfg.platformName) + log.info('using lookaside %s' % self._ccfg.lookaside) + + mirrorDir = util.join(cfg.configPath, 'mirrors') + if os.path.exists(mirrorDir): + self._ccfg.mirrorDirs.insert(0, mirrorDir) + + self._cache.conaryConfigCache[conaryCfgFile] = self._ccfg self._mcfg = None mcfgfn = util.join(cfg.configPath, 'mirror.conf') - if os.path.exists(mcfgfn): + if mcfgfn in self._cache.conaryConfigCache: + self._mcfg = self._cache.conaryConfigCache[mcfgfn] + elif os.path.exists(mcfgfn): self._mcfg = mirror.MirrorFileConfiguration() self._mcfg.read(mcfgfn) + self._cache.conaryConfigCache[mcfgfn] = self._mcfg self._client = conaryclient.ConaryClient(self._ccfg) self._repos = self._client.getRepos() @@ -87,23 +128,15 @@ self._checkoutCache = {} self._cacheDir = tempfile.mkdtemp( + dir=self._cache.sharedTmpDir, prefix='conaryhelper-%s-' % cfg.platformName) - # caches source names and versions for binaries past into - # getSourceVersions. - # binTroveSpec: sourceTroveSpec - self._sourceVersionCache = {} + def clearCache(self): + """ + Clear the trove query cache. + """ - # Keep a cache of all binary versions that have been looked up in - # getBinaryVersions to avoid lots of expensive getTroveVersionsByLabel - # calls. - # frzenset(labels): binTroveNVFSet - self._binaryVersionCache = {} - - # Cache cloned from information - # srcNVF: destNVF - self._clonedFromCache = {} - self._labelClonedFromCache = {} + self._cache.clear() def getConaryConfig(self): """ @@ -307,7 +340,7 @@ def tiFunc(ti, nvf): return (nvf[0], ti, nvf[2]) - return self._cacheTroveInfo(troveSpecs, self._clonedFromCache, + return self._cacheTroveInfo(troveSpecs, self._cache.clonedFromCache, trove._TROVEINFO_TAG_CLONEDFROM, tiFunc=tiFunc) def getClonedFromForLabel(self, label): @@ -320,8 +353,8 @@ if hasattr(label, 'label'): label = label.label() - if label in self._labelClonedFromCache: - return self._labelClonedFromCache[label] + if label in self._cache.labelClonedFromCache: + return self._cache.labelClonedFromCache[label] req = {None: {label: None}} binTrvMap = self._repos.getTroveVersionsByLabel(req) @@ -341,7 +374,7 @@ assert len(cfMap[f]) == 1 ret[f] = list(t)[0] - self._labelClonedFromCache[label] = ret + self._cache.labelClonedFromCache[label] = ret return ret def getSourceVersions(self, binTroveSpecs): @@ -356,7 +389,7 @@ def tiFunc(ti, nvf): return (ti, nvf[1].getSourceVersion(), None) - return self._cacheTroveInfo(binTroveSpecs, self._sourceVersionCache, + return self._cacheTroveInfo(binTroveSpecs, self._cache.sourceVersionCache, trove._TROVEINFO_TAG_SOURCENAME, tiFunc=tiFunc) def getBinaryVersions(self, srcTroveSpecs, labels=None, latest=True, @@ -391,8 +424,8 @@ # need to be refreshed and/or expired. # Check the cache before going on. - if labels in self._binaryVersionCache: - binTrvSpecs = self._binaryVersionCache[labels] + if labels in self._cache.binaryVersionCache: + binTrvSpecs = self._cache.binaryVersionCache[labels] else: # get all binary trove specs for the specified labels req = {None: dict([ (x, None) for x in labels ])} @@ -409,7 +442,7 @@ binTrvSpecs.add((n, v, f)) # Populate cache. - self._binaryVersionCache[labels] = binTrvSpecs + self._cache.binaryVersionCache[labels] = binTrvSpecs # get a map of source trove specs to binary trove specs srcVerMap = self.getSourceVersions(binTrvSpecs) From elliot at rpath.com Thu Mar 18 16:26:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:39 +0000 Subject: mirrorball: clear the trove cache after sanity checking Message-ID: <201003182026.o2IKQdXU026616@scc.eng.rpath.com> changeset: 91af75c991fc user: Elliot Peele date: Thu, 18 Mar 2010 16:06:07 -0400 clear the trove cache after sanity checking diff --git a/updatebot/errata.py b/updatebot/errata.py --- a/updatebot/errata.py +++ b/updatebot/errata.py @@ -500,6 +500,8 @@ log.info('? updateReplacesPackages %s %s' % ( updateId, pkgName)) + # Clear the cache since it would be dirty at this point. + updater._conaryhelper.clearCache() # Fail if there are any errors. assert not errors From elliot at rpath.com Thu Mar 18 16:26:42 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:42 +0000 Subject: mirrorball: pass down includeBuildLabel Message-ID: <201003182026.o2IKQgSj026647@scc.eng.rpath.com> changeset: 42a853a87258 user: Elliot Peele date: Thu, 18 Mar 2010 16:07:01 -0400 pass down includeBuildLabel better label handling diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -200,7 +200,7 @@ return self._conaryhelper.getBinaryVersions(req, labels=labels) def getSourceVersionMapFromBinaryVersion(self, (n, v, f), labels=None, - latest=False): + latest=False, includeBuildLabel=False): """ Find a mapping of source to binaries, given a single binary name, version, and flavor. @@ -214,7 +214,8 @@ """ return self._conaryhelper.getSourceVersionMapFromBinaryVersion( - (n, v, f), labels=labels, latest=latest) + (n, v, f), labels=labels, latest=latest, + includeBuildLabel=includeBuildLabel) def getBinaryVersionsFromSourcePackage(self, srcPkg): """ @@ -226,11 +227,14 @@ @rtype set([(str, conary.versions.Version, conary.deps.deps.Flavor), ]) """ + labels = self._cfg.platformSearchPath or None + # Find the conary source trove srcName = '%s:source' % srcPkg.name srcSpec = (srcName, srcPkg.getConaryVersion(), None) srcTrvs = [ (x, y, None) for x, y, z in - self._conaryhelper.findTrove(srcSpec, getLeaves=False) ] + self._conaryhelper.findTrove(srcSpec, getLeaves=False, + labels=labels) ] # Get a mapping of binaries srcMap = self._conaryhelper.getBinaryVersions(srcTrvs, From elliot at rpath.com Thu Mar 18 16:26:48 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:48 +0000 Subject: mirrorball: add method for querying groups Message-ID: <201003182026.o2IKQm2b026679@scc.eng.rpath.com> changeset: 631e0fa319ca user: Elliot Peele date: Thu, 18 Mar 2010 16:10:21 -0400 add method for querying groups switch to cvc builder diff --git a/updatebot/groupmgr/single.py b/updatebot/groupmgr/single.py --- a/updatebot/groupmgr/single.py +++ b/updatebot/groupmgr/single.py @@ -72,18 +72,15 @@ # Make sure there are groups defined assert self._groups - toBuild = set() - builder = None + pkgMap = {} for group in self._groups.itervalues(): - # Select the builder from the first instance to build all groups. - if not builder: - builder = group._builder + pkgMap.update(group.build()) - # Get the build job for each group. - trvSpecs = group.getBuildJob() - assert len(trvSpecs) == 1 - toBuild.add(trvSpecs[0]) + return pkgMap - # Build all groups in separate jobs, waiting until all have completed - # to commit. - return builder.buildmany(toBuild, lateCommit=True) + def hasGroups(self): + """ + Add method for checking if a manager has any groups defined. + """ + + return bool(self._groups) From elliot at rpath.com Thu Mar 18 16:26:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:49 +0000 Subject: mirrorball: log each sanity checking step, mostly for timing Message-ID: <201003182026.o2IKQnVl026708@scc.eng.rpath.com> changeset: 14ae78f6eb3a user: Elliot Peele date: Thu, 18 Mar 2010 16:10:49 -0400 log each sanity checking step, mostly for timing diff --git a/updatebot/groupmgr/sanity.py b/updatebot/groupmgr/sanity.py --- a/updatebot/groupmgr/sanity.py +++ b/updatebot/groupmgr/sanity.py @@ -53,16 +53,19 @@ for name, group in groups.iteritems(): log.info('checking consistentcy of %s' % name) try: + log.info('checking name version conflict') self._checkNameVersionConflict(group) except NameVersionConflictsFoundError, e: errors.append((group, e)) try: + log.info('checking latest versions') self._checkLatestVersion(group) except OldVersionsFoundError, e: errors.append((group, e)) try: + log.info('checking removals') self._checkRemovals(group, errataState) except ExpectedRemovalValidationFailedError, e: errors.append((group, e)) @@ -186,7 +189,6 @@ labels=self._cfg.platformSearchPath, latest=False) oldVersions |= set(itertools.chain(*srcMap.itervalues())) - errors = {} for name, found in foundTroves.iteritems(): assert name in pkgs From elliot at rpath.com Thu Mar 18 16:26:51 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:51 +0000 Subject: mirrorball: 1. use the buildlabel when querying Message-ID: <201003182026.o2IKQpmJ026737@scc.eng.rpath.com> changeset: c82200400c31 user: Elliot Peele date: Thu, 18 Mar 2010 16:12:26 -0400 1. use the buildlabel when querying 2. handle case when there are no matching groups diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py --- a/updatebot/groupmgr/manager.py +++ b/updatebot/groupmgr/manager.py @@ -180,7 +180,7 @@ if self._sourceVersion: labels = (self._sourceVersion.trailingLabel(), ) else: - labels = (self._cfg.topSourceGroup[1], ) + labels = (self._helper.getConaryConfig().buildLabel, ) # Get a mapping of all source version to binary versions for all # existing binary versions. @@ -196,11 +196,14 @@ # source version. srcVersion = self._helper.findTrove( ('%s:source' % self._sourceName, version, None), - labels=labels)[0][1] + labels=labels) + + if not srcVersion: + return False # Check to see if the latest source version is in the map of # binary versions. - return srcVersion in srcVersions + return srcVersion[0][1] in srcVersions @commit def getBuildJob(self): From elliot at rpath.com Thu Mar 18 16:26:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:52 +0000 Subject: mirrorball: fully baked version of errata group handling Message-ID: <201003182026.o2IKQqcK026768@scc.eng.rpath.com> changeset: 056cf3ae2d03 user: Elliot Peele date: Thu, 18 Mar 2010 16:13:02 -0400 fully baked version of errata group handling diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -20,6 +20,7 @@ import pickle import logging import tempfile +import itertools from conary import versions from conary.deps import deps @@ -368,6 +369,22 @@ raise TargetVersionNotFoundError(version=version, updateId=updateId) + # Lookup any places we need to use old versions ahead of time. + multiVersionExceptions = {} + if updateId in self._cfg.useOldVersion: + log.info('looking up multiple version exception information') + oldVersions = self._cfg.useOldVersion[updateId] + for oldVersion in oldVersions: + srcMap = self._updater.getSourceVersionMapFromBinaryVersion( + oldVersion, labels=self._cfg.platformSearchPath, + latest=False, includeBuildLabel=True) + + multiVersionExceptions.update(dict([ (x[0], x[1]) + for x in itertools.chain(self._updater.getTargetVersions( + itertools.chain(*srcMap.itervalues()) + )[0]) + ])) + # Now that we know that the packages that are part of this update # should be on the target label we can separate things into # advisories. @@ -380,6 +397,13 @@ srcPkgs = self._errata.getAdvisoryPackages(advisory) assert srcPkgs + targetGrp = groupmgr.SingleGroupManager(groupNames[advisory], + self._cfg, targetGroup=True) + + if targetGrp.hasBinaryVersion(): + log.info('%s: found existing version, skipping' % advisory) + continue + grp = mgr.newGroup(groupNames[advisory]) grp.setVersion(version) grp.setErrataState(updateId) @@ -390,47 +414,60 @@ binTrvSpecs = \ self._updater.getBinaryVersionsFromSourcePackage(srcPkg) - # FIXME: Do something here to figure out what version should - # have been promoted from binTrvSpecs. Note that all - # built versions of a package will end up in - # binTrvSpecs, even if they didn't make it into a - # group. - - targetSpecs, failed = self._updater.getTargetVersions(binTrvSpecs) + targetSpecs, failed = self._updater.getTargetVersions( + binTrvSpecs) binTrvs.update(set(targetSpecs)) failures.setdefault(advisory, list()).extend(failed) + # Handle attaching an update that was caused by changes that we + # made outside of the normal update stream to an existing + # advisory. + if advisory in self._cfg.extendAdvisory: + pkgs = self._cfg.extendAdvisory[advisory] + for nvf in pkgs: + srcMap = \ + self._updater.getSourceVersionMapFromBinaryVersion( + nvf, labels=self._cfg.platformSearchPath, + latest=False, includeBuildLabel=True) + assert len(srcMap) == 1 + targetVersions = self._updater.getTargetVersions( + srcMap.values()[0])[0] + binTrvs.update(set(targetVersions)) + binPkgs = {} for n, v, f in binTrvs: - binPkgs.setdefault(n.split(':')[0], dict()).setdefault(v, set()).add(f) + binPkgs.setdefault(n.split(':')[0], + dict()).setdefault(v, set()).add(f) for n, vMap in binPkgs.iteritems(): + # Handle the case where a package has been rebuilt for some + # reason, but we need to use the old version of the package. + if len(vMap) > 1 and n in multiVersionExceptions: + vMap = dict((x, y) for x, y in vMap.iteritems() + if x == multiVersionExceptions[n]) + assert len(vMap) == 1 - # FIXME: If there are two versions of the package that have - # been promoted I would expect it to something like - # the case of setroubleshoot where we had to rebuild - # due to our deps changing. This should be mentioned - # in the config and provide an advisory to tie the - # new version to. - for v, flvs in vMap.iteritems(): log.info('%s: adding package %s=%s' % (advisory, n, v)) for f in flvs: log.info('%s: %s' % (advisory, f)) grp.addPackage(n, v, flvs) - log.info('%s: would be building groups here' % advisory) - #log.info('building groups') - #trvMap = mgr.build() + # Make sure there are groups to build. + if not mgr.hasGroups(): + log.info('%s: groups already built and promoted' % updateId) + continue - log.info('%s: would be promoting groups here' % advisory) - #log.info('promoting groups') + log.info('%s: building groups' % updateId) + trvMap = mgr.build() + + log.info('%s: promoting groups' % updateId) # Setting expected to an empty tuple since we don't expect anything # other than groups to be promoted. - #expected = tuple() - #toPromote = self._flatenSetDict(trvMap) - #promoted = self._updater.publish(toPromote, expected, - # self._cfg.targetLabel) + expected = tuple() + toPromote = self._flattenSetDict(trvMap) + promoted = self._updater.publish(toPromote, expected, + self._cfg.targetLabel) import epdb; epdb.st() From elliot at rpath.com Thu Mar 18 16:26:54 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:54 +0000 Subject: mirrorball: add config option for binding a conary package to an errata Message-ID: <201003182026.o2IKQsQw026798@scc.eng.rpath.com> changeset: 55049ba5ef3e user: Elliot Peele date: Thu, 18 Mar 2010 16:20:33 -0400 add config option for binding a conary package to an errata diff --git a/updatebot/config.py b/updatebot/config.py --- a/updatebot/config.py +++ b/updatebot/config.py @@ -336,6 +336,9 @@ # move the entire update bucket. reorderAdvisory = (CfgList(CfgAdvisoryOrder), []) + # advisory trovespec + extendAdvisory = (CfgDict(CfgList(CfgTroveSpec)), {}) + # fromUpdateId toUpdateId sourceNevra # Sometimes multiple versions of the same package are released as part of a # single advisory. This does not fit the conary model of doing things, so we From elliot at rpath.com Thu Mar 18 16:26:55 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:55 +0000 Subject: mirrorball: when promoting fork before actually promoting any contents, this works around Message-ID: <201003182026.o2IKQthY026825@scc.eng.rpath.com> changeset: 0895339dff9c user: Elliot Peele date: Thu, 18 Mar 2010 16:21:43 -0400 when promoting fork before actually promoting any contents, this works around a memory leak in conary diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -41,6 +41,7 @@ from updatebot import conaryhelper from updatebot import OrderedBot from updatebot import UpdateBotConfig +from updatebot.lib.watchdog import forknwait slog = log.addRootLogger() cfg = UpdateBotConfig() @@ -94,6 +95,8 @@ targetGroupvers = helper.findTrove((cfg.topGroup[0], cfg.targetLabel, None), getLeaves=False) targetLatest = getUpstreamVersionMap(targetGroupvers) +searchPath = [ x for x in itertools.chain((helper._ccfg.buildLabel, ), cfg.platformSearchPath) ] + # Get all updates after the first bucket. missing = False for updateId, bucket in bot._errata.iterByIssueDate(current=1): @@ -132,12 +135,13 @@ # Find expected promote packages. csrcTrvs = [ ('%s:source' % x.name, x.getConaryVersion(), None) for x in bucket ] - srcTrvs = helper.findTroves(csrcTrvs) + srcTrvs = helper.findTroves(csrcTrvs, labels=searchPath) # Map source versions to binary versions. slog.info('retrieving binary trove map') srcTrvMap = helper.getBinaryVersions([ (x, y, None) for x, y, z in - itertools.chain(*srcTrvs.itervalues()) ]) + itertools.chain(*srcTrvs.itervalues()) ], + labels=searchPath) # These are the binary trove specs that we expect to be promoted. expected = [ x for x in itertools.chain(*srcTrvMap.itervalues()) ] @@ -145,12 +149,19 @@ # Get list of extra troves from the config extra = cfg.extraExpectedPromoteTroves.get(updateId, []) - # Create and validate promote changeset - packageList = helper.promote(toPromote, expected, cfg.sourceLabel, - cfg.targetLabel, commit=True, - extraExpectedPromoteTroves=extra) + @forknwait + def promote(): + # Create and validate promote changeset + packageList = helper.promote(toPromote, expected, cfg.sourceLabel, + cfg.targetLabel, commit=True, + extraExpectedPromoteTroves=extra) + return 1 + + rc = promote() # Update latest map for the next loop latestMap = updateLatestMap() + versions.clearVersionCache() + import epdb; epdb.st() From elliot at rpath.com Thu Mar 18 16:26:56 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:56 +0000 Subject: mirrorball: add script to build errata groups Message-ID: <201003182026.o2IKQvsF026854@scc.eng.rpath.com> changeset: bed887e05d0e user: Elliot Peele date: Thu, 18 Mar 2010 16:22:04 -0400 add script to build errata groups diff --git a/scripts/order_import.py b/scripts/order_errata_groups.py copy from scripts/order_import.py copy to scripts/order_errata_groups.py --- a/scripts/order_import.py +++ b/scripts/order_errata_groups.py @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2009 rPath, Inc. +# Copyright (c) 2009-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -61,6 +61,6 @@ errata.fetch() bot = ordered.Bot(cfg, errata) -pkgMap, failures = bot.create() +bot.createErrataGroups() import epdb; epdb.st() From elliot at rpath.com Thu Mar 18 16:26:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:58 +0000 Subject: mirrorball: add script for memory profiling pkgsources Message-ID: <201003182026.o2IKQwnr026885@scc.eng.rpath.com> changeset: fd7a39fa119f user: Elliot Peele date: Thu, 18 Mar 2010 16:22:21 -0400 add script for memory profiling pkgsources diff --git a/scripts/pkgsource b/scripts/memprof copy from scripts/pkgsource copy to scripts/memprof --- a/scripts/pkgsource +++ b/scripts/memprof @@ -35,13 +35,14 @@ cfg.read(mirrorballDir + '/config/%s/updatebotrc' % sys.argv[1] ) pkgSource = pkgsource.PackageSource(cfg) + +from guppy import hpy +hp = hpy() +hp.setrelheap() + pkgSource.load() -import epdb; epdb.st() - -updates = [] -for path, client in pkgSource._clients.iteritems(): - if 'Updates' in path: - updates.extend(client.getUpdateInfo()) +heap = hp.heap() +first = heap[0] import epdb; epdb.st() From elliot at rpath.com Thu Mar 18 16:26:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Thu, 18 Mar 2010 20:26:58 +0000 Subject: mirrorball: branch merge Message-ID: <201003182026.o2IKQwS3026912@scc.eng.rpath.com> changeset: 470d679b56a0 user: Elliot Peele date: Thu, 18 Mar 2010 16:25:31 -0400 branch merge diff --git a/repomd/patchesxml.py b/repomd/patchesxml.py --- a/repomd/patchesxml.py +++ b/repomd/patchesxml.py @@ -62,7 +62,8 @@ Parser for patch element of patches.xml. """ - __slots__ = ('id', 'checksum', 'checksumType', 'location') + __slots__ = ('id', 'checksum', 'checksumType', 'location', '_parser', + 'parseChildren') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. diff --git a/repomd/patchxml.py b/repomd/patchxml.py --- a/repomd/patchxml.py +++ b/repomd/patchxml.py @@ -35,7 +35,8 @@ __slots__ = ('name', 'summary', 'description', 'version', 'release', 'requires', 'recommends', 'rebootNeeded', 'licenseToConfirm', 'packageManager', 'category', - 'packages', 'provides', 'supplements', 'conflicts',) + 'packages', 'provides', 'supplements', 'conflicts', + 'obsoletes') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. diff --git a/repomd/updateinfoxml.py b/repomd/updateinfoxml.py --- a/repomd/updateinfoxml.py +++ b/repomd/updateinfoxml.py @@ -72,7 +72,8 @@ """ __slots__ = ('status', 'emailfrom', 'type', 'id', 'title', 'release', - 'issued', 'references', 'description', 'pkglist', 'packages') + 'issued', 'references', 'description', 'pkglist', 'packages', + 'summary', 'packages') # All attributes are defined in __init__ by iterating over __slots__, # this confuses pylint. @@ -136,6 +137,14 @@ SlotNode.addChild(self, child) +class _Reference(SlotNode): + """ + Prepesent a single reference. + """ + + __slots__ = ('href', 'id', 'title', 'type') + + class _Collection(SlotNode): """ Represents a pkglist collection in updateinfo.xml. @@ -218,6 +227,7 @@ self._databinder.registerType(_Updates, name='updates') self._databinder.registerType(_Update, name='update') self._databinder.registerType(_References, name='references') + self._databinder.registerType(_Reference, name='reference') self._databinder.registerType(_Collection, name='collection') self._databinder.registerType(_UpdateInfoPackage, name='package') diff --git a/scripts/auto_update b/scripts/auto_update --- a/scripts/auto_update +++ b/scripts/auto_update @@ -15,6 +15,7 @@ import os from updatebot import bot, config +from updatebot.cmdline import display def validatePlatform(platform, configDir): validPlatforms = os.listdir(configDir) @@ -39,7 +40,13 @@ cfg = config.UpdateBotConfig() cfg.read(os.path.join(configDir, platform, 'updatebotrc')) obj = bot.Bot(cfg) - return obj.update() + trvMap = obj.update() + + if trvMap: + print 'Updated the following troves:' + print display.displayTroveMap(trvMap) + + return 0 if __name__ == '__main__': import sys diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -317,7 +317,7 @@ # Make sure troveSpecs is an iterable of three tuples. if (len(troveSpecs) == 3 and - not isinstance(troveSpecs[0], (list, set, tuple))): + not isinstance(list(troveSpecs)[0], (list, set, tuple))): # Assume that (n,v,f) was passed in troveSpecs = [ troveSpecs, ] diff --git a/updatebot/display.py b/updatebot/cmdline/display.py rename from updatebot/display.py rename to updatebot/cmdline/display.py --- a/updatebot/display.py +++ b/updatebot/cmdline/display.py @@ -33,3 +33,21 @@ ret.append(' ' * indent + '\'%s\',' % name) return '\n'.join(ret) + +def displayTroveMap(trvMap, indent=4): + """ + Format a mapping of source to binaries. + @param trvMap dictionary of troves from a build + @type trvMap: {srcTrvs: set((n, v, f), ..)} + @return formatted string + """ + + tab = ' ' * indent + + ret = [] + for src, bins in sorted(trvMap.iteritems()): + ret.append('%s=%s' % (src[0], src[1])) + for bin in sorted(bins): + ret.append('%s%s=%s[%s]' % (tab, bin[0], bin[1], bin[2])) + + return '\n'.join(ret) From elliot at rpath.com Mon Mar 22 12:05:13 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:13 +0000 Subject: mirrorball: add missing doc string Message-ID: <201003221605.o2MG5DTr023348@scc.eng.rpath.com> changeset: 331d623928cd user: Elliot Peele date: Mon, 22 Mar 2010 11:40:21 -0400 add missing doc string diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -406,6 +406,9 @@ @param includeBuildLabel: search the build label in addition to specified labels. @type includeBuildLabel: boolean + @param missingOk: If False, raise an error when binaries can not be + found for a given source trove. + @type missingOk: boolean @return {srcTrvSpec: [binTrvSpec, binTrvSpec, ... ]} """ From elliot at rpath.com Mon Mar 22 12:05:14 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:14 +0000 Subject: mirrorball: add simple command line interface Message-ID: <201003221605.o2MG5E3T023394@scc.eng.rpath.com> changeset: f79780be162f user: Elliot Peele date: Mon, 22 Mar 2010 11:45:24 -0400 add simple command line interface diff --git a/updatebot/cmdline/simple.py b/updatebot/cmdline/simple.py new file mode 100644 --- /dev/null +++ b/updatebot/cmdline/simple.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2008 rPath, Inc. +# +# This program is distributed under the terms of the Common Public License, +# version 1.0. A copy of this license should have been distributed with this +# source file in a file called LICENSE. If it is not present, the license +# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. +# +# This program is distributed in the hope that it will be useful, but +# without any warranty; without even the implied warranty of merchantability +# or fitness for a particular purpose. See the Common Public License for +# full details. +# + +import os + +from updatebot import log +from updatebot import config + +def validatePlatform(platform, configDir): + validPlatforms = os.listdir(configDir) + if platform not in validPlatforms: + print ('Invalid platform %s... Please select from the following ' + 'available platforms %s' % (platform, ', '.join(validPlatforms))) + return False + return True + +def usage(argv): + print 'usage: %s ' % argv[0] + return 1 + +def main(argv, workerFunc, configDir='/etc/mirrorball', enableLogging=True): + if enableLogging: + log.setRootLogger() + + if len(argv) != 2: + return usage(argv) + + platform = argv[1] + if not validatePlatform(platform, configDir): + return 1 + + cfg = config.UpdateBotConfig() + cfg.read(os.path.join(configDir, platform, 'updatebotrc')) + + rc = workerFunc(cfg) + return rc From elliot at rpath.com Mon Mar 22 12:05:14 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:14 +0000 Subject: mirrorball: 1. Refactor getBinaryVersionsFromSourcePackage into Message-ID: <201003221605.o2MG5Euj023373@scc.eng.rpath.com> changeset: 7f3b0fb78eb4 user: Elliot Peele date: Mon, 22 Mar 2010 11:45:05 -0400 1. Refactor getBinaryVersionsFromSourcePackage into getBinaryVersionsFromSourcePackages for requesting multiple source packages. 2. Expose extraExpectedPromoteTroves flag for promote. 3. Add getUpstreamVersionMap method for retrieving a mapping of upstream version to binary versions, given a trove spec. 4. Add getSourceVersionMapFromTroveSpec method for retrieving a source map from a trove spec. diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -227,25 +227,50 @@ @rtype set([(str, conary.versions.Version, conary.deps.deps.Flavor), ]) """ - labels = self._cfg.platformSearchPath or None + binMap = self.getBinaryVersionsFromSourcePackages((srcPkg, )) + assert srcPkg in binMap + assert binMap[srcPkg] + return binMap[srcPkg] - # Find the conary source trove - srcName = '%s:source' % srcPkg.name - srcSpec = (srcName, srcPkg.getConaryVersion(), None) - srcTrvs = [ (x, y, None) for x, y, z in - self._conaryhelper.findTrove(srcSpec, getLeaves=False, - labels=labels) ] + def getBinaryVersionsFromSourcePackages(self, srcPkgs): + """ + Get a list of all packages in the conary repository that were built from + the given source package. + @param srcPkgs: itertable of source package objects + @type srcPkgs: list(repomd.packagexml._Package, ...) + @return dict of set of binary trove specs + @rtype dict((repomd.packagexml._Package, + set([(str, conary.versions.Version, conary.deps.deps.Flavor), ])) + """ + + labels = copy.copy(self._cfg.platformSearchPath) or list() + labels.insert(0, self._conaryhelper.getConaryConfig().buildLabel) + + # Find the conary source troves + sources = dict([ (('%s:source' % x.name, x.getConaryVersion(), None), x) + for x in srcPkgs ]) + srcTrvMap = self._conaryhelper.findTroves(sources.keys(), labels=labels, + getLeaves=False) + + srcTrvs = {} + for src, cSrcs in srcTrvMap.iteritems(): + srcPkg = sources[src] + for n, v, f in cSrcs: + srcTrvs[(n, v, None)] = srcPkg # Get a mapping of binaries - srcMap = self._conaryhelper.getBinaryVersions(srcTrvs, - labels=self._cfg.platformSearchPath, - includeBuildLabel=True, - latest=False, missingOk=True) + srcMap = self._conaryhelper.getBinaryVersions(srcTrvs.keys(), + labels=labels, latest=False, missingOk=True) # Return binary versions bins = [ x for x in itertools.chain(*srcMap.itervalues()) ] - assert bins - return bins + + binMap = {} + for srcTrv, binaries in srcMap.iteritems(): + assert binaries + binMap.setdefault(srcTrvs[srcTrv], set()).update(set(binaries)) + + return binMap def getTargetVersions(self, binTrvSpecs): """ @@ -815,7 +840,8 @@ return self._conaryhelper.getBuildRequires(pkgName) - def publish(self, trvLst, expected, targetLabel, checkPackageList=True): + def publish(self, trvLst, expected, targetLabel, checkPackageList=True, + extraExpectedPromoteTroves=None): """ Publish a group and its contents to a target label. @param trvLst: list of troves to publish @@ -826,6 +852,12 @@ @type targetLabel: conary Label object @param checkPackageList: verify list of packages being promoted or not. @type checkPackageList: boolean + @param extraExpectedPromoteTroves: list of trove nvfs that are expected + to be promoted, but are only filtered + by name, rather than version and + flavor. + @type extraExpectedPromoteTroves: list of name, version, flavor tuples + where version and flavor may be None. """ return self._conaryhelper.promote( @@ -834,7 +866,8 @@ self._cfg.sourceLabel, targetLabel, checkPackageList=checkPackageList, - extraPromoteTroves=self._cfg.extraPromoteTroves + extraPromoteTroves=self._cfg.extraPromoteTroves, + extraExpectedPromoteTroves=extraExpectedPromoteTroves, ) def mirror(self, fullTroveSync=False): @@ -878,3 +911,70 @@ """ return self._conaryhelper.isOnBuildLabel(version) + + def getUpstreamVersionMap(self, trvSpec, latest=True): + """ + Get a mapping of all binary versions in the repository that match + the specified trove spec, keyed by upstream version. + @param trvSpec: trove tuple of name, version, and flavor + @type trvSpec: tuple(str, str, str) + @param latest: return only the latest versions of each upstream version. + @type latest: boolean + @return map of upstream version to nvfs + @rtype dict(revision=[(str, conary.versions.Version, + conary.deps.deps.Flavor), ...]) + """ + + # get the mapping of source spec to binary spec. + srcMap = self.getSourceVersionMapFromTroveSpec(trvSpec) + + # If not requesting latest versions, go ahead and return mapping of + # upstream versions to binary versions. + if not latest: + return dict([ (x[1].trailingRevision().getVersion(), y) + for x, y in srcMap.iteritems() ]) + + # Build a mapping for version filtering. + upverMap = {} + for src, bins in srcMap.iteritems(): + upver = src[1].trailingRevision().getVersion() + vMap = upverMap.setdefault(upver, dict()) + for n, v, f in bins: + vMap.setdefault(v, set()).add((n, v, f)) + + # Filter out the latest versions. + latestMap = {} + for upver, vMap in upverMap.iteritems(): + recent = None + for v, nvfs in vMap.iteritems(): + if recent is None: + recent = v + continue + if recent < v: + recent = v + + # Store the latest versions that we need to promote + assert upver not in latestMap + latestMap[upver] = vMap[recent] + + return latestMap + + def getSourceVersionMapFromTroveSpec(self, trvSpec): + """ + Get a mapping of source to binary versions for all troves matching the + specified trove spec. + @param trvSpec: trove tuple of name, version, and flavor + @type trvSpec: tuple(str, str, str) + @return map of source trove to binary troves + @rtype dict((str, conary.versions.Version, None)=[(str, + conary.versions.Version, conary.deps.deps.Flavor), ...]) + """ + + if not trvSpec[0].endswith(':source'): + srcSpec = ('%s:source' % trvSpec[0], trvSpec[1], None) + else: + srcSpec = (trvSpec[0], trvSpec[1], None) + + srcTrvs = self._conaryhelper.findTrove(srcSpec, getLeaves=False) + srcMap = self._conaryhelper.getBinaryVersions(srcTrvs, missingOk=True) + return srcMap From elliot at rpath.com Mon Mar 22 12:05:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:15 +0000 Subject: mirrorball: 1. integrate promote script into ordered bot Message-ID: <201003221605.o2MG5FqY023433@scc.eng.rpath.com> changeset: c575a6eaf549 user: Elliot Peele date: Mon, 22 Mar 2010 11:58:22 -0400 1. integrate promote script into ordered bot 2. refactory errata groups to share code with promote diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -27,11 +27,14 @@ from updatebot import errata from updatebot import groupmgr +from updatebot.lib import watchdog from updatebot.bot import Bot as BotSuperClass from updatebot.errors import UnknownRemoveSourceError from updatebot.errors import PlatformNotImportedError from updatebot.errors import TargetVersionNotFoundError +from updatebot.errors import PromoteMissingVersionError +from updatebot.errors import PromoteFlavorMismatchError from updatebot.errors import PlatformAlreadyImportedError log = logging.getLogger('updatebot.ordered') @@ -331,7 +334,72 @@ return updateSet def promote(self): - pass + """ + Promote binary groups from the devel label to the production lable in + the order that they were built. + """ + + # laod package source + self._pkgSource.load() + + log.info('querying repository for all group versions') + sourceLatest = self._updater.getUpstreamVersionMap(cfg.topGroup) + + log.info('querying target label for all group versions') + targetLatest = self._updater.getUpstreamVersionMap( + (cfg.topGroup[0], cfg.targetLabel, None)) + + # Get all updates after the first bucket. + missing = False + for updateId, bucket in self._errata.iterByIssueDate(current=1): + upver = self._errata.getBucketVersion(updateId) + + # Don't try to promote buckets that have already been promoted. + if upver in targetLatest: + log.info('%s found on target label, skipping' % upver) + continue + + # Make sure version has been imported. + if upver not in sourceLatest: + missing = upver + continue + + # If we find a missing version and then find a version in the + # repository report an error. + if missing: + log.critical('found missing version %s' % missing) + raise PromoteMissingVersionError(missing=missing, next=upver) + + log.info('starting promote of %s' % upver) + + # Get conary versions to promote + toPromote = sourceLatest[upver] + + # Make sure we have the expected number of flavors + if len(set(x[2] for x in toPromote)) != len(cfg.groupFlavors): + slog.error('did not find expected number of flavors') + raise PromoteFlavorMismatchError(cfgFlavors=cfg.groupFlavors, + troves=topPromote, version=topPromote[0][1]) + + # Find excepted promote packages. + srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages(bucket) + exceptions = self._getOldVersionExceptions(updateId) + + # These are the binary trove specs that we expect to be promoted. + expected = self._filterBinPkgSet( + itertools.chain(*srcPkgMap.itervalues()), exceptions) + + # Get list of extra troves from the config + extra = cfg.extraExpectedPromoteTroves.get(updateId, []) + + def promote(): + # Create and validate promote changeset + packageList = self._updater.publish(toPromote, expected, + cfg.targetLabel, extraExpectedPromoteTroves=extra) + return 0 + + rc = watchdog.waitOnce(promote) + def createErrataGroups(self): """ @@ -353,7 +421,6 @@ targetGroup = groupmgr.GroupManager(self._cfg, targetGroup=True) targetErrataState = targetGroup.getErrataState() - failures = {} for updateId, updates in self._errata.iterByIssueDate(current=0): # Stop if the updateId is greater than the state of the latest group # on the production label. @@ -370,20 +437,13 @@ updateId=updateId) # Lookup any places we need to use old versions ahead of time. - multiVersionExceptions = {} - if updateId in self._cfg.useOldVersion: - log.info('looking up multiple version exception information') - oldVersions = self._cfg.useOldVersion[updateId] - for oldVersion in oldVersions: - srcMap = self._updater.getSourceVersionMapFromBinaryVersion( - oldVersion, labels=self._cfg.platformSearchPath, - latest=False, includeBuildLabel=True) - - multiVersionExceptions.update(dict([ (x[0], x[1]) - for x in itertools.chain(self._updater.getTargetVersions( - itertools.chain(*srcMap.itervalues()) - )[0]) - ])) + multiVersionExceptions = dict([ + (x[0], x[1]) for x in itertools.chain( + self._updater.getTargetVersions(itertools.chain( + *self._getOldVersionExceptions(updateId).itervalues() + ))[0] + ) + ]) # Now that we know that the packages that are part of this update # should be on the target label we can separate things into @@ -409,50 +469,34 @@ grp.setErrataState(updateId) log.info('%s: finding built packages' % advisory) + binTrvMap = \ + self._updater.getBinaryVersionsFromSourcePackages(srcPkgs) + binTrvs = set() - for srcPkg in srcPkgs: - binTrvSpecs = \ - self._updater.getBinaryVersionsFromSourcePackage(srcPkg) - + for srcPkg, binTrvSpecs in binTrvMap.iteritems(): targetSpecs, failed = self._updater.getTargetVersions( binTrvSpecs) binTrvs.update(set(targetSpecs)) - failures.setdefault(advisory, list()).extend(failed) # Handle attaching an update that was caused by changes that we # made outside of the normal update stream to an existing # advisory. - if advisory in self._cfg.extendAdvisory: - pkgs = self._cfg.extendAdvisory[advisory] - for nvf in pkgs: - srcMap = \ - self._updater.getSourceVersionMapFromBinaryVersion( - nvf, labels=self._cfg.platformSearchPath, - latest=False, includeBuildLabel=True) - assert len(srcMap) == 1 - targetVersions = self._updater.getTargetVersions( - srcMap.values()[0])[0] - binTrvs.update(set(targetVersions)) + for nvf in self._cfg.extendAdvisory.get(advisory, ()): + srcMap = self._updater.getSourceVersionMapFromBinaryVersion( + nvf, labels=self._cfg.platformSearchPath, + latest=False, includeBuildLabel=True) + assert len(srcMap) == 1 + targetVersions = self._updater.getTargetVersions( + srcMap.values()[0])[0] + binTrvs.update(set(targetVersions)) - binPkgs = {} - for n, v, f in binTrvs: - binPkgs.setdefault(n.split(':')[0], - dict()).setdefault(v, set()).add(f) - - for n, vMap in binPkgs.iteritems(): - # Handle the case where a package has been rebuilt for some - # reason, but we need to use the old version of the package. - if len(vMap) > 1 and n in multiVersionExceptions: - vMap = dict((x, y) for x, y in vMap.iteritems() - if x == multiVersionExceptions[n]) - - assert len(vMap) == 1 - - for v, flvs in vMap.iteritems(): - log.info('%s: adding package %s=%s' % (advisory, n, v)) - for f in flvs: - log.info('%s: %s' % (advisory, f)) - grp.addPackage(n, v, flvs) + prev = None + for n, v, f in self._filterBinPkgSet(binTrvs, multiVersionExceptions): + if prev != (n, v): + log.info('%s: adding package %s=%s' % (advisory, n, v) + prev = (n, v) + log.info('%s: %s' % (advisory, f)) + grp.addPackages(n, v, f) # Make sure there are groups to build. if not mgr.hasGroups(): @@ -470,4 +514,35 @@ promoted = self._updater.publish(toPromote, expected, self._cfg.targetLabel) - import epdb; epdb.st() + def _getOldVersionExceptions(self, updateId): + versionExceptions = {} + if updateId in self._cfg.useOldVersion: + log.info('looking up old version exception information') + for oldVersion in self._cfg.useOldVersion[updateId]: + srcMap = self._updater.getSourceVersionMapFromBinaryVersion( + oldVersion, labels=self._cfg.platformSearchPath, + latest=False, includeBuildLabel=True) + versionExceptions.update(srcMap) + + return versionExceptions + + def _filterBinPkgSet(self, binSet, exceptions): + binPkgs = {} + for n, v, f in binSet: + binPkgs.setdefault(n, dict()).setdefault(v, set()).add(f) + + uniqueSet = set() + for n, vMap in binPkgs.iteritems(): + # Handle the case where a package has been rebuilt for some + # reason, but we need to use the old version of the package. + pkgName = n.split(':')[0] + if len(vMap) > 1 and pkgName in exceptions: + vMap = dict((x, y) for x, y in vMap.iteritems() + if x == exceptions[pkgName]) + + assert len(vMap) == 1 + + for v, flvs in vMap.iteritems(): + uniqueSet.update(set((n, v, f) for f in flvs)) + + return uniqueSet From elliot at rpath.com Mon Mar 22 12:05:15 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:15 +0000 Subject: mirrorball: clean up promote script Message-ID: <201003221605.o2MG5GBC023457@scc.eng.rpath.com> changeset: 75799b02efb6 user: Elliot Peele date: Mon, 22 Mar 2010 11:59:58 -0400 clean up promote script diff --git a/scripts/order_promote.py b/scripts/order_promote.py --- a/scripts/order_promote.py +++ b/scripts/order_promote.py @@ -18,7 +18,6 @@ import os import sys -import itertools sys.path.insert(0, os.environ['HOME'] + '/hg/conary') sys.path.insert(0, os.environ['HOME'] + '/hg/xobj/py') @@ -27,9 +26,6 @@ from conary.lib import util sys.excepthook = util.genExcepthook() -from conary import versions -from conary.conaryclient import cmdline - mbdir = os.path.abspath('../') sys.path.insert(0, mbdir) @@ -38,10 +34,8 @@ import rhnmirror from updatebot import log -from updatebot import conaryhelper from updatebot import OrderedBot from updatebot import UpdateBotConfig -from updatebot.lib.watchdog import forknwait slog = log.addRootLogger() cfg = UpdateBotConfig() @@ -54,114 +48,6 @@ errata.fetch() bot = OrderedBot(cfg, errata) -bot._pkgSource.load() - -helper = conaryhelper.ConaryHelper(cfg) - -def getUpstreamVersionMap(groupvers): - # Turn groupvers into something we can work with. - grpVerMap = {} - for n, v, f in groupvers: - upver = v.trailingRevision().getVersion() - grpVerMap.setdefault(upver, dict()).setdefault(v, set()).add((n, v, f)) - - # Find the latest conary versions of all upstream versions - latestMap = {} - for upver, vers in grpVerMap.iteritems(): - latest = None - for v, nvfs in vers.iteritems(): - if latest is None: - latest = v - continue - if latest < v: - latest = v - - # Store the latest versions that we need to promote - assert upver not in latestMap - latestMap[upver] = vers[latest] - return latestMap - -def updateLatestMap(): - # Get all of the binary versions of the top level group - slog.info('querying repository for all group versions') - groupvers = helper.findTrove(cfg.topGroup, getLeaves=False) - latestMap = getUpstreamVersionMap(groupvers) - return latestMap - -latestMap = updateLatestMap() - -# Get all target versions -slog.info('querying target label for all group versions') -targetGroupvers = helper.findTrove((cfg.topGroup[0], cfg.targetLabel, None), getLeaves=False) -targetLatest = getUpstreamVersionMap(targetGroupvers) - -searchPath = [ x for x in itertools.chain((helper._ccfg.buildLabel, ), cfg.platformSearchPath) ] - -# Get all updates after the first bucket. -missing = False -for updateId, bucket in bot._errata.iterByIssueDate(current=1): - upver = bot._errata.getBucketVersion(updateId) - - if upver in targetLatest: - slog.info('%s found on target label, skipping' % upver) - continue - - # make sure version has been imported - if upver not in latestMap: - missing = upver - continue - - slog.info('starting promote of %s' % upver) - - # If we find a missing version and then find a version in the - # repository report an error. - if missing: - slog.critical('found missing version %s' % missing) - raise RuntimeError - - # Get conary versions to promote - topGroups = latestMap[upver] - toPromote = [] - for n, v, f in topGroups: - toPromote.append((n, v, f)) - toPromote.append(('group-rhel-packages', v, f)) - toPromote.append(('group-rhel-standard', v, f)) - - # Make sure we have the expected number of flavors - if len(topGroups) != len(cfg.groupFlavors): - slog.error('did not find expected number of flavors') - raise RuntimeError - - # Find expected promote packages. - csrcTrvs = [ ('%s:source' % x.name, x.getConaryVersion(), None) - for x in bucket ] - srcTrvs = helper.findTroves(csrcTrvs, labels=searchPath) - - # Map source versions to binary versions. - slog.info('retrieving binary trove map') - srcTrvMap = helper.getBinaryVersions([ (x, y, None) for x, y, z in - itertools.chain(*srcTrvs.itervalues()) ], - labels=searchPath) - - # These are the binary trove specs that we expect to be promoted. - expected = [ x for x in itertools.chain(*srcTrvMap.itervalues()) ] - - # Get list of extra troves from the config - extra = cfg.extraExpectedPromoteTroves.get(updateId, []) - - @forknwait - def promote(): - # Create and validate promote changeset - packageList = helper.promote(toPromote, expected, cfg.sourceLabel, - cfg.targetLabel, commit=True, - extraExpectedPromoteTroves=extra) - return 1 - - rc = promote() - - # Update latest map for the next loop - latestMap = updateLatestMap() - - versions.clearVersionCache() +bot.promote() import epdb; epdb.st() From elliot at rpath.com Mon Mar 22 12:05:16 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:16 +0000 Subject: mirrorball: make sure platform has been created before promoting Message-ID: <201003221605.o2MG5GHI023487@scc.eng.rpath.com> changeset: b8e2edd00fe8 user: Elliot Peele date: Mon, 22 Mar 2010 12:03:54 -0400 make sure platform has been created before promoting diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -339,6 +339,11 @@ the order that they were built. """ + # Get current timestamp + current = self._groupmgr.getErrataState() + if current is None: + raise PlatformNotImportedError + # laod package source self._pkgSource.load() @@ -400,7 +405,6 @@ rc = watchdog.waitOnce(promote) - def createErrataGroups(self): """ Create groups for each advisory that only contain the versions of From elliot at rpath.com Mon Mar 22 12:05:17 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:17 +0000 Subject: mirrorball: add ordered promote related errors Message-ID: <201003221605.o2MG5HYk023514@scc.eng.rpath.com> changeset: 292e38fe6eb3 user: Elliot Peele date: Mon, 22 Mar 2010 12:04:10 -0400 add ordered promote related errors diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -224,6 +224,26 @@ _template = ('Expected to promote %(expected)s, actually tried to promote' ' %(actual)s.') +class PromoteMissingVersionError(PromoteFailedError): + """ + PromoteMissingVersionError, raised when ordered promote finds a group that + should have already been promoted to the target label. + """ + + _params = ['missing', 'next'] + _template = ('Expected to find group verison %(missing)s before %(next)s ' + 'on the target label.') + +class PromoteFlavorMismatchError(PromoteFailedError): + """ + PromoteFlavorMismatchError, raised when the number of flavors found to + promote does not match the number of flavors that should have been built. + """ + + _params = ['cfgFlavors', 'troves', 'version'] + _template = ('The number of configured group flavors did not match the ' + 'number of flavors found in the repository for %(version)s') + class MirrorFailedError(UnhandledUpdateError): """ MirrorFailedError, raised when the mirror process fails. From elliot at rpath.com Mon Mar 22 12:05:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:05:18 +0000 Subject: mirrorball: update/add auto scripts to fit new model and order bits Message-ID: <201003221605.o2MG5IVS023543@scc.eng.rpath.com> changeset: f0a263c37b85 user: Elliot Peele date: Mon, 22 Mar 2010 12:04:43 -0400 update/add auto scripts to fit new model and order bits diff --git a/scripts/auto_update b/scripts/auto_ordered_errata_groups copy from scripts/auto_update copy to scripts/auto_ordered_errata_groups --- a/scripts/auto_update +++ b/scripts/auto_ordered_errata_groups @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,38 +13,21 @@ # full details. # -import os -from updatebot import bot, config +import rhnmirror + +from updatebot import OrderedBot from updatebot.cmdline import display +from updatebot.cmdline.simple import main -def validatePlatform(platform, configDir): - validPlatforms = os.listdir(configDir) - if platform not in validPlatforms: - print ('Invalid platform %s... Please select from the following available ' - 'platforms %s' % (platform, ', '.join(validPlatforms))) - return False - return True +def errataGroups(cfg): + mcfg = rhnmirror.MirrorConfig() + mcfg.read(os.path.join(cfg.configPath, 'erratarc')) -def usage(argv): - print 'usage: %s ' % argv[0] - return 1 + errata = rhnmirror.Errata(mcfg) + errata.fetch() -def main(argv, configDir='/etc/mirrorball'): - if len(argv) != 2: - return usage(argv) - - platform = argv[1] - if not validatePlatform(platform, configDir): - return 1 - - cfg = config.UpdateBotConfig() - cfg.read(os.path.join(configDir, platform, 'updatebotrc')) - obj = bot.Bot(cfg) - trvMap = obj.update() - - if trvMap: - print 'Updated the following troves:' - print display.displayTroveMap(trvMap) + bot = OrderedBot(cfg, errata) + bot.createErrataGroups() return 0 @@ -53,10 +36,4 @@ from conary.lib import util sys.excepthook = util.genExcepthook(debug=False) - from updatebot import log - - # Initialize logging - log.addRootLogger() - - # Do update - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv, errataGroups)) diff --git a/scripts/auto_update b/scripts/auto_ordered_promote copy from scripts/auto_update copy to scripts/auto_ordered_promote --- a/scripts/auto_update +++ b/scripts/auto_ordered_promote @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,38 +13,21 @@ # full details. # -import os -from updatebot import bot, config +import rhnmirror + +from updatebot import OrderedBot from updatebot.cmdline import display +from updatebot.cmdline.simple import main -def validatePlatform(platform, configDir): - validPlatforms = os.listdir(configDir) - if platform not in validPlatforms: - print ('Invalid platform %s... Please select from the following available ' - 'platforms %s' % (platform, ', '.join(validPlatforms))) - return False - return True +def promote(cfg): + mcfg = rhnmirror.MirrorConfig() + mcfg.read(os.path.join(cfg.configPath, 'erratarc')) -def usage(argv): - print 'usage: %s ' % argv[0] - return 1 + errata = rhnmirror.Errata(mcfg) + errata.fetch() -def main(argv, configDir='/etc/mirrorball'): - if len(argv) != 2: - return usage(argv) - - platform = argv[1] - if not validatePlatform(platform, configDir): - return 1 - - cfg = config.UpdateBotConfig() - cfg.read(os.path.join(configDir, platform, 'updatebotrc')) - obj = bot.Bot(cfg) - trvMap = obj.update() - - if trvMap: - print 'Updated the following troves:' - print display.displayTroveMap(trvMap) + bot = OrderedBot(cfg, errata) + bot.promote() return 0 @@ -53,10 +36,4 @@ from conary.lib import util sys.excepthook = util.genExcepthook(debug=False) - from updatebot import log - - # Initialize logging - log.addRootLogger() - - # Do update - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv, promote)) diff --git a/scripts/auto_update b/scripts/auto_ordered_update copy from scripts/auto_update copy to scripts/auto_ordered_update --- a/scripts/auto_update +++ b/scripts/auto_ordered_update @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,34 +13,21 @@ # full details. # -import os -from updatebot import bot, config +import rhnmirror + +from updatebot import OrderedBot from updatebot.cmdline import display +from updatebot.cmdline.simple import main -def validatePlatform(platform, configDir): - validPlatforms = os.listdir(configDir) - if platform not in validPlatforms: - print ('Invalid platform %s... Please select from the following available ' - 'platforms %s' % (platform, ', '.join(validPlatforms))) - return False - return True +def update(cfg): + mcfg = rhnmirror.MirrorConfig() + mcfg.read(os.path.join(cfg.configPath, 'erratarc')) -def usage(argv): - print 'usage: %s ' % argv[0] - return 1 + errata = rhnmirror.Errata(mcfg) + errata.fetch() -def main(argv, configDir='/etc/mirrorball'): - if len(argv) != 2: - return usage(argv) - - platform = argv[1] - if not validatePlatform(platform, configDir): - return 1 - - cfg = config.UpdateBotConfig() - cfg.read(os.path.join(configDir, platform, 'updatebotrc')) - obj = bot.Bot(cfg) - trvMap = obj.update() + bot = OrderedBot(cfg, errata) + trvMap = bot.update() if trvMap: print 'Updated the following troves:' @@ -53,10 +40,4 @@ from conary.lib import util sys.excepthook = util.genExcepthook(debug=False) - from updatebot import log - - # Initialize logging - log.addRootLogger() - - # Do update - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv, update)) diff --git a/scripts/auto_update b/scripts/auto_update --- a/scripts/auto_update +++ b/scripts/auto_update @@ -1,6 +1,6 @@ #!/usr/bin/python # -# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2008-2010 rPath, Inc. # # This program is distributed under the terms of the Common Public License, # version 1.0. A copy of this license should have been distributed with this @@ -13,34 +13,13 @@ # full details. # -import os -from updatebot import bot, config +from updatebot import Bot from updatebot.cmdline import display +from updatebot.cmdline.simple import main -def validatePlatform(platform, configDir): - validPlatforms = os.listdir(configDir) - if platform not in validPlatforms: - print ('Invalid platform %s... Please select from the following available ' - 'platforms %s' % (platform, ', '.join(validPlatforms))) - return False - return True - -def usage(argv): - print 'usage: %s ' % argv[0] - return 1 - -def main(argv, configDir='/etc/mirrorball'): - if len(argv) != 2: - return usage(argv) - - platform = argv[1] - if not validatePlatform(platform, configDir): - return 1 - - cfg = config.UpdateBotConfig() - cfg.read(os.path.join(configDir, platform, 'updatebotrc')) - obj = bot.Bot(cfg) - trvMap = obj.update() +def update(cfg): + bot = Bot(cfg) + trvMap = bot.update() if trvMap: print 'Updated the following troves:' @@ -53,10 +32,5 @@ from conary.lib import util sys.excepthook = util.genExcepthook(debug=False) - from updatebot import log - - # Initialize logging - log.addRootLogger() - # Do update - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv, update)) From elliot at rpath.com Mon Mar 22 12:14:49 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:14:49 +0000 Subject: mirrorball: fix syntax error Message-ID: <201003221614.o2MGEnUr023757@scc.eng.rpath.com> changeset: 89ff20ae4dae user: Elliot Peele date: Mon, 22 Mar 2010 12:14:47 -0400 fix syntax error diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -497,7 +497,7 @@ prev = None for n, v, f in self._filterBinPkgSet(binTrvs, multiVersionExceptions): if prev != (n, v): - log.info('%s: adding package %s=%s' % (advisory, n, v) + log.info('%s: adding package %s=%s' % (advisory, n, v)) prev = (n, v) log.info('%s: %s' % (advisory, f)) grp.addPackages(n, v, f) From elliot at rpath.com Mon Mar 22 12:25:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 16:25:43 +0000 Subject: mirrorball: fix typo Message-ID: <201003221625.o2MGPhUU023958@scc.eng.rpath.com> changeset: 31f10b8bbaed user: Elliot Peele date: Mon, 22 Mar 2010 12:25:41 -0400 fix typo diff --git a/updatebot/cmdline/simple.py b/updatebot/cmdline/simple.py --- a/updatebot/cmdline/simple.py +++ b/updatebot/cmdline/simple.py @@ -31,7 +31,7 @@ def main(argv, workerFunc, configDir='/etc/mirrorball', enableLogging=True): if enableLogging: - log.setRootLogger() + log.addRootLogger() if len(argv) != 2: return usage(argv) From elliot at rpath.com Mon Mar 22 13:30:45 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 17:30:45 +0000 Subject: mirrorball: fix formating to fit in 80 colums Message-ID: <201003221730.o2MHUktm027593@scc.eng.rpath.com> changeset: bba3b9c1d183 user: Elliot Peele date: Mon, 22 Mar 2010 13:30:42 -0400 fix formating to fit in 80 colums diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -348,11 +348,11 @@ self._pkgSource.load() log.info('querying repository for all group versions') - sourceLatest = self._updater.getUpstreamVersionMap(cfg.topGroup) + sourceLatest = self._updater.getUpstreamVersionMap(self._cfg.topGroup) log.info('querying target label for all group versions') targetLatest = self._updater.getUpstreamVersionMap( - (cfg.topGroup[0], cfg.targetLabel, None)) + (self._cfg.topGroup[0], self._cfg.targetLabel, None)) # Get all updates after the first bucket. missing = False @@ -381,13 +381,15 @@ toPromote = sourceLatest[upver] # Make sure we have the expected number of flavors - if len(set(x[2] for x in toPromote)) != len(cfg.groupFlavors): + if len(set(x[2] for x in toPromote)) != len(self._cfg.groupFlavors): slog.error('did not find expected number of flavors') - raise PromoteFlavorMismatchError(cfgFlavors=cfg.groupFlavors, - troves=topPromote, version=topPromote[0][1]) + raise PromoteFlavorMismatchError( + cfgFlavors=self._cfg.groupFlavors, troves=topPromote, + version=topPromote[0][1]) # Find excepted promote packages. - srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages(bucket) + srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages( + bucket) exceptions = self._getOldVersionExceptions(updateId) # These are the binary trove specs that we expect to be promoted. @@ -395,12 +397,12 @@ itertools.chain(*srcPkgMap.itervalues()), exceptions) # Get list of extra troves from the config - extra = cfg.extraExpectedPromoteTroves.get(updateId, []) + extra = self._cfg.extraExpectedPromoteTroves.get(updateId, []) def promote(): # Create and validate promote changeset packageList = self._updater.publish(toPromote, expected, - cfg.targetLabel, extraExpectedPromoteTroves=extra) + self._cfg.targetLabel, extraExpectedPromoteTroves=extra) return 0 rc = watchdog.waitOnce(promote) @@ -495,7 +497,8 @@ binTrvs.update(set(targetVersions)) prev = None - for n, v, f in self._filterBinPkgSet(binTrvs, multiVersionExceptions): + for n, v, f in self._filterBinPkgSet(binTrvs, + multiVersionExceptions): if prev != (n, v): log.info('%s: adding package %s=%s' % (advisory, n, v)) prev = (n, v) From elliot at rpath.com Mon Mar 22 13:45:46 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 17:45:46 +0000 Subject: mirrorball: add logfile handling Message-ID: <201003221745.o2MHjkPV028136@scc.eng.rpath.com> changeset: f65790b5d263 user: Elliot Peele date: Mon, 22 Mar 2010 13:45:44 -0400 add logfile handling diff --git a/updatebot/cmdline/simple.py b/updatebot/cmdline/simple.py --- a/updatebot/cmdline/simple.py +++ b/updatebot/cmdline/simple.py @@ -26,20 +26,25 @@ return True def usage(argv): - print 'usage: %s ' % argv[0] + print 'usage: %s [logfile]' % argv[0] return 1 def main(argv, workerFunc, configDir='/etc/mirrorball', enableLogging=True): + if len(argv) < 2 or len(argv) > 3: + return usage(argv) + + logFile = None + if len(argv) == 3: + logFile = argv[2] + if enableLogging: - log.addRootLogger() - - if len(argv) != 2: - return usage(argv) + log.addRootLogger(logFile=logFile) platform = argv[1] if not validatePlatform(platform, configDir): return 1 + cfg = config.UpdateBotConfig() cfg.read(os.path.join(configDir, platform, 'updatebotrc')) From elliot at rpath.com Mon Mar 22 17:58:27 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 21:58:27 +0000 Subject: mirrorball: add missing import Message-ID: <201003222158.o2MLwRHs001699@scc.eng.rpath.com> changeset: 3b933d8f6c7f user: Elliot Peele date: Mon, 22 Mar 2010 17:58:25 -0400 add missing import diff --git a/scripts/auto_ordered_update b/scripts/auto_ordered_update --- a/scripts/auto_ordered_update +++ b/scripts/auto_ordered_update @@ -13,6 +13,8 @@ # full details. # +import os + import rhnmirror from updatebot import OrderedBot From elliot at rpath.com Mon Mar 22 18:26:10 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 22 Mar 2010 22:26:10 +0000 Subject: mirrorball: add more missing imports Message-ID: <201003222226.o2MMQAaQ002208@scc.eng.rpath.com> changeset: f39f0488aea3 user: Elliot Peele date: Mon, 22 Mar 2010 18:26:08 -0400 add more missing imports diff --git a/scripts/auto_ordered_errata_groups b/scripts/auto_ordered_errata_groups --- a/scripts/auto_ordered_errata_groups +++ b/scripts/auto_ordered_errata_groups @@ -13,6 +13,8 @@ # full details. # +import os + import rhnmirror from updatebot import OrderedBot diff --git a/scripts/auto_ordered_promote b/scripts/auto_ordered_promote --- a/scripts/auto_ordered_promote +++ b/scripts/auto_ordered_promote @@ -13,6 +13,8 @@ # full details. # +import os + import rhnmirror from updatebot import OrderedBot From elliot at rpath.com Fri Mar 26 10:57:43 2010 From: elliot at rpath.com (Elliot Peele) Date: Fri, 26 Mar 2010 14:57:43 +0000 Subject: mirrorball: expose mirror interface in the bot, add script for auto mirroring Message-ID: <201003261457.o2QEvhrB019082@scc.eng.rpath.com> changeset: 80e9335e0237 user: Elliot Peele date: Fri, 26 Mar 2010 10:57:40 -0400 expose mirror interface in the bot, add script for auto mirroring diff --git a/scripts/auto_update b/scripts/auto_mirror copy from scripts/auto_update copy to scripts/auto_mirror --- a/scripts/auto_update +++ b/scripts/auto_mirror @@ -17,20 +17,13 @@ from updatebot.cmdline import display from updatebot.cmdline.simple import main -def update(cfg): +def mirror(cfg): bot = Bot(cfg) - trvMap = bot.update() - - if trvMap: - print 'Updated the following troves:' - print display.displayTroveMap(trvMap) - - return 0 + return bot.mirror() if __name__ == '__main__': import sys from conary.lib import util sys.excepthook = util.genExcepthook(debug=False) - # Do update - sys.exit(main(sys.argv, update)) + sys.exit(main(sys.argv, mirror)) diff --git a/updatebot/bot.py b/updatebot/bot.py --- a/updatebot/bot.py +++ b/updatebot/bot.py @@ -258,3 +258,10 @@ trvMap.update(parentPkgMap) return trvMap + + def mirror(self, fullTroveSync=False): + """ + Mirror platform contents to production repository. + """ + + return self._updater.mirror(fullTroveSync=fullTroveSync) From elliot at rpath.com Fri Mar 26 14:21:01 2010 From: elliot at rpath.com (Elliot Peele) Date: Fri, 26 Mar 2010 18:21:01 +0000 Subject: mirrorball: only calculate upstream platform packages when there is actually an upstream Message-ID: <201003261821.o2QIL10K000341@scc.eng.rpath.com> changeset: 98dbf2b5e05b user: Elliot Peele date: Fri, 26 Mar 2010 14:20:57 -0400 only calculate upstream platform packages when there is actually an upstream platform, otherwise this just leads to cofusing logs diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -584,26 +584,29 @@ for x in pkgs if not self._fltrPkg(x) ] ) - # Find all of the binaries that match the upstream platform sources. - log.info('looking up binary versions of all parent platform packages') - parentPkgMap = self.getBinaryVersions(parentPackages, - labels=self._cfg.platformSearchPath) + # Handle parent packages if we are a child platform. + pkgMap = {} + if parentPackages: + # Find all of the binaries that match the upstream platform sources. + log.info('looking up binary versions of all parent platform packages') + parentPkgMap = self.getBinaryVersions(parentPackages, + labels=self._cfg.platformSearchPath) - # Find all of the binaries that match the pre-built sources. - log.info('looking up binary version information for all prebuilt ' - 'packages') - preBuiltPackageMap = self.getBinaryVersions(preBuiltPackages) + # Find all of the binaries that match the pre-built sources. + log.info('looking up binary version information for all prebuilt ' + 'packages') + preBuiltPackageMap = self.getBinaryVersions(preBuiltPackages) - # Combine the two package maps by name where pre built packages - # override parent packages. - parentNames = dict([ (x[0], x) for x in parentPkgMap ]) - preBuiltNames = dict([ (x[0], x) for x in preBuiltPackageMap ]) + # Combine the two package maps by name where pre built packages + # override parent packages. + parentNames = dict([ (x[0], x) for x in parentPkgMap ]) + preBuiltNames = dict([ (x[0], x) for x in preBuiltPackageMap ]) - parentNames.update(preBuiltNames) - parentPkgMap.update(preBuiltPackageMap) + parentNames.update(preBuiltNames) + parentPkgMap.update(preBuiltPackageMap) - pkgMap = dict([ (parentNames[x], parentPkgMap[parentNames[x]]) - for x in parentNames ]) + pkgMap = dict([ (parentNames[x], parentPkgMap[parentNames[x]]) + for x in parentNames ]) return toBuild, pkgMap, fail From elliot at rpath.com Fri Mar 26 15:38:52 2010 From: elliot at rpath.com (Elliot Peele) Date: Fri, 26 Mar 2010 19:38:52 +0000 Subject: mirrorball: avoid trying to hash a set Message-ID: <201003261938.o2QJcqLb002953@scc.eng.rpath.com> changeset: 47da69d77029 user: Elliot Peele date: Fri, 26 Mar 2010 15:38:50 -0400 avoid trying to hash a set diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -375,7 +375,7 @@ log.warn('found binary without matching source name %s' % list(bins)[0].name) - broken.add((nevra, bins)) + broken.add((nevra, tuple(bins))) # Raise an exception if this ever happens. We can figure out the right # thing to do then, purhaps on a case by case basis. From elliot at rpath.com Sun Mar 28 17:12:17 2010 From: elliot at rpath.com (Elliot Peele) Date: Sun, 28 Mar 2010 21:12:17 +0000 Subject: mirrorball: add missing import Message-ID: <201003282112.o2SLCHZe028042@scc.eng.rpath.com> changeset: 1f9978c46cca user: Elliot Peele date: Sun, 28 Mar 2010 16:55:59 -0400 add missing import diff --git a/updatebot/pkgsource/yumsource.py b/updatebot/pkgsource/yumsource.py --- a/updatebot/pkgsource/yumsource.py +++ b/updatebot/pkgsource/yumsource.py @@ -25,6 +25,8 @@ from updatebot.lib import util from updatebot.pkgsource.common import BasePackageSource +from updatebot.errors import CanNotFindSourceForBinariesError + log = logging.getLogger('updatebot.pkgsource') def loaded(func): From elliot at rpath.com Sun Mar 28 17:12:18 2010 From: elliot at rpath.com (Elliot Peele) Date: Sun, 28 Mar 2010 21:12:18 +0000 Subject: mirrorball: If something looks like it might be a kernel module, raise an error rather Message-ID: <201003282112.o2SLCI1r028099@scc.eng.rpath.com> changeset: 19b273338c1c user: Elliot Peele date: Sun, 28 Mar 2010 17:10:26 -0400 If something looks like it might be a kernel module, raise an error rather than trying to build it in the default set of flavors diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -42,6 +42,7 @@ from updatebot.lib import util from updatebot.errors import JobFailedError from updatebot.errors import CommitFailedError +from updatebot.errors import UnhandledKernelModule from updatebot.errors import FailedToRetrieveChangesetError from updatebot.errors import ChangesetValidationFailedError @@ -356,6 +357,11 @@ for context, flavor in self._cfg.packageFlavors[name]: troves.append((name, version, flavor, context)) + # Check if this looks like a kernel module source rpm that wasn't + # handled by the last two checks. + elif '-kmod' in name: + raise UnhandledKernelModule(name=name) + # All other packages. else: # Build all packages as x86 and x86_64. diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -77,6 +77,7 @@ _params = ['jobId', 'why'] _template = 'rMake job %(jobId)s failed: %(why)s' + class JobNotCompleteError(UpdateBotError): """ JobNotCompleteError, raised when the build dispatcher thinks that the job @@ -86,6 +87,20 @@ _params = ['jobId', ] _template = 'Build job not complete %(jobId)s' + +class UnhandledKernelModule(UpdateBotError): + """ + UnhandledKernelModule, raised when trying to create a build job with a + package that looks as if it might be a kernel module that does not have + special flavors definied. + """ + + _param = ['name', ] + _template = ('Attempted to create build job containing %(name)s, which ' + 'appears to be a kernel module, without kernel flavors defined. Please ' + 'define the set of flavors this kernel module should be built in.') + + class UnhandledUpdateError(UpdateBotError): """ UnhandledUpdateError, raised when the bot finds a state that it does not From elliot at rpath.com Sun Mar 28 17:12:19 2010 From: elliot at rpath.com (Elliot Peele) Date: Sun, 28 Mar 2010 21:12:19 +0000 Subject: mirrorball: process package flavors before kernel flavor handing in case a kernel module Message-ID: <201003282112.o2SLCJNC028128@scc.eng.rpath.com> changeset: cdfed41bb75b user: Elliot Peele date: Sun, 28 Mar 2010 17:12:11 -0400 process package flavors before kernel flavor handing in case a kernel module does not need to be or can't be built in all of the same flavors as the kernel diff --git a/updatebot/build/build.py b/updatebot/build/build.py --- a/updatebot/build/build.py +++ b/updatebot/build/build.py @@ -335,6 +335,11 @@ for flv in self._cfg.groupFlavors: troves.append((name, version, flv)) + # Handle special package flavors when specified. + elif name in self._cfg.packageFlavors: + for context, flavor in self._cfg.packageFlavors[name]: + troves.append((name, version, flavor, context)) + # Kernels are special. elif ((name == 'kernel' or name in self._cfg.kernelModules or @@ -352,11 +357,6 @@ str(flavor).replace('kernel', name)) troves.append((name, version, flavor, context)) - # Handle special package flavors when specified. - elif name in self._cfg.packageFlavors: - for context, flavor in self._cfg.packageFlavors[name]: - troves.append((name, version, flavor, context)) - # Check if this looks like a kernel module source rpm that wasn't # handled by the last two checks. elif '-kmod' in name: diff --git a/updatebot/lib/util.py b/updatebot/lib/util.py --- a/updatebot/lib/util.py +++ b/updatebot/lib/util.py @@ -139,7 +139,8 @@ for path in paths: basePath = os.path.basename(path) if (basePath.startswith('kmod-') or - basePath.startswith('kernel-module')): + basePath.startswith('kernel-module') or + '-kmod' in basePath): return True return False From elliot at rpath.com Mon Mar 29 00:51:39 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 29 Mar 2010 04:51:39 +0000 Subject: mirrorball: 1. filter out empty flavors for sources Message-ID: <201003290451.o2T4pgw0000742@scc.eng.rpath.com> changeset: 96d324a66a1f user: Elliot Peele date: Mon, 29 Mar 2010 00:51:36 -0400 1. filter out empty flavors for sources 2. search on correct label diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -979,5 +979,8 @@ srcSpec = (trvSpec[0], trvSpec[1], None) srcTrvs = self._conaryhelper.findTrove(srcSpec, getLeaves=False) - srcMap = self._conaryhelper.getBinaryVersions(srcTrvs, missingOk=True) + filteredSrcTrvs = [ (x, y, None) for x, y, z in srcTrvs ] + assert filteredSrcTrvs + srcMap = self._conaryhelper.getBinaryVersions(filteredSrcTrvs, + missingOk=True, labels=[filteredSrcTrvs[0][1].trailingLabel(), ]) return srcMap From elliot at rpath.com Mon Mar 29 11:12:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 29 Mar 2010 15:12:53 +0000 Subject: mirrorball: fix typo Message-ID: <201003291512.o2TFCru0007211@scc.eng.rpath.com> changeset: 8be0204c6b93 user: Elliot Peele date: Mon, 29 Mar 2010 11:12:37 -0400 fix typo diff --git a/updatebot/errors.py b/updatebot/errors.py --- a/updatebot/errors.py +++ b/updatebot/errors.py @@ -568,7 +568,7 @@ version, and release can not be found. """ - _parms = ['count', ] + _params = ['count', ] _template = ('Could not find %(count) sources for matching binary ' 'packages. This generally means that there is a binary package with a ' 'source of a different name and a source can not be found with a ' From elliot at rpath.com Mon Mar 29 11:12:53 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 29 Mar 2010 15:12:53 +0000 Subject: mirrorball: add missing import Message-ID: <201003291512.o2TFCrSg007238@scc.eng.rpath.com> changeset: 9dad4799be14 user: Elliot Peele date: Mon, 29 Mar 2010 11:12:50 -0400 add missing import diff --git a/updatebot/update.py b/updatebot/update.py --- a/updatebot/update.py +++ b/updatebot/update.py @@ -17,6 +17,7 @@ """ import os +import copy import logging import itertools From elliot at rpath.com Mon Mar 29 14:16:58 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 29 Mar 2010 18:16:58 +0000 Subject: mirrorball: add reporting for promote and errata groups Message-ID: <201003291816.o2TIGwLh009797@scc.eng.rpath.com> changeset: 45861914fc68 user: Elliot Peele date: Mon, 29 Mar 2010 14:16:41 -0400 add reporting for promote and errata groups diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -197,6 +197,10 @@ # detected. self._updater.sanityCheckSource(srpm) + log.info('starting update run') + + count = 0 + startime = time.time() updateSet = {} for updateId, updates in self._errata.iterByIssueDate(current=current): start = time.time() @@ -330,6 +334,11 @@ time.localtime(updateId)) totalTime = time.time() - start log.info('published update %s in %s seconds' % (advTime, totalTime)) + count += 1 + + log.info('update completed') + log.info('applied %s updates in %s seconds' + % (count, time.time() - startime)) return updateSet @@ -354,6 +363,11 @@ targetLatest = self._updater.getUpstreamVersionMap( (self._cfg.topGroup[0], self._cfg.targetLabel, None)) + log.info('starting promote') + + count = 0 + startime = time.time() + # Get all updates after the first bucket. missing = False for updateId, bucket in self._errata.iterByIssueDate(current=1): @@ -406,6 +420,13 @@ return 0 rc = watchdog.waitOnce(promote) + if rc: + break + count += 1 + + log.info('promote complete') + log.info('promoted %s groups in %s seconds' + % (count, time.time() - startime)) def createErrataGroups(self): """ @@ -427,6 +448,11 @@ targetGroup = groupmgr.GroupManager(self._cfg, targetGroup=True) targetErrataState = targetGroup.getErrataState() + log.info('starting errata group processing') + + count = 0 + startime = time.time() + for updateId, updates in self._errata.iterByIssueDate(current=0): # Stop if the updateId is greater than the state of the latest group # on the production label. @@ -521,6 +547,12 @@ promoted = self._updater.publish(toPromote, expected, self._cfg.targetLabel) + count += 1 + + log.info('completed errata group processing') + log.info('processed %s errata groups in %s seconds' + % (count, time.time() - startime)) + def _getOldVersionExceptions(self, updateId): versionExceptions = {} if updateId in self._cfg.useOldVersion: From elliot at rpath.com Mon Mar 29 14:16:59 2010 From: elliot at rpath.com (Elliot Peele) Date: Mon, 29 Mar 2010 18:16:59 +0000 Subject: mirrorball: make sure trove list is a list Message-ID: <201003291816.o2TIGx2A009828@scc.eng.rpath.com> changeset: c6e8d35dfbf8 user: Elliot Peele date: Mon, 29 Mar 2010 14:16:53 -0400 make sure trove list is a list diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py --- a/updatebot/conaryhelper.py +++ b/updatebot/conaryhelper.py @@ -1026,6 +1026,9 @@ else: extraExpectedPromoteTroves = set(extraExpectedPromoteTroves) + # make sure trvLst is a list. + trvLst = list(trvLst) + # Get the label that the group is on. fromLabel = trvLst[0][1].trailingLabel() From elliot at rpath.com Tue Mar 30 16:24:33 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 30 Mar 2010 20:24:33 +0000 Subject: mirrorball: rework package flavor handling when adding to groups Message-ID: <201003302024.o2UKOXUI003315@scc.eng.rpath.com> changeset: 97671e15bcd4 user: Elliot Peele date: Tue, 30 Mar 2010 11:50:33 -0400 rework package flavor handling when adding to groups rework some of the flavor handling when adding packages to groups with non standard flavors. diff --git a/updatebot/groupmgr/manager.py b/updatebot/groupmgr/manager.py --- a/updatebot/groupmgr/manager.py +++ b/updatebot/groupmgr/manager.py @@ -284,6 +284,18 @@ x86 = deps.parseFlavor('is: x86') x86_64 = deps.parseFlavor('is: x86_64') + # Count the flavors for later use. + flvCount = {x86: 0, x86_64: 0, plain: 0} + for flavor in flavors: + if flavor.satisfies(x86): + flvCount[x86] += 1 + elif flavor.satisfies(x86_64): + flvCount[x86_64] += 1 + elif flavor.freeze() == '': + flvCount[plain] += 1 + else: + raise UnsupportedTroveFlavorError(name=name, flavor=flavor) + if len(flavors) == 1: flavor = flavors[0] # noarch package, add unconditionally @@ -303,24 +315,17 @@ return - elif len(flavors) == 2: + elif (len(flavors) == 2 and + not [ x for x, y in flvCount.iteritems() if y > 1 ]): # This is most likely a normal package with both x86 and x86_64, but # lets make sure anyway. - flvCount = {x86: 0, x86_64: 0, plain: 0} - for flavor in flavors: - if flavor.satisfies(x86): - flvCount[x86] += 1 - elif flavor.satisfies(x86_64): - flvCount[x86_64] += 1 - elif flavor.freeze() == '': - flvCount[plain] += 1 - else: - raise UnsupportedTroveFlavorError(name=name, flavor=flavor) - # make sure there is only one instance of x86 and once instance of + # An exception for python/perl where x86_64 packages are noarch, but + # x86 packages are flavored. + assert (flvCount[x86_64] > 0) ^ (flvCount[plain] > 0) + # make sure there is only one instance of x86 and one instance of # x86_64 in the flavor list. - assert (flvCount[x86_64] > 0) ^ (flvCount[plain] > 0) - assert len([ x for x, y in flvCount.iteritems() if y != 1 ]) == 1 + assert len([ x for x, y in flvCount.iteritems() if y == 0 ]) == 1 # In this case just add the package unconditionally self.add(name, version=version) From elliot at rpath.com Tue Mar 30 16:24:33 2010 From: elliot at rpath.com (Elliot Peele) Date: Tue, 30 Mar 2010 20:24:33 +0000 Subject: mirrorball: unique packages by name, not component Message-ID: <201003302024.o2UKOYtq003346@scc.eng.rpath.com> changeset: 4e2552b89cea user: Elliot Peele date: Tue, 30 Mar 2010 16:24:30 -0400 unique packages by name, not component log when adding contents to groups diff --git a/updatebot/ordered.py b/updatebot/ordered.py --- a/updatebot/ordered.py +++ b/updatebot/ordered.py @@ -78,6 +78,9 @@ assert len(vf) == 1 version = vf.keys()[0] flavors = list(vf[version]) + log.info('adding %s=%s' % (name, version)) + for f in flavors: + log.info('\t%s' % f) self._groupmgr.addPackage(name, version, flavors) def _savePackages(self, pkgMap, fn=None): @@ -522,14 +525,18 @@ srcMap.values()[0])[0] binTrvs.update(set(targetVersions)) - prev = None - for n, v, f in self._filterBinPkgSet(binTrvs, - multiVersionExceptions): - if prev != (n, v): - log.info('%s: adding package %s=%s' % (advisory, n, v)) - prev = (n, v) - log.info('%s: %s' % (advisory, f)) - grp.addPackages(n, v, f) + # Group unique versions by flavor + nvfMap = {} + for n, v, f in self._filterBinPkgSet(binTrvs, multiVersionExceptions): + n = n.split(':')[0] + nvfMap.setdefault((n, v), set()).add(f) + + # Add packages to group model. + for (n, v), flvs in nvfMap.iteritems(): + log.info('%s: adding package %s=%s' % (advisory, n, v)) + for f in flvs: + log.info('%s: %s' % (advisory, f)) + grp.addPackage(n, v, flvs) # Make sure there are groups to build. if not mgr.hasGroups():