From elliot at rpath.com  Thu Sep  1 00:27:20 2011
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 01 Sep 2011 04:27:20 +0000
Subject: mirrorball: combine all results into a single list
Message-ID: <201109010427.p814RK9N015837@scc.eng.rpath.com>

changeset:   8994b0dc462d
user:        Elliot Peele <elliot at bentlogic.net>
date:        Thu, 01 Sep 2011 00:12:09 -0400

combine all results into a single list

diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1088,7 +1088,7 @@
         results = []
         kwargs['checkPackageList'] = False
         for job in self._iterPromoteJobList(trvLst):
-            results.append(self._promote(job, set(), *args, **kwargs))
+            results.extend(self._promote(job, set(), *args, **kwargs))
         return results
 
     def _promote(self, trvLst, expected, sourceLabels, targetLabel,

From elliot at rpath.com  Thu Sep  1 00:27:21 2011
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 01 Sep 2011 04:27:21 +0000
Subject: mirrorball: finish up promote dispatcher, needs testing
Message-ID: <201109010427.p814RLAm015867@scc.eng.rpath.com>

changeset:   d142c81bbfa8
user:        Elliot Peele <elliot at bentlogic.net>
date:        Thu, 01 Sep 2011 00:23:03 -0400

finish up promote dispatcher, needs testing

diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -40,6 +40,7 @@
 from rmake.cmdline import monitor
 
 from updatebot.lib import util
+from updatebot.conaryhelper import ConaryHelper
 from updatebot.errors import JobFailedError
 from updatebot.errors import CommitFailedError
 from updatebot.errors import UnhandledKernelModule
@@ -52,8 +53,9 @@
 from updatebot.build.jobs import LocalDispatcher
 from updatebot.build.jobs import OrderedCommitDispatcher
 from updatebot.build.dispatcher import Dispatcher
+from updatebot.build.dispatcher import RebuildDispatcher
+from updatebot.build.dispatcher import PromoteDispatcher
 from updatebot.build.dispatcher import NonCommittalDispatcher
-from updatebot.build.dispatcher import RebuildDispatcher
 from updatebot.build.callbacks import StatusOnlyDisplay
 
 log = logging.getLogger('updatebot.build')
@@ -157,6 +159,8 @@
 
         self._asyncDispatcher = OrderedCommitDispatcher(self, 30)
 
+        self._conaryhelper = ConaryHelper(self._cfg)
+
     def build(self, troveSpecs):
         """
         Build a list of troves.
@@ -188,7 +192,9 @@
         """
 
         workers = 30
-        if not lateCommit:
+        if self._cfg.updateMode == 'current':
+            dispatcher = PromoteDispatcher(self, workers)
+        elif not lateCommit:
             dispatcher = Dispatcher(self, workers)
         else:
             dispatcher = NonCommittalDispatcher(self, workers)
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -469,14 +469,30 @@
         """
 
         # Find jobs in the Committed state that need to be promoted
-        toPromote = []
-        for jobId, (trove, state, result) in self._jobs.iteritems():
-            # not ready to be promoted
-            if state != JobStatus.JOB_COMMITTED:
-                continue
+        if self._promoteSlots:
+            toPromote = []
+            for jobId, (trove, state, result) in self._jobs.iteritems():
+                # not ready to be promoted
+                if state != JobStatus.JOB_COMMITTED:
+                    continue
 
-            toPromote.append((jobId, result))
+                toPromote.append((jobId, result))
+                self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
 
-        if self._promoteSlots and toPromote:
-            self._promoteSlots -= 1
-            self._promoter.promoteJob(toPromote)
+            if toPromote:
+                self._promoteSlots -= 1
+                self._promoter.promoteJob(toPromote)
+
+        # Gather results
+        for result in self._promoter.getStatus():
+            self._promoteSlots += 1
+            for jobId, trvLst in result:
+                self._jobs[jobId][2] = trvLst
+                self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
+
+        # Gather errors
+        for jobs, error in self._promoter.getErrors():
+            self._promoteSlots += 1
+            for jobId in jobs:
+                self._jobs[jobId][1] = JobStatus.ERROR_PROMOTE_FAILURE
+                self._failures.append((jobId, error))
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -17,6 +17,7 @@
 """
 
 import os
+import itertools
 
 from rmake.cmdline import monitor
 
@@ -147,22 +148,46 @@
 
     threadType = WorkerTypes.PROMOTE
 
-    def __init__(self, status, (helper, targetLabel, trvLst)):
+    def __init__(self, status, (helper, targetLabel, jobs)):
         AbstractWorker.__init__(self, status)
 
         self.helper = helper
         self.targetLabel = targetLabel
-        self.trvLst = trvLst
+        self.jobs = jobs
+        self.workerId = (x[0] for x in jobs)
 
     def work(self):
         """
         Promote the specified list of troves.
         """
 
+        # Let's be smart about promoting troves and batch all of the jobs
+        # together. This means we need to keep track of what we put in and
+        # what we get back so that we can map the output to a jobId.
+
+        # Build mapping of nvf tuple to jobId
+        jobMap = {}
+        for jobId, built in self.jobs.iteritems():
+            for bintrv in itertools.chain(*built.itervalues()):
+                jobMap[bintrv] = jobId
+
+        # Assume that all troves are on the same source label.
         srcLabel = self.trvLst[0][1].trailingLabel()
+
+        # Promote
         result = self.helper.promote(self.trvLst, set(), srcLabel,
             self.targetLabel, checkPackageList=False)
-        self.status.put((MessageTypes.DATA, (self.trvLst, result)))
+
+        # Map the results back to source label troves
+        clonedFrom = self.helper.getClonedFrom(result)
+
+        # Map jobIds back to promoted troves.
+        resultMap = {}
+        for spec, jobId in jobMap.iteritems():
+            resultMap.setdefault(jobId, set()).add(clonedFrom.get(spec))
+
+        # Send back all of the results.
+        self.status.put((MessageTypes.DATA, tuple(resultMap.items())))
 
 
 class JobStarter(AbstractStatusMonitor):

From elliot at rpath.com  Thu Sep  1 00:27:21 2011
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 01 Sep 2011 04:27:21 +0000
Subject: mirrorball: branch merge
Message-ID: <201109010427.p814RL4C015895@scc.eng.rpath.com>

changeset:   c4dcf9036fe0
user:        Elliot Peele <elliot at bentlogic.net>
date:        Thu, 01 Sep 2011 00:27:05 -0400

branch merge

diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -40,6 +40,7 @@
 from rmake.cmdline import monitor
 
 from updatebot.lib import util
+from updatebot.conaryhelper import ConaryHelper
 from updatebot.errors import JobFailedError
 from updatebot.errors import CommitFailedError
 from updatebot.errors import UnhandledKernelModule
@@ -52,8 +53,9 @@
 from updatebot.build.jobs import LocalDispatcher
 from updatebot.build.jobs import OrderedCommitDispatcher
 from updatebot.build.dispatcher import Dispatcher
+from updatebot.build.dispatcher import RebuildDispatcher
+from updatebot.build.dispatcher import PromoteDispatcher
 from updatebot.build.dispatcher import NonCommittalDispatcher
-from updatebot.build.dispatcher import RebuildDispatcher
 from updatebot.build.callbacks import StatusOnlyDisplay
 
 log = logging.getLogger('updatebot.build')
@@ -157,6 +159,8 @@
 
         self._asyncDispatcher = OrderedCommitDispatcher(self, 30)
 
+        self._conaryhelper = ConaryHelper(self._cfg)
+
     def build(self, troveSpecs):
         """
         Build a list of troves.
@@ -188,7 +192,9 @@
         """
 
         workers = 30
-        if not lateCommit:
+        if self._cfg.updateMode == 'current':
+            dispatcher = PromoteDispatcher(self, workers)
+        elif not lateCommit:
             dispatcher = Dispatcher(self, workers)
         else:
             dispatcher = NonCommittalDispatcher(self, workers)
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -469,14 +469,30 @@
         """
 
         # Find jobs in the Committed state that need to be promoted
-        toPromote = []
-        for jobId, (trove, state, result) in self._jobs.iteritems():
-            # not ready to be promoted
-            if state != JobStatus.JOB_COMMITTED:
-                continue
+        if self._promoteSlots:
+            toPromote = []
+            for jobId, (trove, state, result) in self._jobs.iteritems():
+                # not ready to be promoted
+                if state != JobStatus.JOB_COMMITTED:
+                    continue
 
-            toPromote.append((jobId, result))
+                toPromote.append((jobId, result))
+                self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
 
-        if self._promoteSlots and toPromote:
-            self._promoteSlots -= 1
-            self._promoter.promoteJob(toPromote)
+            if toPromote:
+                self._promoteSlots -= 1
+                self._promoter.promoteJob(toPromote)
+
+        # Gather results
+        for result in self._promoter.getStatus():
+            self._promoteSlots += 1
+            for jobId, trvLst in result:
+                self._jobs[jobId][2] = trvLst
+                self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
+
+        # Gather errors
+        for jobs, error in self._promoter.getErrors():
+            self._promoteSlots += 1
+            for jobId in jobs:
+                self._jobs[jobId][1] = JobStatus.ERROR_PROMOTE_FAILURE
+                self._failures.append((jobId, error))
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -17,6 +17,7 @@
 """
 
 import os
+import itertools
 
 from rmake.cmdline import monitor
 
@@ -147,22 +148,46 @@
 
     threadType = WorkerTypes.PROMOTE
 
-    def __init__(self, status, (helper, targetLabel, trvLst)):
+    def __init__(self, status, (helper, targetLabel, jobs)):
         AbstractWorker.__init__(self, status)
 
         self.helper = helper
         self.targetLabel = targetLabel
-        self.trvLst = trvLst
+        self.jobs = jobs
+        self.workerId = (x[0] for x in jobs)
 
     def work(self):
         """
         Promote the specified list of troves.
         """
 
+        # Let's be smart about promoting troves and batch all of the jobs
+        # together. This means we need to keep track of what we put in and
+        # what we get back so that we can map the output to a jobId.
+
+        # Build mapping of nvf tuple to jobId
+        jobMap = {}
+        for jobId, built in self.jobs.iteritems():
+            for bintrv in itertools.chain(*built.itervalues()):
+                jobMap[bintrv] = jobId
+
+        # Assume that all troves are on the same source label.
         srcLabel = self.trvLst[0][1].trailingLabel()
+
+        # Promote
         result = self.helper.promote(self.trvLst, set(), srcLabel,
             self.targetLabel, checkPackageList=False)
-        self.status.put((MessageTypes.DATA, (self.trvLst, result)))
+
+        # Map the results back to source label troves
+        clonedFrom = self.helper.getClonedFrom(result)
+
+        # Map jobIds back to promoted troves.
+        resultMap = {}
+        for spec, jobId in jobMap.iteritems():
+            resultMap.setdefault(jobId, set()).add(clonedFrom.get(spec))
+
+        # Send back all of the results.
+        self.status.put((MessageTypes.DATA, tuple(resultMap.items())))
 
 
 class JobStarter(AbstractStatusMonitor):
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1088,7 +1088,7 @@
         results = []
         kwargs['checkPackageList'] = False
         for job in self._iterPromoteJobList(trvLst):
-            results.append(self._promote(job, set(), *args, **kwargs))
+            results.extend(self._promote(job, set(), *args, **kwargs))
         return results
 
     def _promote(self, trvLst, expected, sourceLabels, targetLabel,

From elliot at rpath.com  Thu Sep  1 00:49:14 2011
From: elliot at rpath.com (Elliot Peele)
Date: Thu, 01 Sep 2011 04:49:14 +0000
Subject: mirrorball: remove the testsuite that hasn't worked in quite
 awhile
Message-ID: <201109010449.p814nEF8016431@scc.eng.rpath.com>

changeset:   d483146dd9b9
user:        Elliot Peele <elliot at bentlogic.net>
date:        Thu, 01 Sep 2011 00:39:00 -0400

remove the testsuite that hasn't worked in quite awhile

diff --git a/test/Makefile b/test/Makefile
deleted file mode 100644
--- a/test/Makefile
+++ /dev/null
@@ -1,10 +0,0 @@
-all:
-	for dir in `find . -type d -name '*test'`; do \
-relativelink=`echo "$$dir" | sed s,/[^/]*,/..,g | sed 's,./,,'`; \
-ln -fs $${relativelink}/testsetup.py $$dir; \
-done
-
-clean:
-	for dir in `find . -type d -name '*test'`; do \
-            rm -f $${dir}/testsetup.py*;\
-        done
diff --git a/test/__init__.py b/test/__init__.py
deleted file mode 100644
diff --git a/test/archive/Makefile b/test/archive/Makefile
deleted file mode 100644
--- a/test/archive/Makefile
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
-#
-# All Rights Reserved
-#
-arch_files = arch/ppc arch/x86 arch/x86_64
-
-use_files =	use/bootstrap\
-		use/builddocs\
-		use/buildtests\
-		use/desktop\
-		use/dietlibc\
-		use/emacs\
-		use/gcj\
-		use/gdbm\
-		use/gnome\
-		use/gtk\
-		use/ipv6\
-		use/kde\
-		use/krb\
-		use/ldap\
-		use/pam\
-		use/pcre\
-		use/perl\
-		use/python\
-		use/qt\
-		use/readline\
-		use/sasl\
-		use/ssl\
-		use/tcl\
-		use/tk\
-		use/X
-
-dist_files = $(arch_files) $(use_files) Makefile
-
-all: default-all
-
-dist: default-dist
-
-clean: default-clean
-
-include ../../Make.rules
-# vim: set sts=8 sw=8 noexpandtab :
diff --git a/test/archive/arch/ppc b/test/archive/arch/ppc
deleted file mode 100644
--- a/test/archive/arch/ppc
+++ /dev/null
@@ -1,10 +0,0 @@
-Name ppc
-archProp LE True
-archProp BE False
-archProp bits32 False
-archProp bits64 True
-macro unamearch ppc
-macro targetarch powerpc
-
-[ppc64]
-
diff --git a/test/archive/arch/x86 b/test/archive/arch/x86
deleted file mode 100644
--- a/test/archive/arch/x86
+++ /dev/null
@@ -1,46 +0,0 @@
-Name x86
-archProp LE True
-archProp BE False
-archProp bits32 True
-archProp bits64 False
-macro targetarch i386
-macro unamearch i386
-macro optflags -O2
-
-[i486]
-macro targetarch i486
-macro unamearch i486
-
-[i586]
-macro targetarch i586
-macro unamearch i586
-subsumes i486
-
-[i686]
-macro targetarch i686
-macro unamearch i686
-subsumes i586, i486
-
-[cmov]
-
-[sse]
-
-[sse2]
-subsumes sse
-
-[sse3]
-subsumes sse2,sse
-
-[mmx]
-
-[mmxext]
-subsumes mmx
-
-[3dnow]
-buildName threednow
-
-[3dnowext]
-buildName threednowext
-subsumes 3dnow
-
-[nx]
diff --git a/test/archive/arch/x86_64 b/test/archive/arch/x86_64
deleted file mode 100644
--- a/test/archive/arch/x86_64
+++ /dev/null
@@ -1,17 +0,0 @@
-archProp LE True
-archProp BE False
-archProp bits32 False
-archProp bits64 True
-macro lib lib64
-
-[nx]
-
-[sse3]
-
-[3dnow]
-buildName threednow
-
-[3dnowext]
-buildName threednowext
-subsumes 3dnow
-
diff --git a/test/archive/authdb b/test/archive/authdb
deleted file mode 100644
Binary file test/archive/authdb has changed
diff --git a/test/archive/control-files/ascii.control b/test/archive/control-files/ascii.control
deleted file mode 100644
--- a/test/archive/control-files/ascii.control
+++ /dev/null
@@ -1,17 +0,0 @@
-Package: ascii
-Version: 3.8-2
-Section: utils
-Priority: optional
-Architecture: i386
-Depends: libc6 (>= 2.3.4-1)
-Installed-Size: 72
-Maintainer: Florian Ernst <florian at debian.org>
-Description: interactive ASCII name and synonym chart
- The ascii utility provides easy conversion between various byte representations
- and the American Standard Code for Information Interchange (ASCII) character
- table.  It knows about a wide variety of hex, binary, octal, Teletype mnemonic,
- ISO/ECMA code point, slang names, XML entity names, and other representations.
- Given any one on the command line, it will try to display all others.  Called
- with no arguments it displays a handy small ASCII chart.
- .
- Homepage: http://www.catb.org/~esr/ascii/
diff --git a/test/archive/distcache-1.4.5-2.src.rpm b/test/archive/distcache-1.4.5-2.src.rpm
deleted file mode 100644
Binary file test/archive/distcache-1.4.5-2.src.rpm has changed
diff --git a/test/archive/key.asc b/test/archive/key.asc
deleted file mode 100644
--- a/test/archive/key.asc
+++ /dev/null
@@ -1,13 +0,0 @@
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1.4.1 (GNU/Linux)
-
-mIsEQwCyngEEAJ3MC9HwzDve2JzvEdhS/eWfjLjtE1Hco/BxZEri4oz4XTScjmbm
-gPGrk84F2Pxb3sqNP3pESjKZO8yNYj5pF9ToVO3uzWKIRk3XGuu1MwheKO0ruOKS
-6wUJo1x2scrAHY5YoOFAQ1SGHWdX8qAHoupDEIO/35bHMeoD1BQeDmGVAAYptC1S
-U0EgMTAyNCB3aXRoIEl0ZXJhdGVkIFMySyBTSEEgQ2hlY2tzdW0gQ0FTVDWItAQT
-AQIAHgUCQwCyngIbAwYLCQgHAwIDFQIDAxYCAQIeAQIXgAAKCRDHwy/B+U5AXijK
-A/0UbwMU5Dg2EMekqgUp4mirm+fFjV+S5fRZnY4Ngj8wXN7BbEiVl9RTBt6O+abA
-EXWYNw85Nlkq77W49qz/XewymaiF9Q6ELJ1XQBKM7bwKRmmwO2dQS6eCQaRfOQ9X
-DeG01oWQDg+ISriSc9WlTdf+aP2VieB2wK55bUw4NWijtg==
-=OcwF
------END PGP PUBLIC KEY BLOCK-----
diff --git a/test/archive/macros b/test/archive/macros
deleted file mode 100644
--- a/test/archive/macros
+++ /dev/null
@@ -1,120 +0,0 @@
-# These are the default macro definitions used when cooking packages using
-# cvc.  You can override these values in individual recipes, as well as on
-# the command line.
-
-lib                 = 'lib'
-# NOTE: lib is an architecture-specific macro, 
-# and is defined in /etc/conary/arch for some architectures, specifically 
-# 64bit systems.
-
-prefix		    = '/usr'
-sysconfdir	    = '/etc'
-initdir		    = '%(sysconfdir)s/init.d'
-exec_prefix	    = '%(prefix)s'
-bindir		    = '%(exec_prefix)s/bin'
-
-essentialbindir	    = '/bin'
-sbindir		    = '%(exec_prefix)s/sbin'
-essentialsbindir    = '/sbin'
-libdir		    = '%(exec_prefix)s/%(lib)s'
-essentiallibdir	    = '/%(lib)s'
-libexecdir	    = '%(exec_prefix)s/libexec'
-localstatedir	    = '/var'
-servicedir          = '/srv'
-cachedir	    = '%(localstatedir)s/cache'
-sharedstatedir	    = '%(prefix)s/com'
-includedir	    = '%(prefix)s/include'
-datadir		    = '%(prefix)s/share'
-mandir		    = '%(datadir)s/man'
-infodir		    = '%(datadir)s/info'
-docdir		    = '%(datadir)s/doc'
-thisdocdir          = '%(docdir)s/%(name)s-%(version)s'
-tagdescriptiondir   = '%(sysconfdir)s/conary/tags'
-taghandlerdir       = '%(libexecdir)s/conary/tags'
-tagdatadir          = '%(datadir)s/conary/tags'
-testdir	            = '%(localstatedir)s/conary/tests'
-thistestdir	    = '%(testdir)s/%(name)s-%(version)s'
-debuglibdir         = '/usr/lib/debug' # no %(prefix)s or %(lib)s!
-debugsrcdir         = '/usr/src/debug' # no %(prefix)s!
-userinfodir         = '%(sysconfdir)s/conary/userinfo'
-groupinfodir        = '%(sysconfdir)s/conary/groupinfo'
-buildlogpath        = '%(debugsrcdir)s/buildlogs/%(name)s-%(version)s-log.bz2'
-buildxmlpath        = '%(debugsrcdir)s/buildlogs/%(name)s-%(version)s-xml.bz2'
-## Java Macros
-# Root directory where all Java VMs/SDK/JREs are installed.
-jvmdir              = '%(libdir)s/jvm'
-# Directory where arch and version independent jars are installed.
-javadir             = '%(datadir)s/java'
-# Root directory where all javadoc is installed. Also already in RH macros.
-javadocdir          = '%(datadir)s/javadoc'
-thisjavadocdir      = '%(javadocdir)s/%(name)s-%(version)s'
-# special component prefixes that the whole system needs to share
-krbprefix	    = '%(exec_prefix)s/kerberos'
-x11prefix	    = '%(exec_prefix)s/X11R6'
-# programs/options (empty ones are for documentation)
-cc		    = 'gcc'
-cxx		    = 'g++'
-cxxflags            = ''    # cxx specific flags
-optflags            = '-O2'
-dbgflags            = '-g' # for debuginfo
-cflags              = '%(optflags)s %(dbgflags)s' 
-cppflags	    = '' 
-ldflags		    = '%(dbgflags)s'
-mflags		    = '' # make flags
-parallelmflags      = ''
-classpath           = ''
-os		    = 'linux'
-debugedit           = 'debugedit'
-strip               = 'eu-strip' # eu-strip for debuginfo, "strip -g" else
-strip_archive       = 'strip -g' # eu-strip segfaults on ar
-monodis             = '%(bindir)s/monodis' # tool to determine mono dependencies
-ruby                = '%(bindir)s/ruby'
-
-# reference macros filled in from the recipe
-archive_name       = '%(name)s'
-archive_version    = '%(version)s'
-# overridden in conary
-major_version      = '%(version)s'
-
-# filled in at cook time
-buildbranch         = ''
-buildlabel          = ''
-
-# cross compiling macros
-# most of the macros beyond this point will only be of interest if you
-# are cross compiling.
-
-# targetarch, buildarch, and hostarch are determined internally by 
-# conary (although you can override them using command line options)
-
-targetvendor        = 'unknown'
-targetos            = 'linux'
-target		    = '%(targetarch)s-%(targetvendor)s-%(targetos)s'
-
-# the platform on which the created binaries should be run
-# (different from host only when the resulting binary is a cross-compiler)
-hostvendor          = 'unknown'
-hostos              = 'linux'
-host		    = '%(hostarch)s-%(hostvendor)s-%(hostos)s'
-
-# build is the system on which the binaries are being run
-buildvendor         = 'unknown'
-buildos             = 'linux'
-build		    = '%(buildarch)s-%(buildvendor)s-%(buildos)s'
-
-# in a default non-cross-compiling build, the build system cc is just the 
-# regular cc.  These will be overridden by conary when cross-compiling
-
-buildcc             = '%(cc)s'
-buildcxx            = '%(cxx)s'
-
-# The following macros will be set when cross-compiling unless they are set
-# here.
-#crossdir       = 'cross-target-%(target)s'
-#crossprefix	= '/opt/%(crossdir)s'
-#sysroot        = '%(crossprefix)s/sys-root'
-#headerpath     = '%(sysroot)s/%(includedir)s'
-#hostmajorarch  = 
-
-# the following dummy value is for testing macro overrides
-dummyMacro      = 'wrong'
diff --git a/test/archive/policy/fake.py b/test/archive/policy/fake.py
deleted file mode 100644
--- a/test/archive/policy/fake.py
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.opensource.org/licenses/cpl.php.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import errno
-import os
-import shutil
-import stat
-
-from conary.lib import util
-from conary.build import macros, policy
-from conary.build.use import Use
-
-
-class Foo(policy.DestdirPolicy):
-    """
-        Fake policy
-    """
-    invariantinclusions = [ '.*' ]
-
-    def __init__(self, *args, **keywords):
-        policy.DestdirPolicy.__init__(self, *args, **keywords)
-
-    def doFile(self, path):
-        raise RuntimeError('This fake policy always breaks.')
diff --git a/test/archive/pubring.gpg b/test/archive/pubring.gpg
deleted file mode 100644
Binary file test/archive/pubring.gpg has changed
diff --git a/test/archive/recipeTemplates/default b/test/archive/recipeTemplates/default
deleted file mode 100644
--- a/test/archive/recipeTemplates/default
+++ /dev/null
@@ -1,10 +0,0 @@
-#
-# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
-#
-
-class %(upperName)s(PackageRecipe):
-    name = '%(name)s'
-    version = ''
-
-    def setup(r):
-        pass
diff --git a/test/archive/recipeTemplates/rpath b/test/archive/recipeTemplates/rpath
deleted file mode 100644
--- a/test/archive/recipeTemplates/rpath
+++ /dev/null
@@ -1,12 +0,0 @@
-#
-# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
-# This file is distributed under the terms of the MIT License.
-# A copy is available at http://www.rpath.com/permanent/mit-license.html
-#
-
-class %(upperName)s(PackageRecipe):
-    name = '%(name)s'
-    version = ''
-
-    def setup(r):
-        pass
diff --git a/test/archive/recipeTemplates/test b/test/archive/recipeTemplates/test
deleted file mode 100644
--- a/test/archive/recipeTemplates/test
+++ /dev/null
@@ -1,12 +0,0 @@
-# %(contactName)s
-# %(contact)s
-# %(year)s
-# %(name)s
-# %(upperName)s
-
-class %(upperName)s(PackageRecipe):
-    name = '%(name)s'
-    version = '0'
-
-    def setup(r):
-        r.Create('%%(prefix)s/%(name)s')
diff --git a/test/archive/secring.gpg b/test/archive/secring.gpg
deleted file mode 100644
Binary file test/archive/secring.gpg has changed
diff --git a/test/archive/tmpwatch-2.9.0-2.src.rpm b/test/archive/tmpwatch-2.9.0-2.src.rpm
deleted file mode 100644
Binary file test/archive/tmpwatch-2.9.0-2.src.rpm has changed
diff --git a/test/archive/use/X b/test/archive/use/X
deleted file mode 100644
diff --git a/test/archive/use/bootstrap b/test/archive/use/bootstrap
deleted file mode 100644
--- a/test/archive/use/bootstrap
+++ /dev/null
@@ -1,2 +0,0 @@
-sense disallowed
-buildRequired False
diff --git a/test/archive/use/builddocs b/test/archive/use/builddocs
deleted file mode 100644
--- a/test/archive/use/builddocs
+++ /dev/null
@@ -1,2 +0,0 @@
-sense preferred
-buildRequired False
diff --git a/test/archive/use/buildtests b/test/archive/use/buildtests
deleted file mode 100644
--- a/test/archive/use/buildtests
+++ /dev/null
@@ -1,3 +0,0 @@
-name buildtests
-sense Preferred
-buildRequired False
diff --git a/test/archive/use/cross b/test/archive/use/cross
deleted file mode 100644
--- a/test/archive/use/cross
+++ /dev/null
@@ -1,2 +0,0 @@
-sense disallowed
-buildRequired True
diff --git a/test/archive/use/desktop b/test/archive/use/desktop
deleted file mode 100644
diff --git a/test/archive/use/dietlibc b/test/archive/use/dietlibc
deleted file mode 100644
diff --git a/test/archive/use/emacs b/test/archive/use/emacs
deleted file mode 100644
diff --git a/test/archive/use/gcj b/test/archive/use/gcj
deleted file mode 100644
diff --git a/test/archive/use/gnome b/test/archive/use/gnome
deleted file mode 100644
diff --git a/test/archive/use/gtk b/test/archive/use/gtk
deleted file mode 100644
diff --git a/test/archive/use/ipv6 b/test/archive/use/ipv6
deleted file mode 100644
diff --git a/test/archive/use/kde b/test/archive/use/kde
deleted file mode 100644
diff --git a/test/archive/use/krb b/test/archive/use/krb
deleted file mode 100644
diff --git a/test/archive/use/ldap b/test/archive/use/ldap
deleted file mode 100644
diff --git a/test/archive/use/nptl b/test/archive/use/nptl
deleted file mode 100644
diff --git a/test/archive/use/pam b/test/archive/use/pam
deleted file mode 100644
--- a/test/archive/use/pam
+++ /dev/null
@@ -1,1 +0,0 @@
-sense required
diff --git a/test/archive/use/pcre b/test/archive/use/pcre
deleted file mode 100644
diff --git a/test/archive/use/perl b/test/archive/use/perl
deleted file mode 100644
diff --git a/test/archive/use/python b/test/archive/use/python
deleted file mode 100644
diff --git a/test/archive/use/qt b/test/archive/use/qt
deleted file mode 100644
diff --git a/test/archive/use/readline b/test/archive/use/readline
deleted file mode 100644
diff --git a/test/archive/use/sasl b/test/archive/use/sasl
deleted file mode 100644
diff --git a/test/archive/use/ssl b/test/archive/use/ssl
deleted file mode 100644
--- a/test/archive/use/ssl
+++ /dev/null
@@ -1,3 +0,0 @@
-name ssl
-sense required
-buildRequired True
diff --git a/test/archive/use/tcl b/test/archive/use/tcl
deleted file mode 100644
diff --git a/test/archive/use/tk b/test/archive/use/tk
deleted file mode 100644
diff --git a/test/bootstrap.py b/test/bootstrap.py
deleted file mode 100644
--- a/test/bootstrap.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import os
-import sys
-
-testUtilDir = os.environ.get('TESTUTILS_PATH', '../../testutils')
-if os.path.exists(testUtilDir):
-    sys.path.insert(0, testUtilDir)
-
-try:
-    import testrunner
-except ImportError:
-    raise RuntimeError('Could not find testrunner - set TESTUTILS_PATH')
diff --git a/test/functionaltest/Makefile b/test/functionaltest/Makefile
deleted file mode 100644
--- a/test/functionaltest/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-MYDIR=$(shell basename `pwd`)
-
-test:
-	cd .. && ./testsuite.py $(MYDIR)
-
-debug:
-	cd .. && ./testsuite.py $(MYDIR) --debug
-
-coverage:
-	rm -rf annotate;
-	cd .. && ./testsuite.py $(MYDIR) --coverage
-	mv ../annotate .
-
diff --git a/test/functionaltest/__init__.py b/test/functionaltest/__init__.py
deleted file mode 100644
diff --git a/test/functionaltest/testsetup.py b/test/functionaltest/testsetup.py
deleted file mode 120000
--- a/test/functionaltest/testsetup.py
+++ /dev/null
@@ -1,1 +0,0 @@
-../testsetup.py
\ No newline at end of file
diff --git a/test/slehelp.py b/test/slehelp.py
deleted file mode 100644
--- a/test/slehelp.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsuite
-testsuite.setup()
-
-import os
-from rmake_test import rmakehelp
-
-from updatebot import config
-
-class Helper(rmakehelp.RmakeHelper):
-    def setUp(self):
-        rmakehelp.RmakeHelper.setUp(self)
-        self.updateBotCfg = config.UpdateBotConfig()
-        self.updateBotCfg.configPath = self.cfg.root
-        self.cfg.user = ('test', 'test')
-        self.writeFile(self.cfg.root + '/conaryrc', '')
-        self.writeFile(self.cfg.root + '/rmakerc', '')
-        os.chdir(self.workDir)
-
-    def initializeFlavor(self):
-        pass
-
-    def prepopulateKeyCache(self, keyCache):
-        pass
diff --git a/test/smoketest/Makefile b/test/smoketest/Makefile
deleted file mode 100644
--- a/test/smoketest/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-MYDIR=$(shell basename `pwd`)
-
-test:
-	cd .. && ./testsuite.py $(MYDIR)
-
-debug:
-	cd .. && ./testsuite.py $(MYDIR) --debug
-
-coverage:
-	rm -rf annotate;
-	cd .. && ./testsuite.py $(MYDIR) --coverage
-	mv ../annotate .
-
diff --git a/test/smoketest/__init__.py b/test/smoketest/__init__.py
deleted file mode 100644
diff --git a/test/smoketest/testsetup.py b/test/smoketest/testsetup.py
deleted file mode 120000
--- a/test/smoketest/testsetup.py
+++ /dev/null
@@ -1,1 +0,0 @@
-../testsetup.py
\ No newline at end of file
diff --git a/test/testsetup.py b/test/testsetup.py
deleted file mode 100644
--- a/test/testsetup.py
+++ /dev/null
@@ -1,22 +0,0 @@
-import os
-import sys
-
-
-curDir = os.path.dirname(__file__)
-for dirlevel in range(10):
-    testsuitePath = os.path.realpath(curDir + '/..' * dirlevel)
-    if os.path.exists(testsuitePath + '/testsuite.py'):
-        break
-else:
-    raise RuntimeError('Could not find testsuite.py!')
-if not testsuitePath in sys.path:
-    sys.path.insert(0, testsuitePath)
-
-
-import testsuite
-testsuite.setup()
-
-
-def main():
-    if sys._getframe(1).f_globals['__name__'] == '__main__':
-        testsuite.main()
diff --git a/test/testsuite.py b/test/testsuite.py
deleted file mode 100755
--- a/test/testsuite.py
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/python
-# -*- mode: python -*-
-#
-# Copyright (c) 2006-2007 rPath, Inc.  All Rights Reserved.
-#
-# W0603: using the global statement
-#pylint: disable-msg=W0603
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import sys
-import os
-import pwd
-import bootstrap
-
-archivePath = None
-testPath = None
-pluginPath = None
-
-conaryDir = None
-_setupPath = None
-_individual = False
-
-def isIndividual():
-    return _individual
-
-def setup():
-    global _setupPath
-    if _setupPath:
-        return _setupPath
-    global testPath
-    global archivePath
-    global conaryDir
-    global pluginPath
-
-    def setPathFromEnv(variable, directory):
-        parDir = os.path.dirname(os.path.realpath(__file__))
-        if not directory.startswith('/'):
-            parDir = os.path.dirname(parDir) + '/' + directory
-        else:
-            parDir = directory
-        thisPath = os.getenv(variable, parDir)
-        os.environ[variable] = thisPath
-        if thisPath in sys.path:
-            sys.path.remove(thisPath)
-        sys.path.insert(0, thisPath)
-        return thisPath
-
-    testutilsPath = setPathFromEnv('TESTUTILS_PATH', '../testutils')
-    conaryDir = setPathFromEnv('CONARY_PATH', '../conary')
-    conaryTestPath = setPathFromEnv('CONARY_TEST_PATH', '../conary-test-2.0')
-    setPathFromEnv('CONARY_POLICY_PATH', '/usr/lib/conary/policy')
-    mirrorballPath = setPathFromEnv('SLEESTACK_PATH', '')
-
-    rmakePath = setPathFromEnv('RMAKE_PATH', '../rmake')
-    rmakeTestPath = setPathFromEnv('RMAKE_TEST_PATH', '../rmake-private')
-    xmllibPath = setPathFromEnv('XMLLIB_PATH', '../rpath-xmllib')
-    pluginPath = os.path.realpath(rmakeTestPath + '/rmake_plugins')
-
-    # Insert the following paths into the python path and sys path in
-    # listed order.
-    paths = (mirrorballPath, rmakePath, conaryDir, conaryTestPath,
-             rmakeTestPath, xmllibPath, testutilsPath)
-    pythonPath = os.environ.get('PYTHONPATH', "")
-    for p in reversed(paths):
-        if p in sys.path:
-            sys.path.remove(p)
-        sys.path.insert(0, p)
-    for p in paths:
-        if p not in pythonPath:
-            pythonPath = os.pathsep.join((pythonPath, p))
-    os.environ['PYTHONPATH'] = pythonPath
-
-    if isIndividual():
-        serverDir = '/tmp/conary-server'
-        #pylint: disable-msg=E1103
-        if os.path.exists(serverDir) and not os.path.access(serverDir, os.W_OK):
-            serverDir = serverDir + '-' + pwd.getpwuid(os.getuid())[0]
-        os.environ['SERVER_FILE_PATH'] = serverDir
-
-    from testrunner import resources, testhelp
-    testPath = testhelp.getTestPath()
-    archivePath = testPath + '/archive'
-    resources.archivePath = archivePath
-
-    resources.pluginPath = pluginPath
-    resources.rmakePath = rmakePath
-
-    # W0621 - redefining name util, W0404 - reimporting util
-    #pylint: disable-msg=W0621,W0404
-    from conary.lib import util
-    sys.excepthook = util.genExcepthook(True)
-
-    # import tools normally expected in testsuite.
-    sys.modules[__name__].context = testhelp.context
-    sys.modules[__name__].TestCase = testhelp.TestCase
-    sys.modules[__name__].findPorts = testhelp.findPorts
-    sys.modules[__name__].SkipTestException = testhelp.SkipTestException
-
-    _setupPath = testPath
-    return testPath
-
-
-_individual = False
-
-
-def getCoverageDirs(*_):
-    codePath = os.environ['SLEESTACK_PATH'].rstrip('/')
-    coverageDirs = [ 'aptmd', 'repomd', 'rpmutils', 'updatebot' ]
-    return [ os.path.join(codePath, x) for x in coverageDirs ]
-
-def getCoverageExclusions(*_):
-    return ['test/.*']
-
-def sortTests(tests):
-    order = {'rbuild_test.smoketest': 0, 
-             'rbuild_test.unit_test': 1,
-             'rbuild_test.functionaltest': 2}
-    maxNum = len(order)
-    tests = [ (test, test.index('test')) for test in tests]
-    tests = sorted((order.get(test[:index+4], maxNum), test)
-                   for (test, index) in tests)
-    tests = [ x[1] for x in tests ]
-    return tests
-
-def main(argv=None, individual=True):
-    from testrunner import testhelp
-    handlerClass = testhelp.getHandlerClass(testhelp.ConaryTestSuite,
-                                            getCoverageDirs,
-                                            getCoverageExclusions,
-                                            sortTests)
-    global _individual
-    _individual = individual
-    if argv is None:
-        argv = list(sys.argv)
-    topdir = testhelp.getTestPath()
-    cwd = os.getcwd()
-    if cwd != topdir and cwd not in sys.path:
-        sys.path.insert(0, cwd)
-
-    handler = handlerClass(individual=individual, topdir=topdir,
-                           testPath=testPath, conaryDir=conaryDir)
-    results = handler.main(argv)
-    if results is None:
-        sys.exit(0)
-    sys.exit(not results.wasSuccessful())
-
-if __name__ == '__main__':
-    setup()
-    # unused import coveragehook
-    # redefining util from outer scope
-    #pylint: disable-msg=W0611,W0621
-    from conary.lib import util
-    from conary.lib import coveragehook
-    sys.excepthook = util.genExcepthook(True)
-    main(sys.argv, individual=False)
diff --git a/test/unit_test/Makefile b/test/unit_test/Makefile
deleted file mode 100644
--- a/test/unit_test/Makefile
+++ /dev/null
@@ -1,13 +0,0 @@
-MYDIR=$(shell basename `pwd`)
-
-test:
-	cd .. && ./testsuite.py $(MYDIR)
-
-debug:
-	cd .. && ./testsuite.py $(MYDIR) --debug
-
-coverage:
-	rm -rf annotate;
-	cd .. && ./testsuite.py $(MYDIR) --coverage
-	mv ../annotate .
-
diff --git a/test/unit_test/__init__.py b/test/unit_test/__init__.py
deleted file mode 100644
diff --git a/test/unit_test/aptmdtest/__init__.py b/test/unit_test/aptmdtest/__init__.py
deleted file mode 100644
diff --git a/test/unit_test/aptmdtest/packagestest.py b/test/unit_test/aptmdtest/packagestest.py
deleted file mode 100644
--- a/test/unit_test/aptmdtest/packagestest.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testrunner import resources
-
-import os
-import StringIO
-
-from aptmd import packages
-
-import slehelp
-
-class PackagesTest(slehelp.Helper):
-    def testMetadataParse1(self):
-        ctrlFile = os.path.join(resources.archivePath, 'control-files',
-            'ascii.control')
-
-        parser = packages.PackagesParser()
-        parser.parse(ctrlFile)
-        self.failUnlessEqual(parser._curObj.name, 'ascii')
-        self.failUnlessEqual(parser._curObj.version, '3.8')
-        self.failUnlessEqual(parser._curObj.release, '2')
-        self.failUnlessEqual(parser._curObj.arch, 'i386')
-        self.failUnlessEqual(parser._curObj.get('installed-size'), '72')
-        self.failUnlessEqual(parser._curObj.summary, 'interactive ASCII name and synonym chart')
-        self.failUnlessEqual(parser._curObj.description, """\
- The ascii utility provides easy conversion between various byte representations
- and the American Standard Code for Information Interchange (ASCII) character
- table.  It knows about a wide variety of hex, binary, octal, Teletype mnemonic,
- ISO/ECMA code point, slang names, XML entity names, and other representations.
- Given any one on the command line, it will try to display all others.  Called
- with no arguments it displays a handy small ASCII chart.
- .
- Homepage: http://www.catb.org/~esr/ascii/
-""")
-
-testsetup.main()
diff --git a/test/unit_test/aptmdtest/testsetup.py b/test/unit_test/aptmdtest/testsetup.py
deleted file mode 120000
--- a/test/unit_test/aptmdtest/testsetup.py
+++ /dev/null
@@ -1,1 +0,0 @@
-../../testsetup.py
\ No newline at end of file
diff --git a/test/unit_test/rpmutiltest/__init__.py b/test/unit_test/rpmutiltest/__init__.py
deleted file mode 100644
diff --git a/test/unit_test/rpmutiltest/testsetup.py b/test/unit_test/rpmutiltest/testsetup.py
deleted file mode 120000
--- a/test/unit_test/rpmutiltest/testsetup.py
+++ /dev/null
@@ -1,1 +0,0 @@
-../../testsetup.py
\ No newline at end of file
diff --git a/test/unit_test/rpmutiltest/vercmptest.py b/test/unit_test/rpmutiltest/vercmptest.py
deleted file mode 100644
--- a/test/unit_test/rpmutiltest/vercmptest.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testutils import mock
-
-import slehelp
-
-from rpmutils import rpmvercmp
-
-class VerCmpTest(slehelp.Helper):
-    def testVerCmp(self):
-        def _test(a, b, expected):
-            result = rpmvercmp(a, b)
-            self.failUnlessEqual(result, expected)
-
-        _test('1', '1', 0)
-        _test('1', '2', -1)
-        _test('2', '1', 1)
-        _test('a', 'a', 0)
-        _test('a', 'b', -1)
-        _test('b', 'a', 1)
-        _test('1.2', '1.3', -1)
-        _test('1.3', '1.1', 1)
-        _test('1.a', '1.a', 0)
-        _test('1.a', '1.b', -1)
-        _test('1.b', '1.a', 1)
-        _test('1.2+', '1.2', 0)
-        _test('1.0010', '1.0', 1)
-        _test('1.05', '1.5', 0)
-        _test('1.0', '1', 1)
-        _test('2.50', '2.5', 1)
-        _test('fc4', 'fc.4', 0)
-        _test('FC5', 'fc5', -1)
-        _test('2a', '2.0', -1)
-        _test('1.0', '1.fc4', 1)
-        _test('3.0.0_fc', '3.0.0.fc', 0)
-        _test('1++', '1_', 0)
-        _test('+', '_', -1)
-        _test('_', '+', 1)
-
-testsetup.main()
diff --git a/test/unit_test/testsetup.py b/test/unit_test/testsetup.py
deleted file mode 120000
--- a/test/unit_test/testsetup.py
+++ /dev/null
@@ -1,1 +0,0 @@
-../testsetup.py
\ No newline at end of file
diff --git a/test/unit_test/updatebottest/__init__.py b/test/unit_test/updatebottest/__init__.py
deleted file mode 100644
diff --git a/test/unit_test/updatebottest/advisetest.py b/test/unit_test/updatebottest/advisetest.py
deleted file mode 100644
--- a/test/unit_test/updatebottest/advisetest.py
+++ /dev/null
@@ -1,106 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testutils import mock
-
-import slehelp
-
-from updatebot import advise
-from updatebot import errors
-
-class AdviseTest(slehelp.Helper):
-    def testCheck(self):
-        mockRpmSource = mock.MockObject()
-        mockPatchSource = mock.MockObject()
-        mockHasException = mock.MockObject()
-        mockIsSecurity = mock.MockObject()
-        mockMkAdvisory = mock.MockObject()
-
-        mockSrcPkg1 = mock.MockObject()
-        mockSrcPkg2 = mock.MockObject()
-        mockSrcPkg3 = mock.MockObject()
-
-        mockBinPkg1 = mock.MockObject()
-        mockBinPkg2 = mock.MockObject()
-        mockBinPkg3 = mock.MockObject()
-
-        mockPatch1 = mock.MockObject()
-        mockPatch2 = mock.MockObject()
-
-        srcPkgMap = {mockSrcPkg1: [mockSrcPkg1, mockBinPkg1],
-                     mockSrcPkg2: [mockSrcPkg2, mockBinPkg2],
-                     mockSrcPkg3: [mockSrcPkg3, mockBinPkg3]}
-        mockRpmSource._mock.set(srcPkgMap=srcPkgMap)
-
-        pkgMap = {mockBinPkg1: set([mockPatch1, ]), }
-        mockPatchSource._mock.set(pkgMap=pkgMap)
-
-        input = [(None, mockSrcPkg1),
-                 (None, mockSrcPkg2),
-                 (None, mockSrcPkg3)]
-
-        advisor = advise.Advisor(self.updateBotCfg, mockRpmSource,
-                                 mockPatchSource)
-        self.mock(advisor, '_hasException', mockHasException)
-        self.mock(advisor, '_isSecurity', mockIsSecurity)
-        self.mock(advisor, '_mkAdvisory', mockMkAdvisory)
-
-        mockHasException._mock.setReturn(True, mockBinPkg2)
-        mockHasException._mock.setReturn(False, mockBinPkg3)
-        mockIsSecurity._mock.setReturn(False, mockBinPkg3)
-
-        expected = {(None, mockSrcPkg1): set([mockPatch1, ]),}
-
-        # test normal case
-        result = advisor.check(input)
-        self.failUnlessEqual(result, None)
-        self.failUnlessEqual(advisor._cache, expected)
-        mockHasException._mock.assertCalled(mockBinPkg2)
-        mockHasException._mock.assertCalled(mockBinPkg3)
-        mockIsSecurity._mock.assertCalled(mockBinPkg3)
-
-        # test exception case
-        mockHasException._mock.setReturn(False, mockBinPkg2)
-        mockIsSecurity._mock.setReturn(True, mockBinPkg2)
-        self.failUnlessRaises(errors.NoAdvisoryFoundError, advisor.check, input)
-        mockHasException._mock.assertCalled(mockBinPkg2)
-        mockIsSecurity._mock.assertCalled(mockBinPkg2)
-
-    def testHasException(self):
-        binPkg = mock.MockObject(stableReturnValues=True)
-        mockCfg = mock.MockObject(stableReturnValues=True)
-        mockCfg._mock.set(advisoryException=[['foo', None]])
-
-        advisor = advise.Advisor(mockCfg, None, None)
-
-        binPkg._mock.set(location='foo/bar')
-        result = advisor._hasException(binPkg)
-        self.failUnlessEqual(result, True)
-
-        binPkg._mock.set(location='bar/foo')
-        result = advisor._hasException(binPkg)
-        self.failUnlessEqual(result, False)
-
-    def testIsSecurity(self):
-        advisor = advise.Advisor(self.updateBotCfg, None, None)
-        binPkg = mock.MockObject(stableReturnValues=True)
-        binPkg._mock.set(location='foo-Updates/bar')
-        result = advisor._isSecurity(binPkg)
-        self.failUnlessEqual(result, True)
-        binPkg._mock.set(location='foo-Online/bar')
-        result = advisor._isSecurity(binPkg)
-        self.failUnlessEqual(result, False)
-
-testsetup.main()
diff --git a/test/unit_test/updatebottest/buildtest.py b/test/unit_test/updatebottest/buildtest.py
deleted file mode 100644
--- a/test/unit_test/updatebottest/buildtest.py
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testutils import mock
-
-import slehelp
-
-from rmake.cmdline import monitor, commit
-
-from updatebot import build
-from updatebot import errors
-
-class BuilderTest(slehelp.Helper):
-    def testBuild(self):
-        jobId = 1
-
-        mockVersion1 = mock.MockObject()
-        mockVersion2 = mock.MockObject()
-        mockFlavor1 = mock.MockObject()
-        mockFlavor2 = mock.MockObject()
-
-        trvSpecs = [('foo', mockVersion1, mockFlavor1),
-                    ('group-foo', mockVersion2, mockFlavor2),
-                   ]
-
-        contexts = ['a', 'b']
-
-        self.mock(self.updateBotCfg, 'archContexts', contexts)
-
-        troves = [('foo', mockVersion1, mockFlavor1, 'a'),
-                  ('foo', mockVersion1, mockFlavor1, 'b'),
-                  ('group-foo', mockVersion2, mockFlavor2),
-                 ]
-
-        trvMap = {('foo', mockVersion1, mockFlavor1, 'a'):
-                    [('foo', mockVersion1, mockFlavor1), ],
-                  ('foo', mockVersion1, mockFlavor1, 'b'):
-                    [('foo:lib', mockVersion1, mockFlavor1),
-                     ('foo', mockVersion1, mockFlavor1), ],
-                  ('group-foo', mockVersion2, mockFlavor2, None):
-                    [('group-foo-devel', mockVersion2, mockFlavor2), ],
-                 }
-
-        ret = {('foo', mockVersion1, None):
-                set([('foo', mockVersion1, mockFlavor1),
-                     ('foo:lib', mockVersion1, mockFlavor1),
-                    ]),
-               ('group-foo', mockVersion2, None):
-                set([('group-foo-devel', mockVersion2, mockFlavor2),
-                    ]),
-              }
-
-        mockStartJob = mock.MockObject()
-        mockMonitorJob = mock.MockObject()
-        mockSanityCheckJob = mock.MockObject()
-        mockCommitJob = mock.MockObject()
-
-        mockStartJob._mock.setReturn(jobId, troves)
-        mockCommitJob._mock.setReturn(trvMap, jobId)
-
-        builder = build.Builder(self.updateBotCfg)
-        self.mock(builder, '_startJob', mockStartJob)
-        self.mock(builder, '_monitorJob', mockMonitorJob)
-        self.mock(builder, '_sanityCheckJob', mockSanityCheckJob)
-        self.mock(builder, '_commitJob', mockCommitJob)
-
-        result = builder.build(trvSpecs)
-        self.failUnlessEqual(result, ret)
-        mockStartJob._mock.assertCalled(troves)
-        mockMonitorJob._mock.assertCalled(jobId)
-        mockSanityCheckJob._mock.assertCalled(jobId)
-        mockCommitJob._mock.assertCalled(jobId)
-
-    def testStartJob(self):
-        trvSpecs = (('foo', '', ''), )
-
-        mockJob = mock.MockObject()
-        mockHelper = mock.MockObject(stableReturnValues=True)
-        mockHelper.createBuildJob._mock.setReturn(mockJob, trvSpecs)
-        mockHelper.buildJob._mock.setReturn(1, mockJob)
-
-        builder = build.Builder(self.updateBotCfg)
-        builder._helper = mockHelper
-        jobId = builder._startJob(trvSpecs)
-
-        mockHelper.createBuildJob._mock.assertCalled(trvSpecs)
-        mockHelper.buildJob._mock.assertCalled(mockJob)
-        self.failUnlessEqual(jobId, 1)
-
-    def testMonitorJob(self):
-        mockMonitor = mock.MockObject()
-
-        builder = build.Builder(self.updateBotCfg)
-        self.mock(build.monitor, 'monitorJob', mockMonitor)
-
-        builder._monitorJob(1)
-
-        mockMonitor._mock.assertCalled(builder._helper.client, 1,
-            exitOnFinish=True, displayClass=build._StatusOnlyDisplay)
-
-    def testSanityCheckJob(self):
-        jobId = 1
-
-        mockJob = mock.MockObject(stableReturnValues=True)
-        mockHelper = mock.MockObject(stableReturnValues=True)
-        mockHelper.getJob._mock.setReturn(mockJob, jobId)
-
-        builder = build.Builder(self.updateBotCfg)
-        builder._helper = mockHelper
-
-        mockJob.isFailed._mock.setReturn(True)
-        self.failUnlessRaises(errors.JobFailedError, builder._sanityCheckJob, jobId)
-        mockHelper.getJob._mock.assertCalled(jobId)
-        mockJob.isFailed._mock.assertCalled()
-        mockJob.isFinished._mock.assertNotCalled()
-        mockJob.iterBuiltTroves._mock.assertNotCalled()
-        mockJob.isFailed._mock.setReturn(False)
-
-        mockJob.isFinished._mock.setReturn(False)
-        self.failUnlessRaises(errors.JobFailedError, builder._sanityCheckJob, jobId)
-        mockHelper.getJob._mock.assertCalled(jobId)
-        mockJob.isFailed._mock.assertCalled()
-        mockJob.isFinished._mock.assertCalled()
-        mockJob.iterBuiltTroves._mock.assertNotCalled()
-        mockJob.isFinished._mock.setReturn(True)
-
-        mockJob.iterBuiltTroves._mock.setReturn([])
-        self.failUnlessRaises(errors.JobFailedError, builder._sanityCheckJob, jobId)
-        mockHelper.getJob._mock.assertCalled(jobId)
-        mockJob.isFailed._mock.assertCalled()
-        mockJob.isFinished._mock.assertCalled()
-        mockJob.iterBuiltTroves._mock.assertCalled()
-        mockJob.iterBuiltTroves._mock.setReturn([1, 2, 3])
-
-    def testCommitJob(self):
-        jobId = 1
-
-        sampleData = {'foo': {
-                        ('foo', '1.0', 'is:x86'): [
-                            ('foo:source', '1.0-1', ''),
-                            ('foo:config', '1.0-1-1', 'is:x86'),
-                            ('foo:runtime', '1.0-1-1', 'is:x86'), ]
-                        },
-                      'bar': {
-                        ('bar', '1.0', 'is:x86_64'): [
-                            ('bar:source', '1.0-1', ''),
-                            ('bar:runtime', '1.0-1-1', 'is:x86_64'), ]
-                        },
-                     }
-
-        expected = {}
-        for value in sampleData.itervalues():
-            for buildTrove, committedTrove in value.iteritems():
-                expected[buildTrove] = committedTrove
-
-        mockJob = mock.MockObject(stableReturnValues=True)
-        mockHelper = mock.MockObject(stableReturnValues=True)
-        mockHelper.getJob._mock.setReturn(mockJob, jobId)
-        mockCommitJobs = mock.MockObject(stableReturnValues=True)
-        mockBuildCfg = mock.MockObject(stableReturnValues=True)
-        mockBuildCfg._mock.set(reposName='foo.example.com')
-
-        self.mock(commit, 'commitJobs', mockCommitJobs)
-
-        builder = build.Builder(self.updateBotCfg)
-        builder._helper = mockHelper
-        builder._rmakeCfg = mockBuildCfg
-
-        mockCommitJobs._mock.setDefaultReturn((True, sampleData))
-        trvMap = builder._commitJob(jobId)
-        self.failUnlessEqual(trvMap, expected)
-        mockHelper.getJob._mock.assertCalled(jobId)
-        mockHelper.client.startCommit._mock.assertCalled([jobId, ])
-        mockHelper.client.commitSucceeded._mock.assertCalled(sampleData)
-
-        mockCommitJobs._mock.setDefaultReturn((False, sampleData))
-        self.failUnlessRaises(errors.CommitFailedError, builder._commitJob, jobId)
-        mockHelper.getJob._mock.assertCalled(jobId)
-        mockHelper.client.startCommit._mock.assertCalled([jobId, ])
-        mockHelper.client.commitFailed._mock.assertCalled([jobId, ], sampleData)
-        mockHelper.client.commitSucceeded._mock.assertNotCalled()
-
-
-testsetup.main()
diff --git a/test/unit_test/updatebottest/conaryhelpertest.py b/test/unit_test/updatebottest/conaryhelpertest.py
deleted file mode 100644
--- a/test/unit_test/updatebottest/conaryhelpertest.py
+++ /dev/null
@@ -1,406 +0,0 @@
-#!/usr/bin/python
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testutils import mock
-
-import os
-import slehelp
-
-from conary import trove
-from conary import versions
-from conary.deps import deps
-
-from updatebot import util
-from updatebot import errors
-from updatebot import conaryhelper
-
-class ConaryHelperTest(slehelp.Helper):
-    label = 'foo.rpath.com at rpath:foo-devel'
-    findTroveReturn = (
-        ('group-foo',
-         versions.VersionFromString('/' + label + '/1.0-1-1'),
-         deps.parseFlavor('is:x86'),
-        ),
-        ('group-foo',
-         versions.VersionFromString('/' + label + '/1.0-1-1'),
-         deps.parseFlavor('is:x86 x86_64'),
-        ),
-        ('group-foo',
-         versions.VersionFromString('/' + label + '/1.0-2-1'),
-         deps.parseFlavor('is:x86'),
-        ),
-        ('group-foo',
-         versions.VersionFromString('/' + label + '/1.0-2-1'),
-         deps.parseFlavor('is:x86 x86_64'),
-        ),
-    )
-
-    findTroveReturn[0][1].setTimeStamps((1.0, ))
-    findTroveReturn[1][1].setTimeStamps((1.0, ))
-    findTroveReturn[2][1].setTimeStamps((2.0, ))
-    findTroveReturn[3][1].setTimeStamps((2.0, ))
-
-    def testGetConaryConfig(self):
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        ccfg = helper.getConaryConfig()
-        self.failUnless(ccfg is helper._ccfg)
-
-    def testGetSourceTroves(self):
-        group = ('group-foo', self.label, None)
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        helper._ccfg.buildLabel = self.label
-
-        mockRepos = mock.MockObject(stableReturnValues=True)
-        mockRepos.findTrove._mock.setReturn(self.findTroveReturn,
-            helper._ccfg.buildLabel, group)
-        self.mock(helper, '_repos', mockRepos)
-
-        mockFindLatest = mock.MockObject(stableReturnValues=True)
-        mockFindLatest._mock.setReturn(self.findTroveReturn[2:],
-                                       self.findTroveReturn)
-        self.mock(helper, '_findLatest', mockFindLatest)
-
-        getSourceTrovesReturn1 = set([
-            ('foo:source',
-             versions.VersionFromString('/' + self.label + '/1.0-1'),
-             None
-            ),
-        ])
-        getSourceTrovesReturn2 = set([
-            ('bar:source',
-             versions.VersionFromString('/' + self.label + '/1.0-1'),
-             None
-            ),
-        ])
-        mockGetSourceTroves = mock.MockObject(stableReturnValues=True)
-        mockGetSourceTroves._mock.setReturn(getSourceTrovesReturn1,
-                                            self.findTroveReturn[2])
-        mockGetSourceTroves._mock.setReturn(getSourceTrovesReturn2,
-                                            self.findTroveReturn[3])
-        self.mock(helper, '_getSourceTroves', mockGetSourceTroves)
-
-        # normal case
-        srcTrvs = helper.getSourceTroves(group)
-        self.failUnlessEqual(type(set()), type(srcTrvs))
-        self.failUnless(srcTrvs.issuperset(getSourceTrovesReturn1))
-        self.failUnless(srcTrvs.issuperset(getSourceTrovesReturn2))
-        mockRepos.findTrove._mock.assertCalled(helper._ccfg.buildLabel, group)
-        mockFindLatest._mock.assertCalled(self.findTroveReturn)
-        mockGetSourceTroves._mock.assertCalled(self.findTroveReturn[2])
-        mockGetSourceTroves._mock.assertCalled(self.findTroveReturn[3])
-
-        # exception case
-        mockFindLatest._mock.setReturn(self.findTroveReturn[1:],
-                                       self.findTroveReturn)
-        self.failUnlessRaises(errors.TooManyFlavorsFoundError,
-                              helper.getSourceTroves, group)
-        mockRepos.findTrove._mock.assertCalled(helper._ccfg.buildLabel, group)
-        mockFindLatest._mock.assertCalled(self.findTroveReturn)
-        mockGetSourceTroves._mock.assertNotCalled()
-
-    def testFindLatest(self):
-        test1 = list(self.findTroveReturn)
-        testret1 = self.findTroveReturn[2:]
-
-        test2 = list(self.findTroveReturn)
-        test2.reverse()
-        testret2 = test2[:2]
-
-        test3 = [self.findTroveReturn[2], self.findTroveReturn[0],
-                 self.findTroveReturn[3], self.findTroveReturn[1]]
-        testret3 = [test1[2], test1[3]]
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        for test, result in ((test1, testret1),
-                             (test2, testret2),
-                             (test3, testret3)):
-            ret = helper._findLatest(test)
-            for res in result:
-                self.failUnless(res in ret)
-
-    def testInternalGetSourceTroves(self):
-        trvSpec = self.findTroveReturn[3]
-        cl = [ (trvSpec[0], (None, None), (trvSpec[1], trvSpec[2]), True) ]
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-
-        mockChangeSet = mock.MockObject()
-        mockClient = mock.MockObject(stableReturnValues=True)
-        mockClient.createChangeSet._mock.setReturn(mockChangeSet, cl,
-            withFiles=False, withFileContents=False, recurse=False)
-        self.mock(helper, '_client', mockClient)
-
-        mockSourceVersion = mock.MockObject()
-        mockVersion = mock.MockObject(stableReturnValues=True)
-        mockVersion.getSourceVersion._mock.setReturn(mockSourceVersion)
-
-        nvfs = [('foo:runtime', mockVersion, None),
-                ('bar:runtime', mockVersion, None),
-                ('baz:devel', mockVersion, None)]
-
-        mockTrove = mock.MockObject(stableReturnValues=True)
-        mockTrove.iterTroveList._mock.setReturn(nvfs, weakRefs=True,
-                                                      strongRefs=True)
-
-        mockGetTrove = mock.MockObject(stableReturnValues=True)
-        mockGetTrove._mock.setReturn(mockTrove, mockChangeSet, *trvSpec)
-        self.mock(helper, '_getTrove', mockGetTrove)
-
-        mockSource1 = mock.MockObject(stableReturnValues=True)
-        mockSource1._mock.setReturn('foo:source')
-        mockSource2 = mock.MockObject(stableReturnValues=True)
-        mockSource2._mock.setReturn('bar:source')
-        mockSource3 = mock.MockObject(stableReturnValues=True)
-        mockSource3._mock.setReturn('baz:source')
-
-        sources = [mockSource1, mockSource2, mockSource3]
-
-        mockRepos = mock.MockObject(stableReturnValues=True)
-        mockRepos.getTroveInfo._mock.setReturn(sources,
-            trove._TROVEINFO_TAG_SOURCENAME, nvfs)
-        self.mock(helper, '_repos', mockRepos)
-
-        expectedResult = set()
-        expectedResult.add(('foo:source', mockSourceVersion, None))
-        expectedResult.add(('bar:source', mockSourceVersion, None))
-        expectedResult.add(('baz:source', mockSourceVersion, None))
-
-        result = helper._getSourceTroves(trvSpec)
-        self.failUnlessEqual(result, expectedResult)
-        mockClient.createChangeSet._mock.assertCalled(cl, withFiles=False,
-            withFileContents=False, recurse=False)
-        mockGetTrove._mock.assertCalled(mockChangeSet, *trvSpec)
-        mockTrove.iterTroveList._mock.assertCalled(weakRefs=True,
-                                                   strongRefs=True)
-        mockRepos.getTroveInfo._mock.assertCalled(
-            trove._TROVEINFO_TAG_SOURCENAME, nvfs)
-        mockVersion.getSourceVersion._mock.assertCalled()
-        mockVersion.getSourceVersion._mock.assertCalled()
-        mockVersion.getSourceVersion._mock.assertCalled()
-        mockSource1._mock.assertCalled()
-        mockSource2._mock.assertCalled()
-        mockSource3._mock.assertCalled()
-
-    def testGetTrove(self):
-        trvSpec = self.findTroveReturn[3]
-        mockTroveCs = mock.MockObject()
-        mockCs = mock.MockObject(stableReturnValues=True)
-        mockCs.getNewTroveVersion._mock.setReturn(mockTroveCs, *trvSpec)
-
-        mockTrove = mock.MockObject()
-        mockTroveTrove = mock.MockObject(stableReturnValues=True)
-        mockTroveTrove._mock.setReturn(mockTrove, mockTroveCs,
-                                       skipIntegrityChecks=True)
-        self.mock(trove, 'Trove', mockTroveTrove)
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        result = helper._getTrove(mockCs, *trvSpec)
-
-        self.failUnless(result is mockTrove)
-        mockCs.getNewTroveVersion._mock.assertCalled(*trvSpec)
-        mockTroveTrove._mock.assertCalled(mockTroveCs,
-                                          skipIntegrityChecks=True)
-
-    def testGetManifest(self):
-        trvName = 'foo'
-        mockCheckout = mock.MockObject(stableReturnValues=True)
-        mockCheckout._mock.setReturn('foo', trvName)
-
-        expected = ['bar', 'baz']
-
-        os.mkdir('foo')
-        fh = open('foo/manifest', 'w')
-        fh.write('\n'.join(expected))
-        fh.close()
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        self.mock(helper, '_checkout', mockCheckout)
-
-        result = helper.getManifest(trvName)
-        self.failUnlessEqual(expected, result)
-        self.failUnless(not os.path.exists('foo'))
-        mockCheckout._mock.assertCalled(trvName)
-
-        self.failUnlessRaises(errors.NoManifestFoundError,
-                              helper.getManifest, trvName)
-        mockCheckout._mock.assertCalled(trvName)
-
-    def testSetManifest(self):
-        trvName = 'foo'
-        manifestLst = ['bar', 'baz']
-        commitMessage = 'foobar'
-
-        mockTrove = mock.MockObject(stableReturnValues=True)
-        mockRmTree = mock.MockObject(stableReturnValues=True)
-        mockCommit = mock.MockObject(stableReturnValues=True)
-        mockNewPkg = mock.MockObject(stableReturnValues=True)
-        mockCheckout = mock.MockObject(stableReturnValues=True)
-        mockGetVersionsByName = mock.MockObject(stableReturnValues=False)
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        self.mock(helper, '_commit', mockCommit)
-        self.mock(helper, '_newpkg', mockNewPkg)
-        self.mock(helper, '_checkout', mockCheckout)
-        self.mock(helper, '_getVersionsByName', mockGetVersionsByName)
-        self.mock(util, 'rmtree', mockRmTree)
-
-        # new pkg case
-        os.mkdir('foo')
-        mockGetVersionsByName._mock.setReturn([], '%s:source' % trvName)
-        mockNewPkg._mock.setReturn('foo', trvName)
-
-        result = helper.setManifest(trvName, manifestLst, commitMessage)
-        self.failUnlessEqual(result, [])
-        self.failUnless(os.path.exists('foo/manifest'))
-        self.failUnlessEqual(open('foo/manifest').read(), 'bar\nbaz\n')
-        mockGetVersionsByName._mock.assertCalled('%s:source' % trvName)
-        mockNewPkg._mock.assertCalled(trvName)
-        mockCheckout._mock.assertNotCalled()
-        mockCommit._mock.assertCalled('foo', commitMessage)
-        mockRmTree._mock.assertCalled('foo')
-
-        # checkout case
-        os.mkdir('bar')
-        mockGetVersionsByName._mock.setReturn(['bar', ], '%s:source' % trvName)
-        mockCheckout._mock.setReturn('bar', trvName)
-
-        result = helper.setManifest(trvName, manifestLst, commitMessage)
-        self.failUnlessEqual(result, ['bar', ])
-        self.failUnless(os.path.exists('bar/manifest'))
-        self.failUnlessEqual(open('bar/manifest').read(), 'bar\nbaz\n')
-        mockGetVersionsByName._mock.assertCalled('%s:source' % trvName)
-        mockNewPkg._mock.assertNotCalled()
-        mockCheckout._mock.assertCalled(trvName)
-        mockCommit._mock.assertCalled('bar', commitMessage)
-        mockRmTree._mock.assertCalled('bar')
-
-    def testCheckout(self):
-        mockTempFile = mock.MockObject(stableReturnValues=True)
-        mockTempFile.mkdtemp._mock.setReturn('foodir', prefix='conaryhelper-')
-
-        mockCheckin = mock.MockObject(stableReturnValues=True)
-        self.mock(conaryhelper, 'checkin', mockCheckin)
-        self.mock(conaryhelper, 'tempfile', mockTempFile)
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        result = helper._checkout('foo')
-        self.failUnlessEqual(result, 'foodir')
-        mockCheckin.checkout._mock.assertCalled(helper._repos, helper._ccfg,
-                                                'foodir', ['foo', ])
-        mockTempFile.mkdtemp._mock.assertCalled(prefix='conaryhelper-')
-
-    def testCommit(self):
-        mockCheckin = mock.MockObject()
-        self.mock(conaryhelper, 'checkin', mockCheckin)
-
-        os.mkdir('foo')
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        result = helper._commit('foo', 'foocommit')
-        self.failUnlessEqual(result, None)
-        mockCheckin.commit._mock.assertCalled(helper._repos, helper._ccfg,
-                                              'foocommit')
-
-    def testNewPkg(self):
-        mockTempFile = mock.MockObject(stableReturnValues=True)
-        mockTempFile.mkdtemp._mock.setReturn('foodir', prefix='conaryhelper-')
-        mockCheckin = mock.MockObject()
-        self.mock(conaryhelper, 'checkin', mockCheckin)
-        self.mock(conaryhelper, 'tempfile', mockTempFile)
-
-        os.mkdir('foodir')
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        result = helper._newpkg('foo')
-        self.failUnlessEqual(result, 'foodir/foo')
-        mockTempFile.mkdtemp._mock.assertCalled(prefix='conaryhelper-')
-        mockCheckin.newTrove._mock.assertCalled(helper._repos, helper._ccfg,
-                                                'foo')
-
-    def testGetVersionByName(self):
-        pkgName = 'foo'
-        mockCfg = mock.MockObject()
-        mockLabel = mock.MockObject()
-        mockRepos = mock.MockObject()
-        mockVersion = mock.MockObject()
-
-        mockCfg._mock.set(buildLabel=mockLabel)
-        mockRepos.getTroveLeavesByLabel._mock.setReturn(
-            {pkgName: {mockVersion: None}},
-            {pkgName: {mockLabel: None}})
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        helper._repos = mockRepos
-        helper._ccfg = mockCfg
-
-        result = helper._getVersionsByName(pkgName)
-        self.failUnlessEqual(result, [mockVersion, ])
-        mockRepos.getTroveLeavesByLabel._mock.assertCalled(
-            {pkgName: {mockLabel: None}})
-
-        mockRepos.getTroveLeavesByLabel._mock.setReturn(
-            {},
-            {pkgName: {mockLabel: None}})
-
-        result = helper._getVersionsByName(pkgName)
-        self.failUnlessEqual(result, [])
-        mockRepos.getTroveLeavesByLabel._mock.assertCalled(
-            {pkgName: {mockLabel: None}})
-
-    def testPromote(self):
-        mockFromLabel = mock.MockObject()
-        mockToLabel = mock.MockObject()
-        mockVersion = mock.MockObject()
-        mockCs = mock.MockObject()
-        mockCsNewTrove = mock.MockObject()
-        mockRepos = mock.MockObject()
-
-        trvLst = [('foo', mockVersion, None), ]
-
-        mockVersion.trailingLabel._mock.setReturn(mockFromLabel)
-        mockCs.iterNewTroveList._mock.setReturn([mockCsNewTrove, ])
-        mockRepos.commitChangeSet._mock.setReturn(None, mockCs)
-
-        def getMockClient(returnValue):
-            mockClient = mock.MockObject()
-            mockClient.createSiblingCloneChangeSet._mock.setReturn(
-                returnValue, {mockFromLabel: mockToLabel}, trvLst,
-                cloneSources=True)
-            return mockClient
-
-        helper = conaryhelper.ConaryHelper(self.updateBotCfg)
-        helper._repos = mockRepos
-
-        # normal case
-        expected = ('foo', None, None)
-        helper._client = getMockClient((True, mockCs))
-        mockCsNewTrove.getNewNameVersionFlavor._mock.setReturn(expected)
-        result = helper.promote(trvLst, trvLst, mockToLabel)
-        self.failUnlessEqual(result, [expected, ])
-        mockVersion.trailingLabel._mock.assertCalled()
-        mockCs.iterNewTroveList._mock.assertCalled()
-        mockCsNewTrove.getNewNameVersionFlavor._mock.assertCalled()
-        mockRepos.commitChangeSet._mock.assertCalled(mockCs)
-
-        # test PromoteFailedError exception
-        helper._client = getMockClient((False, mockCs))
-        self.failUnlessRaises(errors.PromoteFailedError, helper.promote,
-                              trvLst, trvLst, mockToLabel)
-
-        # test PromoteMismatchError exception
-        helper._client = getMockClient((True, mockCs))
-        self.failUnlessRaises(errors.PromoteMismatchError, helper.promote,
-                              trvLst, [('bar', None, None), ], mockToLabel)
-
-testsetup.main()
diff --git a/test/unit_test/updatebottest/errortest.py b/test/unit_test/updatebottest/errortest.py
deleted file mode 100644
--- a/test/unit_test/updatebottest/errortest.py
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-
-import slehelp
-
-from updatebot import errors
-
-class ErrorTest(slehelp.Helper):
-    def testRepr(self):
-        error = errors.UpdateBotError()
-        self.failUnlessEqual(repr(error), 'updatebot.errors.UpdateBotError()')
-
-
-testsetup.main()
diff --git a/test/unit_test/updatebottest/patchsourcetest.py b/test/unit_test/updatebottest/patchsourcetest.py
deleted file mode 100644
--- a/test/unit_test/updatebottest/patchsourcetest.py
+++ /dev/null
@@ -1,98 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testutils import mock
-
-import slehelp
-
-from updatebot import patchsource
-
-class PatchSourceTest(slehelp.Helper):
-    def testLoadFromClient(self):
-        mockPatch = mock.MockObject()
-        mockClient = mock.MockObject(stableReturnValues=True)
-        mockClient.getPatchDetail._mock.setReturn([mockPatch, ])
-
-        mockLoadOne = mock.MockObject(stableReturnValues=True)
-        mockLoadOne._mock.setReturn(None, mockPatch, '/foo')
-
-        patchSource = patchsource.PatchSource(self.updateBotCfg)
-        self.mock(patchSource, '_loadOne', mockLoadOne)
-
-        result = patchSource.loadFromClient(mockClient, '/foo')
-        self.failUnlessEqual(result, None)
-        mockClient.getPatchDetail._mock.assertCalled()
-        mockLoadOne._mock.assertCalled(mockPatch, '/foo')
-
-    def testLoadOne(self):
-        mockPackage = mock.MockObject(stableReturnValues=True)
-        mockPackage._mock.set(location='foo/bar')
-
-        mockPatch = mock.MockObject(stableReturnValues=True)
-        mockPatch._mock.set(packages=[mockPackage, ])
-
-        mockFilterPatch = mock.MockObject()
-        mockFilterPatch._mock.setReturn(False, mockPatch)
-
-        expectedResult = {mockPackage: set([mockPatch, ])}
-
-        patchSource = patchsource.PatchSource(self.updateBotCfg)
-        self.mock(patchSource, '_filterPatch', mockFilterPatch)
-
-        result = patchSource._loadOne(mockPatch, 'baz')
-        self.failUnlessEqual(result, None)
-        self.failUnlessEqual(patchSource.pkgMap, expectedResult)
-        self.failUnlessEqual(mockPackage.location, 'baz/foo/bar')
-        mockFilterPatch._mock.assertCalled(mockPatch)
-
-        mockFilterPatch._mock.setReturn(True, mockPatch)
-        result = patchSource._loadOne(mockPatch, 'baz')
-        self.failUnlessEqual(result, None)
-        self.failUnlessEqual(patchSource.pkgMap, expectedResult)
-        mockFilterPatch._mock.assertCalled(mockPatch)
-
-    def testFilterPatch(self):
-        mockConfig = mock.MockObject()
-        mockFilter = mock.MockObject()
-        mockPatch = mock.MockObject()
-
-        mockPatch._mock.set(summary='foo')
-        mockPatch._mock.set(description='bar')
-
-        mockConfig._mock.set(patchFilter=[(None, mockFilter), ])
-
-        patchSource = patchsource.PatchSource(mockConfig)
-
-        mockFilter.match._mock.setReturn(True, 'foo')
-        result = patchSource._filterPatch(mockPatch)
-        self.failUnlessEqual(result, True)
-        mockFilter.match._mock.assertCalled('foo')
-
-        mockFilter.match._mock.setReturn(False, 'foo')
-        mockFilter.match._mock.setReturn(True, 'bar')
-        result = patchSource._filterPatch(mockPatch)
-        self.failUnlessEqual(result, True)
-        mockFilter.match._mock.assertCalled('foo')
-        mockFilter.match._mock.assertCalled('bar')
-
-        mockFilter.match._mock.setReturn(False, 'foo')
-        mockFilter.match._mock.setReturn(False, 'bar')
-        result = patchSource._filterPatch(mockPatch)
-        self.failUnlessEqual(result, False)
-        mockFilter.match._mock.assertCalled('foo')
-        mockFilter.match._mock.assertCalled('bar')
-
-
-testsetup.main()
diff --git a/test/unit_test/updatebottest/testsetup.py b/test/unit_test/updatebottest/testsetup.py
deleted file mode 120000
--- a/test/unit_test/updatebottest/testsetup.py
+++ /dev/null
@@ -1,1 +0,0 @@
-../../testsetup.py
\ No newline at end of file
diff --git a/test/unit_test/updatebottest/updatetest.py b/test/unit_test/updatebottest/updatetest.py
deleted file mode 100644
--- a/test/unit_test/updatebottest/updatetest.py
+++ /dev/null
@@ -1,323 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testutils import mock
-
-import slehelp
-
-from updatebot import util
-from updatebot import errors
-from updatebot import update
-
-class UpdateTest(slehelp.Helper):
-    def testGetUpdates(self):
-        mockRpmSource = mock.MockObject(stableReturnValues=True)
-        updater = update.Updater(self.updateBotCfg, mockRpmSource)
-
-        updateTrove = (('foo:source', None, None), mock.MockObject())
-        adviseTrove = (('bar:source', None, None), mock.MockObject())
-
-        mockFindUpdatableTroves = mock.MockObject(stableReturnValues=True)
-        mockFindUpdatableTroves._mock.setReturn([updateTrove, adviseTrove],
-                                                self.updateBotCfg.topGroup)
-        self.mock(updater, '_findUpdatableTroves', mockFindUpdatableTroves)
-
-        mockSanitizeTrove = mock.MockObject(stableReturnValues=True)
-        mockSanitizeTrove._mock.setReturn(True, *updateTrove)
-        mockSanitizeTrove._mock.setReturn(False, *adviseTrove)
-        self.mock(updater, '_sanitizeTrove', mockSanitizeTrove)
-
-        toAdvise, toUpdate = updater.getUpdates()
-        self.failUnlessEqual(len(toAdvise), 2)
-        self.failUnless(updateTrove in toAdvise)
-        self.failUnless(adviseTrove in toAdvise)
-        self.failUnlessEqual(len(toUpdate), 1)
-        self.failUnless(updateTrove in toUpdate)
-        mockFindUpdatableTroves._mock.assertCalled(self.updateBotCfg.topGroup)
-        mockSanitizeTrove._mock.assertCalled(*updateTrove)
-        mockSanitizeTrove._mock.assertCalled(*adviseTrove)
-
-    def testFindUpdatableTroves(self):
-        trvSpec = ('group-foo', 'foo.rpath.com at rpath:foo-devel', None)
-        mockRpmSource = mock.MockObject(stableReturnValues=True)
-        updater = update.Updater(self.updateBotCfg, mockRpmSource)
-
-        mockFooVersion = mock.MockObject(stableReturnValues=True)
-        mockBarVersion = mock.MockObject(stableReturnValues=True)
-        mockBazVersion = mock.MockObject(stableReturnValues=True)
-
-        mockFooRevision = mock.MockObject(stableReturnValues=True)
-        mockBarRevision = mock.MockObject(stableReturnValues=True)
-        mockBazRevision = mock.MockObject(stableReturnValues=True)
-
-        mockFooVersion.trailingRevision._mock.setReturn(mockFooRevision)
-        mockBarVersion.trailingRevision._mock.setReturn(mockBarRevision)
-        mockBazVersion.trailingRevision._mock.setReturn(mockBazRevision)
-
-        troves = [('info-foo:source', None, None),
-                  ('group-foo:source', None, None),
-                  ('foo:source', mockFooVersion, None),
-                  ('bar:source', mockBarVersion, None),
-                  ('baz:source', mockBazVersion, None)]
-
-        mockConaryHelper = mock.MockObject(stableReturnValues=True)
-        mockConaryHelper.getSourceTroves._mock.setReturn(troves, trvSpec)
-        self.mock(updater, '_conaryhelper', mockConaryHelper)
-
-        mockFooSrpm = mock.MockObject(stableReturnValues=True)
-        mockBarSrpm = mock.MockObject(stableReturnValues=True)
-        mockBazSrpm = mock.MockObject(stableReturnValues=True)
-
-        mockGetLatestSource = mock.MockObject(stableReturnValues=True)
-        mockGetLatestSource._mock.setReturn(mockFooSrpm, 'foo')
-        mockGetLatestSource._mock.setReturn(mockBarSrpm, 'bar')
-        mockGetLatestSource._mock.setReturn(mockBazSrpm, 'baz')
-        self.mock(updater, '_getLatestSource', mockGetLatestSource)
-
-        mockSrpmToConaryVersion = mock.MockObject(stableReturnValues=True)
-        self.mock(util, 'srpmToConaryVersion', mockSrpmToConaryVersion)
-
-        # srpm is newer than conary repo
-        mockFooRevision._mock.set(version='0')
-        mockSrpmToConaryVersion._mock.setReturn('1', mockFooSrpm)
-
-        # srpm is same as conary repo
-        mockBarRevision._mock.set(version='2')
-        mockSrpmToConaryVersion._mock.setReturn('2', mockBarSrpm)
-
-        # srpm is older than conary repo
-        mockBazRevision._mock.set(version='4')
-        mockSrpmToConaryVersion._mock.setReturn('3', mockBazSrpm)
-
-        expectedResult = [(('foo', mockFooVersion, None), mockFooSrpm),
-                          (('baz', mockBazVersion, None), mockBazSrpm)]
-
-        result = updater._findUpdatableTroves(trvSpec)
-        self.failUnlessEqual(result, expectedResult)
-        mockConaryHelper.getSourceTroves._mock.assertCalled(trvSpec)
-        mockGetLatestSource._mock.assertCalled('foo')
-        mockGetLatestSource._mock.assertCalled('bar')
-        mockGetLatestSource._mock.assertCalled('baz')
-        mockSrpmToConaryVersion._mock.assertCalled(mockFooSrpm)
-        mockSrpmToConaryVersion._mock.assertCalled(mockBarSrpm)
-        mockSrpmToConaryVersion._mock.assertCalled(mockBazSrpm)
-        mockFooVersion.trailingRevision._mock.assertCalled()
-        mockBarVersion.trailingRevision._mock.assertCalled()
-        mockBazVersion.trailingRevision._mock.assertCalled()
-
-    def testGetLatestSource(self):
-        mockRpmSource = mock.MockObject(stableReturnValues=True)
-        updater = update.Updater(self.updateBotCfg, mockRpmSource)
-
-        mockPackageVerCmp = mock.MockObject()
-        self.mock(util, 'packagevercmp', mockPackageVerCmp)
-
-        mockList = mock.MockObject(stableReturnValues=True)
-        mockList.sort._mock.setReturn(None, mockPackageVerCmp)
-        mockList[-1] = None
-        mockElement = mockList[-1]
-
-        mockRpmSource._mock.set(srcNameMap={'foo': mockList})
-
-        result = updater._getLatestSource('foo')
-        self.failUnless(result is mockElement)
-        mockList.sort._mock.assertCalled(mockPackageVerCmp)
-
-    def testSanitizeTrove(self):
-        trvSpec = ('foo', None, None)
-        mockRpmSource = mock.MockObject(stableReturnValues=True)
-        updater = update.Updater(self.updateBotCfg, mockRpmSource)
-
-        newBinPkg1 = mock.MockObject(stableReturnValues=True)
-        newBinPkg2 = mock.MockObject(stableReturnValues=True)
-
-        newBinPkg1._mock.set(name='foo')
-        newBinPkg2._mock.set(name='foo-devel')
-
-        oldBinPkg1 = mock.MockObject(stableReturnValues=True)
-        oldBinPkg2 = mock.MockObject(stableReturnValues=True)
-        oldBinPkg3 = mock.MockObject(stableReturnValues=True)
-        oldBinPkg4 = mock.MockObject(stableReturnValues=True)
-
-        oldBinPkg1._mock.set(name='foo')
-        oldBinPkg1._mock.set(epoch='0')
-        oldBinPkg1._mock.set(version='1')
-        oldBinPkg1._mock.set(arch='x86')
-
-        oldBinPkg2._mock.set(name='foo-devel')
-        oldBinPkg2._mock.set(epoch='0')
-        oldBinPkg2._mock.set(version='1')
-        oldBinPkg2._mock.set(arch='x86')
-
-        oldBinPkg3._mock.set(name='bar')
-        oldBinPkg3._mock.set(epoch='0')
-        oldBinPkg3._mock.set(version='1')
-        oldBinPkg3._mock.set(arch='x86_64')
-
-        oldBinPkg4._mock.set(name='bar')
-        oldBinPkg4._mock.set(epoch='0')
-        oldBinPkg4._mock.set(version='0')
-        oldBinPkg4._mock.set(arch='x86_64')
-
-        newSrcPkg = mock.MockObject()
-        oldSrcPkg = mock.MockObject()
-        oldSrcPkg2 = mock.MockObject()
-
-        newSrcPkg._mock.set(version='1')
-        newSrcPkg._mock.set(epoch='0')
-
-        srcPkgMap = {newSrcPkg: [newBinPkg1,
-                                 newBinPkg2],
-                     oldSrcPkg: [oldBinPkg1,
-                                 oldBinPkg2,
-                                 oldBinPkg3],
-                     oldSrcPkg2: [oldBinPkg4, ]}
-
-        locationMap = {'a': oldBinPkg1,
-                       'b': oldBinPkg2,
-                       'c': oldBinPkg3}
-
-        binPkgMap = {oldBinPkg1: oldSrcPkg,
-                     oldBinPkg2: oldSrcPkg,
-                     oldBinPkg3: oldSrcPkg,
-                     oldBinPkg4: oldSrcPkg2}
-
-        binNameMap = {'foo': [oldBinPkg1, ],
-                      'foo-devel': [oldBinPkg2, ],
-                      'bar': [oldBinPkg3,
-                              oldBinPkg4]}
-
-        mockRpmSource._mock.set(srcPkgMap=srcPkgMap)
-        mockRpmSource._mock.set(locationMap=locationMap)
-        mockRpmSource._mock.set(binPkgMap=binPkgMap)
-        mockRpmSource._mock.set(binNameMap=binNameMap)
-
-        mockConaryHelper = mock.MockObject(stableReturnValues=True)
-        mockConaryHelper.getManifest._mock.setReturn(['a', 'b'], 'foo')
-        self.mock(updater, '_conaryhelper', mockConaryHelper)
-
-        mockPackageVerCmp = mock.MockObject(stableReturnValue=True)
-        self.mock(util, 'packagevercmp', mockPackageVerCmp)
-
-        # need to update repository
-        mockPackageVerCmp._mock.setReturn(1, newSrcPkg, oldSrcPkg)
-        result1 = updater._sanitizeTrove(trvSpec, newSrcPkg)
-        self.failUnlessEqual(result1, True)
-        mockConaryHelper.getManifest._mock.assertCalled('foo')
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-
-        # only need to send advisory
-        mockPackageVerCmp._mock.setReturn(0, newSrcPkg, oldSrcPkg)
-        result1 = updater._sanitizeTrove(trvSpec, newSrcPkg)
-        self.failUnlessEqual(result1, False)
-        mockConaryHelper.getManifest._mock.assertCalled('foo')
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-
-        # test update goes backwards exception
-        mockPackageVerCmp._mock.setReturn(-1, newSrcPkg, oldSrcPkg)
-        self.failUnlessRaises(errors.UpdateGoesBackwardsError,
-            updater._sanitizeTrove, trvSpec, newSrcPkg)
-        mockConaryHelper.getManifest._mock.assertCalled('foo')
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-
-        # test remove package without exception
-        mockConaryHelper.getManifest._mock.setReturn(['a', 'c'], 'foo')
-        mockPackageVerCmp._mock.setReturn(0, newSrcPkg, oldSrcPkg)
-        mockPackageVerCmp._mock.setReturn(1, oldBinPkg3, oldBinPkg4)
-        mockPackageVerCmp._mock.setReturn(-1, oldBinPkg4, oldBinPkg3)
-        result = updater._sanitizeTrove(trvSpec, newSrcPkg)
-        self.failUnlessEqual(result, False)
-        mockConaryHelper.getManifest._mock.assertCalled('foo')
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-        self.failUnless(oldBinPkg3 in srcPkgMap[newSrcPkg])
-        # reset srcPkgMap
-        srcPkgMap[newSrcPkg].remove(oldBinPkg3)
-
-        # test remove package with exception
-        oldBinPkg4._mock.set(version='2')
-        mockConaryHelper.getManifest._mock.setReturn(['a', 'c'], 'foo')
-        mockPackageVerCmp._mock.setReturn(0, newSrcPkg, oldSrcPkg)
-        mockPackageVerCmp._mock.setReturn(-1, oldBinPkg3, oldBinPkg4)
-        mockPackageVerCmp._mock.setReturn(1, oldBinPkg4, oldBinPkg3)
-        self.failUnlessRaises(errors.UpdateRemovesPackageError,
-            updater._sanitizeTrove, trvSpec, newSrcPkg)
-        mockConaryHelper.getManifest._mock.assertCalled('foo')
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-        mockPackageVerCmp._mock.assertCalled(newSrcPkg, oldSrcPkg)
-
-    def testUpdate(self):
-        nvf = ('foo', None, None)
-        manifest = ['a', 'b', ]
-        mockSourcePkg = mock.MockObject()
-        mockGetManifestFromRpmSource = mock.MockObject()
-        mockConaryHelper = mock.MockObject()
-
-        mockGetManifestFromRpmSource._mock.setReturn(manifest, mockSourcePkg)
-        mockConaryHelper.setManifest._mock.setReturn(None, nvf[0], manifest,
-            commitMessage=self.updateBotCfg.commitMessage)
-
-        updater = update.Updater(self.updateBotCfg, None)
-        self.mock(updater, '_getManifestFromRpmSource',
-            mockGetManifestFromRpmSource)
-        self.mock(updater, '_conaryhelper', mockConaryHelper)
-
-        result = updater.update(nvf, mockSourcePkg)
-        self.failUnlessEqual(result, None)
-        mockGetManifestFromRpmSource._mock.assertCalled(mockSourcePkg)
-        mockConaryHelper.setManifest._mock.assertCalled(nvf[0], manifest,
-            commitMessage=self.updateBotCfg.commitMessage)
-
-    def testGetManifestFromRpmSource(self):
-        mockSourcePkg = mock.MockObject()
-        mockRpmSource = mock.MockObject()
-        mockGetLatest = mock.MockObject()
-
-        mockPkg = mock.MockObject()
-        mockPkg._mock.set(location='a')
-
-        srcPkgMap = {mockSourcePkg: [mockPkg, ]}
-        mockRpmSource._mock.set(srcPkgMap=srcPkgMap)
-
-        mockGetLatest._mock.setReturn([mockPkg, ], [mockPkg, ])
-
-        updater = update.Updater(self.updateBotCfg, mockRpmSource)
-        self.mock(updater, '_getLatestOfAvailableArches', mockGetLatest)
-
-        expected = ['a', ]
-        result = updater._getManifestFromRpmSource(mockSourcePkg)
-        self.failUnlessEqual(result, expected)
-        mockGetLatest._mock.assertCalled([mockPkg, ])
-
-    def testPublish(self):
-        mockTrvLst = mock.MockObject()
-        mockExpected = mock.MockObject()
-        mockLabel = mock.MockObject()
-        mockConaryHelper = mock.MockObject()
-        mockReturn = mock.MockObject()
-
-        mockConaryHelper.promote._mock.setReturn(mockReturn, mockTrvLst,
-            mockExpected, mockLabel)
-
-        updater = update.Updater(self.updateBotCfg, None)
-        self.mock(updater, '_conaryhelper', mockConaryHelper)
-
-        result = updater.publish(mockTrvLst, mockExpected, mockLabel)
-        self.failUnlessEqual(result, mockReturn)
-
-testsetup.main()
diff --git a/test/unit_test/updatebottest/utiltest.py b/test/unit_test/updatebottest/utiltest.py
deleted file mode 100644
--- a/test/unit_test/updatebottest/utiltest.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#
-# Copyright (c) 2008 rPath, Inc.
-#
-# This program is distributed under the terms of the Common Public License,
-# version 1.0. A copy of this license should have been distributed with this
-# source file in a file called LICENSE. If it is not present, the license
-# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
-#
-# This program is distributed in the hope that it will be useful, but
-# without any warranty; without even the implied warranty of merchantability
-# or fitness for a particular purpose. See the Common Public License for
-# full details.
-#
-
-import testsetup
-from testutils import mock
-
-import slehelp
-
-from updatebot import util
-
-class UtilTest(slehelp.Helper):
-    def testJoin(self):
-        def t(input, output):
-            result = util.join(*input)
-            self.failUnlessEqual(result, output)
-
-        t(['/foo', 'bar'],
-          '/foo/bar')
-        t(['/foo/../', 'bar/'],
-          '/bar')
-        t(['/foo', '/bar'],
-          '/foo/bar')
-
-    def testSrpmToConaryVersion(self):
-        mockSrcPkg = mock.MockObject()
-
-        mockSrcPkg._mock.set(version='a-b')
-        mockSrcPkg._mock.set(release='b-c')
-
-        expected = 'a_b_b_c'
-        result = util.srpmToConaryVersion(mockSrcPkg)
-        self.failUnlessEqual(result, expected)
-
-        mockSrcPkg._mock.set(version='a_b')
-        result = util.srpmToConaryVersion(mockSrcPkg)
-        self.failUnlessEqual(result, expected)
-
-    def testPackagevercmp(self):
-        def t(expected, kw1, kw2):
-            mockPkg1 = mock.MockObject()
-            mockPkg2 = mock.MockObject()
-
-            for kw in kw1.iterkeys():
-                mockPkg1._mock.set(**{kw:kw1[kw]})
-            for kw in kw2.iterkeys():
-                mockPkg2._mock.set(**{kw:kw2[kw]})
-
-            result = util.packagevercmp(mockPkg1, mockPkg2)
-            self.failUnlessEqual(result, expected)
-
-        t(1, {'epoch': '1'},
-             {'epoch': '0'})
-        t(-1, {'epoch': '0'},
-              {'epoch': '1'})
-        t(1, {'epoch': '0',
-              'version': '1'},
-             {'epoch': '0',
-              'version': '0'})
-        t(-1, {'epoch': '0',
-              'version': '0'},
-             {'epoch': '0',
-              'version': '1'})
-        t(1, {'epoch': '0',
-              'version': '0',
-              'release': '1'},
-             {'epoch': '0',
-              'version': '0',
-              'release': '0'})
-        t(-1, {'epoch': '0',
-              'version': '0',
-              'release': '0'},
-             {'epoch': '0',
-              'version': '0',
-              'release': '1'})
-        t(0, {'epoch': '0',
-              'version': '0',
-              'release': '0'},
-             {'epoch': '0',
-              'version': '0',
-              'release': '0'})
-
-
-testsetup.main()

From elliot at rpath.com  Tue Sep  6 14:00:14 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 06 Sep 2011 18:00:14 +0000
Subject: mirrorball: check for the correct status
Message-ID: <201109061800.p86I0Eui004424@scc.eng.rpath.com>

changeset:   583de91d06d2
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 06 Sep 2011 14:00:11 -0400

check for the correct status

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -473,7 +473,7 @@
             toPromote = []
             for jobId, (trove, state, result) in self._jobs.iteritems():
                 # not ready to be promoted
-                if state != JobStatus.JOB_COMMITTED:
+                if state != buildjob.JOB_STATE_COMMITTED:
                     continue
 
                 toPromote.append((jobId, result))

From elliot at rpath.com  Tue Sep  6 14:37:16 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 06 Sep 2011 18:37:16 +0000
Subject: mirrorball: fix incompatable data structure problems
Message-ID: <201109061837.p86IbG8C006288@scc.eng.rpath.com>

changeset:   7bcbd5ab2acd
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 06 Sep 2011 14:37:04 -0400

fix incompatable data structure problems

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -476,7 +476,7 @@
                 if state != buildjob.JOB_STATE_COMMITTED:
                     continue
 
-                toPromote.append((jobId, result))
+                toPromote.append((jobId, tuple(result.items())))
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
 
             if toPromote:
@@ -486,8 +486,8 @@
         # Gather results
         for result in self._promoter.getStatus():
             self._promoteSlots += 1
-            for jobId, trvLst in result:
-                self._jobs[jobId][2] = trvLst
+            for jobId, promoted in result:
+                self._jobs[jobId][2] = promoted
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
 
         # Gather errors
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -167,24 +167,29 @@
 
         # Build mapping of nvf tuple to jobId
         jobMap = {}
-        for jobId, built in self.jobs.iteritems():
-            for bintrv in itertools.chain(*built.itervalues()):
-                jobMap[bintrv] = jobId
+        for jobId, built in self.jobs:
+            for srcTrv, binTrvs in dict(built).iteritems():
+                for binTrv in binTrvs:
+                    jobMap.setdefault(binTrv, list()).append(jobId, srcTrv)
+
+        # Get the list of binary troves to promote.
+        trvLst = jobMap.keys()
 
         # Assume that all troves are on the same source label.
-        srcLabel = self.trvLst[0][1].trailingLabel()
+        srcLabel = trvLst[0][1].trailingLabel()
 
         # Promote
-        result = self.helper.promote(self.trvLst, set(), srcLabel,
-            self.targetLabel, checkPackageList=False)
+        result = self.helper.promote(trvLst, set(), srcLabel, self.targetLabel,
+            checkPackageList=False)
 
         # Map the results back to source label troves
         clonedFrom = self.helper.getClonedFrom(result)
 
         # Map jobIds back to promoted troves.
         resultMap = {}
-        for spec, jobId in jobMap.iteritems():
-            resultMap.setdefault(jobId, set()).add(clonedFrom.get(spec))
+        for spec, (jobId, srcTrv) in jobMap.iteritems():
+            resultMap.setdefault(jobId, dict()).setdefault(srcTrv,
+                set()).add(clonedFrom.get(spec))
 
         # Send back all of the results.
         self.status.put((MessageTypes.DATA, tuple(resultMap.items())))

From elliot at rpath.com  Tue Sep  6 14:37:17 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 06 Sep 2011 18:37:17 +0000
Subject: mirrorball: branch merge
Message-ID: <201109061837.p86IbHrJ006319@scc.eng.rpath.com>

changeset:   8705f09a6543
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 06 Sep 2011 14:37:15 -0400

branch merge

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -476,7 +476,7 @@
                 if state != buildjob.JOB_STATE_COMMITTED:
                     continue
 
-                toPromote.append((jobId, result))
+                toPromote.append((jobId, tuple(result.items())))
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
 
             if toPromote:
@@ -486,8 +486,8 @@
         # Gather results
         for result in self._promoter.getStatus():
             self._promoteSlots += 1
-            for jobId, trvLst in result:
-                self._jobs[jobId][2] = trvLst
+            for jobId, promoted in result:
+                self._jobs[jobId][2] = promoted
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
 
         # Gather errors
diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -167,24 +167,29 @@
 
         # Build mapping of nvf tuple to jobId
         jobMap = {}
-        for jobId, built in self.jobs.iteritems():
-            for bintrv in itertools.chain(*built.itervalues()):
-                jobMap[bintrv] = jobId
+        for jobId, built in self.jobs:
+            for srcTrv, binTrvs in dict(built).iteritems():
+                for binTrv in binTrvs:
+                    jobMap.setdefault(binTrv, list()).append(jobId, srcTrv)
+
+        # Get the list of binary troves to promote.
+        trvLst = jobMap.keys()
 
         # Assume that all troves are on the same source label.
-        srcLabel = self.trvLst[0][1].trailingLabel()
+        srcLabel = trvLst[0][1].trailingLabel()
 
         # Promote
-        result = self.helper.promote(self.trvLst, set(), srcLabel,
-            self.targetLabel, checkPackageList=False)
+        result = self.helper.promote(trvLst, set(), srcLabel, self.targetLabel,
+            checkPackageList=False)
 
         # Map the results back to source label troves
         clonedFrom = self.helper.getClonedFrom(result)
 
         # Map jobIds back to promoted troves.
         resultMap = {}
-        for spec, jobId in jobMap.iteritems():
-            resultMap.setdefault(jobId, set()).add(clonedFrom.get(spec))
+        for spec, (jobId, srcTrv) in jobMap.iteritems():
+            resultMap.setdefault(jobId, dict()).setdefault(srcTrv,
+                set()).add(clonedFrom.get(spec))
 
         # Send back all of the results.
         self.status.put((MessageTypes.DATA, tuple(resultMap.items())))

From elliot at rpath.com  Tue Sep  6 15:02:29 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 06 Sep 2011 19:02:29 +0000
Subject: mirrorball: make results hashable
Message-ID: <201109061902.p86J2Th0008311@scc.eng.rpath.com>

changeset:   f14286a47163
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 06 Sep 2011 15:02:27 -0400

make results hashable

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -476,7 +476,10 @@
                 if state != buildjob.JOB_STATE_COMMITTED:
                     continue
 
-                toPromote.append((jobId, tuple(result.items())))
+                # Make result hashable
+                res = tuple([ (x, tuple(y)) for x, y in result.iteritems() ])
+
+                toPromote.append((jobId, res))
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
 
             if toPromote:

From elliot at rpath.com  Tue Sep  6 15:53:35 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 06 Sep 2011 19:53:35 +0000
Subject: mirrorball: must pass in tuple
Message-ID: <201109061953.p86JrZ3w009784@scc.eng.rpath.com>

changeset:   83a2f8a092d1
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 06 Sep 2011 15:52:20 -0400

must pass in tuple

diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -170,7 +170,7 @@
         for jobId, built in self.jobs:
             for srcTrv, binTrvs in dict(built).iteritems():
                 for binTrv in binTrvs:
-                    jobMap.setdefault(binTrv, list()).append(jobId, srcTrv)
+                    jobMap.setdefault(binTrv, list()).append((jobId, srcTrv))
 
         # Get the list of binary troves to promote.
         trvLst = jobMap.keys()

From elliot at rpath.com  Tue Sep  6 15:53:35 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 06 Sep 2011 19:53:35 +0000
Subject: mirrorball: using a generator as an id was probably a bad idea
Message-ID: <201109061953.p86JrZIc009815@scc.eng.rpath.com>

changeset:   21581d6d2e17
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 06 Sep 2011 15:53:33 -0400

using a generator as an id was probably a bad idea

diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -154,7 +154,7 @@
         self.helper = helper
         self.targetLabel = targetLabel
         self.jobs = jobs
-        self.workerId = (x[0] for x in jobs)
+        self.workerId = tuple([ x[0] for x in jobs ])
 
     def work(self):
         """

From elliot at rpath.com  Wed Sep  7 13:41:11 2011
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 07 Sep 2011 17:41:11 +0000
Subject: mirrorball: stop trying to hash lists
Message-ID: <201109071741.p87HfB7f014477@scc.eng.rpath.com>

changeset:   f712454ff48a
user:        Elliot Peele <https://issues.rpath.com/>
date:        Wed, 07 Sep 2011 13:41:05 -0400

stop trying to hash lists

diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1027,7 +1027,7 @@
             # conary promote code.
             if n.endswith(':source'):
                 continue
-            data.setdefault(n.split(':'), dict()).setdefault(v, set()).add(f)
+            data.setdefault(n.split(':')[0], dict()).setdefault(v, set()).add(f)
 
         while data:
             job = []

From elliot at rpath.com  Wed Sep  7 14:33:37 2011
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 07 Sep 2011 18:33:37 +0000
Subject: mirrorball: add methods for getting nevras
Message-ID: <201109071833.p87IXbol016768@scc.eng.rpath.com>

changeset:   7aaa4f2967ad
user:        Elliot Peele <https://issues.rpath.com/>
date:        Wed, 07 Sep 2011 14:33:34 -0400

add methods for getting nevras

diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -77,6 +77,9 @@
         self.conaryConfigCache = {}
         self.sharedTmpDir = None
 
+        self.nevraCache = {}
+        self.labelNevraCache = {}
+
 
 class ConaryHelper(object):
     """
@@ -298,7 +301,7 @@
 
         return self.getSourceVersions(list(troves))
 
-    def _cacheTroveInfo(self, troveSpecs, cache, tiType, tiFunc=None):
+    def _cacheTroveInfo(self, troveSpecs, cache, tiType, tiFunc=None, missingOk=False):
         """
         Retrieve a bit of trove info for a listed trove specs and cache the
         results.
@@ -329,11 +332,14 @@
         for i, nvf in enumerate(uncached):
             # If this trove doesn't have this piece of trove info, log a warning
             # and skip over it.
-            if tiLst[i] is None:
+            if tiLst[i] is None and not missingOk:
                 log.warn('found missing trove info for %s, skipping' % (nvf, ))
                 continue
 
-            ti = tiLst[i]()
+            ti = tiLst[i]
+            if callable(ti):
+                ti = ti()
+
             if tiFunc:
                 ti = tiFunc(ti, nvf)
             tiMap.setdefault(ti, set()).add(nvf)
@@ -344,6 +350,60 @@
 
         return tiMap
 
+    def getNevras(self, troveSpecs):
+        """
+        Get a mapping of nvf to nevra for all specified trove specs.
+        @param troveSpecs: list of trove specs
+        @type troveSpecs: list((name, conary.versions.Version,
+            conary.deps.deps.Flavor), ..)
+        @return map of trove spec to nevra, map contains None if no nevra was
+            found.
+        """
+
+        def tiFunc(ti, nvf):
+            if ti is None:
+                return None
+
+            return (ti.rpm.name(), ti.rpm.epoch(), ti.rpm.version(),
+                ti.rpm.release(), ti.rpm.arch())
+
+        self._cacheTroveInfo(troveSpecs, self._cache.nevraCache,
+            trove._TROVEINFO_TAG_CAPSULE, tiFunc=tiFunc)
+
+        results = {}
+        for spec in troveSpecs:
+            results[spec] = self._cache.nevraCache.get(spec, None)
+
+        return results
+
+    def getNevrasForLabel(self, label):
+        """
+        Query an entire label for nevra information.
+        @param label: conary label
+        @type label: conary.versions.Label
+        """
+
+        if hasattr(label, 'label'):
+            label = label.label()
+
+        if label in self._cache.labelNevraCache:
+            return self._cache.labelClonedFromCache[label]
+
+        req = {None: {label: None}}
+        binTrvMap = self._repos.getTroveVersionsByLabel(req)
+
+        binTrvs = set()
+        for n, vMap in binTrvMap.iteritems():
+            for v, flvs in vMap.iteritems():
+                if n.endswith(':source'):
+                    binTrvs.add((n, v, None))
+                else:
+                    binTrvs.update(set((n, v, x) for x in flvs))
+
+        nevras = self.getNevras(binTrvs)
+        self._cache.labelNevraCache[label] = nevras
+        return nevras
+
     def getClonedFrom(self, troveSpecs):
         """
         Get a mapping of cloned from trove info for a list of trove specs in the

From elliot at rpath.com  Wed Sep  7 15:21:25 2011
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 07 Sep 2011 19:21:25 +0000
Subject: mirrorball: it's ok if nevra info is missing for some troves,
	not all of them contain capsules
Message-ID: <201109071921.p87JLP0M019822@scc.eng.rpath.com>

changeset:   c4fc5deb3242
user:        Elliot Peele <https://issues.rpath.com/>
date:        Wed, 07 Sep 2011 15:09:08 -0400

it's ok if nevra info is missing for some troves, not all of them contain capsules

diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -301,7 +301,8 @@
 
         return self.getSourceVersions(list(troves))
 
-    def _cacheTroveInfo(self, troveSpecs, cache, tiType, tiFunc=None, missingOk=False):
+    def _cacheTroveInfo(self, troveSpecs, cache, tiType, tiFunc=None,
+        missingOk=False):
         """
         Retrieve a bit of trove info for a listed trove specs and cache the
         results.
@@ -368,7 +369,7 @@
                 ti.rpm.release(), ti.rpm.arch())
 
         self._cacheTroveInfo(troveSpecs, self._cache.nevraCache,
-            trove._TROVEINFO_TAG_CAPSULE, tiFunc=tiFunc)
+            trove._TROVEINFO_TAG_CAPSULE, tiFunc=tiFunc, missingOk=True)
 
         results = {}
         for spec in troveSpecs:

From elliot at rpath.com  Wed Sep  7 15:21:25 2011
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 07 Sep 2011 19:21:25 +0000
Subject: mirrorball: cleanup
Message-ID: <201109071921.p87JLPD1019853@scc.eng.rpath.com>

changeset:   985d16f8ba62
user:        Elliot Peele <https://issues.rpath.com/>
date:        Wed, 07 Sep 2011 15:21:20 -0400

cleanup

diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -170,7 +170,7 @@
         for jobId, built in self.jobs:
             for srcTrv, binTrvs in dict(built).iteritems():
                 for binTrv in binTrvs:
-                    jobMap.setdefault(binTrv, list()).append((jobId, srcTrv))
+                    jobMap[binTrv] = (jobId, srcTrv)
 
         # Get the list of binary troves to promote.
         trvLst = jobMap.keys()
@@ -179,8 +179,8 @@
         srcLabel = trvLst[0][1].trailingLabel()
 
         # Promote
-        result = self.helper.promote(trvLst, set(), srcLabel, self.targetLabel,
-            checkPackageList=False)
+        result = self.helper.promote(trvLst, set(), [srcLabel, ],
+            self.targetLabel, checkPackageList=False)
 
         # Map the results back to source label troves
         clonedFrom = self.helper.getClonedFrom(result)
@@ -189,7 +189,7 @@
         resultMap = {}
         for spec, (jobId, srcTrv) in jobMap.iteritems():
             resultMap.setdefault(jobId, dict()).setdefault(srcTrv,
-                set()).add(clonedFrom.get(spec))
+                set()).add(sorted(clonedFrom.get(spec))[-1])
 
         # Send back all of the results.
         self.status.put((MessageTypes.DATA, tuple(resultMap.items())))

From elliot at rpath.com  Wed Sep  7 15:25:59 2011
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 07 Sep 2011 19:25:59 +0000
Subject: mirrorball: use workerId as an identifier instead of the job
Message-ID: <201109071925.p87JPxII020042@scc.eng.rpath.com>

changeset:   d8844e50b295
user:        Elliot Peele <https://issues.rpath.com/>
date:        Wed, 07 Sep 2011 15:25:57 -0400

use workerId as an identifier instead of the job

diff --git a/updatebot/build/common.py b/updatebot/build/common.py
--- a/updatebot/build/common.py
+++ b/updatebot/build/common.py
@@ -149,17 +149,17 @@
         elif isinstance(job, set):
             job = frozenset(job)
 
-        if job in self._workers:
-            log.critical('job already being monitored: %s' % (job, ))
-            import epdb; epdb.st()
-            return
-
         args = list(self._threadArgs)
         args.append(job)
 
         worker = self.workerClass(self._status, args)
 
-        self._workers[job] = worker
+        if worker.workerId in self._workers:
+            log.critical('job already being monitored: %s' % (job, ))
+            import epdb; epdb.st()
+            return
+
+        self._workers[worker.workerId] = worker
         worker.daemon = True
         worker.start()
 

From elliot at rpath.com  Thu Sep 15 20:20:00 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 00:20:00 +0000
Subject: mirrorball: fix typo
Message-ID: <201109160020.p8G0K0BC019297@scc.eng.rpath.com>

changeset:   1d5726c6a07b
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 12 Sep 2011 21:48:52 -0400

fix typo

diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -388,7 +388,7 @@
             label = label.label()
 
         if label in self._cache.labelNevraCache:
-            return self._cache.labelClonedFromCache[label]
+            return self._cache.labelNevraCache[label]
 
         req = {None: {label: None}}
         binTrvMap = self._repos.getTroveVersionsByLabel(req)

From elliot at rpath.com  Thu Sep 15 20:20:00 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 00:20:00 +0000
Subject: mirrorball: whitespace cleanup
Message-ID: <201109160020.p8G0K0d8019328@scc.eng.rpath.com>

changeset:   b971fae9c679
user:        Elliot Peele <https://issues.rpath.com/>
date:        Wed, 14 Sep 2011 18:31:49 -0400

whitespace cleanup

diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -384,7 +384,8 @@
             if line in self._pkgSource.locationMap:
                 binPkg = self._pkgSource.locationMap[line]
                 srcPkg = self._pkgSource.binPkgMap[binPkg]
-            elif line.strip().endswith('.src.rpm') and self._cfg.synthesizeSources:
+            elif (line.strip().endswith('.src.rpm') and
+                  self._cfg.synthesizeSources):
                 # this is a fake source.  Move on.
                 continue
             else:
@@ -855,7 +856,8 @@
 
         # Take the basename of all paths in the manifest since the same rpm will
         # be in different repositories for each platform.
-        baseManifest = sorted([ dropArchRE.sub('', os.path.basename(x)) for x in manifest ])
+        baseManifest = sorted([ dropArchRE.sub('', os.path.basename(x))
+            for x in manifest ])
         parentBaseManifest = sorted([ dropArchRE.sub('', os.path.basename(x))
                                       for x in parentManifest ])
         # baseManifest = sorted([ os.path.basename(x) for x in manifest ])
@@ -863,21 +865,36 @@
         #                               for x in parentManifest ])
 
         if baseManifest != parentBaseManifest:
-            if srcPkg.getFileName() in self._cfg.expectParentManifestDifferences:
-                if srcPkg.getFileName() in baseManifest and srcPkg.getFileName() in parentBaseManifest:
-                    log.info('%s: found expected difference in manifests between parent and child platforms, ignoring parent platform' % srcPkg)
+            if (srcPkg.getFileName() in
+                self._cfg.expectParentManifestDifferences):
+
+                if (srcPkg.getFileName() in baseManifest and
+                    srcPkg.getFileName() in parentBaseManifest):
+
+                    log.info('%s: found expected difference in manifests '
+                        'between parent and child platforms, ignoring parent '
+                        'platform' % srcPkg)
                     return None
                 else:
                     # This is basically an assertion.
-                    log.error('%s: unexpected manifest error between parent and child platforms: %s not found in both manifests' % (srcName, srcPkg))
-                    raise ParentPlatformManifestInconsistencyError(srcPkg=srcPkg, manifest=manifest, parentManifest=parentManifest)
+                    log.error('%s: unexpected manifest error between parent '
+                        'and child platforms: %s not found in both manifests'
+                        % (srcName, srcPkg))
+                    raise ParentPlatformManifestInconsistencyError(
+                        srcPkg=srcPkg, manifest=manifest,
+                        parentManifest=parentManifest)
+
             if self._cfg.ignoreAllParentManifestDifferences:
-                log.warn('%s: found matching parent trove, but manifests differ; soldiering onward to madness--thank goodness this is a dry run...' % srcPkg)
+                log.warn('%s: found matching parent trove, but manifests '
+                    'differ; soldiering onward to madness--thank goodness this '
+                    'is a dry run...' % srcPkg)
                 import epdb ; epdb.st()
                 return None
             else:
-                log.error('%s: found matching parent trove, but manifests differ' % srcPkg)
-                raise ParentPlatformManifestInconsistencyError(srcPkg=srcPkg, manifest=manifest, parentManifest=parentManifest)
+                log.error('%s: found matching parent trove, but manifests '
+                    'differ' % srcPkg)
+                raise ParentPlatformManifestInconsistencyError(srcPkg=srcPkg,
+                    manifest=manifest, parentManifest=parentManifest)
 
         return srcVersion
 

From elliot at rpath.com  Thu Sep 15 20:20:01 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 00:20:01 +0000
Subject: mirrorball: allow downstream consumers of the bot class to pass
	in a list of already well formed nvf -> srcPkg pairs to be updated from
	-> to.
Message-ID: <201109160020.p8G0K1Ga019355@scc.eng.rpath.com>

changeset:   693f27f1b6b1
user:        Elliot Peele <https://issues.rpath.com/>
date:        Thu, 15 Sep 2011 20:11:01 -0400

allow downstream consumers of the bot class to pass in a list of already well formed nvf -> srcPkg pairs to be updated from -> to.

diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -179,7 +179,7 @@
         return trvMap, failed
 
     def update(self, force=None, updatePkgs=None, expectedRemovals=None,
-        allowPackageDowngrades=None):
+        allowPackageDowngrades=None, updateTroves=None):
         """
         Update the conary repository from the yum repositories.
         @param force: list of packages to update without exception
@@ -192,14 +192,16 @@
                                        from/to.
         @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
         @type expectedRemovals: set of package names
+        @param updateTroves: overrides the value of updatePkgs. Set of (n, v, f)
+            tuple to update from and source package to update to.
+        @type updateTroves: set(((n, v, f), srcPkg))
         """
 
         if force is not None:
             self._cfg.disableUpdateSanity = True
             assert isinstance(force, list)
 
-        updateTroves = None
-        if updatePkgs:
+        if updatePkgs and not updateTroves:
             updateTroves = set(((x.name, None, None), x) for x in updatePkgs)
 
         start = time.time()

From elliot at rpath.com  Thu Sep 15 20:20:01 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 00:20:01 +0000
Subject: mirrorball: use empty string for null epochs instead of None
Message-ID: <201109160020.p8G0K1UF019382@scc.eng.rpath.com>

changeset:   f8f9e26f20d9
user:        Elliot Peele <https://issues.rpath.com/>
date:        Thu, 15 Sep 2011 20:12:14 -0400

use empty string for null epochs instead of None

diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -365,7 +365,12 @@
             if ti is None:
                 return None
 
-            return (ti.rpm.name(), ti.rpm.epoch(), ti.rpm.version(),
+            if ti.rpm.epoch() is None:
+                epoch = ''
+            else:
+                epoch = str(ti.rpm.epoch())
+
+            return (ti.rpm.name(), epoch, ti.rpm.version(),
                 ti.rpm.release(), ti.rpm.arch())
 
         self._cacheTroveInfo(troveSpecs, self._cache.nevraCache,

From elliot at rpath.com  Thu Sep 15 20:20:02 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 00:20:02 +0000
Subject: mirrorball: cleanup and working package builds
Message-ID: <201109160020.p8G0K2t9019411@scc.eng.rpath.com>

changeset:   f4f3bbaae741
user:        Elliot Peele <https://issues.rpath.com/>
date:        Thu, 15 Sep 2011 20:12:38 -0400

cleanup and working package builds

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -20,32 +20,65 @@
 import pickle
 import logging
 import tempfile
-import itertools
 
 from conary import versions
 from conary.deps import deps
 
-from updatebot import errata
 from updatebot import groupmgr
-from updatebot import conaryhelper
 from updatebot.lib import util
-from updatebot.lib import watchdog
 from updatebot.bot import Bot as BotSuperClass
 
-from updatebot.errors import SourceNotImportedError
 from updatebot.errors import UnknownRemoveSourceError
 from updatebot.errors import PlatformNotImportedError
-from updatebot.errors import TargetVersionNotFoundError
-from updatebot.errors import PromoteMissingVersionError
-from updatebot.errors import PromoteFlavorMismatchError
 from updatebot.errors import PlatformAlreadyImportedError
-from updatebot.errors import FoundModifiedNotImportedErrataError
-from updatebot.errors import UnhandledUpdateError
 
-# FIXME: update the logger for current mode
-#log = logging.getLogger('updatebot.ordered')
 log = logging.getLogger('updatebot.current')
 
+class UpdateSet(object):
+    """
+    Basic structure for iterating over a set of update packages.
+    """
+
+    def __init__(self, updatePkgs):
+        self._updatePkgs = updatePkgs
+
+    def __len__(self):
+        return len(self._updatePkgs)
+
+    def __iter__(self):
+        """
+        Update pacakges in NEVRA order if we can.
+        """
+
+        data = {}
+        for srcPkg in self._updatePkgs:
+            data.setdefault(srcPkg.name, set()).add(srcPkg)
+
+        while data:
+            job = []
+            toRemove = []
+            for n, nevras in data.iteritems():
+                nevra = sorted(nevras)[0]
+                nevras.remove(nevra)
+                job.append(nevra)
+
+                if not nevras:
+                    toRemove.append(n)
+
+            for n in toRemove:
+                data.pop(n)
+
+            yield job
+
+    def filterPkgs(self, fltr):
+        if not fltr:
+            return
+        self._updatePkgs = fltr(self._updatePkgs)
+
+    def pop(self):
+        return self._updatePkgs.pop()
+
+
 class Bot(BotSuperClass):
     """
     Implement package driven create/update interface.
@@ -55,18 +88,13 @@
 
     _create = BotSuperClass.create
     _update = BotSuperClass.update
-    # REMOVED errataSource from __init__
+
     def __init__(self, cfg):
         BotSuperClass.__init__(self, cfg)
-        # FIXME: remove errata stuff
-        #self._errata = errata.ErrataFilter(self._cfg, self._ui, self._pkgSource,
-        #    errataSource)
-        
+
         self._groupmgr = groupmgr.GroupManager(self._cfg, self._ui,
             useMap=self._pkgSource.useMap)
-        
-        self._conaryhelper = conaryhelper.ConaryHelper(self._cfg)
-        
+
         if self._cfg.platformSearchPath:
             self._parentGroup = groupmgr.GroupManager(self._cfg, self._ui,
                                                       parentGroup=True)
@@ -212,14 +240,14 @@
 
         for binSet in pkgMap.itervalues():
             pkgs = {}
-            
+
             addGroup = False
             if not binSet:
                 log.warn('Empty bin set. BAD.')
                 import epdb; epdb.st()
- 
+
             nevraMap = self._conaryhelper.getNevras(binSet)
-            
+
             for conaryVer, nevra in nevraMap.iteritems():
                 if nevra:
                     curPkg = mkNevraObj(nevra)
@@ -259,114 +287,12 @@
                     for f in flavors:
                         log.info('\t%s' % f)
                     group.addPackage(name, version, flavors)
- 
+
 
         # FOR TESTING WE SHOULD INSPECT THE PKGMAP HERE
         print "REMOVE LINE AFTER TESTING"
         import epdb; epdb.st()
 
-
-    def _getAllPkgVersionsLabel(self):
-        """
-        get all versions of a package from the buildLabel 
-        and return 2 dictionaries
-        allPackageVersions
-        allSourceVersions
-        """
-
-        log.info('querying buildLabel %s for all committed packages' %
-                 self._updater._conaryhelper._ccfg.buildLabel)
-
-        allPackageVersions = {}
-
-        # Get all versions of all on buildLabel committed to repo.
-        allPkgVersions = self._updater._conaryhelper._repos.getTroveVersionsByLabel(
-                                {None: {self._updater._conaryhelper._ccfg.buildLabel: None}})
-
-        for binary, versions in allPkgVersions.iteritems():
-
-            if ( binary.endswith(':debuginfo') or 
-                binary.endswith(':source')):
-                 continue
-
-            allPackageVersions[binary] = [ x.versions[1].version 
-                                                for x in versions.keys() ]
-
-
-        allSourceVersions = {}
-
-        # Build dict of all source versions found in repo.
-        for source, versions in allPkgVersions.iteritems():
-            if source.endswith(':source'):
-                allSourceVersions[source.replace(':source', '')] = [ x.versions[1].version 
-                                                                        for x in versions.keys() ]
-
-
-        return allPackageVersions, allSourceVersions
-
-
-
-    def _diffRepos(self, allVersions, pkgSrcType=None, debug=0):
-        """
-        Diff the yum/rpm repo vs the label and return lists
-        @pkgSrcType = 'bin' or None -- At this time no need for binary  
-        @allSource = self._pkgSource.srcNameMap or self._pkgSource.binNameMap
-        @toUpdate = [(rpm)] set of rpms to be updated
-        """
-        
-        allSource = self._pkgSource.srcNameMap
-        
-        if pkgSrcType:
-            allSource = self._pkgSource.binNameMap
-            
-        toUpdate = []
-
-        for pkgName, pkgSet in allSource.iteritems():
-            if len(pkgSet) == 0:
-                continue
-
-            pkgList = list(pkgSet)
-
-            # Some attempt to retain order...
-            pkgList.sort(util.rpmvercmp)
-
-            for pkgPkg in pkgList:
-                onLabel = []
-                log.info('Working on %s package' % pkgPkg)
-
-                version = util.srpmToConaryVersion(pkgPkg)
-                #version = pkgPkg.getConaryVersion() # Only works on src
-             
-                # FAST lookup
-                if pkgPkg.name in allVersions:
-                    onLabel = [ x for x in allVersions[pkgPkg.name] if
-                                                x == version ] 
-
-                # This is not the way to do this
-                # Use for debuging only 
-                # maybe look up by sha1 sum as well...               
-                # SLOW  -- Really this is slow
-                if debug: 
-                    log.debug('looking up %s on the label' % pkgPkg)
-                    onLabel = self._conaryhelper.findTrove((pkgPkg.name, 
-                                               version, None))
-
-                if onLabel:
-                    log.info('MATCH:version %s of %s  already on the label'
-                                % (version, pkgPkg.name))
-                    continue
-
-                log.info('Version %s %s not on label adding to update'
-                            % (version, pkgPkg.name))
-                if pkgPkg.name not in allVersions:
-                    log.warn('%s is a new pkg adding to create' % pkgPkg.name)
-                    toUpdate.append(pkgPkg)
-                else:
-                    log.info('%s added to update list' % pkgPkg)
-                    toUpdate.append(pkgPkg)
-                    
-        return toUpdate
-
     def _removeSource(self, updateId, group):
         # Handle the case of entire source being obsoleted, this causes all
         # binaries from that source to be removed from the group model.
@@ -430,27 +356,144 @@
 
         return pkgMap
 
+    def _getLatestNevraPackageMap(self, nevraMap):
+        """
+        Take a mapping of nvf to nevra and transform it into nevra to latest
+        package nvf.
+        """
+
+        nevras = {}
+        for (n, v, f), nevra in nevraMap.iteritems():
+            # Skip nvfs that don't have a nevra
+            if not nevra:
+                continue
+
+            # Skip sources
+            if v.isSourceVersion():
+                continue
+
+            # Repack nevra subbing '0' for '', since yum metadata doesn't
+            # represent epochs of None.
+            if nevra[1] == '':
+                nevra = (nevra[0], '0', nevra[2], nevra[3], nevra[4])
+
+            pkgNvf = (n.split(':')[0], v, f)
+            nevras.setdefault(nevra, set()).add(pkgNvf)
+
+        ret = {}
+        for nevra, pkgs in nevras.iteritems():
+            ret[nevra] = sorted(pkgs)[-1]
+
+        return ret
+
+    def _getPromotePackages(self):
+        """
+        Get the packages that have beeen built on the buildLabel, but have yet
+        to be promoted to the target label. You should only run accross these if
+        the promote step in the package build failed.
+        """
+
+        helper = self._updater._conaryhelper
+
+        # Get all of the nevras for the target and source labels.
+        log.info('querying target nevras')
+        targetNevras = helper.getNevrasForLabel(self._cfg.targetLabel)
+        log.info('querying source nevras')
+        sourceNevras = helper.getNevrasForLabel(helper._ccfg.buildLabel)
+
+        # Massage nevra maps into nevra -> latest package nvf
+        targetLatest = self._getLatestNevraPackageMap(targetNevras)
+        sourceLatest = self._getLatestNevraPackageMap(sourceNevras)
+
+        # Now we need to map the target nvfs back the source nvfs they were
+        # cloned from.
+        log.info('querying target cloned from information')
+        targetClonedFrom = helper.getClonedFromForLabel(self._cfg.targetLabel)
+
+        # Now diff the two maps. We are looking for things that have been
+        # updated on the source label, that have not made it to the target
+        # label.
+        toPromote = set()
+        for nevra, nvf in sourceLatest.iteritems():
+            # nevra has not been promoted.
+            if nevra not in targetLatest:
+                toPromote.add(nvf)
+            # the conary package containing this nevra has been rebuilt.
+            elif nvf not in targetClonedFrom:
+                toPromote.add(nvf)
+
+        return toPromote
+
+    def _getUpdatePackages(self):
+        """
+        Compare the upstream yum repository and the conary repository to figure
+        out what sources need to be updated.
+        """
+
+        # Get a mapping of nvf -> nevra
+        sourceNevras = self._updater._conaryhelper.getNevrasForLabel(
+            self._updater._conaryhelper._ccfg.buildLabel)
+
+        # Reverse the map and filter out old conary versions.
+        sourceLatest = self._getLatestNevraPackageMap(sourceNevras)
+
+        # Iterate over all of the available source rpms to find any versions
+        # that have not been imported into the conary repository.
+        toUpdate = set()
+        toUpdateMap = {}
+        for binPkg, srcPkg in self._pkgSource.binPkgMap.iteritems():
+            # Skip over source packages
+            if binPkg.arch == 'src':
+                continue
+
+            if binPkg.getNevra() not in sourceLatest:
+                toUpdateMap.setdefault(srcPkg, set()).add(binPkg)
+                toUpdate.add(srcPkg)
+
+        return UpdateSet(toUpdate)
+
+    def _getPreviousNVFForSrcPkg(self, srcPkg):
+        """
+        Figure out what this source package was meant to update.
+        """
+
+        # Get a mapping of nvf -> nevra
+        sourceNevras = self._updater._conaryhelper.getNevrasForLabel(
+            self._updater._conaryhelper._ccfg.buildLabel)
+
+        # Reverse the map and filter out old conary versions.
+        sourceLatest = self._getLatestNevraPackageMap(sourceNevras)
+
+        # Find the previous nevra
+        srcPkgs = sorted(self._pkgSource.srcNameMap.get(srcPkg.name))
+        idx = srcPkgs.index(srcPkg) - 1
+
+        # This is a new package
+        if idx < 0:
+            return (srcPkg.name, None, None)
+
+        binPkgs = [ x for x in self._pkgSource.srcPkgMap[srcPkgs[idx]]
+            if x.arch != 'src' ]
+
+        binPkg = binPkgs[0]
+
+        binNVF = sourceLatest[binPkg.getNevra()]
+        sourceVersions = self._updater._conaryhelper.getSourceVersions(
+            [binNVF, ]).keys()
+
+        assert len(sourceVersions) == 1
+        return sourceVersions[0]
 
     def update(self, *args, **kwargs):
         """
         Handle update case.
         """
 
-        # Load specific kwargs
-        restoreFile = kwargs.pop('restoreFile', None)
-        checkMissingPackages = kwargs.pop('checkMissingPackages', True)
-        
-        # Generate an updateId for the group so we can build several 
-        # groups a day without stomping on the name
-        #updateId = int(time.time())
-        groupId = int(time.time())
-
-        # Generate the updateID for config file.
-        # Not sure how to handle this yet so we are basing it on the day
-        # This would make it easier to control groups built on the 
-        # same day...
-        updateId = int(time.mktime(time.strptime(time.strftime('%Y%m%d', 
-                                        time.gmtime(time.time())), '%Y%m%d')))
+        # We don't use these for anything in current update mode, but need to
+        # pop them off the kwargs dict so as to not confuse the lower level
+        # update code.
+        kwargs.pop('checkMissingPackages', None)
+        kwargs.pop('restoreFile', None)
 
         # Get current group
         group = self._groupmgr.getGroup()
@@ -460,31 +503,73 @@
         if current is None:
             raise PlatformNotImportedError
 
+        # Get the latest errata state, increment if the source has been built.
+        if group.hasBinaryVersion():
+            group.errataState += 1
+        updateId = group.errataState
 
         # Load package source.
         self._pkgSource.load()
 
         log.info('starting update run')
 
-        count = 0
         startime = time.time()
-        updateSet = {}
-        
-        # Almost completely unnecessary now... need to remove at some point.
-        allPackageVersions, allSourceVersions = self._getAllPkgVersionsLabel()
 
-        # Using this to keep track of what the label looked like before we 
-        # promote all the pkgs we are building and for building groups
-        allNevrasOnLabel = self._conaryhelper.getNevrasForLabel(
-                                            self._conaryhelper._ccfg.buildLabel)
+        # Figure out what packages still need to be promoted.
+        promotePkgs = self._getPromotePackages()
 
-        # Diff the repos based on the label 
-        # FIXME: I believe seperating srcUpdate and update is not neccessary 
-        srcUpdate = self._diffRepos(allVersions=allSourceVersions, debug=0)
+        # Go ahead and promote any packages that didn't get promoted during the
+        # last run or have been rebuilt since then.
+        log.info('found %s packages that need to be promoted' %
+            len(promotePkgs))
+        self._updater.publish(promotePkgs, promotePkgs, self._cfg.targetLabel)
 
-        # TESTING
-        #print "need to check toUpdate"
-        #import epdb ; epdb.st() 
+        # Figure out what packages need to be updated.
+        updatePkgs = self._getUpdatePackages()
+
+        # remove packages from config
+        removePackages = self._cfg.updateRemovesPackages.get(updateId, [])
+        removeObsoleted = self._cfg.removeObsoleted.get(updateId, [])
+        removeReplaced = self._cfg.updateReplacesPackages.get(updateId, [])
+
+        # take the union of the three lists to get a unique list of packages
+        # to remove.
+        expectedRemovals = (set(removePackages) |
+                            set(removeObsoleted) |
+                            set(removeReplaced))
+
+        # Update package set.
+        if updatePkgs:
+            updatePkgs.filterPkgs(kwargs.pop('fltr', None))
+
+            for updateSet in updatePkgs:
+                updateTroves = set([ (self._getPreviousNVFForSrcPkg(x), x)
+                    for x in updateSet])
+                self._update(*args, updateTroves=updateTroves, updatePkgs=True,
+                    expectedRemovals=expectedRemovals, **kwargs)
+                # The NEVRA maps will be changing every time through. Make sure
+                # the clear the cache.
+                self._updater._conaryhelper.clearCache()
+
+    def buildgroups(self):
+        """
+        Find the latest packages on the production label by nevra and build a
+        group, taking into account any packages that would have been obsoleted
+        along the way.
+        """
+
+        # Get current group
+        group = self._groupmgr.getGroup()
+
+        # Get current timestamp
+        current = group.errataState
+        if current is None:
+            raise PlatformNotImportedError
+
+        # Get the latest errata state, increment if the source has been built.
+        if group.hasBinaryVersion():
+            group.errataState += 1
+        updateId = group.errataState
 
         # remove packages from config
         removePackages = self._cfg.updateRemovesPackages.get(updateId, [])
@@ -492,8 +577,8 @@
         # FIXME: NEED A FIX for this... since group and pkgs are built seperate
         # once the new package is built it should be added to the group...
         # however if the group build fails and the pkg is not replacement 
-        # package is not added it wouldn't be added again until a new new version
-        # comes around... bad
+        # package is not added it wouldn't be added again until a new new
+        # version comes around... bad
         removeReplaced = self._cfg.updateReplacesPackages.get(updateId, [])
 
         # take the union of the three lists to get a unique list of packages
@@ -511,74 +596,6 @@
         # Get the list of package that are allowed to be downgraded.
         allowDowngrades = self._cfg.allowPackageDowngrades.get(updateId, [])
 
-        updates = set()
-        
-        # Create the update set to pass to the updater
-        if srcUpdate:
-            updates.update(srcUpdate)    
-        
-        # If recovering from a failure, restore the pkgMap from disk.
-        pkgMap = {}
-        if restoreFile:
-            pkgMap = self._restorePackages(restoreFile)
-            restoreFile = None
-
-        # Filter out anything that has already been built from the list
-        # of updates.
-        upMap = dict([ (x.name, x) for x in updates ])
-        for n, v, f in pkgMap:
-            if n in upMap:
-                updates.remove(upMap[n])
-
-        # Update package set.
-        if updates:
-            fltr = kwargs.pop('fltr', None)
-            if fltr:
-                updates = fltr(updates)
-
-            # FIXME: For testing Only build one pkg. 
-            # Might add a config option to try and take human bites
-            upDates = set()
-            upDates.add([ x for x in updates ][0])
-            # This is to check what we are building before we build it
-            
-
-            # FOR TESTING WE SHOULD INSPECT THE PKGMAP HERE
-            #print "REMOVE LINE AFTER ALTERNATE REPO SETUP"
-            #import epdb; epdb.st()
-            
-            # FIXME: For testing purpose we are taking human bites
-            pkgMap.update(self._update(*args, updatePkgs=upDates,
-                expectedRemovals=expectedRemovals,
-                allowPackageDowngrades=allowDowngrades, **kwargs))
-
-            #pkgMap.update(self._update(*args, updatePkgs=updates,
-            #    expectedRemovals=expectedRemovals,
-            #    allowPackageDowngrades=allowDowngrades, **kwargs))
-
-        # FOR TESTING WE SHOULD INSPECT THE PKGMAP HERE
-        #print "REMOVE LINE AFTER TESTING"
-        #import epdb; epdb.st()
-
-        # Save package map in case we run into trouble later.
-        self._savePackages(pkgMap)
-
-        # Start Group Build
-
-        # Get current group
-        group = self._groupmgr.getGroup()
-
-        # Check to see if there is a binary version if the current group.
-        # This handles restarts where the group failed to build, but we don't
-        # want to rebuild all of the packages again.
-        if not group.hasBinaryVersion():
-            group.build()
-
-        # Get current timestamp
-        current = group.errataState
-        if current is None:
-            raise PlatformNotImportedError        
-
         # Generate grpPkgMap
 
         #grpPkgNameMap = [ (x.name, x.version, x.flavor) for x in group.iterpackages() ]
@@ -620,11 +637,9 @@
 
         # Get timestamp version.
         # Changing the group version to be more granular than just day
-        # This is to avoid building the same group over and over on the 
-        # same day... to change back groupId == updateId
-        version = time.strftime('%Y.%m.%d_%H%M.%S',time.gmtime(groupId))
-        if not version:
-            version = 'unknown.%s' % groupId
+        # This is to avoid building the same group over and over on the
+        # same day...
+        version = time.strftime('%Y.%m.%d_%H%M.%S', time.gmtime(time.time()))
 
         # Build groups.
         log.info('setting version %s' % version)
@@ -638,7 +653,6 @@
             log.info('promoting group %s ' % group.version)
             # Add promote code here
 
-
         updateSet.update(pkgMap)
 
         # Report timings
@@ -652,12 +666,10 @@
         # already-built job id's.  (See long comment above.)
         #import epdb ; epdb.st()
 
-
-
         # TESTING
         print "need to check updates set"
-        import epdb ; epdb.st() 
- 
+        import epdb ; epdb.st()
+
         # BELOW THIS LINE WILL BE REMOVED
 
         # FIXME: remove errata crap
@@ -669,361 +681,6 @@
 
         return updateSet
 
-    def promote(self, enforceAllExpected=True, checkMissingPackages=True):
-        """
-        Promote binary groups from the devel label to the production lable in
-        the order that they were built.
-        """
-
-        # Get current timestamp
-        current = self._groupmgr.latest.errataState
-        if current is None:
-            raise PlatformNotImportedError
-
-        # laod package source
-        self._pkgSource.load()
-
-        if checkMissingPackages:
-            # Ensure no packages are missing from repository.
-            self._errata.sanityCheckOrder()
-            missingPackages, missingOrder = self._checkMissingPackages()
-            if len(missingPackages):
-                raise UnhandledUpdateError(why='missing %s ordered source packages from repository' % len(missingPackages))
-
-        log.info('querying repository for all group versions')
-        sourceLatest = self._updater.getUpstreamVersionMap(self._cfg.topGroup)
-
-        log.info('querying target label for all group versions')
-        targetLatest = self._updater.getUpstreamVersionMap(
-            (self._cfg.topGroup[0], self._cfg.targetLabel, None))
-
-        log.info('querying target label for cloned from information')
-        # The targetSpec list tells us if the latest version of each group has
-        # been promoted to the target label.
-        sourceSpecs = [ x for x in itertools.chain(*sourceLatest.itervalues()) ]
-        targetSpecs, failed = self._updater.getTargetVersions(sourceSpecs,
-            logErrors=False)
-        targetSpecMap = {}
-        for spec in targetSpecs:
-            ver = spec[1].trailingRevision().getVersion()
-            targetSpecMap.setdefault(ver, set()).add(spec)
-
-        log.info('starting promote')
-
-        count = 0
-        startime = time.time()
-
-        # Get all updates after the first bucket.
-        missing = False
-        for updateId, bucket in self._errata.iterByIssueDate(current=-1):
-            upver = self._errata.getBucketVersion(updateId)
-
-            if updateId <= self._cfg.errataPromoteAfter:
-                log.info('version %s (%s) at or below promotion timestamp threshold (%s), skipping' % (upver, updateId, self._cfg.errataPromoteAfter))
-                continue
-            # FOR SLES FIX 
-            #if upver == '2011.07.13_0800.00': 
-            #    import epdb; epdb.st()    
-            #if upver >= '2011.07.18_0400.00': 
-              #  import epdb; epdb.st()    
-            # Don't try to promote buckets that have already been promoted.
-            if upver in targetLatest and upver in targetSpecMap:
-                log.info('%s found on target label, skipping' % upver)
-                continue
-            elif upver not in targetSpecMap:
-                # FIXME: This log message is confusing.
-                log.info('%s found on target label, but newer version '
-                         'available on source, will repromote' % upver)
-
-            # Make sure version has been imported.
-            if upver not in sourceLatest:
-                missing = upver
-                continue
-
-            # If we find a missing version and then find a version in the
-            # repository report an error.
-            if missing:
-                log.critical('found missing version %s' % missing)
-                raise PromoteMissingVersionError(missing=missing, next=upver)
-
-            log.info('starting promote of %s' % upver)
-
-            # Get conary versions to promote
-            toPromote = sourceLatest[upver]
-
-            # Make sure we have the expected number of flavors
-            if len(set(x[2] for x in toPromote)) != len(self._cfg.groupFlavors):
-                log.error('did not find expected number of flavors')
-                raise PromoteFlavorMismatchError(
-                    cfgFlavors=self._cfg.groupFlavors, troves=toPromote,
-                    version=toPromote[0][1])
-
-            # Find excepted promote packages.
-            srcPkgMap = self._updater.getBinaryVersionsFromSourcePackages(
-                bucket)
-            exceptions = dict([ (x[0], x[1]) for x in itertools.chain(
-                *self._getOldVersionExceptions(updateId).itervalues()) ])
-
-
-            advisories = [ x['name'] for x in
-                           self._errata.getUpdateDetail(updateId) ]
-
-            for advisory in advisories:
-                # Handle attaching an update that was caused by changes that we
-                # made outside of the normal update stream to an existing
-                # advisory.
-                for nvf in self._cfg.extendAdvisory.get(advisory, ()):
-                    srcMap = self._updater.getSourceVersionMapFromBinaryVersion(
-                        nvf, labels=self._cfg.platformSearchPath,
-                        latest=False, includeBuildLabel=True)
-                    assert len(srcMap) == 1
-                    srcPkgMap.update(srcMap)
-
-            # These are the binary trove specs that we expect to be promoted.
-            expected = self._filterBinPkgSet(
-                itertools.chain(*srcPkgMap.itervalues()), exceptions)
-
-            # Get list of extra troves from the config
-            extra = self._cfg.extraExpectedPromoteTroves.get(updateId, [])
-
-            def promote():
-                # Create and validate promote changeset
-                packageList = self._updater.publish(toPromote, expected,
-                    self._cfg.targetLabel, extraExpectedPromoteTroves=extra,
-                    enforceAllExpected=enforceAllExpected)
-                return 0
-
-            rc = watchdog.waitOnce(promote)
-            if rc:
-                break
-            count += 1
-
-        log.info('promote complete')
-        log.info('promoted %s groups in %s seconds'
-            % (count, time.time() - startime))
-
-    def createErrataGroups(self, rebuildGroups=False):
-        """
-        Create groups for each advisory that only contain the versions of
-        packages that were included in that advisory. Once created, promote
-        to production branch.
-        """
-
-        # Get current timestamp
-        current = self._groupmgr.latest.errataState
-        if current is None:
-            raise PlatformNotImportedError
-
-        # Load package source.
-        self._pkgSource.load()
-
-        # Get latest errataState from the targetLabel so that we can
-        # fence group building based on the target label state.
-        targetGroup = groupmgr.GroupManager(self._cfg, self._ui,
-                                            targetGroup=True)
-        targetErrataState = targetGroup.latest.errataState
-
-        log.info('starting errata group processing')
-
-        count = 0
-        startime = time.time()
-
-        for updateId, updates in self._errata.iterByIssueDate(current=0):
-            # Stop if the updateId is greater than the state of the
-            # latest group on the production label.
-            if updateId > targetErrataState:
-                log.info('current updateId (%s) is newer than target label '
-                    'contents' % updateId)
-                break
-
-            # Make sure the group representing the current updateId has been
-            # imported and promoted to the production label.
-            version = self._errata.getBucketVersion(updateId)
-            if not targetGroup.hasBinaryVersion(sourceVersion=version):
-                raise TargetVersionNotFoundError(version=version,
-                                                 updateId=updateId)
-
-            log.info('%s: looking up version exceptions' % updateId)
-            # Lookup any places we need to use old versions ahead of time.
-            multiVersionExceptions = dict([
-                (x[0], x[1]) for x in itertools.chain(
-                    self._updater.getTargetVersions(itertools.chain(
-                *self._getOldVersionExceptions(updateId).itervalues()
-                    ))[0]
-                )
-            ])
-
-            # Now that we know that the packages that are part of this update
-            # should be on the target label we can separate things into
-            # advisories.
-            mgr = groupmgr.ErrataGroupManagerSet(self._cfg, self._ui)
-            groupNames = self._errata.getNames(updateId)
-            for advInfo in self._errata.getUpdateDetail(updateId):
-                advisory = advInfo['name']
-                log.info('%s: processing' % advisory)
-
-                srcPkgs = self._errata.getAdvisoryPackages(advisory)
-                if advisory in self._cfg.brokenErrata:
-                    # We expect srcPkgs to be empty for known-broken errata.
-                    log.warning('%s: skipping broken advisory' % advisory)
-                    continue
-                else:
-                    assert srcPkgs
-
-                targetGrp = groupmgr.SingleGroupManager(groupNames[advisory],
-                    self._cfg, self._ui, targetGroup=True)
-
-                if targetGrp.hasBinaryVersion() and not rebuildGroups:
-                    log.info('%s: found existing version, skipping' % advisory)
-                    continue
-
-                grp = mgr.newGroup(groupNames[advisory]).getGroup()
-                grp.version = version
-                grp.errataState = updateId
-
-                log.info('%s: finding built packages' % advisory)
-                binTrvMap = \
-                    self._updater.getBinaryVersionsFromSourcePackages(srcPkgs)
-
-                binTrvs = set()
-                for srcPkg, binTrvSpecs in binTrvMap.iteritems():
-                    targetSpecs, failed = self._updater.getTargetVersions(
-                        binTrvSpecs)
-                    binTrvs.update(set(targetSpecs))
-
-                # Handle attaching an update that was caused by changes that we
-                # made outside of the normal update stream to an existing
-                # advisory.
-                for nvf in self._cfg.extendAdvisory.get(advisory, ()):
-                    srcMap = self._updater.getSourceVersionMapFromBinaryVersion(
-                        nvf, labels=self._cfg.platformSearchPath,
-                        latest=False, includeBuildLabel=True)
-                    assert len(srcMap) == 1
-                    targetVersions = self._updater.getTargetVersions(
-                        srcMap.values()[0])[0]
-                    binTrvs.update(set(targetVersions))
-
-                # Group unique versions by flavor
-                nvfMap = {}
-                for n, v, f in self._filterBinPkgSet(binTrvs, multiVersionExceptions):
-                    # Taghandler components were moved to common label
-                    # and packages were rebuilt, so omit them here to
-                    # prevent older versions of packages being pulled in
-                    # (raising exceptions during commit) due solely to
-                    # their taghandler components (which are missing in
-                    # the newer versions).
-                    if n.endswith(':tagdescription') or n.endswith(':taghandler'):
-                        continue
-                    n = n.split(':')[0]
-                    nvfMap.setdefault((n, v), set()).add(f)
-
-                if rebuildGroups and targetGrp.hasBinaryVersion():
-                    # For comparing with rebuilt group model.
-                    oldGrp = {}
-                    for x in grp.iterpackages():
-                        if x.flavor != None:
-                            log.info('%s: found flavoring for %s=%s: %s' % (advisory, x.name, x.version, x.flavor))
-                        oldGrp.setdefault(x.name,set()).add((x.version,
-                                                             x.flavor))
-
-                    packagesToRemove = set()
-                    for packageToRemove in grp.iterpackages():
-                        packagesToRemove.add(packageToRemove.name)
-                    for packageToRemove in packagesToRemove:
-                        grp.removePackage(packageToRemove)
-
-                    # Should be empty:
-                    if len([ x for x in grp.iterpackages() ]):
-                        log.error('%s: group model not empty after pre-rebuild package removal' % advisory)
-                        # Change to assertion?
-                        import epdb ; epdb.st()
-
-                # Add packages to group model.
-                for (n, v), flvs in nvfMap.iteritems():
-                    log.info('%s: adding package %s=%s' % (advisory, n, v))
-                    for f in flvs:
-                        log.info('%s: %s' % (advisory, f))
-                    grp.addPackage(n, v, flvs)
-
-                if rebuildGroups and targetGrp.hasBinaryVersion():
-                    # Sanity-checking craziness, could use a haircut, perhaps
-                    # a move to its own function.
-                    newGrp = {}
-                    for x in grp.iterpackages():
-                        newGrp.setdefault(x.name,set()).add((x.version,
-                                                             x.flavor))
-
-                    # Compare old & new.
-                    for oldPkg, oldVF in oldGrp.iteritems():
-                        for oldV, oldF in oldVF:
-                            try:
-                                if not newGrp[oldPkg]:
-                                    raise KeyError
-                            except KeyError:
-                                log.error('%s: missing package %s in new group model' % (advisory, oldPkg))
-                                import epdb ; epdb.st()
-                                continue
-                            found = False
-                            oldVt = versions.ThawVersion(oldV).trailingRevision().getVersion()
-                            for newVF in newGrp[oldPkg]:
-                                if not oldF:
-                                    newVt = versions.ThawVersion(newVF[0]).trailingRevision().getVersion()
-                                    if oldVt == newVt:
-                                        log.info('%s: found matching version %s for %s in new group model' % (advisory, newVt, oldPkg))
-                                        if oldV != newVF[0]:
-                                            log.warning('%s: note difference in %s source/build versions due to rebuilt package: %s != %s' % (advisory, oldPkg, oldV, newVF[0]))
-                                        found = True
-                                        del newGrp[oldPkg]
-                                        break
-                                else:
-                                    newVt = versions.ThawVersion(newVF[0]).trailingRevision().getVersion()
-                                    if oldVt == newVt and oldF == newVF[1]:
-                                        log.info('%s: found matching version ' % advisory + '%s and flavor %s ' % (newVt, newVF[1]) + 'for %s in new group model' % oldPkg)
-                                        if oldV != newVF[0]:
-                                            log.warning('%s: note difference in %s source/build versions due to rebuilt package: %s != %s' % (advisory, oldPkg, oldV, newVF[0]))
-                                        found = True
-                                        newGrp[oldPkg].remove((newVF[0], oldF))
-                                        if not len(newGrp[oldPkg]):
-                                            log.info('%s: all versions/flavors matched for %s' % (advisory, oldPkg))
-                                            del newGrp[oldPkg]
-                                        break
-                            if not found:
-                                log.error('%s: cannot find %s for %s in new group model' % (advisory, oldV, oldPkg))
-                                import epdb ; epdb.st()
-                    
-                    if len(newGrp):
-                        log.error('%s: new group model has extra packages: %s' % (advisory, newGrp))
-                        import epdb ; epdb.st()
-                    else:
-                        log.info('%s: new group model package versions verified to match old group model' % advisory)
-                        log.info('%s: (note flavor match between old and new models may have been left unchecked for some packages)' % advisory)
-                    
-            # Make sure there are groups to build.
-            if not mgr.hasGroups():
-                log.info('%s: groups already built and promoted' % updateId)
-                continue
-
-            log.info('%s: committing group sources' % updateId)
-
-            mgr.commit()
-
-            log.info('%s: building groups' % updateId)
-            trvMap = mgr.build()
-
-            log.info('%s: promoting groups' % updateId)
-            # Setting expected to an empty tuple since we don't expect anything
-            # other than groups to be promoted.
-            expected = tuple()
-            toPromote = self._flattenSetDict(trvMap)
-            promoted = self._updater.publish(toPromote, expected,
-                                             self._cfg.targetLabel)
-
-            count += 1
-
-        log.info('completed errata group processing')
-        log.info('processed %s errata groups in %s seconds'
-            % (count, time.time() - startime))
-
     def _getOldVersionExceptions(self, updateId):
         versionExceptions = {}
         if updateId in self._cfg.useOldVersion:

From elliot at rpath.com  Fri Sep 16 01:32:07 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 05:32:07 +0000
Subject: mirrorball: more cleanup
Message-ID: <201109160532.p8G5W7Zb027465@scc.eng.rpath.com>

changeset:   5fec78502094
user:        Elliot Peele <https://issues.rpath.com/>
date:        Thu, 15 Sep 2011 21:24:42 -0400

more cleanup

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2009-2010 rPath, Inc.
+# Copyright (c) 2009-2011 rPath, Inc.
 #
 # This program is distributed under the terms of the Common Public License,
 # version 1.0. A copy of this license should have been distributed with this
@@ -13,16 +13,12 @@
 #
 
 """
-Module for doing updates ordered by errata information.
+Module for doing updates by importing all of the available packages and then
+building groups of the latest versions by nevra.
 """
 
 import time
-import pickle
 import logging
-import tempfile
-
-from conary import versions
-from conary.deps import deps
 
 from updatebot import groupmgr
 from updatebot.lib import util
@@ -361,6 +357,7 @@
         sourceNevras = helper.getNevrasForLabel(helper._ccfg.buildLabel)
 
         # Massage nevra maps into nevra -> latest package nvf
+        log.info('building latest nevra maps')
         targetLatest = self._getLatestNevraPackageMap(targetNevras)
         sourceLatest = self._getLatestNevraPackageMap(sourceNevras)
 
@@ -389,6 +386,8 @@
         out what sources need to be updated.
         """
 
+        log.info('finding updates')
+
         # Get a mapping of nvf -> nevra
         sourceNevras = self._updater._conaryhelper.getNevrasForLabel(
             self._updater._conaryhelper._ccfg.buildLabel)
@@ -519,7 +518,7 @@
                 self._updater._conaryhelper.clearCache()
 
         log.info('completed package update of %s packages in %s seconds'
-            % (len(updatePkgs), starttime-time.time()))
+            % (len(updatePkgs), time.time()-starttime))
 
     def buildgroups(self):
         """
@@ -658,30 +657,3 @@
                 versionExceptions.update(srcMap)
 
         return versionExceptions
-
-    def _filterBinPkgSet(self, binSet, exceptions):
-        binPkgs = {}
-        for n, v, f in binSet:
-            binPkgs.setdefault(n, dict()).setdefault(v, set()).add(f)
-
-        uniqueSet = set()
-        for n, vMap in binPkgs.iteritems():
-            # Handle the case where a package has been rebuilt for some
-            # reason, but we need to use the old version of the package.
-            pkgName = n.split(':')[0]
-            if len(vMap) > 1:
-                if pkgName in exceptions:
-                    log.info('using old version of %s' % n)
-                    vMap = dict((x, y) for x, y in vMap.iteritems()
-                                if x == exceptions[pkgName])
-                else:
-                    log.info('found multiple versions of %s, using latest' % n)
-                    v = sorted(vMap)[-1]
-                    vMap = { v: vMap[v], }
-
-            assert len(vMap) == 1
-
-            for v, flvs in vMap.iteritems():
-                uniqueSet.update(set((n, v, f) for f in flvs))
-
-        return uniqueSet

From elliot at rpath.com  Fri Sep 16 01:32:06 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 05:32:06 +0000
Subject: mirrorball: remove unused methods, add comments, and some logging
Message-ID: <201109160532.p8G5W71U027447@scc.eng.rpath.com>

changeset:   a2df8e5521f1
user:        Elliot Peele <https://issues.rpath.com/>
date:        Thu, 15 Sep 2011 21:14:58 -0400

remove unused methods, add comments, and some logging

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -138,51 +138,6 @@
         if addPackages or removePackages:
             group.modifyContents(additions=addPackages, removals=removePackages)
 
-    def _savePackages(self, pkgMap, fn=None):
-        """
-        Save the package map to a file.
-        """
-
-        if fn is None:
-            fn = tempfile.mktemp()
-
-        log.info('saving package map to file: %s' % fn)
-
-        # freeze contents
-        frzPkgs = dict([ ((x[0], x[1].freeze(), x[2]),
-                          set([ (z[0], z[1].freeze(), z[2].freeze())
-                                for z in y]))
-                          for x, y in pkgMap.iteritems() ])
-
-        # pickle frozen contents
-        pickle.dump(frzPkgs, open(fn, 'w'))
-
-    def _restorePackages(self, fn):
-        """
-        Restore the frozen form of the package map.
-        """
-
-        log.info('restoring package map from file: %s' % fn)
-
-        thawVersion = versions.ThawVersion
-
-        def thawFlavor(flv):
-            if flv is None:
-                return flv
-            else:
-                return deps.ThawFlavor(flv)
-
-        # load pickle
-        frzPkgs = pickle.load(open(fn))
-
-        # thaw versions and flavors
-        pkgMap = dict([ ((x[0], thawVersion(x[1]), thawFlavor(x[2])),
-                         set([ (z[0], thawVersion(z[1]), thawFlavor(z[2]))
-                               for z in y ]))
-                        for x, y in frzPkgs.iteritems() ])
-
-        return pkgMap
-
     def create(self, *args, **kwargs):
         """
         Handle initial import case.
@@ -197,7 +152,11 @@
             raise PlatformAlreadyImportedError
 
         self._pkgSource.load()
-        toCreate = self._errata.getInitialPackages()
+
+        # FIXME: Need to determine the initial set of packages to import. Maybe
+        #        we find the first time every nevra appears or maybe we just
+        #        import all of the packages we can see and build a latest group?
+        toCreate = set()
 
         fltr = kwargs.pop('fltr', None)
         if fltr:
@@ -457,6 +416,9 @@
         Figure out what this source package was meant to update.
         """
 
+        # FIXME: Should make this take a list of srcPkgs so that we can
+        #        avoid multiple repository calls.
+
         # Get a mapping of nvf -> nevra
         sourceNevras = self._updater._conaryhelper.getNevrasForLabel(
             self._updater._conaryhelper._ccfg.buildLabel)
@@ -513,7 +475,7 @@
 
         log.info('starting update run')
 
-        startime = time.time()
+        starttime = time.time()
 
         # Figure out what packages still need to be promoted.
         promotePkgs = self._getPromotePackages()
@@ -543,14 +505,22 @@
             updatePkgs.filterPkgs(kwargs.pop('fltr', None))
 
             for updateSet in updatePkgs:
+                log.info('building set of update troves')
                 updateTroves = set([ (self._getPreviousNVFForSrcPkg(x), x)
                     for x in updateSet])
+
+                log.info('running update')
                 self._update(*args, updateTroves=updateTroves, updatePkgs=True,
                     expectedRemovals=expectedRemovals, **kwargs)
+
                 # The NEVRA maps will be changing every time through. Make sure
                 # the clear the cache.
+                log.info('dumping conaryhelper cache')
                 self._updater._conaryhelper.clearCache()
 
+        log.info('completed package update of %s packages in %s seconds'
+            % (len(updatePkgs), starttime-time.time()))
+
     def buildgroups(self):
         """
         Find the latest packages on the production label by nevra and build a
@@ -574,11 +544,6 @@
         # remove packages from config
         removePackages = self._cfg.updateRemovesPackages.get(updateId, [])
         removeObsoleted = self._cfg.removeObsoleted.get(updateId, [])
-        # FIXME: NEED A FIX for this... since group and pkgs are built seperate
-        # once the new package is built it should be added to the group...
-        # however if the group build fails and the pkg is not replacement 
-        # package is not added it wouldn't be added again until a new new
-        # version comes around... bad
         removeReplaced = self._cfg.updateReplacesPackages.get(updateId, [])
 
         # take the union of the three lists to get a unique list of packages
@@ -586,6 +551,7 @@
         expectedRemovals = (set(removePackages) |
                             set(removeObsoleted) |
                             set(removeReplaced))
+
         # The following packages are expected to exist and must be removed
         # (removeObsoleted may be mentioned for buckets where the package
         # is not in the model, in order to support later adding the ability

From elliot at rpath.com  Fri Sep 16 01:32:07 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 16 Sep 2011 05:32:07 +0000
Subject: mirrorball: wait until job is committed and has results
Message-ID: <201109160532.p8G5W77M027498@scc.eng.rpath.com>

changeset:   4a0bc4e3cd78
user:        Elliot Peele <https://issues.rpath.com/>
date:        Fri, 16 Sep 2011 01:31:45 -0400

wait until job is committed and has results

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -476,18 +476,13 @@
                 if state != buildjob.JOB_STATE_COMMITTED:
                     continue
 
+                # It might take a few iterations through the loop for the
+                # result to show up.
+                if not result:
+                    continue
+
                 # Make result hashable
-                # FIXME:
-                # Ran into an instance where the results were none... \
-                # this is probably bad but for now we are going to
-                # ignore it and move on
-                if result:
-                    res = tuple([ (x, tuple(y)) for x, y in result.iteritems() ])
-                else:
-                    log.error('jobId %s returned None for result' % jobId)
-                    log.warn('This is probably bad FIXME')
-                    import epdb; epdb.st()
-                    continue
+                res = tuple([ (x, tuple(y)) for x, y in result.iteritems() ])
 
                 toPromote.append((jobId, res))
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTING

From elliot at rpath.com  Mon Sep 19 12:32:03 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 16:32:03 +0000
Subject: mirrorball: only log builds that are not in the "completed" state
Message-ID: <201109191632.p8JGW3gA002230@scc.eng.rpath.com>

changeset:   b164227b9257
user:        Elliot Peele <https://issues.rpath.com/>
date:        Fri, 16 Sep 2011 15:24:29 -0400

only log builds that are not in the "completed" state

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -56,7 +56,9 @@
         Check if all jobs are complete.
         """
 
-        log.info('waiting for the following jobs: %s' % ','.join(map(str, self._jobs.keys())))
+        log.info('waiting for the following jobs: %s' %
+            ', '.join([ '%s:%s' % (x, self._jobs[x][1]) for x in self._jobs
+            if self._jobs[x][1] not in self._completed]))
 
         if not len(self._jobs):
             return False
@@ -435,11 +437,6 @@
         JobStatus.JOB_PROMOTED,
     )
 
-    _slotdone = (
-        buildjob.JOB_STATE_FAILED,
-        buildjob.JOB_STATE_BUILT
-    )
-
     _promoterClass = JobPromoter
 
     def __init__(self, builder, maxSlots):

From elliot at rpath.com  Mon Sep 19 12:32:03 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 16:32:03 +0000
Subject: mirrorball: add nevra class
Message-ID: <201109191632.p8JGW3cR002258@scc.eng.rpath.com>

changeset:   2e1cfcacaf96
user:        Elliot Peele <https://issues.rpath.com/>
date:        Sat, 17 Sep 2011 00:18:56 -0400

add nevra class

diff --git a/rpmutils/__init__.py b/rpmutils/__init__.py
--- a/rpmutils/__init__.py
+++ b/rpmutils/__init__.py
@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2008-2009 rPath, Inc.
+# Copyright (c) 2008-2011 rPath, Inc.
 #
 # This program is distributed under the terms of the Common Public License,
 # version 1.0. A copy of this license should have been distributed with this
@@ -16,7 +16,8 @@
 Module that provides common utility functions for interacting with rpms.
 """
 
-__ALL__ = ('rpmvercmp', 'readHeader', )
+__ALL__ = ('rpmvercmp', 'readHeader', 'NEVRA')
 
 from rpmutils.vercmp import rpmvercmp
 from rpmutils.header import readHeader
+from rpmutils.nevra import NEVRA
diff --git a/rpmutils/nevra.py b/rpmutils/nevra.py
new file mode 100644
--- /dev/null
+++ b/rpmutils/nevra.py
@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2011 rPath, Inc.
+#
+# This program is distributed under the terms of the Common Public License,
+# version 1.0. A copy of this license should have been distributed with this
+# source file in a file called LICENSE. If it is not present, the license
+# is always available at http://www.rpath.com/permanent/licenses/CPL-1.0.
+#
+# This program is distributed in the hope that it will be useful, but
+# without any warranty; without even the implied warranty of merchantability
+# or fitness for a particular purpose. See the Common Public License for
+# full details.
+#
+
+"""
+Module for defining NEVRA datastructure.
+"""
+
+from collections import namedtuple
+
+from rpmutils import rpmvercmp
+
+class NEVRA(namedtuple('nevra', 'name epoch version release arch')):
+    """
+    Class to represent an RPM NEVRA.
+    """
+
+    def __cmp__(self, other):
+        nameCmp = cmp(self.name, other.name)
+        if nameCmp != 0:
+            return nameCmp
+
+        epochCmp = rpmvercmp(self.epoch, other.epoch)
+        if epochCmp != 0:
+            return epochCmp
+
+        versionCmp = rpmvercmp(self.version, other.version)
+        if versionCmp != 0:
+            return versionCmp
+
+        releaseCmp = rpmvercmp(self.release, other.release)
+        if releaseCmp != 0:
+            return releaseCmp
+
+        archCmp = cmp(self.arch, other.arch)
+        if archCmp != 0:
+            return archCmp
+
+        return 0
+

From elliot at rpath.com  Mon Sep 19 12:32:04 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 16:32:04 +0000
Subject: mirrorball: checkpoint group build code
Message-ID: <201109191632.p8JGW40D002302@scc.eng.rpath.com>

changeset:   7c031d836fed
user:        Elliot Peele <https://issues.rpath.com/>
date:        Sat, 17 Sep 2011 01:12:01 -0400

checkpoint group build code

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -19,9 +19,13 @@
 
 import time
 import logging
+import itertools
+
+from conary.trovetup import TroveTuple
+
+from rpmutils import NEVRA
 
 from updatebot import groupmgr
-from updatebot.lib import util
 from updatebot.bot import Bot as BotSuperClass
 
 from updatebot.errors import UnknownRemoveSourceError
@@ -139,6 +143,8 @@
         Handle initial import case.
         """
 
+        raise NotImplementedError
+
         group = self._groupmgr.getGroup()
         if group.errataState == 'None':
             group.errataState = None
@@ -177,77 +183,6 @@
 
         return pkgMap, failures
 
-    def _addNewPackages(self, allNevrasOnLabel, pkgMap, group):
-        """
-        Add pkgMap to group.
-        """
-        from repomd.packagexml import _Package as SrcPkg
-        def mkNevraObj(nevra):
-            # positive this code is somewhere in the mess... 
-            # no time to look for it  
-            nvr = SrcPkg()
-            nvr.name, nvr.epoch, nvr.version, nvr.release, nvr.arch = nevra
-            if nvr.epoch is None:
-                nvr.epoch = ''
-            elif not isinstance(nvr.epoch, str):
-                nvr.epoch = str(nvr.epoch)
-            return nvr
-
-        for binSet in pkgMap.itervalues():
-            pkgs = {}
-
-            addGroup = False
-            if not binSet:
-                log.warn('Empty bin set. BAD.')
-                import epdb; epdb.st()
-
-            nevraMap = self._conaryhelper.getNevras(binSet)
-
-            for conaryVer, nevra in nevraMap.iteritems():
-                if nevra:
-                    curPkg = mkNevraObj(nevra)
-                    labelNevra = [ mkNevraObj(labelNevra) for labelVer, 
-                                    labelNevra in allNevrasOnLabel.iteritems() 
-                                        if labelVer[0] == conaryVer[0] ] 
-                    labelNevra.sort(util.packagevercmp)
-                    labelPkg = labelNevra[-1]
-                    if group.hasPackage(curPkg.name):
-                        if util.packagevercmp(curPkg, labelPkg) == 1:
-                            log.info('%s newer than what was on label' % 
-                                    "_".join([curPkg.name,curPkg.version,curPkg.release]))
-                            addGroup = True
-                        else:
-                            log.info('later version found on label')
-                    else:
-                        log.info('new package adding to group')
-                        addGroup = True
-
-            if addGroup:
-
-                for n, v, f in binSet:
-                    if ':' in n:
-                        continue
-                    elif n not in pkgs:
-                        pkgs[n] = {v: set([f, ])}
-                    elif v not in pkgs[n]:
-                        pkgs[n][v] = set([f, ])
-                    else:
-                        pkgs[n][v].add(f)
-
-                for name, vf in pkgs.iteritems():
-                    assert len(vf) == 1
-                    version = vf.keys()[0]
-                    flavors = list(vf[version])
-                    log.info('adding %s=%s' % (name, version))
-                    for f in flavors:
-                        log.info('\t%s' % f)
-                    group.addPackage(name, version, flavors)
-
-
-        # FOR TESTING WE SHOULD INSPECT THE PKGMAP HERE
-        print "REMOVE LINE AFTER TESTING"
-        import epdb; epdb.st()
-
     def _removeSource(self, updateId, group):
         # Handle the case of entire source being obsoleted, this causes all
         # binaries from that source to be removed from the group model.
@@ -311,6 +246,35 @@
 
         return pkgMap
 
+    def _getNevrasForLabel(self, label):
+        """
+        Get a mapping of pkg nvf -> nevra for a given label. Use this method
+        instead of the conary helper directly so that we are using NEVRA objects
+        and already map epochs of None to 0 to match yum metadata.
+        """
+
+        nevraMap = self._updater._conaryhelper.getNevrasForLabel(label)
+
+        newMap = {}
+        for nvf, nevra in nevraMap.iteritems():
+            # Skip nvfs that don't have a nevra
+            if not nevra:
+                continue
+
+            # Skip sources.
+            if nvf[1].isSourceVersion():
+                continue
+
+            # Repack nevra subbing '0' for '', since yum metadata doesn't
+            # represent epochs of None.
+            if nevra[1] == '':
+                nevra = (nevra[0], '0', nevra[2], nevra[3], nevra[4])
+
+            pkgNvf = (nvf[0].split(':')[0], nvf[1], nvf[2])
+            newMap[pkgNvf] = NEVRA(*nevra)
+
+        return newMap
+
     def _getLatestNevraPackageMap(self, nevraMap):
         """
         Take a mapping of nvf to nevra and transform it into nevra to latest
@@ -318,22 +282,8 @@
         """
 
         nevras = {}
-        for (n, v, f), nevra in nevraMap.iteritems():
-            # Skip nvfs that don't have a nevra
-            if not nevra:
-                continue
-
-            # Skip sources
-            if v.isSourceVersion():
-                continue
-
-            # Repack nevra subbing '0' for '', since yum metadata doesn't
-            # represent epochs of None.
-            if nevra[1] == '':
-                nevra = (nevra[0], '0', nevra[2], nevra[3], nevra[4])
-
-            pkgNvf = (n.split(':')[0], v, f)
-            nevras.setdefault(nevra, set()).add(pkgNvf)
+        for nvf, nevra in nevraMap.iteritems():
+            nevras.setdefault(nevra, set()).add(nvf)
 
         ret = {}
         for nevra, pkgs in nevras.iteritems():
@@ -352,9 +302,9 @@
 
         # Get all of the nevras for the target and source labels.
         log.info('querying target nevras')
-        targetNevras = helper.getNevrasForLabel(self._cfg.targetLabel)
+        targetNevras = self._getNevrasForLabel(self._cfg.targetLabel)
         log.info('querying source nevras')
-        sourceNevras = helper.getNevrasForLabel(helper._ccfg.buildLabel)
+        sourceNevras = self._getNevrasForLabel(helper._ccfg.buildLabel)
 
         # Massage nevra maps into nevra -> latest package nvf
         log.info('building latest nevra maps')
@@ -389,7 +339,7 @@
         log.info('finding updates')
 
         # Get a mapping of nvf -> nevra
-        sourceNevras = self._updater._conaryhelper.getNevrasForLabel(
+        sourceNevras = self._getNevrasForLabel(
             self._updater._conaryhelper._ccfg.buildLabel)
 
         # Reverse the map and filter out old conary versions.
@@ -419,7 +369,7 @@
         #        avoid multiple repository calls.
 
         # Get a mapping of nvf -> nevra
-        sourceNevras = self._updater._conaryhelper.getNevrasForLabel(
+        sourceNevras = self._getNevrasForLabel(
             self._updater._conaryhelper._ccfg.buildLabel)
 
         # Reverse the map and filter out old conary versions.
@@ -429,22 +379,32 @@
         srcPkgs = sorted(self._pkgSource.srcNameMap.get(srcPkg.name))
         idx = srcPkgs.index(srcPkg) - 1
 
+        while idx > 0:
+            binPkgs = [ x for x in self._pkgSource.srcPkgMap[srcPkgs[idx]]
+                if x.arch != 'src' ]
+
+            # Apparently we have managed to not import (or maybe just build)
+            # some sub packages, so let's iterate over all of the until we find
+            # one that matches.
+            for binPkg in binPkgs:
+                if binPkg not in sourceLatest:
+                    continue
+
+                binNVF = sourceLatest[binPkg.getNevra()]
+                sourceVersions = self._updater._conaryhelper.getSourceVersions(
+                    [binNVF, ]).keys()
+
+                assert len(sourceVersions) == 1
+                return sourceVersions[0]
+
+            # Apparently we managed to skip importing a source package, or maybe
+            # it just didn't get built. Fall back to the previous source.
+            idx -= 1
+
         # This is a new package
         if idx < 0:
             return (srcPkg.name, None, None)
 
-        binPkgs = [ x for x in self._pkgSource.srcPkgMap[srcPkgs[idx]]
-            if x.arch != 'src' ]
-
-        binPkg = binPkgs[0]
-
-        binNVF = sourceLatest[binPkg.getNevra()]
-        sourceVersions = self._updater._conaryhelper.getSourceVersions(
-            [binNVF, ]).keys()
-
-        assert len(sourceVersions) == 1
-        return sourceVersions[0]
-
     def update(self, *args, **kwargs):
         """
         Handle update case.
@@ -500,6 +460,7 @@
                             set(removeReplaced))
 
         # Update package set.
+        pkgMap = {}
         if updatePkgs:
             updatePkgs.filterPkgs(kwargs.pop('fltr', None))
 
@@ -509,8 +470,9 @@
                     for x in updateSet])
 
                 log.info('running update')
-                self._update(*args, updateTroves=updateTroves, updatePkgs=True,
-                    expectedRemovals=expectedRemovals, **kwargs)
+                pkgMap.update(self._update(*args, updateTroves=updateTroves,
+                    updatePkgs=True, expectedRemovals=expectedRemovals,
+                    **kwargs))
 
                 # The NEVRA maps will be changing every time through. Make sure
                 # the clear the cache.
@@ -520,6 +482,68 @@
         log.info('completed package update of %s packages in %s seconds'
             % (len(updatePkgs), time.time()-starttime))
 
+        return pkgMap
+
+    def _addNewPackages(self, group):
+        """
+        Find all of the packages from the buildLabel that are newer that those
+        in the group. Handle any obsoletes along the way.
+        """
+
+        # FIXME: Right now we are going to deal with the buildLabel, we probably
+        #        want to switch to the target label once a script has been
+        #        written to rewrite existing group models to map them onto
+        #        target label versions.
+
+        # Make sure nothing is cached before we get started.
+        self._updater._conaryhelper.clearCache()
+
+        # Get a mapping of nvf -> nevra
+        nevraMap = self._getNevrasForLabel(
+             self._updater._conaryhelper._ccfg.buildLabel)
+
+        # Reverse the map and filter out old conary versions.
+        latest = self._getLatestNevraPackageMap(nevraMap)
+
+        # index by name, will need this later
+        names = {}
+        for nevra, nvf in latest.iteritems():
+            names.setdefault(nevra[0], dict())[nevra] = nvf
+
+        toAdd = {}
+        for pkg in group.iterpackages():
+            nvf = TroveTuple(pkg.name, pkg.version, pkg.flavor)
+
+            assert nvf in nevraMap
+
+            # Get the current nevra
+            nevra = nevraMap[nvf]
+
+            # Now we need to find all versions of this package that
+            # are equal to or newer than the current nevra.
+            pkgs = names.get(nevra.name)
+            nevras = sorted(pkgs)
+            idx = nevras.index(nevra)
+
+            updates = {}
+            while idx < len(nevras):
+                updates[nevras[idx]] = pkgs[nevras[idx]]
+                idx += 1
+
+            ##
+            # FIXME: Obsolete handling goes here
+            #
+            # For each new verison figure out if there are any obsoletes that
+            # need to be handled, then handle them.
+            ##
+
+            # For now just pick the latest one and add it to the group.
+            foo = updates[sorted(updates)[-1]]
+            toAdd.setdefault((foo.name, foo.version), set()).add(foo.flavor)
+
+        for (name, version), flavors in toAdd.iteritems():
+            group.addPackages(name, version, flavors)
+
     def buildgroups(self):
         """
         Find the latest packages on the production label by nevra and build a
@@ -527,6 +551,8 @@
         along the way.
         """
 
+        starttime = time.time()
+
         # Get current group
         group = self._groupmgr.getGroup()
 
@@ -563,12 +589,6 @@
 
         # Generate grpPkgMap
 
-        #grpPkgNameMap = [ (x.name, x.version, x.flavor) for x in group.iterpackages() ]
-        # FIXME: should take the list of troves in the group and 
-        # query the label to see if newer nevra versions exist 
-        # Then add them to the group
-
-        # FIXME: Not implemented yet
         # Keep Obsoleted
         keepObsolete = set(self._cfg.keepObsolete)
         keepObsoleteSource = set(self._cfg.keepObsoleteSource)
@@ -588,14 +608,8 @@
             for pkg in removeObsoleted:
                 group.removePackage(pkg, missingOk=True)
 
-        pkgMap = self._useOldVersions(updateId, pkgMap)
-
-        # Make sure built troves are part of the group.
-        self._addNewPackages(allNevrasOnLabel, pkgMap, group)
-
-        # FOR TESTING WE SHOULD INSPECT THE PKGMAP HERE
-        print "Check the pkgMap now"
-        import epdb; epdb.st()
+        # Find and add new packages
+        self._addNewPackages(group)
 
         # Modify any extra groups to match config.
         self._modifyGroups(updateId, group)
@@ -613,38 +627,19 @@
         group = group.commit()
         grpTrvMap = group.build()
 
-        # Add promote for group if success
-        if grpTrvMap:
-            log.info('promoting group %s ' % group.version)
-            # Add promote code here
-
-        updateSet.update(pkgMap)
+        # Promote groups
+        log.info('promoting group %s ' % group.version)
+        toPromote = [ x for x in itertools.chain(grpTrvMap.itervalues()) ]
+        promoted = self._updater.publish(toPromote, toPromote)
 
         # Report timings
         advTime = time.strftime('%m-%d-%Y %H:%M:%S',
                                     time.localtime(updateId))
-        totalTime = time.time() - startime
-        log.info('published update %s in %s seconds' % (advTime, totalTime))
-        count += 1
+        totalTime = time.time() - starttime
+        log.info('published group update %s in %s seconds'
+            % (advTime, totalTime))
 
-        # Set this breakpoint when you're committing a list of
-        # already-built job id's.  (See long comment above.)
-        #import epdb ; epdb.st()
-
-        # TESTING
-        print "need to check updates set"
-        import epdb ; epdb.st()
-
-        # BELOW THIS LINE WILL BE REMOVED
-
-        # FIXME: remove errata crap
-
-
-        log.info('update completed')
-        log.info('applied %s updates in %s seconds'
-            % (count, time.time() - startime))
-
-        return updateSet
+        return promoted
 
     def _getOldVersionExceptions(self, updateId):
         versionExceptions = {}

From elliot at rpath.com  Mon Sep 19 12:32:04 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 16:32:04 +0000
Subject: mirrorball: add support for comparing non NEVRA nevra tuples
Message-ID: <201109191632.p8JGW4Fq002281@scc.eng.rpath.com>

changeset:   c66bac623734
user:        Elliot Peele <https://issues.rpath.com/>
date:        Sat, 17 Sep 2011 01:11:51 -0400

add support for comparing non NEVRA nevra tuples

diff --git a/rpmutils/nevra.py b/rpmutils/nevra.py
--- a/rpmutils/nevra.py
+++ b/rpmutils/nevra.py
@@ -26,6 +26,9 @@
     """
 
     def __cmp__(self, other):
+        if isinstance(other, tuple):
+            other = self.__class__(*other)
+
         nameCmp = cmp(self.name, other.name)
         if nameCmp != 0:
             return nameCmp

From elliot at rpath.com  Mon Sep 19 13:13:03 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 17:13:03 +0000
Subject: mirrorball: hash(binPkg) != hash(binPkg.getNevra())
Message-ID: <201109191713.p8JHD3wo003784@scc.eng.rpath.com>

changeset:   37496a0c4248
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 13:12:59 -0400

hash(binPkg) != hash(binPkg.getNevra())

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -379,15 +379,15 @@
         srcPkgs = sorted(self._pkgSource.srcNameMap.get(srcPkg.name))
         idx = srcPkgs.index(srcPkg) - 1
 
-        while idx > 0:
+        while idx >= 0:
             binPkgs = [ x for x in self._pkgSource.srcPkgMap[srcPkgs[idx]]
                 if x.arch != 'src' ]
 
             # Apparently we have managed to not import (or maybe just build)
-            # some sub packages, so let's iterate over all of the until we find
+            # some sub packages, so let's iterate over all of them until we find
             # one that matches.
             for binPkg in binPkgs:
-                if binPkg not in sourceLatest:
+                if binPkg.getNevra() not in sourceLatest:
                     continue
 
                 binNVF = sourceLatest[binPkg.getNevra()]
@@ -405,6 +405,8 @@
         if idx < 0:
             return (srcPkg.name, None, None)
 
+        assert False, 'How did we get here?'
+
     def update(self, *args, **kwargs):
         """
         Handle update case.
@@ -571,12 +573,6 @@
         removeObsoleted = self._cfg.removeObsoleted.get(updateId, [])
         removeReplaced = self._cfg.updateReplacesPackages.get(updateId, [])
 
-        # take the union of the three lists to get a unique list of packages
-        # to remove.
-        expectedRemovals = (set(removePackages) |
-                            set(removeObsoleted) |
-                            set(removeReplaced))
-
         # The following packages are expected to exist and must be removed
         # (removeObsoleted may be mentioned for buckets where the package
         # is not in the model, in order to support later adding the ability
@@ -587,8 +583,6 @@
         # Get the list of package that are allowed to be downgraded.
         allowDowngrades = self._cfg.allowPackageDowngrades.get(updateId, [])
 
-        # Generate grpPkgMap
-
         # Keep Obsoleted
         keepObsolete = set(self._cfg.keepObsolete)
         keepObsoleteSource = set(self._cfg.keepObsoleteSource)

From elliot at rpath.com  Mon Sep 19 15:51:55 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 19:51:55 +0000
Subject: mirrorball: Thaw the flavor first
Message-ID: <201109191951.p8JJptix009123@scc.eng.rpath.com>

changeset:   eeeef944c8ed
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 15:50:49 -0400

Thaw the flavor first

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -22,6 +22,7 @@
 import itertools
 
 from conary.trovetup import TroveTuple
+from conary.deps.deps import ThawFlavor
 
 from rpmutils import NEVRA
 
@@ -514,7 +515,7 @@
 
         toAdd = {}
         for pkg in group.iterpackages():
-            nvf = TroveTuple(pkg.name, pkg.version, pkg.flavor)
+            nvf = TroveTuple(pkg.name, pkg.version, ThawFlavor(pkg.flavor))
 
             assert nvf in nevraMap
 

From elliot at rpath.com  Mon Sep 19 15:51:56 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 19:51:56 +0000
Subject: mirrorball: ThawFlavor doesn't support unicode
Message-ID: <201109191951.p8JJpur5009155@scc.eng.rpath.com>

changeset:   02857218f46c
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 15:51:53 -0400

ThawFlavor doesn't support unicode

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -515,7 +515,8 @@
 
         toAdd = {}
         for pkg in group.iterpackages():
-            nvf = TroveTuple(pkg.name, pkg.version, ThawFlavor(pkg.flavor))
+            flavor = ThawFlavor(str(pkg.flavor))
+            nvf = TroveTuple(pkg.name, pkg.version, flavor)
 
             assert nvf in nevraMap
 

From elliot at rpath.com  Mon Sep 19 16:40:54 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 20:40:54 +0000
Subject: mirrorball: Remove any packages that don't have nevras
Message-ID: <201109192040.p8JKesUZ010435@scc.eng.rpath.com>

changeset:   74e4dc301590
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 16:40:52 -0400

Remove any packages that don't have nevras

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -514,10 +514,19 @@
             names.setdefault(nevra[0], dict())[nevra] = nvf
 
         toAdd = {}
+        toRemove = set()
         for pkg in group.iterpackages():
             flavor = ThawFlavor(str(pkg.flavor))
             nvf = TroveTuple(pkg.name, pkg.version, flavor)
 
+            # Older groups contain packages that do not reference any
+            # capsules. This was a side affect of how groups are managed in
+            # ordered mode. Since we don't need to do that anymore, just
+            # remove the package. Hopefully they won't creep back in.
+            if nvf not in nevraMap:
+                toRemove.add(nvf)
+                continue
+
             assert nvf in nevraMap
 
             # Get the current nevra
@@ -545,6 +554,9 @@
             foo = updates[sorted(updates)[-1]]
             toAdd.setdefault((foo.name, foo.version), set()).add(foo.flavor)
 
+        for nvf in toRemove:
+            group.removePackages(nvf.name, flavor=nvf.flavor)
+
         for (name, version), flavors in toAdd.iteritems():
             group.addPackages(name, version, flavors)
 
diff --git a/updatebot/groupmgr/group.py b/updatebot/groupmgr/group.py
--- a/updatebot/groupmgr/group.py
+++ b/updatebot/groupmgr/group.py
@@ -350,7 +350,7 @@
         raise UnhandledPackageAdditionError(name=name)
 
     @require_write
-    def removePackage(self, name, missingOk=False):
+    def removePackage(self, name, flavor=None, missingOk=False):
         """
         Remove a given trove from the package group contents.
         """
@@ -358,8 +358,12 @@
         if self._pkgGroupName not in self._groups:
             return
 
-        return self._groups[self._pkgGroupName].remove(name,
-             missingOk=missingOk)
+        group = self._groups[self._pkgGroupName]
+
+        if flavor:
+            group.removePackageFlavor(name, flavor.freeze())
+        else:
+            group.remove(name, missingOk=missingOk)
 
     def hasPackage(self, name):
         """

From elliot at rpath.com  Mon Sep 19 17:15:41 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 21:15:41 +0000
Subject: mirrorball: expose keepRemovedPackages hire up in the stack and
	map to the keepRemoved config option
Message-ID: <201109192115.p8JLFf2v013739@scc.eng.rpath.com>

changeset:   b2adcadd93b6
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 17:13:56 -0400

expose keepRemovedPackages hire up in the stack and map to the keepRemoved config option

diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -179,7 +179,8 @@
         return trvMap, failed
 
     def update(self, force=None, updatePkgs=None, expectedRemovals=None,
-        allowPackageDowngrades=None, updateTroves=None):
+        allowPackageDowngrades=None, updateTroves=None,
+        keepRemovedPackages=None):
         """
         Update the conary repository from the yum repositories.
         @param force: list of packages to update without exception
@@ -195,6 +196,10 @@
         @param updateTroves: overrides the value of updatePkgs. Set of (n, v, f)
             tuple to update from and source package to update to.
         @type updateTroves: set(((n, v, f), srcPkg))
+        @param keepRemovedPackages: list of package nevras to keep even though
+                                    they have been removed in the latest version
+                                    of the source.
+        @type keepRemovedPackages: list(nevra, nevra, ...)
         """
 
         if force is not None:
@@ -214,7 +219,8 @@
         toAdvise, toUpdate = self._updater.getUpdates(
             updateTroves=updateTroves,
             expectedRemovals=expectedRemovals,
-            allowPackageDowngrades=allowPackageDowngrades)
+            allowPackageDowngrades=allowPackageDowngrades,
+            keepRemovedPackages=keepRemovedPackages)
 
         # If forcing an update, make sure that all packages are listed in
         # toAdvise and toUpdate as needed.
diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -462,6 +462,9 @@
                             set(removeObsoleted) |
                             set(removeReplaced))
 
+        # Packages that would otherwise be removed
+        keepRemoved = self._cfg.keepRemoved.get(updateId, [])
+
         # Update package set.
         pkgMap = {}
         if updatePkgs:
@@ -475,6 +478,7 @@
                 log.info('running update')
                 pkgMap.update(self._update(*args, updateTroves=updateTroves,
                     updatePkgs=True, expectedRemovals=expectedRemovals,
+                    keepRemovedPackages=keepRemoved,
                     **kwargs))
 
                 # The NEVRA maps will be changing every time through. Make sure
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -56,7 +56,7 @@
         self._conaryhelper = conaryhelper.ConaryHelper(self._cfg)
 
     def getUpdates(self, updateTroves=None, expectedRemovals=None,
-        allowPackageDowngrades=None):
+        allowPackageDowngrades=None, keepRemovedPackages=None):
         """
         Find all packages that need updates and/or advisories from a top level
         binary group.
@@ -68,6 +68,10 @@
         @param allowPackageDowngrades: list of source nevra tuples to downgrade
                                        from/to.
         @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
+        @param keepRemovedPackages: list of package nevras to keep even though
+                                    they have been removed in the latest version
+                                    of the source.
+        @type keepRemovedPackages: list(nevra, nevra, ...)
         @return list of packages to send advisories for and list of packages
                 to update
         """
@@ -88,7 +92,8 @@
             # Will raise exception if any errors are found, halting execution.
             if self._sanitizeTrove(nvf, srpm,
                     expectedRemovals=expectedRemovals,
-                    allowPackageDowngrades=allowPackageDowngrades):
+                    allowPackageDowngrades=allowPackageDowngrades,
+                    keepRemovedPackages=keepRemovedPackages):
                 toUpdate.append((nvf, srpm))
                 toAdvise.append((nvf, srpm))
 

From elliot at rpath.com  Mon Sep 19 17:15:41 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 21:15:41 +0000
Subject: mirrorball: add a bit of debugging, singleish package promotes
	shouldn't take more than 10 minutes
Message-ID: <201109192115.p8JLFftG013771@scc.eng.rpath.com>

changeset:   01fe4e4214c2
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 17:15:30 -0400

add a bit of debugging, singleish package promotes shouldn't take more than 10 minutes

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -447,6 +447,8 @@
         self._promoter = self._promoterClass((self._builder._conaryhelper,
             self._builder._cfg.targetLabel))
 
+        self._status = {}
+
     def _jobDone(self):
         # Override the job done method from the parent to hook into the
         # build loop. This is kinda dirty, but I don't really have a
@@ -484,16 +486,25 @@
                 toPromote.append((jobId, res))
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
 
+                self._status[jobId] = time.time()
+
             if toPromote:
                 self._promoteSlots -= 1
                 self._promoter.promoteJob(toPromote)
 
+        # If anything has been promoting for more than 10 minutes, let us poke
+        # around and figure out why.
+        for jobId, startTime in self._status.iteritems():
+            if time.time() - startTime > (60 * 10):
+                import epdb; epdb.st()
+
         # Gather results
         for result in self._promoter.getStatus():
             self._promoteSlots += 1
             for jobId, promoted in result:
                 self._jobs[jobId][2] = promoted
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
+                self._status.pop(jobId)
 
         # Gather errors
         for jobs, error in self._promoter.getErrors():
@@ -501,3 +512,4 @@
             for jobId in jobs:
                 self._jobs[jobId][1] = JobStatus.ERROR_PROMOTE_FAILURE
                 self._failures.append((jobId, error))
+                self._status.pop(jobId)

From elliot at rpath.com  Mon Sep 19 17:15:42 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 21:15:42 +0000
Subject: mirrorball: branch merge
Message-ID: <201109192115.p8JLFghY013798@scc.eng.rpath.com>

changeset:   0b40c401281b
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 17:15:39 -0400

branch merge

diff --git a/updatebot/bot.py b/updatebot/bot.py
--- a/updatebot/bot.py
+++ b/updatebot/bot.py
@@ -179,7 +179,8 @@
         return trvMap, failed
 
     def update(self, force=None, updatePkgs=None, expectedRemovals=None,
-        allowPackageDowngrades=None, updateTroves=None):
+        allowPackageDowngrades=None, updateTroves=None,
+        keepRemovedPackages=None):
         """
         Update the conary repository from the yum repositories.
         @param force: list of packages to update without exception
@@ -195,6 +196,10 @@
         @param updateTroves: overrides the value of updatePkgs. Set of (n, v, f)
             tuple to update from and source package to update to.
         @type updateTroves: set(((n, v, f), srcPkg))
+        @param keepRemovedPackages: list of package nevras to keep even though
+                                    they have been removed in the latest version
+                                    of the source.
+        @type keepRemovedPackages: list(nevra, nevra, ...)
         """
 
         if force is not None:
@@ -214,7 +219,8 @@
         toAdvise, toUpdate = self._updater.getUpdates(
             updateTroves=updateTroves,
             expectedRemovals=expectedRemovals,
-            allowPackageDowngrades=allowPackageDowngrades)
+            allowPackageDowngrades=allowPackageDowngrades,
+            keepRemovedPackages=keepRemovedPackages)
 
         # If forcing an update, make sure that all packages are listed in
         # toAdvise and toUpdate as needed.
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -447,6 +447,8 @@
         self._promoter = self._promoterClass((self._builder._conaryhelper,
             self._builder._cfg.targetLabel))
 
+        self._status = {}
+
     def _jobDone(self):
         # Override the job done method from the parent to hook into the
         # build loop. This is kinda dirty, but I don't really have a
@@ -484,16 +486,25 @@
                 toPromote.append((jobId, res))
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
 
+                self._status[jobId] = time.time()
+
             if toPromote:
                 self._promoteSlots -= 1
                 self._promoter.promoteJob(toPromote)
 
+        # If anything has been promoting for more than 10 minutes, let us poke
+        # around and figure out why.
+        for jobId, startTime in self._status.iteritems():
+            if time.time() - startTime > (60 * 10):
+                import epdb; epdb.st()
+
         # Gather results
         for result in self._promoter.getStatus():
             self._promoteSlots += 1
             for jobId, promoted in result:
                 self._jobs[jobId][2] = promoted
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
+                self._status.pop(jobId)
 
         # Gather errors
         for jobs, error in self._promoter.getErrors():
@@ -501,3 +512,4 @@
             for jobId in jobs:
                 self._jobs[jobId][1] = JobStatus.ERROR_PROMOTE_FAILURE
                 self._failures.append((jobId, error))
+                self._status.pop(jobId)
diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -462,6 +462,9 @@
                             set(removeObsoleted) |
                             set(removeReplaced))
 
+        # Packages that would otherwise be removed
+        keepRemoved = self._cfg.keepRemoved.get(updateId, [])
+
         # Update package set.
         pkgMap = {}
         if updatePkgs:
@@ -478,6 +481,7 @@
                 
                 pkgMap.update(self._update(*args, updateTroves=updateTroves,
                     updatePkgs=True, expectedRemovals=expectedRemovals,
+                    keepRemovedPackages=keepRemoved,
                     **kwargs))
 
                 # The NEVRA maps will be changing every time through. Make sure
diff --git a/updatebot/update.py b/updatebot/update.py
--- a/updatebot/update.py
+++ b/updatebot/update.py
@@ -56,7 +56,7 @@
         self._conaryhelper = conaryhelper.ConaryHelper(self._cfg)
 
     def getUpdates(self, updateTroves=None, expectedRemovals=None,
-        allowPackageDowngrades=None):
+        allowPackageDowngrades=None, keepRemovedPackages=None):
         """
         Find all packages that need updates and/or advisories from a top level
         binary group.
@@ -68,6 +68,10 @@
         @param allowPackageDowngrades: list of source nevra tuples to downgrade
                                        from/to.
         @type allowPackageDowngrades: list(list(from srcNevra, to srcNevra), )
+        @param keepRemovedPackages: list of package nevras to keep even though
+                                    they have been removed in the latest version
+                                    of the source.
+        @type keepRemovedPackages: list(nevra, nevra, ...)
         @return list of packages to send advisories for and list of packages
                 to update
         """
@@ -88,7 +92,8 @@
             # Will raise exception if any errors are found, halting execution.
             if self._sanitizeTrove(nvf, srpm,
                     expectedRemovals=expectedRemovals,
-                    allowPackageDowngrades=allowPackageDowngrades):
+                    allowPackageDowngrades=allowPackageDowngrades,
+                    keepRemovedPackages=keepRemovedPackages):
                 toUpdate.append((nvf, srpm))
                 toAdvise.append((nvf, srpm))
 

From elliot at rpath.com  Mon Sep 19 19:02:04 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 23:02:04 +0000
Subject: mirrorball: don't try to readd all of the group content
Message-ID: <201109192302.p8JN24NI016838@scc.eng.rpath.com>

changeset:   0cdcc9f598c3
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 19:00:12 -0400

don't try to readd all of the group content

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -441,7 +441,7 @@
 
         # Figure out what packages still need to be promoted.
         promotePkgs = self._getPromotePackages()
-        
+
         # Go ahead and promote any packages that didn't get promoted during the
         # last run or have been rebuilt since then.
         log.info('found %s packages that need to be promoted' %
@@ -477,8 +477,6 @@
 
                 log.info('running update')
 
-                #import epdb ; epdb.st()
-                
                 pkgMap.update(self._update(*args, updateTroves=updateTroves,
                     updatePkgs=True, expectedRemovals=expectedRemovals,
                     keepRemovedPackages=keepRemoved,
@@ -557,8 +555,13 @@
             # need to be handled, then handle them.
             ##
 
+            foo = updates[sorted(updates)[-1]]
+            # If the only available package is the one that is already in the
+            # group, skip it and move on.
+            if foo == nvf:
+                continue
+
             # For now just pick the latest one and add it to the group.
-            foo = updates[sorted(updates)[-1]]
             toAdd.setdefault((foo[0], foo[1]), set()).add(foo[2])
 
         for nvf in toRemove:
@@ -638,9 +641,7 @@
         # Build groups.
         log.info('setting version %s' % version)
         group.version = version
-        
-        import epdb ; epdb.st()
-        
+
         group = group.commit()
         grpTrvMap = group.build()
 

From elliot at rpath.com  Mon Sep 19 19:02:04 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 23:02:04 +0000
Subject: mirrorball: sepparate out troves by source rather than binary
Message-ID: <201109192302.p8JN24tA016861@scc.eng.rpath.com>

changeset:   691660824a17
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 19:00:51 -0400

sepparate out troves by source rather than binary

diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1080,28 +1080,27 @@
 
         return verMap
 
-    @staticmethod
-    def _iterPromoteJobList(trvLst):
+    def _iterPromoteJobList(self, trvLst):
         """
         Filter trove list into separate jobs when attempting to promote two
         versions of the same package.
         """
 
+        # Get sources for all of the troves so that we can avoid promoting
+        # binaries built from different versions of the same source in the
+        # same job.
+        sources = self.getSourceVersions(trvLst)
+
         data = {}
-        for n, v, f in trvLst:
-            # Skip sources, they come along for free with the underlying
-            # conary promote code.
-            if n.endswith(':source'):
-                continue
-            data.setdefault(n.split(':')[0], dict()).setdefault(v, set()).add(f)
+        for (n, v, f), binTrvs in sources.iteritems():
+            data.setdefault(n.split(':')[0], dict())[v] = binTrvs
 
         while data:
             job = []
             toRemove = []
             for n, vs in data.iteritems():
                 v = sorted(vs)[0]
-                for f in vs.pop(v):
-                    job.append((n, v, f))
+                job.extend(vs.pop(v))
 
                 if not vs:
                     toRemove.append(n)

From elliot at rpath.com  Mon Sep 19 19:02:05 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 23:02:05 +0000
Subject: mirrorball: wait longer
Message-ID: <201109192302.p8JN25qm016889@scc.eng.rpath.com>

changeset:   afd14b4b202f
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 19:01:12 -0400

wait longer

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -495,7 +495,7 @@
         # If anything has been promoting for more than 10 minutes, let us poke
         # around and figure out why.
         for jobId, startTime in self._status.iteritems():
-            if time.time() - startTime > (60 * 10):
+            if time.time() - startTime > (60 * 20):
                 import epdb; epdb.st()
 
         # Gather results
@@ -504,7 +504,7 @@
             for jobId, promoted in result:
                 self._jobs[jobId][2] = promoted
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
-                self._status.pop(jobId)
+                self._status.pop(jobId, None)
 
         # Gather errors
         for jobs, error in self._promoter.getErrors():
@@ -512,4 +512,4 @@
             for jobId in jobs:
                 self._jobs[jobId][1] = JobStatus.ERROR_PROMOTE_FAILURE
                 self._failures.append((jobId, error))
-                self._status.pop(jobId)
+                self._status.pop(jobId, None)

From elliot at rpath.com  Mon Sep 19 19:02:05 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 23:02:05 +0000
Subject: mirrorball: cleanup
Message-ID: <201109192302.p8JN25kq016919@scc.eng.rpath.com>

changeset:   1dcc5153d153
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 19:01:31 -0400

cleanup

diff --git a/scripts/current_update.py b/scripts/current_update.py
--- a/scripts/current_update.py
+++ b/scripts/current_update.py
@@ -40,40 +40,8 @@
 
 fltr = None
 
-# FIXME: Once update finished 
-# Most of this not needed
-if cfg.platformName == 'rhel':
-    #import rhnmirror
-
-    #mcfg = rhnmirror.MirrorConfig()
-    #mcfg.read(confDir + '/erratarc')
-
-    #errata = rhnmirror.Errata(mcfg)
-    bot = current.Bot(cfg)
-
-    checkMissingPackages = False
-
-else:
-    bot = current.Bot(cfg)
-
-    #if cfg.platformName == 'sles':
-    #    from errata.sles import AdvisoryManager as Errata
-
-    #elif cfg.platformName in ('sles11', 'sles11sp1hae'):
-    #    from errata.sles11 import AdvisoryManager11 as Errata
-
-    #elif cfg.platformName == 'centos':
-    #    from errata.centos import AdvisoryManager as Errata
-
-    #else:
-    #    raise RuntimeError, 'no errata source found for %s' % cfg.platformName
-
-    #errata = Errata(bot._pkgSource)
-    #bot._errata._errata = errata
-
-    checkMissingPackages = True
-
-pkgMap = bot.update(fltr=fltr, restoreFile=restoreFile,
-                    checkMissingPackages=checkMissingPackages)
+bot = current.Bot(cfg)
+pkgMap = bot.update(fltr=fltr)
+#bot.buildgroups()
 
 import epdb; epdb.st()

From elliot at rpath.com  Mon Sep 19 19:02:05 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 19 Sep 2011 23:02:05 +0000
Subject: mirrorball: branch merge
Message-ID: <201109192302.p8JN25Gs016950@scc.eng.rpath.com>

changeset:   7f4532d98288
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 19 Sep 2011 19:02:01 -0400

branch merge

diff --git a/scripts/current_update.py b/scripts/current_update.py
--- a/scripts/current_update.py
+++ b/scripts/current_update.py
@@ -40,40 +40,8 @@
 
 fltr = None
 
-# FIXME: Once update finished 
-# Most of this not needed
-if cfg.platformName == 'rhel':
-    #import rhnmirror
-
-    #mcfg = rhnmirror.MirrorConfig()
-    #mcfg.read(confDir + '/erratarc')
-
-    #errata = rhnmirror.Errata(mcfg)
-    bot = current.Bot(cfg)
-
-    checkMissingPackages = False
-
-else:
-    bot = current.Bot(cfg)
-
-    #if cfg.platformName == 'sles':
-    #    from errata.sles import AdvisoryManager as Errata
-
-    #elif cfg.platformName in ('sles11', 'sles11sp1hae'):
-    #    from errata.sles11 import AdvisoryManager11 as Errata
-
-    #elif cfg.platformName == 'centos':
-    #    from errata.centos import AdvisoryManager as Errata
-
-    #else:
-    #    raise RuntimeError, 'no errata source found for %s' % cfg.platformName
-
-    #errata = Errata(bot._pkgSource)
-    #bot._errata._errata = errata
-
-    checkMissingPackages = True
-
-pkgMap = bot.update(fltr=fltr, restoreFile=restoreFile,
-                    checkMissingPackages=checkMissingPackages)
+bot = current.Bot(cfg)
+pkgMap = bot.update(fltr=fltr)
+#bot.buildgroups()
 
 import epdb; epdb.st()
diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -495,7 +495,7 @@
         # If anything has been promoting for more than 10 minutes, let us poke
         # around and figure out why.
         for jobId, startTime in self._status.iteritems():
-            if time.time() - startTime > (60 * 10):
+            if time.time() - startTime > (60 * 20):
                 import epdb; epdb.st()
 
         # Gather results
@@ -504,7 +504,7 @@
             for jobId, promoted in result:
                 self._jobs[jobId][2] = promoted
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
-                self._status.pop(jobId)
+                self._status.pop(jobId, None)
 
         # Gather errors
         for jobs, error in self._promoter.getErrors():
@@ -512,4 +512,4 @@
             for jobId in jobs:
                 self._jobs[jobId][1] = JobStatus.ERROR_PROMOTE_FAILURE
                 self._failures.append((jobId, error))
-                self._status.pop(jobId)
+                self._status.pop(jobId, None)
diff --git a/updatebot/conaryhelper.py b/updatebot/conaryhelper.py
--- a/updatebot/conaryhelper.py
+++ b/updatebot/conaryhelper.py
@@ -1080,28 +1080,27 @@
 
         return verMap
 
-    @staticmethod
-    def _iterPromoteJobList(trvLst):
+    def _iterPromoteJobList(self, trvLst):
         """
         Filter trove list into separate jobs when attempting to promote two
         versions of the same package.
         """
 
+        # Get sources for all of the troves so that we can avoid promoting
+        # binaries built from different versions of the same source in the
+        # same job.
+        sources = self.getSourceVersions(trvLst)
+
         data = {}
-        for n, v, f in trvLst:
-            # Skip sources, they come along for free with the underlying
-            # conary promote code.
-            if n.endswith(':source'):
-                continue
-            data.setdefault(n.split(':')[0], dict()).setdefault(v, set()).add(f)
+        for (n, v, f), binTrvs in sources.iteritems():
+            data.setdefault(n.split(':')[0], dict())[v] = binTrvs
 
         while data:
             job = []
             toRemove = []
             for n, vs in data.iteritems():
                 v = sorted(vs)[0]
-                for f in vs.pop(v):
-                    job.append((n, v, f))
+                job.extend(vs.pop(v))
 
                 if not vs:
                     toRemove.append(n)
diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -441,7 +441,7 @@
 
         # Figure out what packages still need to be promoted.
         promotePkgs = self._getPromotePackages()
-        
+
         # Go ahead and promote any packages that didn't get promoted during the
         # last run or have been rebuilt since then.
         log.info('found %s packages that need to be promoted' %
@@ -477,8 +477,6 @@
 
                 log.info('running update')
 
-                #import epdb ; epdb.st()
-                
                 pkgMap.update(self._update(*args, updateTroves=updateTroves,
                     updatePkgs=True, expectedRemovals=expectedRemovals,
                     keepRemovedPackages=keepRemoved,
@@ -557,8 +555,13 @@
             # need to be handled, then handle them.
             ##
 
+            foo = updates[sorted(updates)[-1]]
+            # If the only available package is the one that is already in the
+            # group, skip it and move on.
+            if foo == nvf:
+                continue
+
             # For now just pick the latest one and add it to the group.
-            foo = updates[sorted(updates)[-1]]
             toAdd.setdefault((foo[0], foo[1]), set()).add(foo[2])
 
         for nvf in toRemove:
@@ -641,9 +644,7 @@
         # Build groups.
         log.info('setting version %s' % version)
         group.version = version
-        
-        import epdb ; epdb.st()
-        
+
         group = group.commit()
         grpTrvMap = group.build()
 

From elliot at rpath.com  Tue Sep 20 03:50:24 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Sep 2011 07:50:24 +0000
Subject: mirrorball: try not to re-add packages to the group
Message-ID: <201109200750.p8K7oOtd026030@scc.eng.rpath.com>

changeset:   b22875746f18
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 20 Sep 2011 03:49:21 -0400

try not to re-add packages to the group

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -507,6 +507,7 @@
         self._updater._conaryhelper.clearCache()
 
         # Get a mapping of nvf -> nevra
+        log.info('retrieving nevras for build label')
         nevraMap = self._getNevrasForLabel(
              self._updater._conaryhelper._ccfg.buildLabel)
 
@@ -516,7 +517,9 @@
         # index by name, will need this later
         names = {}
         for nevra, nvf in latest.iteritems():
-            names.setdefault(nevra[0], dict())[nevra] = nvf
+            names.setdefault((nevra.name, nevra.arch), dict())[nevra] = nvf
+
+        import epdb; epdb.st()
 
         toAdd = {}
         toRemove = set()
@@ -539,7 +542,7 @@
 
             # Now we need to find all versions of this package that
             # are equal to or newer than the current nevra.
-            pkgs = names.get(nevra.name)
+            pkgs = names.get((nevra.name, nevra.arch))
             nevras = sorted(pkgs)
             idx = nevras.index(nevra)
 
@@ -561,6 +564,11 @@
             if foo == nvf:
                 continue
 
+            if [ x for x in group.iterpackages()
+                 if (foo[0], foo[1].freeze(), foo[2].freeze()) ==
+                    (x.name, x.version, x.flavor) ]:
+                continue
+
             # For now just pick the latest one and add it to the group.
             toAdd.setdefault((foo[0], foo[1]), set()).add(foo[2])
 

From elliot at rpath.com  Tue Sep 20 03:50:24 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Sep 2011 07:50:24 +0000
Subject: mirrorball: namedtuples need to have lt defined?
Message-ID: <201109200750.p8K7oOiT026057@scc.eng.rpath.com>

changeset:   5f450901661b
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 20 Sep 2011 03:49:42 -0400

namedtuples need to have lt defined?

diff --git a/rpmutils/nevra.py b/rpmutils/nevra.py
--- a/rpmutils/nevra.py
+++ b/rpmutils/nevra.py
@@ -25,6 +25,8 @@
     Class to represent an RPM NEVRA.
     """
 
+    __slots__ = ()
+
     def __cmp__(self, other):
         if isinstance(other, tuple):
             other = self.__class__(*other)
@@ -51,3 +53,8 @@
 
         return 0
 
+    def __lt__(self, other):
+        c = self.__cmp__(other)
+        if c == -1:
+            return True
+        return False

From elliot at rpath.com  Tue Sep 20 03:50:24 2011
From: elliot at rpath.com (Elliot Peele)
Date: Tue, 20 Sep 2011 07:50:24 +0000
Subject: mirrorball: remove the status node once the debugger has
 continued
Message-ID: <201109200750.p8K7oO0c026088@scc.eng.rpath.com>

changeset:   60f27fa4ab70
user:        Elliot Peele <https://issues.rpath.com/>
date:        Tue, 20 Sep 2011 03:50:18 -0400

remove the status node once the debugger has continued

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -494,9 +494,13 @@
 
         # If anything has been promoting for more than 10 minutes, let us poke
         # around and figure out why.
+        toRemove = set()
         for jobId, startTime in self._status.iteritems():
             if time.time() - startTime > (60 * 20):
                 import epdb; epdb.st()
+                toRemove.add(jobId)
+        for jobId in toRemove:
+            self._status.pop(jobId, None)
 
         # Gather results
         for result in self._promoter.getStatus():

From elliot at rpath.com  Fri Sep 23 06:05:46 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 23 Sep 2011 10:05:46 +0000
Subject: mirrorball: remove debugging
Message-ID: <201109231005.p8NA5k25019124@scc.eng.rpath.com>

changeset:   34e24596d991
user:        Elliot Peele <https://issues.rpath.com/>
date:        Fri, 23 Sep 2011 06:03:22 -0400

remove debugging

diff --git a/updatebot/build/dispatcher.py b/updatebot/build/dispatcher.py
--- a/updatebot/build/dispatcher.py
+++ b/updatebot/build/dispatcher.py
@@ -447,8 +447,6 @@
         self._promoter = self._promoterClass((self._builder._conaryhelper,
             self._builder._cfg.targetLabel))
 
-        self._status = {}
-
     def _jobDone(self):
         # Override the job done method from the parent to hook into the
         # build loop. This is kinda dirty, but I don't really have a
@@ -484,32 +482,20 @@
                 res = tuple([ (x, tuple(y)) for x, y in result.iteritems() ])
 
                 toPromote.append((jobId, res))
-                self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
-
-                self._status[jobId] = time.time()
 
             if toPromote:
+                for jobId, res in toPromote:
+                    self._jobs[jobId][1] = JobStatus.JOB_PROMOTING
+
                 self._promoteSlots -= 1
                 self._promoter.promoteJob(toPromote)
 
-        # If anything has been promoting for more than 10 minutes, let us poke
-        # around and figure out why.
-        toRemove = set()
-        for jobId, startTime in self._status.iteritems():
-            if time.time() - startTime > (60 * 20):
-                log.info('Completely silly promote times have been reached')
-                #import epdb; epdb.st()
-                #toRemove.add(jobId)
-        for jobId in toRemove:
-            self._status.pop(jobId, None)
-
         # Gather results
         for result in self._promoter.getStatus():
             self._promoteSlots += 1
             for jobId, promoted in result:
                 self._jobs[jobId][2] = promoted
                 self._jobs[jobId][1] = JobStatus.JOB_PROMOTED
-                self._status.pop(jobId, None)
 
         # Gather errors
         for jobs, error in self._promoter.getErrors():
@@ -517,4 +503,3 @@
             for jobId in jobs:
                 self._jobs[jobId][1] = JobStatus.ERROR_PROMOTE_FAILURE
                 self._failures.append((jobId, error))
-                self._status.pop(jobId, None)

From elliot at rpath.com  Fri Sep 23 06:05:45 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 23 Sep 2011 10:05:45 +0000
Subject: mirrorball: add evr for archless comparison
Message-ID: <201109231005.p8NA5kMo019106@scc.eng.rpath.com>

changeset:   3f238088bc23
user:        Elliot Peele <https://issues.rpath.com/>
date:        Fri, 23 Sep 2011 06:02:59 -0400

add evr for archless comparison

diff --git a/rpmutils/nevra.py b/rpmutils/nevra.py
--- a/rpmutils/nevra.py
+++ b/rpmutils/nevra.py
@@ -20,6 +20,13 @@
 
 from rpmutils import rpmvercmp
 
+class EVR(namedtuple('evr', 'epoch version release')):
+    """
+    Class for storing just epoch, version, and release for comparison purposes.
+    """
+    __slots__ = ()
+
+
 class NEVRA(namedtuple('nevra', 'name epoch version release arch')):
     """
     Class to represent an RPM NEVRA.
@@ -58,3 +65,7 @@
         if c == -1:
             return True
         return False
+
+    @property
+    def evr(self):
+        return EVR(self.epoch, self.version, self.release)

From elliot at rpath.com  Fri Sep 23 06:05:46 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 23 Sep 2011 10:05:46 +0000
Subject: mirrorball: remove redundant dictionary creation
Message-ID: <201109231005.p8NA5kIl019161@scc.eng.rpath.com>

changeset:   c6bd12b445c3
user:        Elliot Peele <https://issues.rpath.com/>
date:        Fri, 23 Sep 2011 06:03:45 -0400

remove redundant dictionary creation

diff --git a/updatebot/build/monitor.py b/updatebot/build/monitor.py
--- a/updatebot/build/monitor.py
+++ b/updatebot/build/monitor.py
@@ -168,7 +168,7 @@
         # Build mapping of nvf tuple to jobId
         jobMap = {}
         for jobId, built in self.jobs:
-            for srcTrv, binTrvs in dict(built).iteritems():
+            for srcTrv, binTrvs in built:
                 for binTrv in binTrvs:
                     jobMap[binTrv] = (jobId, srcTrv)
 

From elliot at rpath.com  Fri Sep 23 06:05:46 2011
From: elliot at rpath.com (Elliot Peele)
Date: Fri, 23 Sep 2011 10:05:46 +0000
Subject: mirrorball: group building, now with more working
Message-ID: <201109231005.p8NA5kLa019191@scc.eng.rpath.com>

changeset:   c3603dbb5ba5
user:        Elliot Peele <https://issues.rpath.com/>
date:        Fri, 23 Sep 2011 06:05:39 -0400

group building, now with more working

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -21,11 +21,12 @@
 import logging
 import itertools
 
-from conary.trovetup import TroveTuple
 from conary.deps.deps import ThawFlavor
+from conary.versions import ThawVersion
 
 from rpmutils import NEVRA
 
+from updatebot.lib import util
 from updatebot import groupmgr
 from updatebot.bot import Bot as BotSuperClass
 
@@ -512,110 +513,157 @@
              self._updater._conaryhelper._ccfg.buildLabel)
 
         # Reverse the map and filter out old conary versions.
+        log.info('getting latest nevra packages')
         latest = self._getLatestNevraPackageMap(nevraMap)
 
-        # index by name, will need this later
+        # Get all of the sources for the possible packages
+        log.info('getting source versions for all packages')
+        allSources = self._updater._conaryhelper.getSourceVersions(
+            nevraMap.keys())
+
+        # The above gets us every binary version that was ever built from a
+        # source, we need the latest version of each package that was
+        # generated from a given source.
+        srcPkgs = {}
+        for src, bins in allSources.iteritems():
+            nameMap = {}
+            for n, v, f in bins:
+                nameMap.setdefault(n, dict()).setdefault(v, set()).add(f)
+            reduced = set()
+            for n, vs in nameMap.iteritems():
+                v = sorted(vs)[-1]
+                for f in vs[v]:
+                    reduced.add((n, v, f))
+            srcPkgs[src] = reduced
+
+        toAdd = {}
+
+        ##
+        # Find the latest NEVRAs available.
+        ##
+
+        # index nevras by name/arch so that we can find the latest version
         names = {}
         for nevra, nvf in latest.iteritems():
-            names.setdefault((nevra.name, nevra.arch), dict())[nevra] = nvf
+            names.setdefault(nevra.name, dict()).setdefault(nevra.evr,
+                dict())[nevra] = nvf
 
-        toAdd = {}
-        toRemove = set()
-        
-        #groupPkgMap = [ (str(pkg.name), ThawFlavor(str(pkg.flavor))) 
-        #                    for pkg in group.iterpackages() ]
-        
-        nevras = {}
-        for nvf, nevra in nevraMap.iteritems():
-                nevras.setdefault(nevra, set()).add(nvf)
-        
+        # Find the latest nevras.
+        actualLatest = {}
+        for name, vercmpd in names.iteritems():
+            lt = sorted(vercmpd.keys(), cmp=util.packagevercmp)[-1]
+            for nevra, nvf in vercmpd[lt].iteritems():
+                actualLatest[nvf] = nevra
 
-        #import epdb; epdb.st()
-        
-        binSpecMap = {}
-        removed = {}
-        for xPkg in names.iteritems():
-            for nvf, lPkg in names[xPkg[0]].iteritems():
-                myup = [ x for x,y in latest.iteritems() if x.name == nvf.name]
-                myup.sort()
-                mylatest = myup[-1]
-                allup = [ x for x,y in latest.iteritems() 
-                        if (x.name, x.epoch, x.version, x.release) 
-                        == (mylatest.name, mylatest.epoch, 
-                            mylatest.version, mylatest.release) ]
-                for nevra in allup:
-                    
-                    log.info('working %s %s  %s %s %s' 
-                            % (nevra.name, nevra.epoch, 
-                                nevra.version, nevra.release, nevra.arch))
-                    
-                    if latest.has_key(nevra):
-                        binSpecMap.setdefault(nevra, latest[nevra])
-                    else:
-                        log.error('WTF?')
-                        import epdb; epdb.st()
+        ##
+        # Add latest NEVRAs by source to make sure we don't add NEVRAs that
+        # reference more than one source version.
+        ##
 
+        # Map the latest nevra nvfs to conary sources.
+        latestSources = self._updater._conaryhelper.getSourceVersions(
+            actualLatest.keys())
 
-        #import epdb; epdb.st()
-        # All the latest versions of the binaries maped by source.
-        srcSpecMap = self._updater._conaryhelper.getSourceVersions(binSpecMap.values())
-        
-        #nevraSrcSpecMap = self._updater._conaryhelper.getSourceVersions(nevraMap.keys())
+        # Map by name so that we can find duplicate versions
+        srcMap = {}
+        for src, bins in latestSources.iteritems():
+            srcMap.setdefault(src[0], set()).add(src)
 
-        #import epdb; epdb.st()
+        for name, srcs in srcMap.iteritems():
+            # Take the easy way out if there is only one source version of
+            # this name.
+            if len(srcs) == 1:
+                for bin in srcPkgs[list(srcs)[0]]:
+                    toAdd.setdefault((bin[0], bin[1]), set()).add(bin[2])
+                continue
 
-        mylist = [ (x[0], x) for x,y in srcSpecMap.iteritems() ]
-        mylist.sort()
+            # Now we have to work reasonably hard to find a nevra that we can
+            # use to sort source versions since we don't know what source nevra
+            # maps to a particular conary source version.
+            #
+            # FIXME: This is probably a good place for a conary feature to
+            #        source source nevras in trove info in the binary packages.
+            #        We would probably need to fall back to the manifest in the
+            #        source if the trove info wasn't there.
 
-        idx = 0
-        while idx + 1 < len(mylist):
-            if mylist[idx][0] == mylist[idx + 1][0]:
-                log.info('DUPE: %s --> %s' % (mylist[idx], mylist[idx + 1]))
-                srcnvf = mylist[idx][1]
-                srcnvf_ =  mylist[idx + 1][1]
-                binNvfs = srcSpecMap[srcnvf]
-                binNvfs_ = srcSpecMap[srcnvf_]
-                d_one = dict([ ((x[0], nevraMap[x].arch), nevraMap[x]) for x in binNvfs ])
-                d_two =  dict([ ((x[0], nevraMap[x].arch), nevraMap[x]) for x in binNvfs_ ])
-                for x in d_one.keys():
-                    if x in d_two.keys():
-                        d_three = dict([((d_one[x].epoch, d_one[x].version, d_one[x].release), srcnvf), 
-                                    ((d_two[x].epoch, d_two[x].version, d_two[x].release), srcnvf_)])
-                        l_one = d_three.keys()
-                        l_one.sort()
-                        rem = d_three[l_one[0]]
-                        removed.setdefault((rem[0], rem[1]), set()).add(rem[2])
-                        srcSpecMap.pop(rem)
-                        break
+            # find the latest source by nevra
+            fullSrcs = dict([ (x, srcPkgs[x]) for x in srcs ])
+            # find a common binary across all source versions
+            binnames = {}
+            for src, bins in fullSrcs.iteritems():
+                # only consider the sources that we are concerned with.
+                if src not in srcs:
+                    continue
+                for bin in bins:
+                    binnames.setdefault((bin[0], bin[2]), dict())[bin] = src
+            common = None
+            for binname, bind in binnames.iteritems():
+                if len(bind) == len(srcs):
+                    common = bind
+                    break
 
-                #import epdb; epdb.st()
-            idx += 1
+            # If we get here and common is None, that means that there were no
+            # common pacakges between the source versions. This happens in RHEL
+            # with some of the kmod packages. I guess assume that the conary
+            # source version is good enough for sorting?
+            if common is None:
+                lts = sorted(srcs)[-1]
 
-        import epdb; epdb.st()
+            else:
+                # now lookup the nevras for the versions of this binary so
+                # that we can determine which source is latest.
+                nvmap = {}
+                for bin, src in common.iteritems():
+                    nvmap[nevraMap[bin]] = src
 
-        for src in srcSpecMap:
-            for foo in srcSpecMap[src]:
-                if [ x for x in group.iterpackages()
-                        if (foo[0], foo[1].freeze(), foo[2].freeze()) ==
-                            (x.name, x.version, x.flavor) ]:
-                    log.info('found %s %s %s in the group skipping...' %
-                                (foo[0], foo[1], foo[2]))
-                    continue
+                lts = nvmap[sorted(nvmap)[-1]]
 
-                log.info('adding %s %s %s to the group' % (foo[0], foo[1], foo[2]))
-                toAdd.setdefault((foo[0], foo[1]), set()).add(foo[2])
-                           
-        import epdb; epdb.st()
+            for bin in fullSrcs.get(lts):
+                toAdd.setdefault((bin[0], bin[1]), set()).add(bin[2])
 
+        ##
+        # Now to remove all of the things that are already in the group from
+        # the toAdd dict.
+        ##
 
-        for nvf in toRemove:
-            group.removePackage(nvf.name, flavor=nvf.flavor)
+        grpPkgs = {}
+        for pkg in group.iterpackages():
+            name = str(pkg.name)
+            version = ThawVersion(str(pkg.version))
+            flavor = ThawFlavor(str(pkg.flavor))
+
+            grpPkgs.setdefault((name, version), set()).add(flavor)
+
+        for nv, fs in grpPkgs.iteritems():
+            if nv in toAdd and toAdd[nv] == fs:
+                toAdd.pop(nv)
+
+        ##
+        # Check to make sure we aren't adding back a package that was previously
+        # removed.
+        ##
+
+        newPkgs = dict([ (x[0], x) for x in toAdd
+            if x[0] not in [ y[0] for y in grpPkgs ] ])
+
+        removeObsoleted = set([ x for x in
+            itertools.chain(*self._cfg.removeObsoleted.values()) ])
+        updateRemovesPackage = set([ x for x in
+            itertools.chain(*self._cfg.updateRemovesPackages.values()) ])
+
+        removed = removeObsoleted | updateRemovesPackage
+
+        for name in removed:
+            if name in newPkgs:
+                toAdd.pop(newPkgs[name])
+
+        ##
+        # Actually add the packages to the group model.
+        ##
 
         for (name, version), flavors in toAdd.iteritems():
             group.addPackage(name, version, flavors)
 
-        import epdb; epdb.st()
-
     def buildgroups(self):
         """
         Find the latest packages on the production label by nevra and build a
@@ -691,7 +739,6 @@
         # Build groups.
         log.info('setting version %s' % version)
         group.version = version
-        import epdb; epdb.st()
         group = group.commit()
         grpTrvMap = group.build()
 

From elliot at rpath.com  Mon Sep 26 10:30:35 2011
From: elliot at rpath.com (Elliot Peele)
Date: Mon, 26 Sep 2011 14:30:35 +0000
Subject: mirrorball: several group modification fixups
Message-ID: <201109261430.p8QEUZjp003325@scc.eng.rpath.com>

changeset:   54af7693e4ec
user:        Elliot Peele <https://issues.rpath.com/>
date:        Mon, 26 Sep 2011 10:30:33 -0400

several group modification fixups

diff --git a/updatebot/current.py b/updatebot/current.py
--- a/updatebot/current.py
+++ b/updatebot/current.py
@@ -537,6 +537,7 @@
             srcPkgs[src] = reduced
 
         toAdd = {}
+        toRemove = set()
 
         ##
         # Find the latest NEVRAs available.
@@ -614,9 +615,9 @@
                 # that we can determine which source is latest.
                 nvmap = {}
                 for bin, src in common.iteritems():
-                    nvmap[nevraMap[bin]] = src
+                    nvmap.setdefault(nevraMap[bin], set()).add(src)
 
-                lts = nvmap[sorted(nvmap)[-1]]
+                lts = sorted(nvmap[sorted(nvmap)[-1]])[-1]
 
             for bin in fullSrcs.get(lts):
                 toAdd.setdefault((bin[0], bin[1]), set()).add(bin[2])
@@ -639,6 +640,36 @@
                 toAdd.pop(nv)
 
         ##
+        # Iterate over the group contents, looking for any packages that may
+        # have been rebuilt, but the nevra stayed the same.
+        ##
+
+        for (n, v), fs in grpPkgs.iteritems():
+            for f in fs:
+                # Get the nevra for this name, version, and flavor
+                nevra = nevraMap.get((n, v, f))
+
+                # If the package is already in the toAdd map, skip over it.
+                if [ x for x in toAdd
+                    if x[0] == n and f in toAdd[(x[0], x[1])] ]:
+                    continue
+
+                # Found a package that isn't actually attached to a nevra,
+                # remove it.
+                if nevra is None and not [ x for x in nevraMap
+                    if x[0] == n and x[2] == f ]:
+                    toRemove.add((n, v, f))
+                    continue
+
+                # Now find the latest nvf for this nevra.
+                n2, v2, f2 = latest.get(nevra)
+
+                # If the latest nevra nvf is newer than what is in the
+                # group, replace it.
+                if v < lt[1]:
+                    toAdd.setdefault((n2, v2), set()).add(f2)
+
+        ##
         # Check to make sure we aren't adding back a package that was previously
         # removed.
         ##
@@ -658,10 +689,20 @@
                 toAdd.pop(newPkgs[name])
 
         ##
+        # Remove any packages that were flagged for removal.
+        ##
+
+        for n, v, f in toRemove:
+            log.info('removing %s[%s]' % (n, f))
+            group.removePackage(n, flavor=f)
+
+        ##
         # Actually add the packages to the group model.
         ##
 
         for (name, version), flavors in toAdd.iteritems():
+            for f in flavors:
+                log.info('adding %s=%s[%s]' % (name, version, f))
             group.addPackage(name, version, flavors)
 
     def buildgroups(self):

From elliot at rpath.com  Wed Sep 28 17:18:07 2011
From: elliot at rpath.com (Elliot Peele)
Date: Wed, 28 Sep 2011 21:18:07 +0000
Subject: mirrorball: strip off any component names
Message-ID: <201109282118.p8SLI7wC018528@scc.eng.rpath.com>

changeset:   55b3f5a168c2
user:        Elliot Peele <https://issues.rpath.com/>
date:        Wed, 28 Sep 2011 17:17:33 -0400

strip off any component names

diff --git a/updatebot/build/build.py b/updatebot/build/build.py
--- a/updatebot/build/build.py
+++ b/updatebot/build/build.py
@@ -527,6 +527,9 @@
             # the deps modules in conary.
             name = name.encode()
 
+            # Make sure name is not a component, like a source component.
+            name = name.split(':')[0]
+
             # Build groups in all of the defined flavors. We don't need a
             # context here since groups are all built in a single job.
             if name.startswith('group-'):