request_splitter: switch from build percentage to staging age for merge.

Using build percentage is sub-optimal since many events can reset the
percentage which can result in additional requests being merged into the
staging. A better metric is the time since the first request was added to
the staging (the age of the staging). Unfortunately, this is not trivial to
determine especially given that the original request may be superseded or
unstaged entirely. As such the datetime at which the staging was activated
is stored in the staging pseudometa. A max age is then used to allow for
merging up until that point.

osclib/request_splitter.py

@@ -8,9 +8,9 @@ class RequestSplitter(object):
         self.api = api
         self.requests = requests
         self.in_ring = in_ring
-        self.mergeable_build_percent = 80
         # 55 minutes to avoid two staging bot loops of 30 minutes
         self.request_age_threshold = 55 * 60
+        self.staging_age_max = 8 * 60 * 60
 
         self.requests_ignored = self.api.get_ignored_requests()
 
@@ -170,8 +170,17 @@ class RequestSplitter(object):
         return len(pseudometa['requests']) > 0 and 'splitter_info' in pseudometa
 
     def should_staging_merge(self, status, pseudometa):
-        return (status['overall_state'] == 'building' and
-                self.api.project_status_build_percent(status) <= self.mergeable_build_percent)
+        if 'activated' not in pseudometa['splitter_info']:
+            # No information on the age of the staging.
+            return False
+
+        # Allows for immediate staging when possible while not blocking requests
+        # created shortly after. This method removes the need to wait to create
+        # a larger staging at once while not ending up with lots of tiny
+        # stagings. As such this handles both high and low request backlogs.
+        activated = dateutil.parser.parse(pseudometa['splitter_info']['activated'])
+        delta = datetime.utcnow() - activated
+        return delta.total_seconds() <= self.staging_age_max
 
     def staging_status_load(self, project):
         status = self.api.project_status(project)

osclib/stagingapi.py

@@ -15,6 +15,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from cStringIO import StringIO
+from datetime import datetime
 import json
 import logging
 import urllib2
@@ -691,6 +692,7 @@ class StagingAPI(object):
         data['splitter_info'] = {
             'group': group,
             'strategy': strategy_info,
+            'activated': str(datetime.utcnow()),
         }
         self.set_prj_pseudometa(project, data)