Bug 1289514 - Consume pushlog version 2; r=catlee
Version 2 of the pushlog JSON was rolled out a long time ago -
probably over a year ago. It is the preferred pushlog format
to consume.
So change hgpoller.py to consume it.
The actual code changes were pretty simple: just add a query
string parameter to the URL and process the "pushes" key of the
JSON instead of the root object.
While I was updating mocked JSON in the tests, I changed things
to use 4 space indent instead of 1 space. The data came straight
from hg.mozilla.org.
MozReview-Commit-ID: 1Dzwx9Q61zL
--- a/changes/hgpoller.py
+++ b/changes/hgpoller.py
@@ -39,70 +39,82 @@
#
# ***** END LICENSE BLOCK *****
"""hgpoller provides Pollers to work on single hg repositories as well
as on a group of hg repositories. It's polling the json feed of pushlog,
which of the form
{
- "15092": {
- "date": 1281863455,
- "changesets": [
- {
- "node": "ace72819f4a94b9175519a8fa5a1db654edae098",
- "files": [
- "gfx/thebes/gfxBlur.cpp"
- ],
- "tags": [],
- "author": "Julian Seward <jseward@acm.org>",
- "branch": "default",
- "desc": "Bug 582668 - gfxAlphaBoxBlur::Paint appears to pass garbage down through Cairo. r=roc"
- },
- {
- "node": "43b490ef9dab30db2c4e2706110ad5d524a21597",
- "files": [
- "content/html/document/src/nsHTMLDocument.cpp",
- "dom/interfaces/html/nsIDOMNSHTMLDocument.idl",
- "js/src/xpconnect/src/dom_quickstubs.qsconf"
- ],
- "tags": [],
- "author": "Ms2ger <ms2ger@gmail.com>",
- "branch": "default",
- "desc": "Bug 585877 - remove document.height / document.width. r=sicking, sr=jst"
- },
- {
- "node": "75caf7ab03760f6bc39775cd8c4e097f33161c58",
- "files": [
- "modules/plugin/base/src/nsNPAPIPlugin.cpp"
- ],
- "tags": [],
- "author": "Martin Str\u00e1nsk\u00fd <stransky@redhat.com>",
- "branch": "default",
- "desc": "Bug 574354 - Disable OOP for plugins wrapped by nspluginwrapper. r=josh"
- }
- ],
- "user": "dgottwald@mozilla.com"
- }
+ "lastpushid": 30492,
+ "pushes": {
+ "15092": {
+ "changesets": [
+ {
+ "author": "Julian Seward <jseward@acm.org>",
+ "branch": "default",
+ "desc": "Bug 582668 - gfxAlphaBoxBlur::Paint appears to pass garbage down through Cairo. r=roc",
+ "files": [
+ "gfx/thebes/gfxBlur.cpp"
+ ],
+ "node": "ace72819f4a94b9175519a8fa5a1db654edae098",
+ "parents": [
+ "f2af48b0cd7c22c0af016d33a34ae5dc6e3141ab"
+ ],
+ "tags": []
+ },
+ {
+ "author": "Ms2ger <ms2ger@gmail.com>",
+ "branch": "default",
+ "desc": "Bug 585877 - remove document.height / document.width. r=sicking, sr=jst",
+ "files": [
+ "content/html/document/src/nsHTMLDocument.cpp",
+ "dom/interfaces/html/nsIDOMNSHTMLDocument.idl",
+ "js/src/xpconnect/src/dom_quickstubs.qsconf"
+ ],
+ "node": "43b490ef9dab30db2c4e2706110ad5d524a21597",
+ "parents": [
+ "ace72819f4a94b9175519a8fa5a1db654edae098"
+ ],
+ "tags": []
+ },
+ {
+ "author": "Martin Str\u00e1nsk\u00fd <stransky@redhat.com>",
+ "branch": "default",
+ "desc": "Bug 574354 - Disable OOP for plugins wrapped by nspluginwrapper. r=josh",
+ "files": [
+ "modules/plugin/base/src/nsNPAPIPlugin.cpp"
+ ],
+ "node": "75caf7ab03760f6bc39775cd8c4e097f33161c58",
+ "parents": [
+ "43b490ef9dab30db2c4e2706110ad5d524a21597"
+ ],
+ "tags": []
+ }
+ ],
+ "date": 1281863455,
+ "user": "dgottwald@mozilla.com"
+ }
+ }
}
"""
import time
from twisted.python import log
from twisted.internet import defer, reactor
from twisted.internet.task import LoopingCall
from twisted.web.client import getPage
from buildbot.changes import base, changes
from buildbot.util import json
def _parse_changes(data):
- pushes = json.loads(data).values()
+ pushes = json.loads(data)['pushes'].values()
# Sort by push date
pushes.sort(key=lambda p: p['date'])
return pushes
class Pluggable(object):
'''The Pluggable class implements a forward for Deferred's that
can be thrown away.
@@ -220,17 +232,17 @@ class BaseHgPoller(BasePoller):
log.msg("Polling Hg server at %s" % url)
return getPage(url, timeout=self.timeout)
def _make_url(self):
url = None
if self.pushlogUrlOverride:
url = self.pushlogUrlOverride
else:
- url = "/".join((self.baseURL, 'json-pushes?full=1'))
+ url = "/".join((self.baseURL, 'json-pushes?version=2&full=1'))
args = []
if self.lastChangeset is not None:
args.append('fromchange=' + self.lastChangeset)
if self.tipsOnly:
args.append('tipsonly=1')
if args:
if '?' not in url:
--- a/test/test_hgpoller.py
+++ b/test/test_hgpoller.py
@@ -54,61 +54,61 @@ class TestHTTPServer(object):
# call to server_stop here closes that socket, which results in a
# socket error in the thread, which we catch and then exit.
self.server.server_close()
self.server_thread.join()
class UrlCreation(unittest.TestCase):
def testSimpleUrl(self):
- correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?full=1'
+ correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?version=2&full=1'
poller = hgpoller.BaseHgPoller(
hgURL='https://hg.mozilla.org', branch='mozilla-central')
url = poller._make_url()
self.failUnlessEqual(url, correctUrl)
def testUrlWithLastChangeset(self):
- correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?full=1&fromchange=123456'
+ correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?version=2&full=1&fromchange=123456'
poller = hgpoller.BaseHgPoller(
hgURL='https://hg.mozilla.org', branch='mozilla-central')
poller.lastChangeset = '123456'
url = poller._make_url()
self.failUnlessEqual(url, correctUrl)
def testTipsOnlyUrl(self):
- correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?full=1&tipsonly=1'
+ correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?version=2&full=1&tipsonly=1'
poller = hgpoller.BaseHgPoller(
hgURL='https://hg.mozilla.org', branch='mozilla-central',
tipsOnly=True)
url = poller._make_url()
self.failUnlessEqual(url, correctUrl)
def testTipsOnlyWithLastChangeset(self):
# there's two possible correct URLs in this case
correctUrls = [
- 'https://hg.mozilla.org/releases/mozilla-1.9.1/json-pushes?full=1&fromchange=123456&tipsonly=1',
- 'https://hg.mozilla.org/releases/mozilla-1.9.1/json-pushes?full=1&tipsonly=1&fromchange=123456'
+ 'https://hg.mozilla.org/releases/mozilla-1.9.1/json-pushes?version=2&full=1&fromchange=123456&tipsonly=1',
+ 'https://hg.mozilla.org/releases/mozilla-1.9.1/json-pushes?version=2&full=1&tipsonly=1&fromchange=123456'
]
poller = hgpoller.BaseHgPoller(hgURL='https://hg.mozilla.org',
branch='releases/mozilla-1.9.1', tipsOnly=True)
poller.lastChangeset = '123456'
url = poller._make_url()
self.failUnlessIn(url, correctUrls)
def testOverrideUrl(self):
- correctUrl = 'https://hg.mozilla.org/other_repo/json-pushes?full=1&fromchange=123456'
+ correctUrl = 'https://hg.mozilla.org/other_repo/json-pushes?version=2&full=1&fromchange=123456'
poller = hgpoller.BaseHgPoller(
hgURL='https://hg.mozilla.org', branch='mozilla-central',
- pushlogUrlOverride='https://hg.mozilla.org/other_repo/json-pushes?full=1')
+ pushlogUrlOverride='https://hg.mozilla.org/other_repo/json-pushes?version=2&full=1')
poller.lastChangeset = '123456'
url = poller._make_url()
self.failUnlessEqual(url, correctUrl)
def testUrlWithUnicodeLastChangeset(self):
- correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?full=1&fromchange=123456'
+ correctUrl = 'https://hg.mozilla.org/mozilla-central/json-pushes?version=2&full=1&fromchange=123456'
poller = hgpoller.BaseHgPoller(
hgURL='https://hg.mozilla.org', branch='mozilla-central')
poller.lastChangeset = u'123456'
url = poller._make_url()
self.failUnlessEqual(url, correctUrl)
self.failUnless(isinstance(url, str))
@@ -193,90 +193,84 @@ class TestPolling(unittest.TestCase):
def testHgAllLocalesPoller(self):
url = 'http://localhost:%s' % str(self.server.port)
return self.doPollingTest(hgpoller.HgAllLocalesPoller, hgURL=url,
repositoryIndex='foobar')
validPushlog = """
{
- "15226": {
- "date": 1282358416,
- "changesets": [
- {
- "node": "4c23e51a484f077ea27af3ea4a4ee13da5aeb5e6",
- "files": [
- "embedding/android/GeckoInputConnection.java",
- "embedding/android/GeckoSurfaceView.java",
- "widget/src/android/nsWindow.cpp",
- "widget/src/android/nsWindow.h"
- ],
- "tags": [],
- "author": "Jim Chen <jchen@mozilla.com>",
- "branch": "GECKO20b5pre_20100820_RELBRANCH",
- "desc": "Bug 588456 - Properly commit Android IME composition on blur; r=mwu a=blocking-fennec"
- }
- ],
- "user": "dougt@mozilla.com"
- },
- "15227": {
- "date": 1282362551,
- "changesets": [
- {
- "node": "ee6fb954cbc3de0f76e84cad6bdff452116e1b03",
- "files": [
- "browser/base/content/browser.js",
- "browser/components/privatebrowsing/content/aboutPrivateBrowsing.xhtml",
- "browser/locales/en-US/chrome/overrides/netError.dtd",
- "build/automation.py.in",
- "docshell/resources/content/netError.xhtml",
- "dom/locales/en-US/chrome/netErrorApp.dtd",
- "extensions/cookie/nsPermissionManager.cpp"
- ],
- "tags": [],
- "author": "Bobby Holley <bobbyholley@gmail.com>",
- "branch": "default",
- "desc": "Backout of changesets c866e73f3209 and baff7b7b32bc because of sicking's push-and-run bustage. a=backout"
- },
- {
- "node": "33be08836cb164f9e546231fc59e9e4cf98ed991",
- "files": [
- "modules/libpref/src/init/all.js"
- ],
- "tags": [],
- "author": "Bobby Holley <bobbyholley@gmail.com>",
- "branch": "default",
- "desc": "Bug 563088 - Re-enable image discarding.r=joe,a=blocker"
- }
- ],
- "user": "bobbyholley@stanford.edu"
- }
+ "lastpushid": 30492,
+ "pushes": {
+ "15226": {
+ "changesets": [
+ {
+ "author": "Jim Chen <jchen@mozilla.com>",
+ "branch": "GECKO20b5pre_20100820_RELBRANCH",
+ "desc": "Bug 588456 - Properly commit Android IME composition on blur; r=mwu a=blocking-fennec",
+ "files": [
+ "embedding/android/GeckoInputConnection.java",
+ "embedding/android/GeckoSurfaceView.java",
+ "widget/src/android/nsWindow.cpp",
+ "widget/src/android/nsWindow.h"
+ ],
+ "node": "4c23e51a484f077ea27af3ea4a4ee13da5aeb5e6",
+ "parents": [
+ "935c15d506516a2269cee35a1a80748aaec1ae08"
+ ],
+ "tags": []
+ }
+ ],
+ "date": 1282358416,
+ "user": "dougt@mozilla.com"
+ },
+ "15227": {
+ "changesets": [
+ {
+ "author": "Bobby Holley <bobbyholley@gmail.com>",
+ "branch": "default",
+ "desc": "Backout of changesets c866e73f3209 and baff7b7b32bc because of sicking's push-and-run bustage. a=backout",
+ "files": [
+ "browser/base/content/browser.js",
+ "browser/components/privatebrowsing/content/aboutPrivateBrowsing.xhtml",
+ "browser/locales/en-US/chrome/overrides/netError.dtd",
+ "build/automation.py.in",
+ "docshell/resources/content/netError.xhtml",
+ "dom/locales/en-US/chrome/netErrorApp.dtd",
+ "extensions/cookie/nsPermissionManager.cpp"
+ ],
+ "node": "ee6fb954cbc3de0f76e84cad6bdff452116e1b03",
+ "parents": [
+ "baff7b7b32bc3dd7132cddd3957f6898b5bebfaf"
+ ],
+ "tags": []
+ },
+ {
+ "author": "Bobby Holley <bobbyholley@gmail.com>",
+ "branch": "default",
+ "desc": "Bug 563088 - Re-enable image discarding.r=joe,a=blocker",
+ "files": [
+ "modules/libpref/src/init/all.js"
+ ],
+ "node": "33be08836cb164f9e546231fc59e9e4cf98ed991",
+ "parents": [
+ "ee6fb954cbc3de0f76e84cad6bdff452116e1b03"
+ ],
+ "tags": []
+ }
+ ],
+ "date": 1282362551,
+ "user": "bobbyholley@stanford.edu"
+ }
+ }
}
"""
malformedPushlog = """
-{
- "15226": {
- "date": 1282358416,
- "changesets": [
- {
- "node": "4c23e51a484f077ea27af3ea4a4ee13da5aeb5e6",
- "files": [
- "embedding/android/GeckoInputConnection.java",
- "embedding/android/GeckoSurfaceView.java",
- "widget/src/android/nsWindow.cpp",
- "widget/src/android/nsWindow.h"
- ],
- "tags": [],
- "author": "Jim Chen <jchen@mozilla.com>",
- "branch": "GECKO20b5pre_20100820_RELBRANCH",
- "desc": "Bug 588456 - Properly commit Android IME composition on blur; r=mwu a=blocking-fennec"
- }
- ],
- "user": "dougt@mozilla.com"
+{ "invalid json" }
"""
class PushlogParsing(unittest.TestCase):
def testValidPushlog(self):
pushes = hgpoller._parse_changes(validPushlog)
self.failUnlessEqual(len(pushes), 2)
