Bug 1244861 - Gzip outgoing telemetry pings. r=rnewman draft
authorMichael Comella <michael.l.comella@gmail.com>
Wed, 10 Feb 2016 16:11:34 -0800
changeset 330194 90955bbb73238034ae6ee46c6f0af33fff23d016
parent 329332 103f2dcf2778d0c22db916e96a962bc0d4eb7ac0
child 514129 9fba7479ea07ea9429a5d88c705fcebb73fdff70
push id10709
push usermichael.l.comella@gmail.com
push dateThu, 11 Feb 2016 00:14:39 +0000
reviewersrnewman
bugs1244861
milestone47.0a1
Bug 1244861 - Gzip outgoing telemetry pings. r=rnewman This commit adds the GzipNonChunkedCompressingEntity which is necessary because the telemetry servers don't support chunked uploading, which the built in GzipCompressingEntity does. I tested this on my local device and logs for successful uploads were sent for both the testing gzip server as well as the official telemetry server. My data correctly appears on the former and I did not check the latter. MozReview-Commit-ID: 4bCNiRYyqFD
mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java
mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/GzipNonChunkedCompressingEntity.java
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java
@@ -191,16 +191,17 @@ public class TelemetryUploadService exte
             resource = new BaseResource(ping.getURL());
         } catch (final URISyntaxException e) {
             Log.w(LOGTAG, "URISyntaxException for server URL when creating BaseResource: returning.");
             return;
         }
 
         delegate.setResource(resource);
         resource.delegate = delegate;
+        resource.setShouldCompressUploadedEntity(true, false); // Telemetry servers don't support chunking.
 
         // We're in a background thread so we don't have any reason to do this asynchronously.
         // If we tried, onStartCommand would return and IntentService might stop itself before we finish.
         resource.postBlocking(ping.getPayload());
     }
 
     private static class CorePingResultDelegate extends ResultDelegate {
         public CorePingResultDelegate() {
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java
@@ -25,16 +25,17 @@ import org.mozilla.gecko.background.comm
 import org.mozilla.gecko.sync.ExtendedJSONObject;
 
 import ch.boye.httpclientandroidlib.Header;
 import ch.boye.httpclientandroidlib.HttpEntity;
 import ch.boye.httpclientandroidlib.HttpResponse;
 import ch.boye.httpclientandroidlib.HttpVersion;
 import ch.boye.httpclientandroidlib.client.AuthCache;
 import ch.boye.httpclientandroidlib.client.ClientProtocolException;
+import ch.boye.httpclientandroidlib.client.entity.GzipCompressingEntity;
 import ch.boye.httpclientandroidlib.client.methods.HttpDelete;
 import ch.boye.httpclientandroidlib.client.methods.HttpGet;
 import ch.boye.httpclientandroidlib.client.methods.HttpPatch;
 import ch.boye.httpclientandroidlib.client.methods.HttpPost;
 import ch.boye.httpclientandroidlib.client.methods.HttpPut;
 import ch.boye.httpclientandroidlib.client.methods.HttpRequestBase;
 import ch.boye.httpclientandroidlib.client.methods.HttpUriRequest;
 import ch.boye.httpclientandroidlib.client.protocol.ClientContext;
@@ -75,16 +76,19 @@ public class BaseResource implements Res
 
   protected final URI uri;
   protected BasicHttpContext context;
   protected DefaultHttpClient client;
   public    ResourceDelegate delegate;
   protected HttpRequestBase request;
   public final String charset = "utf-8";
 
+  private boolean shouldGzipCompress = false;
+  private boolean shouldGzipChunk = true; // Default to using GzipCompressingEntity, which is built-in functionality.
+
   /**
    * We have very few writes (observers tend to be installed around sync
    * sessions) and many iterations (every HTTP request iterates observers), so
    * CopyOnWriteArrayList is a reasonable choice.
    */
   protected static final CopyOnWriteArrayList<WeakReference<HttpResponseObserver>>
     httpResponseObservers = new CopyOnWriteArrayList<>();
 
@@ -158,16 +162,34 @@ public class BaseResource implements Res
   }
 
   @Override
   public String getHostname() {
     return this.getURI().getHost();
   }
 
   /**
+   * Causes the Resource to compress the uploaded entity payload in requests with payloads (e.g. post, put)
+   * @param shouldCompress true if the entity should be compressed, false otherwise
+   * @param shouldChunk true if the transfer should be chunked, false otherwise; no effect if compression == false
+   */
+  public void setShouldCompressUploadedEntity(final boolean shouldCompress, final boolean shouldChunk) {
+    shouldGzipCompress = shouldCompress;
+    shouldGzipChunk = shouldChunk;
+  }
+
+  private HttpEntity getMaybeCompressedEntity(final HttpEntity entity) {
+    if (!shouldGzipCompress) {
+      return entity;
+    }
+
+    return shouldGzipChunk ? new GzipCompressingEntity(entity) : new GzipNonChunkedCompressingEntity(entity);
+  }
+
+  /**
    * This shuts up HttpClient, which will otherwise debug log about there
    * being no auth cache in the context.
    */
   private static void addAuthCacheToContext(HttpUriRequest request, HttpContext context) {
     AuthCache authCache = new BasicAuthCache();                // Not thread safe.
     context.setAttribute(ClientContext.AUTH_CACHE, authCache);
   }
 
@@ -360,32 +382,35 @@ public class BaseResource implements Res
   public void delete() {
     Logger.debug(LOG_TAG, "HTTP DELETE " + this.uri.toASCIIString());
     this.go(new HttpDelete(this.uri));
   }
 
   @Override
   public void post(HttpEntity body) {
     Logger.debug(LOG_TAG, "HTTP POST " + this.uri.toASCIIString());
+    body = getMaybeCompressedEntity(body);
     HttpPost request = new HttpPost(this.uri);
     request.setEntity(body);
     this.go(request);
   }
 
   @Override
   public void patch(HttpEntity body) {
     Logger.debug(LOG_TAG, "HTTP PATCH " + this.uri.toASCIIString());
+    body = getMaybeCompressedEntity(body);
     HttpPatch request = new HttpPatch(this.uri);
     request.setEntity(body);
     this.go(request);
   }
 
   @Override
   public void put(HttpEntity body) {
     Logger.debug(LOG_TAG, "HTTP PUT " + this.uri.toASCIIString());
+    body = getMaybeCompressedEntity(body);
     HttpPut request = new HttpPut(this.uri);
     request.setEntity(body);
     this.go(request);
   }
 
   protected static StringEntity stringEntityWithContentTypeApplicationJSON(String s) {
     StringEntity e = new StringEntity(s, "UTF-8");
     e.setContentType("application/json");
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/GzipNonChunkedCompressingEntity.java
@@ -0,0 +1,93 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.net;
+
+import ch.boye.httpclientandroidlib.HttpEntity;
+import ch.boye.httpclientandroidlib.client.entity.GzipCompressingEntity;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Wrapping entity that compresses content when {@link #writeTo writing}.
+ *
+ * This differs from {@link GzipCompressingEntity} in that it does not chunk
+ * the sent data, therefore replacing the "Transfer-Encoding" HTTP header with
+ * the "Content-Length" header required by some servers.
+ *
+ * However, to measure the content length, the gzipped content will be temporarily
+ * stored in memory so be careful what content you send!
+ */
+public class GzipNonChunkedCompressingEntity extends GzipCompressingEntity {
+    private byte[] gzippedContent;
+
+    public GzipNonChunkedCompressingEntity(final HttpEntity entity) {
+        super(entity);
+    }
+
+    /**
+     * @return content length for gzipped content or -1 if there is an error
+     */
+    @Override
+    public long getContentLength() {
+        try {
+            initBuffer();
+        } catch (final IOException e) {
+            // GzipCompressingEntity always returns -1 in which case a 'Content-Length' header is omitted.
+            // Presumably, without it the request will fail (either client-side or server-side).
+            return -1;
+        }
+        return gzippedContent.length;
+    }
+
+    @Override
+    public boolean isChunked() {
+        // "Content-Length" & chunked encoding are mutually exclusive:
+        //   https://en.wikipedia.org/wiki/Chunked_transfer_encoding
+        return false;
+    }
+
+    @Override
+    public InputStream getContent() throws IOException {
+        initBuffer();
+        return new ByteArrayInputStream(gzippedContent);
+    }
+
+    @Override
+    public void writeTo(final OutputStream outstream) throws IOException {
+        initBuffer();
+        outstream.write(gzippedContent);
+    }
+
+    private void initBuffer() throws IOException {
+        if (gzippedContent != null) {
+            return;
+        }
+
+        final ByteArrayOutputStream s = getStreamForGzippedContent(wrappedEntity.getContentLength());
+        try {
+            super.writeTo(s);
+        } finally {
+            s.close();
+        }
+
+        gzippedContent = s.toByteArray();
+    }
+
+    /**
+     * Returns a ByteArrayOutputStream appropriately sized to store the gzipped content.
+     */
+    private static ByteArrayOutputStream getStreamForGzippedContent(final long unzippedContentLength) {
+        // The buffer size needed by the gzipped content should be smaller than this,
+        // but it's more efficient just to allocate one larger buffer than allocate
+        // twice if the gzipped content is too large for the default buffer.
+        final int initialArraySize = (unzippedContentLength <= Integer.MAX_VALUE) ?
+                (int) unzippedContentLength : Integer.MAX_VALUE;
+        return new ByteArrayOutputStream(initialArraySize);
+    }
+}