Bug 1244861 - Gzip outgoing telemetry pings. r=rnewman
This commit adds the GzipNonChunkedCompressingEntity which is necessary because
the telemetry servers don't support chunked uploading, which the built in
GzipCompressingEntity does.
I tested this on my local device and logs for successful uploads were sent for
both the testing gzip server as well as the official telemetry server. My data
correctly appears on the former and I did not check the latter.
MozReview-Commit-ID: 4bCNiRYyqFD
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java
@@ -191,16 +191,17 @@ public class TelemetryUploadService exte
resource = new BaseResource(ping.getURL());
} catch (final URISyntaxException e) {
Log.w(LOGTAG, "URISyntaxException for server URL when creating BaseResource: returning.");
return;
}
delegate.setResource(resource);
resource.delegate = delegate;
+ resource.setShouldCompressUploadedEntity(true, false); // Telemetry servers don't support chunking.
// We're in a background thread so we don't have any reason to do this asynchronously.
// If we tried, onStartCommand would return and IntentService might stop itself before we finish.
resource.postBlocking(ping.getPayload());
}
private static class CorePingResultDelegate extends ResultDelegate {
public CorePingResultDelegate() {
--- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java
@@ -25,16 +25,17 @@ import org.mozilla.gecko.background.comm
import org.mozilla.gecko.sync.ExtendedJSONObject;
import ch.boye.httpclientandroidlib.Header;
import ch.boye.httpclientandroidlib.HttpEntity;
import ch.boye.httpclientandroidlib.HttpResponse;
import ch.boye.httpclientandroidlib.HttpVersion;
import ch.boye.httpclientandroidlib.client.AuthCache;
import ch.boye.httpclientandroidlib.client.ClientProtocolException;
+import ch.boye.httpclientandroidlib.client.entity.GzipCompressingEntity;
import ch.boye.httpclientandroidlib.client.methods.HttpDelete;
import ch.boye.httpclientandroidlib.client.methods.HttpGet;
import ch.boye.httpclientandroidlib.client.methods.HttpPatch;
import ch.boye.httpclientandroidlib.client.methods.HttpPost;
import ch.boye.httpclientandroidlib.client.methods.HttpPut;
import ch.boye.httpclientandroidlib.client.methods.HttpRequestBase;
import ch.boye.httpclientandroidlib.client.methods.HttpUriRequest;
import ch.boye.httpclientandroidlib.client.protocol.ClientContext;
@@ -75,16 +76,19 @@ public class BaseResource implements Res
protected final URI uri;
protected BasicHttpContext context;
protected DefaultHttpClient client;
public ResourceDelegate delegate;
protected HttpRequestBase request;
public final String charset = "utf-8";
+ private boolean shouldGzipCompress = false;
+ private boolean shouldGzipChunk = true; // Default to using GzipCompressingEntity, which is built-in functionality.
+
/**
* We have very few writes (observers tend to be installed around sync
* sessions) and many iterations (every HTTP request iterates observers), so
* CopyOnWriteArrayList is a reasonable choice.
*/
protected static final CopyOnWriteArrayList<WeakReference<HttpResponseObserver>>
httpResponseObservers = new CopyOnWriteArrayList<>();
@@ -158,16 +162,34 @@ public class BaseResource implements Res
}
@Override
public String getHostname() {
return this.getURI().getHost();
}
/**
+ * Causes the Resource to compress the uploaded entity payload in requests with payloads (e.g. post, put)
+ * @param shouldCompress true if the entity should be compressed, false otherwise
+ * @param shouldChunk true if the transfer should be chunked, false otherwise; no effect if compression == false
+ */
+ public void setShouldCompressUploadedEntity(final boolean shouldCompress, final boolean shouldChunk) {
+ shouldGzipCompress = shouldCompress;
+ shouldGzipChunk = shouldChunk;
+ }
+
+ private HttpEntity getMaybeCompressedEntity(final HttpEntity entity) {
+ if (!shouldGzipCompress) {
+ return entity;
+ }
+
+ return shouldGzipChunk ? new GzipCompressingEntity(entity) : new GzipNonChunkedCompressingEntity(entity);
+ }
+
+ /**
* This shuts up HttpClient, which will otherwise debug log about there
* being no auth cache in the context.
*/
private static void addAuthCacheToContext(HttpUriRequest request, HttpContext context) {
AuthCache authCache = new BasicAuthCache(); // Not thread safe.
context.setAttribute(ClientContext.AUTH_CACHE, authCache);
}
@@ -360,32 +382,35 @@ public class BaseResource implements Res
public void delete() {
Logger.debug(LOG_TAG, "HTTP DELETE " + this.uri.toASCIIString());
this.go(new HttpDelete(this.uri));
}
@Override
public void post(HttpEntity body) {
Logger.debug(LOG_TAG, "HTTP POST " + this.uri.toASCIIString());
+ body = getMaybeCompressedEntity(body);
HttpPost request = new HttpPost(this.uri);
request.setEntity(body);
this.go(request);
}
@Override
public void patch(HttpEntity body) {
Logger.debug(LOG_TAG, "HTTP PATCH " + this.uri.toASCIIString());
+ body = getMaybeCompressedEntity(body);
HttpPatch request = new HttpPatch(this.uri);
request.setEntity(body);
this.go(request);
}
@Override
public void put(HttpEntity body) {
Logger.debug(LOG_TAG, "HTTP PUT " + this.uri.toASCIIString());
+ body = getMaybeCompressedEntity(body);
HttpPut request = new HttpPut(this.uri);
request.setEntity(body);
this.go(request);
}
protected static StringEntity stringEntityWithContentTypeApplicationJSON(String s) {
StringEntity e = new StringEntity(s, "UTF-8");
e.setContentType("application/json");
new file mode 100644
--- /dev/null
+++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/GzipNonChunkedCompressingEntity.java
@@ -0,0 +1,93 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.sync.net;
+
+import ch.boye.httpclientandroidlib.HttpEntity;
+import ch.boye.httpclientandroidlib.client.entity.GzipCompressingEntity;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Wrapping entity that compresses content when {@link #writeTo writing}.
+ *
+ * This differs from {@link GzipCompressingEntity} in that it does not chunk
+ * the sent data, therefore replacing the "Transfer-Encoding" HTTP header with
+ * the "Content-Length" header required by some servers.
+ *
+ * However, to measure the content length, the gzipped content will be temporarily
+ * stored in memory so be careful what content you send!
+ */
+public class GzipNonChunkedCompressingEntity extends GzipCompressingEntity {
+ private byte[] gzippedContent;
+
+ public GzipNonChunkedCompressingEntity(final HttpEntity entity) {
+ super(entity);
+ }
+
+ /**
+ * @return content length for gzipped content or -1 if there is an error
+ */
+ @Override
+ public long getContentLength() {
+ try {
+ initBuffer();
+ } catch (final IOException e) {
+ // GzipCompressingEntity always returns -1 in which case a 'Content-Length' header is omitted.
+ // Presumably, without it the request will fail (either client-side or server-side).
+ return -1;
+ }
+ return gzippedContent.length;
+ }
+
+ @Override
+ public boolean isChunked() {
+ // "Content-Length" & chunked encoding are mutually exclusive:
+ // https://en.wikipedia.org/wiki/Chunked_transfer_encoding
+ return false;
+ }
+
+ @Override
+ public InputStream getContent() throws IOException {
+ initBuffer();
+ return new ByteArrayInputStream(gzippedContent);
+ }
+
+ @Override
+ public void writeTo(final OutputStream outstream) throws IOException {
+ initBuffer();
+ outstream.write(gzippedContent);
+ }
+
+ private void initBuffer() throws IOException {
+ if (gzippedContent != null) {
+ return;
+ }
+
+ final ByteArrayOutputStream s = getStreamForGzippedContent(wrappedEntity.getContentLength());
+ try {
+ super.writeTo(s);
+ } finally {
+ s.close();
+ }
+
+ gzippedContent = s.toByteArray();
+ }
+
+ /**
+ * Returns a ByteArrayOutputStream appropriately sized to store the gzipped content.
+ */
+ private static ByteArrayOutputStream getStreamForGzippedContent(final long unzippedContentLength) {
+ // The buffer size needed by the gzipped content should be smaller than this,
+ // but it's more efficient just to allocate one larger buffer than allocate
+ // twice if the gzipped content is too large for the default buffer.
+ final int initialArraySize = (unzippedContentLength <= Integer.MAX_VALUE) ?
+ (int) unzippedContentLength : Integer.MAX_VALUE;
+ return new ByteArrayOutputStream(initialArraySize);
+ }
+}