--- a/mobile/android/base/java/org/mozilla/gecko/GeckoApplication.java
+++ b/mobile/android/base/java/org/mozilla/gecko/GeckoApplication.java
@@ -39,23 +39,23 @@ import org.mozilla.gecko.icons.Icons;
import org.mozilla.gecko.lwt.LightweightTheme;
import org.mozilla.gecko.mdns.MulticastDNSManager;
import org.mozilla.gecko.media.AudioFocusAgent;
import org.mozilla.gecko.media.RemoteManager;
import org.mozilla.gecko.notifications.NotificationClient;
import org.mozilla.gecko.notifications.NotificationHelper;
import org.mozilla.gecko.preferences.DistroSharedPrefsImport;
import org.mozilla.gecko.util.ActivityUtils;
+import org.mozilla.gecko.telemetry.TelemetryBackgroundReceiver;
import org.mozilla.gecko.util.BundleEventListener;
import org.mozilla.gecko.util.EventCallback;
import org.mozilla.gecko.util.GeckoBundle;
import org.mozilla.gecko.util.HardwareUtils;
import org.mozilla.gecko.util.PRNGFixes;
import org.mozilla.gecko.util.ThreadUtils;
-import org.mozilla.gecko.util.UUIDUtil;
import java.io.File;
import java.lang.reflect.Method;
import java.util.UUID;
public class GeckoApplication extends Application
implements ContextGetter {
private static final String LOG_TAG = "GeckoApplication";
@@ -273,16 +273,18 @@ public class GeckoApplication extends Ap
HardwareUtils.init(context);
FilePicker.init(context);
DownloadsIntegration.init();
HomePanelsManager.getInstance().init(context);
GlobalPageMetadata.getInstance().init();
+ TelemetryBackgroundReceiver.getInstance().init(context);
+
// We need to set the notification client before launching Gecko, since Gecko could start
// sending notifications immediately after startup, which we don't want to lose/crash on.
GeckoAppShell.setNotificationListener(new NotificationClient(context));
// This getInstance call will force initialization of the NotificationHelper, but does nothing with the result
NotificationHelper.getInstance(context).init();
MulticastDNSManager.getInstance(context).init();
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryBackgroundReceiver.java
@@ -0,0 +1,352 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.telemetry;
+
+import android.content.BroadcastReceiver;
+import android.content.Context;
+import android.content.Intent;
+import android.content.IntentFilter;
+import android.content.SharedPreferences;
+import android.os.Bundle;
+import android.os.Parcelable;
+import android.support.annotation.Nullable;
+import android.support.v4.content.LocalBroadcastManager;
+import android.util.Log;
+
+import org.mozilla.gecko.AppConstants;
+import org.mozilla.gecko.sync.telemetry.TelemetryContract;
+import org.mozilla.gecko.telemetry.pingbuilders.TelemetrySyncEventPingBuilder;
+import org.mozilla.gecko.telemetry.pingbuilders.TelemetrySyncPingBuilder;
+import org.mozilla.gecko.telemetry.pingbuilders.TelemetrySyncPingBundleBuilder;
+import org.mozilla.gecko.telemetry.schedulers.TelemetryUploadAllPingsImmediatelyScheduler;
+import org.mozilla.gecko.telemetry.schedulers.TelemetryUploadScheduler;
+import org.mozilla.gecko.telemetry.stores.TelemetryJSONFilePingStore;
+import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
+import org.mozilla.gecko.util.ThreadUtils;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.Set;
+
+/**
+ * Receives and processes telemetry broadcasts from background services, namely Sync.
+ * Nomenclature:
+ * - Bundled Sync Ping: a Sync Ping as documented at http://gecko.readthedocs.io/en/latest/toolkit/components/telemetry/telemetry/data/sync-ping.html
+ * as of commit https://github.com/mozilla-services/docs/commit/7eb4b412d3ab5ec46b280eff312ace32e7cf27e6
+ * - Telemetry data: incoming background telemetry, of two types: "sync" and "sync event"
+ * - Local Sync Ping: a persistable representation of incoming telemetry data. Not intended for upload.
+ * See {@link TelemetryLocalPing}
+ *
+ * General flow:
+ * - background telemetry bundles come in, describing syncs or events that happened
+ * - telemetry bundles are transformed into a local pings and persisted
+ * - once there are enough local pings, or another upload condition kicks in, all of the persisted
+ * local pings are bundled into a single outgoing Sync Ping, removed from the local store,
+ * and Sync Ping is persisted and uploaded.
+ *
+ * @author grisha
+ */
+public class TelemetryBackgroundReceiver extends BroadcastReceiver {
+ // NB: spelling is to appease logger's limitation on sizes of tags.
+ private final static String LOG_TAG = "TelemetryBgReceiver";
+
+ private static final String ACTION_BACKGROUND_TELEMETRY = "org.mozilla.gecko.telemetry.BACKGROUND";
+ private static final String SYNC_BUNDLE_STORE_DIR = "sync-ping-data";
+ private static final String SYNC_STORE_DIR = "sync-data";
+ private static final String SYNC_EVENT_STORE_DIR = "sync-event-data";
+ private static final int LOCAL_SYNC_EVENT_PING_THRESHOLD = 100;
+ private static final int LOCAL_SYNC_PING_THRESHOLD = 100;
+ private static final long MAX_TIME_BETWEEN_UPLOADS = 12 * 60 * 60 * 1000; // 12 hours
+
+ private static final String PREF_FILE_BACKGROUND_TELEMETRY = AppConstants.ANDROID_PACKAGE_NAME + ".telemetry.background";
+ private static final String PREF_IDS = "ids";
+ private static final String PREF_LAST_ATTEMPTED_UPLOADED = "last_attempted_upload";
+
+ // We don't currently support passing profile along with background telemetry. Profile is used to
+ // identify where pings are persisted locally.
+ private static final String DEFAULT_PROFILE = "default";
+
+ private static final TelemetryBackgroundReceiver instance = new TelemetryBackgroundReceiver();
+
+ public static TelemetryBackgroundReceiver getInstance() {
+ return instance;
+ }
+
+ public void init(Context context) {
+ LocalBroadcastManager.getInstance(context).registerReceiver(
+ this, new IntentFilter(ACTION_BACKGROUND_TELEMETRY));
+ }
+
+ @Override
+ public void onReceive(final Context context, final Intent intent) {
+ Log.i(LOG_TAG, "Handling background telemetry broadcast");
+
+ if (!intent.hasExtra(TelemetryContract.KEY_TELEMETRY)) {
+ throw new IllegalStateException("Received a background telemetry broadcast without data.");
+ }
+
+ if (!intent.hasExtra(TelemetryContract.KEY_TYPE)) {
+ throw new IllegalStateException("Received a background telemetry broadcast without type.");
+ }
+
+ // We want to know if any of the below code is faulty in non-obvious ways, and as such there
+ // isn't an overarching try/catch to silence the errors.
+ // That is, let's crash here if something goes really wrong, and hope that we'll spot the
+ // error in the crash stats.
+ ThreadUtils.postToBackgroundThread(new Runnable() {
+ @Override
+ public void run() {
+ final String type = intent.getStringExtra(TelemetryContract.KEY_TYPE);
+
+ // Setup local telemetry stores.
+ final TelemetryJSONFilePingStore syncTelemetryStore = new TelemetryJSONFilePingStore(
+ context.getFileStreamPath(SYNC_STORE_DIR), DEFAULT_PROFILE);
+ final TelemetryJSONFilePingStore syncEventTelemetryStore = new TelemetryJSONFilePingStore(
+ context.getFileStreamPath(SYNC_EVENT_STORE_DIR), DEFAULT_PROFILE);
+
+ // Process incoming telemetry.
+ final Bundle telemetryBundle = intent.getParcelableExtra(TelemetryContract.KEY_TELEMETRY);
+ final String uid = telemetryBundle.getString(TelemetryContract.KEY_LOCAL_UID);
+ final String deviceID = telemetryBundle.getString(TelemetryContract.KEY_LOCAL_DEVICE_ID);
+
+ // Transform incoming telemetry into a local ping of correct type (sync vs event).
+ final TelemetryLocalPing localPing;
+ final TelemetryPingStore telemetryStore;
+ switch (type) {
+ case TelemetryContract.KEY_TYPE_SYNC:
+ final ArrayList<Parcelable> devices = telemetryBundle.getParcelableArrayList(TelemetryContract.KEY_DEVICES);
+ final Serializable error = telemetryBundle.getSerializable(TelemetryContract.KEY_ERROR);
+ final Serializable stages = telemetryBundle.getSerializable(TelemetryContract.KEY_STAGES);
+ final long took = telemetryBundle.getLong(TelemetryContract.KEY_TOOK);
+ final boolean didRestart = telemetryBundle.getBoolean(TelemetryContract.KEY_RESTARTED);
+
+ telemetryStore = syncTelemetryStore;
+ TelemetrySyncPingBuilder localPingBuilder = new TelemetrySyncPingBuilder();
+
+ if (uid != null) {
+ localPingBuilder.setUID(uid);
+ }
+
+ if (deviceID != null) {
+ localPingBuilder.setDeviceID(deviceID);
+ }
+
+ if (devices != null) {
+ localPingBuilder.setDevices(devices);
+ }
+
+ if (stages != null) {
+ localPingBuilder.setStages(stages);
+ }
+
+ if (error != null) {
+ localPingBuilder.setError(error);
+ }
+
+ localPing = localPingBuilder
+ .setRestarted(didRestart)
+ .setTook(took)
+ .build();
+ break;
+ case TelemetryContract.KEY_TYPE_EVENT:
+ telemetryStore = syncEventTelemetryStore;
+ localPing = new TelemetrySyncEventPingBuilder()
+ .fromEventTelemetry(
+ (Bundle) intent.getParcelableExtra(
+ TelemetryContract.KEY_TELEMETRY))
+ .build();
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown background telemetry type.");
+ }
+
+ // Persist the incoming telemetry data.
+ try {
+ telemetryStore.storePing(localPing);
+ } catch (IOException e) {
+ Log.e(LOG_TAG, "Could not store incoming telemetry. Attempting to upload already stored telemetry.", e);
+ }
+
+ // Determine if we should try uploading at this point, and attempt to do so.
+ final SharedPreferences sharedPreferences = context.getSharedPreferences(
+ PREF_FILE_BACKGROUND_TELEMETRY, Context.MODE_PRIVATE);
+ final TelemetryPingStore syncPingStore = new TelemetryJSONFilePingStore(
+ context.getFileStreamPath(SYNC_BUNDLE_STORE_DIR), DEFAULT_PROFILE);
+
+ long lastAttemptedSyncPingUpload = sharedPreferences.getLong(PREF_LAST_ATTEMPTED_UPLOADED, 0L);
+ boolean idsChanged = setOrUpdateIDsIfChanged(sharedPreferences, uid, deviceID);
+
+ // Is there a good reason to upload at this time?
+ final String reasonToUpload = reasonToUpload(
+ idsChanged,
+ syncTelemetryStore.getCount(),
+ syncEventTelemetryStore.getCount(),
+ lastAttemptedSyncPingUpload
+ );
+
+ // If we have a reason to upload at this point, bundle up sync and event telemetry
+ // into a Sync Ping.
+ if (reasonToUpload != null) {
+ // Get the IDs of telemetry objects we're about to bundle up.
+ // Note that this may race with other incoming background telemetry.
+ // We may accidentally drop non-bundled local telemetry if it was emitted while
+ // we're processing the current telemetry.
+ // Chances of that happening are very small due to how often background telemetry
+ // is actually emitted: very infrequently on a timescale of disk access.
+ final Set<String> localSyncTelemetryToRemove = syncTelemetryStore.getStoredIDs();
+ final Set<String> localSyncEventTelemetryToRemove = syncEventTelemetryStore.getStoredIDs();
+
+ // Bundle up all that we have in our telemetry stores.
+ final TelemetryOutgoingPing syncPing = new TelemetrySyncPingBundleBuilder()
+ .setSyncStore(syncTelemetryStore)
+ .setSyncEventStore(syncEventTelemetryStore)
+ .setReason(reasonToUpload)
+ .build();
+
+ // Persist a Sync Ping Bundle.
+ boolean bundledSyncPingPersisted = true;
+ try {
+ syncPingStore.storePing(syncPing);
+ } catch (IOException e) {
+ // If we fail to persist a bundled sync ping, we can either attempt to upload it,
+ // or skip the upload. Each choice has its own set of trade-offs.
+ // In short, current approach is to skip an upload. See Bug 1369186.
+ //
+ // If we choose to upload a Sync Ping that failed to persist locally, it becomes
+ // possible to upload the same telemetry multiple times. Since currently we do
+ // not have IDs as part of our sync and event telemetry objects, it is impossible
+ // to guarantee idempotence on the receiver's end. As such, we care to not
+ // upload the same thing multiple times. One way to achieve this involves
+ // creating an additional mapping of Bundled Sync Ping ID to two sets,
+ // {sync telemetry ids} and {event telemetry ids}, and taking care to only include
+ // telemetry in the subsequent pings which has not yet been associated with a
+ // Sync Ping Bundle. Given the fact that this will likely use a SharedPreference
+ // and that we inherently have a concurrent telemetry pipeline, it's quite possible
+ // that we'll get this wrong in some subtle way after considerable effort.
+ //
+ // An alternative is to simply not upload if we failed to persist a Sync Ping.
+ // This side-steps issues of idempotancy, but comes at a risk of taking longer
+ // to upload telemetry, or sometimes not uploading it at all - see Bug 1366045.
+ // However, those issues become likely only if our local JSON storage is failing
+ // frequently. This should not happen under most circumstances, and if it does
+ // happen frequently, we're likely to have a host of other problems.
+ //
+ // Yet another solution is to alter server-side processing of this data such that
+ // a unique ID may be included alongside each sync/event telemetry object. This
+ // will result in a very straightforward client implementation with much better
+ // consistency guarantees.
+ //
+ // See Bug 1368579 for exploring possible "runaway storage" implications of
+ // the current approach.
+ //
+ // Also note that a core assumption here is that storePing never successfully writes
+ // to disk if it throws.
+ Log.e(LOG_TAG, "Unable to write bundled sync ping to disk. Skipping upload.", e);
+ bundledSyncPingPersisted = false;
+ }
+
+ if (bundledSyncPingPersisted) {
+ // It is now safe to delete persisted telemetry which we just bundled up.
+ syncTelemetryStore.onUploadAttemptComplete(localSyncTelemetryToRemove);
+ syncEventTelemetryStore.onUploadAttemptComplete(localSyncEventTelemetryToRemove);
+ }
+ }
+
+ // Kick-off ping upload. If this succeeds, the uploader service will remove persisted
+ // Sync Ping Bundles. Otherwise, we'll attempt another upload next time telemetry is
+ // processed.
+ // If we already have some persisted Sync Pings, that means a previous upload
+ // failed - or, less likely, is in progress and did not yet succeed. It should be safe to
+ // upload. Even if we raced with ourselves and uploaded some of the bundled sync pings more
+ // than once, it's possible to guarantee idempotence on the receiver's end since we
+ // include a unique ID with each ping. However, this depends on the telemetry pipeline
+ // successfully de-duplicating submitted pings. As of Q2 2017, success rate of de-duping
+ // sits around 45%, and is anticipated to be improved to 80-90% sometime by the end of 2017.
+ // Relevant bugs are 1369512, 1357275.
+ // Not uploading here means possibly delaying an already once-failed upload for a long
+ // time. See Bug 1366045 for exploring scheduling options.
+ if (reasonToUpload != null || syncPingStore.getCount() > 0) {
+ // Bump the "last-attempted-uploaded" timestamp, even though we might still fail
+ // to upload. Since we check for presence of pending pings above, if this upload
+ // fails we'll try again whenever next telemetry event happens.
+ sharedPreferences
+ .edit()
+ .putLong(PREF_LAST_ATTEMPTED_UPLOADED, System.currentTimeMillis())
+ .apply();
+
+ final TelemetryUploadScheduler scheduler = new TelemetryUploadAllPingsImmediatelyScheduler();
+ if (scheduler.isReadyToUpload(context, syncPingStore)) {
+ scheduler.scheduleUpload(context, syncPingStore);
+ }
+ }
+ }
+ });
+ }
+
+ // There's no "scheduler" in a classic sense, and so we might end up not uploading pings at all
+ // if there has been no new incoming telemetry data. See Bug 1366045.
+ @Nullable
+ protected static String reasonToUpload(boolean idsChanged, int syncCount, int eventCount, long lastUploadAttempt) {
+ // Whenever any IDs change, upload.
+ if (idsChanged) {
+ return TelemetrySyncPingBundleBuilder.UPLOAD_REASON_IDCHANGE;
+ }
+
+ // Whenever we hit a certain threshold of local persisted telemetry, upload.
+ if (syncCount > LOCAL_SYNC_PING_THRESHOLD || eventCount > LOCAL_SYNC_EVENT_PING_THRESHOLD) {
+ return TelemetrySyncPingBundleBuilder.UPLOAD_REASON_COUNT;
+ }
+
+ final long now = System.currentTimeMillis();
+
+ // If it's the first time we're processing telemetry data, upload ahead of schedule as a way
+ // of saying "we're alive". This might often correspond to sending data about the first sync.
+ if (lastUploadAttempt == 0L) {
+ return TelemetrySyncPingBundleBuilder.UPLOAD_REASON_FIRST;
+ }
+
+ // Wall clock changed significantly; upload because we can't be sure of our timing anymore.
+ // Allow for some wiggle room to account for clocks jumping around insignificantly.
+ final long DRIFT_BUFFER_IN_MS = 60 * 1000L;
+ if ((lastUploadAttempt - now) > DRIFT_BUFFER_IN_MS) {
+ return TelemetrySyncPingBundleBuilder.UPLOAD_REASON_CLOCK_DRIFT;
+ }
+
+ // Upload if we haven't uploaded for some time.
+ if ((now - lastUploadAttempt) >= MAX_TIME_BETWEEN_UPLOADS) {
+ return TelemetrySyncPingBundleBuilder.UPLOAD_REASON_SCHEDULE;
+ }
+
+ // No reason to upload.
+ return null;
+ }
+
+ // This has storage side-effects.
+ private boolean setOrUpdateIDsIfChanged(SharedPreferences prefs, String uid, String deviceID) {
+ final String currentIDsCombined = uid.concat(deviceID);
+ final String previousIDsHash = prefs.getString(PREF_IDS, "");
+
+ // Persist IDs for the first time, declare them as "not changed".
+ if (previousIDsHash.equals("")) {
+ final SharedPreferences.Editor prefsEditor = prefs.edit();
+ prefsEditor.putString(PREF_IDS, currentIDsCombined);
+ prefsEditor.apply();
+ return false;
+ }
+
+ // If IDs are different update local cache and declare them as "changed".
+ if (!previousIDsHash.equals(currentIDsCombined)) {
+ final SharedPreferences.Editor prefsEditor = prefs.edit();
+ prefsEditor.putString(PREF_IDS, currentIDsCombined);
+ prefsEditor.apply();
+ return true;
+ }
+
+ // Nothing changed, and no side-effects took place.
+ return false;
+ }
+
+}
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryDispatcher.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryDispatcher.java
@@ -67,52 +67,52 @@ public class TelemetryDispatcher {
// There are measurements in the core ping (e.g. seq #) that would ideally be atomically updated
// when the ping is stored. However, for simplicity, we use the json store and accept the possible
// loss of data (see bug 1243585 comment 16+ for more).
coreStore = new TelemetryJSONFilePingStore(new File(storePath, CORE_STORE_DIR_NAME), profileName);
uploadAllPingsImmediatelyScheduler = new TelemetryUploadAllPingsImmediatelyScheduler();
}
- private void queuePingForUpload(final Context context, final TelemetryPing ping, final TelemetryPingStore store,
- final TelemetryUploadScheduler scheduler) {
+ private void queuePingForUpload(final Context context, final TelemetryOutgoingPing ping, final TelemetryPingStore store,
+ final TelemetryUploadScheduler scheduler) {
final QueuePingRunnable runnable = new QueuePingRunnable(context, ping, store, scheduler);
ThreadUtils.postToBackgroundThread(runnable); // TODO: Investigate how busy this thread is. See if we want another.
}
/**
* Queues the given ping for upload and potentially schedules upload. This method can be called from any thread.
*/
public void queuePingForUpload(final Context context, final TelemetryCorePingBuilder pingBuilder) {
- final TelemetryPing ping = pingBuilder.build();
+ final TelemetryOutgoingPing ping = pingBuilder.build();
queuePingForUpload(context, ping, coreStore, uploadAllPingsImmediatelyScheduler);
}
- private static class QueuePingRunnable implements Runnable {
+ /* package-private */ static class QueuePingRunnable implements Runnable {
private final Context applicationContext;
- private final TelemetryPing ping;
+ private final TelemetryOutgoingPing ping;
private final TelemetryPingStore store;
private final TelemetryUploadScheduler scheduler;
- public QueuePingRunnable(final Context context, final TelemetryPing ping, final TelemetryPingStore store,
- final TelemetryUploadScheduler scheduler) {
+ /* package-private */ QueuePingRunnable(final Context context, final TelemetryOutgoingPing ping, final TelemetryPingStore store,
+ final TelemetryUploadScheduler scheduler) {
this.applicationContext = context.getApplicationContext();
this.ping = ping;
this.store = store;
this.scheduler = scheduler;
}
@Override
public void run() {
// We block while storing the ping so the scheduled upload is guaranteed to have the newly-stored value.
try {
store.storePing(ping);
} catch (final IOException e) {
// Don't log exception to avoid leaking profile path.
Log.e(LOGTAG, "Unable to write ping to disk. Continuing with upload attempt");
}
- if (scheduler.isReadyToUpload(store)) {
+ if (scheduler.isReadyToUpload(applicationContext, store)) {
scheduler.scheduleUpload(applicationContext, store);
}
}
}
}
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryLocalPing.java
@@ -0,0 +1,34 @@
+/* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.telemetry;
+
+import android.support.annotation.Nullable;
+
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+
+/**
+ * A "local" ping, which is not intended to be uploaded, but simply stored for later processing.
+ * Currently, many instances of local pings are bundled into a Sync Ping at appropriate moments.
+ */
+public class TelemetryLocalPing implements TelemetryPing {
+ private final ExtendedJSONObject payload;
+ private final String docID;
+
+ public TelemetryLocalPing(final ExtendedJSONObject payload, final String docID) {
+ this.payload = payload;
+ this.docID = docID;
+ }
+
+ public ExtendedJSONObject getPayload() { return payload; }
+ public String getDocID() { return docID; }
+
+ // Following the path of least resistance to avoid decoupling a ping from where it should
+ // be uploaded, for a local ping we declare that the path is nullable, and in fact it's always null.
+ @Nullable
+ public String getURLPath() {
+ return null;
+ }
+}
copy from mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryPing.java
copy to mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryOutgoingPing.java
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryPing.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryOutgoingPing.java
@@ -12,23 +12,31 @@ import org.mozilla.gecko.sync.ExtendedJS
*
* The doc ID is used by a Store to manipulate its internal pings and should
* be the same value found in the urlPath.
*
* If you want to create one of these, consider extending
* {@link org.mozilla.gecko.telemetry.pingbuilders.TelemetryPingBuilder}
* or one of its descendants.
*/
-public class TelemetryPing {
- private final String urlPath;
+public class TelemetryOutgoingPing implements TelemetryPing {
private final ExtendedJSONObject payload;
private final String docID;
+ private final String urlPath;
- public TelemetryPing(final String urlPath, final ExtendedJSONObject payload, final String docID) {
- this.urlPath = urlPath;
+ public TelemetryOutgoingPing(final String urlPath, final ExtendedJSONObject payload, final String docID) {
this.payload = payload;
this.docID = docID;
+ this.urlPath = urlPath;
}
public String getURLPath() { return urlPath; }
- public ExtendedJSONObject getPayload() { return payload; }
- public String getDocID() { return docID; }
+
+ @Override
+ public ExtendedJSONObject getPayload() {
+ return payload;
+ }
+
+ @Override
+ public String getDocID() {
+ return docID;
+ }
}
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryPing.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryPing.java
@@ -2,33 +2,13 @@
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
package org.mozilla.gecko.telemetry;
import org.mozilla.gecko.sync.ExtendedJSONObject;
-/**
- * Container for telemetry data and the data necessary to upload it.
- *
- * The doc ID is used by a Store to manipulate its internal pings and should
- * be the same value found in the urlPath.
- *
- * If you want to create one of these, consider extending
- * {@link org.mozilla.gecko.telemetry.pingbuilders.TelemetryPingBuilder}
- * or one of its descendants.
- */
-public class TelemetryPing {
- private final String urlPath;
- private final ExtendedJSONObject payload;
- private final String docID;
-
- public TelemetryPing(final String urlPath, final ExtendedJSONObject payload, final String docID) {
- this.urlPath = urlPath;
- this.payload = payload;
- this.docID = docID;
- }
-
- public String getURLPath() { return urlPath; }
- public ExtendedJSONObject getPayload() { return payload; }
- public String getDocID() { return docID; }
+public interface TelemetryPing {
+ ExtendedJSONObject getPayload();
+ String getDocID();
+ String getURLPath();
}
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java
@@ -9,16 +9,17 @@ import android.content.Context;
import android.content.Intent;
import android.util.Log;
import ch.boye.httpclientandroidlib.HttpHeaders;
import ch.boye.httpclientandroidlib.HttpResponse;
import ch.boye.httpclientandroidlib.client.ClientProtocolException;
import ch.boye.httpclientandroidlib.client.methods.HttpRequestBase;
import ch.boye.httpclientandroidlib.impl.client.DefaultHttpClient;
import org.mozilla.gecko.GeckoProfile;
+import org.mozilla.gecko.Telemetry;
import org.mozilla.gecko.preferences.GeckoPreferences;
import org.mozilla.gecko.restrictions.Restrictable;
import org.mozilla.gecko.restrictions.Restrictions;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import org.mozilla.gecko.sync.net.BaseResource;
import org.mozilla.gecko.sync.net.BaseResourceDelegate;
import org.mozilla.gecko.sync.net.Resource;
import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
@@ -99,16 +100,19 @@ public class TelemetryUploadService exte
if (pingsToUpload.isEmpty()) {
return true;
}
final String serverSchemeHostPort = TelemetryPreferences.getServerSchemeHostPort(context, store.getProfileName());
final HashSet<String> successfulUploadIDs = new HashSet<>(pingsToUpload.size()); // used for side effects.
final PingResultDelegate delegate = new PingResultDelegate(successfulUploadIDs);
for (final TelemetryPing ping : pingsToUpload) {
+ if (!(ping instanceof TelemetryOutgoingPing)) {
+ throw new IllegalStateException("Tried uploading a non-outgoing ping.");
+ }
// TODO: It'd be great to re-use the same HTTP connection for each upload request.
delegate.setDocID(ping.getDocID());
final String url = serverSchemeHostPort + "/" + ping.getURLPath();
uploadPayload(url, ping.getPayload(), delegate);
// There are minimal gains in trying to upload if we already failed one attempt.
if (delegate.hadConnectionError()) {
break;
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryCorePingBuilder.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryCorePingBuilder.java
@@ -10,36 +10,36 @@ import android.content.Context;
import android.content.SharedPreferences;
import android.os.Build;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.support.annotation.WorkerThread;
import android.text.TextUtils;
import android.util.Log;
-import org.mozilla.gecko.AppConstants;
+
import org.mozilla.gecko.GeckoApp;
import org.mozilla.gecko.GeckoProfile;
import org.mozilla.gecko.GeckoSharedPrefs;
import org.mozilla.gecko.Locales;
import org.mozilla.gecko.search.SearchEngine;
import org.mozilla.gecko.sync.ExtendedJSONObject;
-import org.mozilla.gecko.telemetry.TelemetryPing;
+import org.mozilla.gecko.telemetry.TelemetryOutgoingPing;
import org.mozilla.gecko.util.DateUtil;
import org.mozilla.gecko.Experiments;
import org.mozilla.gecko.util.StringUtils;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Locale;
import java.util.concurrent.TimeUnit;
/**
- * Builds a {@link TelemetryPing} representing a core ping.
+ * Builds a {@link TelemetryOutgoingPing} representing a core ping.
*
* See https://gecko.readthedocs.org/en/latest/toolkit/components/telemetry/telemetry/core-ping.html
* for details on the core ping.
*/
public class TelemetryCorePingBuilder extends TelemetryPingBuilder {
private static final String LOGTAG = StringUtils.safeSubstring(TelemetryCorePingBuilder.class.getSimpleName(), 0, 23);
// For legacy reasons, this preference key is not namespaced with "core".
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryLocalPingBuilder.java
@@ -0,0 +1,17 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.telemetry.pingbuilders;
+
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.telemetry.TelemetryLocalPing;
+
+import java.util.UUID;
+
+abstract class TelemetryLocalPingBuilder {
+ final ExtendedJSONObject payload = new ExtendedJSONObject();
+ final String docID = UUID.randomUUID().toString();
+
+ abstract TelemetryLocalPing build();
+}
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryPingBuilder.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetryPingBuilder.java
@@ -3,36 +3,36 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.pingbuilders;
import org.mozilla.gecko.AppConstants;
import org.mozilla.gecko.sync.ExtendedJSONObject;
-import org.mozilla.gecko.telemetry.TelemetryPing;
+import org.mozilla.gecko.telemetry.TelemetryOutgoingPing;
import java.util.Set;
import java.util.UUID;
/**
- * A generic Builder for {@link TelemetryPing} instances. Each overriding class is
+ * A generic Builder for {@link TelemetryOutgoingPing} instances. Each overriding class is
* expected to create a specific type of ping (e.g. "core").
*
* This base class handles the common ping operations under the hood:
* * Validating mandatory fields
* * Forming the server url
*/
abstract class TelemetryPingBuilder {
// In the server url, the initial path directly after the "scheme://host:port/"
private static final String SERVER_INITIAL_PATH = "submit/telemetry";
private final String serverPath;
protected final ExtendedJSONObject payload;
- private final String docID;
+ protected final String docID;
public TelemetryPingBuilder() {
docID = UUID.randomUUID().toString();
serverPath = getTelemetryServerPath(getDocType(), docID);
payload = new ExtendedJSONObject();
}
/**
@@ -41,19 +41,19 @@ abstract class TelemetryPingBuilder {
public abstract String getDocType();
/**
* @return the fields that are mandatory for the resultant ping to be uploaded to
* the server. These will be validated before the ping is built.
*/
public abstract String[] getMandatoryFields();
- public TelemetryPing build() {
+ public TelemetryOutgoingPing build() {
validatePayload();
- return new TelemetryPing(serverPath, payload, docID);
+ return new TelemetryOutgoingPing(serverPath, payload, docID);
}
private void validatePayload() {
final Set<String> keySet = payload.keySet();
for (final String mandatoryField : getMandatoryFields()) {
if (!keySet.contains(mandatoryField)) {
throw new IllegalArgumentException("Builder does not contain mandatory field: " +
mandatoryField);
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncEventPingBuilder.java
@@ -0,0 +1,24 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+package org.mozilla.gecko.telemetry.pingbuilders;
+
+import android.os.Bundle;
+
+import org.mozilla.gecko.telemetry.TelemetryLocalPing;
+
+/**
+ * Local ping builder which understands how to process event data.
+ * This is a placeholder, to be implemented in Bug 1363924.
+ */
+public class TelemetrySyncEventPingBuilder extends TelemetryLocalPingBuilder {
+ public TelemetrySyncEventPingBuilder fromEventTelemetry(Bundle data) {
+ return this;
+ }
+
+ @Override
+ public TelemetryLocalPing build() {
+ throw new UnsupportedOperationException();
+ }
+}
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBuilder.java
@@ -0,0 +1,162 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, you can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+package org.mozilla.gecko.telemetry.pingbuilders;
+
+import android.os.Bundle;
+import android.os.Parcelable;
+import android.support.annotation.NonNull;
+
+import org.json.simple.JSONArray;
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.sync.telemetry.TelemetryContract;
+import org.mozilla.gecko.sync.telemetry.TelemetryStageCollector;
+import org.mozilla.gecko.telemetry.TelemetryLocalPing;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+/**
+ * Local ping builder which understands how to process sync data.
+ * Whenever hashing of data is involved, we expect it to be performed at the time of collection,
+ * somewhere in {@link org.mozilla.gecko.sync.telemetry.TelemetryCollector} and friends.
+ */
+public class TelemetrySyncPingBuilder extends TelemetryLocalPingBuilder {
+ private static final int DATA_FORMAT_VERSION = 1;
+
+ public TelemetrySyncPingBuilder setStages(@NonNull final Serializable data) {
+ HashMap<String, TelemetryStageCollector> stages = castSyncData(data);
+
+ final JSONArray engines = new JSONArray();
+ for (String stageName : stages.keySet()) {
+ final TelemetryStageCollector stage = stages.get(stageName);
+
+ // Skip stages that did nothing.
+ if (stage.inbound == 0 && stage.outbound == 0) {
+ continue;
+ }
+
+ final ExtendedJSONObject stageJSON = new ExtendedJSONObject();
+
+ stageJSON.put("name", stageName);
+ stageJSON.put("took", stage.finished - stage.started);
+
+ // Desktop also includes a "status" field with internal constants as possible values.
+ // Status may be deducted by inspecting 'failureReason', and as such it is omitted here.
+ // Absence of 'failureReason' means that stage succeeded.
+
+ if (stage.inbound > 0) {
+ final ExtendedJSONObject incomingJSON = new ExtendedJSONObject();
+ incomingJSON.put("applied", stage.inbound);
+ if (stage.inboundStored > 0) {
+ incomingJSON.put("succeeded", stage.inboundStored);
+ }
+ if (stage.inboundFailed > 0) {
+ incomingJSON.put("failed", stage.inboundFailed);
+ }
+ if (stage.reconciled > 0) {
+ incomingJSON.put("reconciled", stage.reconciled);
+ }
+ stageJSON.put("incoming", incomingJSON);
+ }
+
+ if (stage.outbound > 0) {
+ final ExtendedJSONObject outgoingJSON = new ExtendedJSONObject();
+ // We specifically do not check if `outboundStored` is greater than zero.
+ // `outbound` schema is simpler than `inbound`, namely there isn't an "attempted
+ // to send" count.
+ // Stage telemetry itself has that data (outbound = outboundStored + outboundFailed),
+ // and so this is our way to relay slightly more information.
+ // e.g. we'll know there's something wrong if `sent = 0` and `failed` is missing.
+ outgoingJSON.put("sent", stage.outboundStored);
+ if (stage.outboundFailed > 0) {
+ outgoingJSON.put("failed", stage.outboundFailed);
+ }
+ stageJSON.put("outgoing", outgoingJSON);
+ }
+
+ // We depend on the error builder from TelemetryCollector to produce the right schema.
+ // Spreading around our schema definition like that is awkward, but, alas, here we are.
+ if (stage.error != null) {
+ stageJSON.put("failureReason", stage.error);
+ }
+
+ addUnchecked(engines, stageJSON);
+ }
+ payload.put("engines", engines);
+ return this;
+ }
+
+ public TelemetrySyncPingBuilder setUID(@NonNull String uid) {
+ payload.put("uid", uid);
+ return this;
+ }
+
+ public TelemetrySyncPingBuilder setDeviceID(@NonNull String deviceID) {
+ payload.put("deviceID", deviceID);
+ return this;
+ }
+
+ public TelemetrySyncPingBuilder setRestarted(boolean didRestart) {
+ if (!didRestart) {
+ return this;
+ }
+
+ payload.put("restarted", true);
+ return this;
+ }
+
+ public TelemetrySyncPingBuilder setDevices(ArrayList<Parcelable> devices) {
+ final JSONArray devicesJSON = new JSONArray();
+
+ for (Parcelable device : devices) {
+ final Bundle deviceBundle = (Bundle) device;
+ final ExtendedJSONObject deviceJSON = new ExtendedJSONObject();
+
+ deviceJSON.put("os", deviceBundle.getString(TelemetryContract.KEY_DEVICE_OS));
+ deviceJSON.put("version", deviceBundle.getString(TelemetryContract.KEY_DEVICE_VERSION));
+ deviceJSON.put("id", deviceBundle.getString(TelemetryContract.KEY_DEVICE_ID));
+
+ addUnchecked(devicesJSON, deviceJSON);
+ }
+
+ if (devicesJSON.size() > 0) {
+ payload.put("devices", devicesJSON);
+ }
+ return this;
+ }
+
+ public TelemetrySyncPingBuilder setError(@NonNull Serializable error) {
+ payload.put("failureReason", (ExtendedJSONObject) error);
+ return this;
+ }
+
+ public TelemetrySyncPingBuilder setTook(long took) {
+ payload.put("took", took);
+ return this;
+ }
+
+ @Override
+ public TelemetryLocalPing build() {
+ payload.put("version", DATA_FORMAT_VERSION);
+ return new TelemetryLocalPing(payload, docID);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void addUnchecked(final JSONArray list, final ExtendedJSONObject obj) {
+ list.add(obj);
+ }
+
+ /**
+ * We broadcast this data via LocalBroadcastManager and control both sides of this code, so it
+ * is acceptable to do an unchecked cast.
+ */
+ @SuppressWarnings("unchecked")
+ private static HashMap<String, TelemetryStageCollector> castSyncData(final Serializable data) {
+ return (HashMap<String, TelemetryStageCollector>) data;
+ }
+}
new file mode 100644
--- /dev/null
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBundleBuilder.java
@@ -0,0 +1,114 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+package org.mozilla.gecko.telemetry.pingbuilders;
+
+import android.os.Build;
+import android.support.annotation.NonNull;
+
+import org.json.simple.JSONArray;
+import org.mozilla.gecko.AppConstants;
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.telemetry.TelemetryOutgoingPing;
+import org.mozilla.gecko.telemetry.TelemetryPing;
+import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Locale;
+import java.util.TimeZone;
+
+/**
+ * Responsible for building a Sync Ping, based on the telemetry docs:
+ * http://gecko.readthedocs.io/en/latest/toolkit/components/telemetry/telemetry/data/sync-ping.html
+ *
+ * This builder takes two stores ('sync' and 'event') and produces a single "sync ping".
+ *
+ * Note that until Bug 1363924, event telemetry will be ignored.
+ *
+ * Sample result will look something like:
+ * {
+ * "syncs": [list of syncs, as produced by the SyncBuilder],
+ * "events": [list of events, as produced by the EventBuilder]
+ * }
+ */
+public class TelemetrySyncPingBundleBuilder extends TelemetryPingBuilder {
+ private static final String PING_TYPE = "sync";
+ private static final int PING_VERSION = 4;
+
+ public static final String UPLOAD_REASON_FIRST = "first";
+ public static final String UPLOAD_REASON_CLOCK_DRIFT = "clockdrift";
+ public static final String UPLOAD_REASON_SCHEDULE = "schedule";
+ public static final String UPLOAD_REASON_IDCHANGE = "idchange";
+ public static final String UPLOAD_REASON_COUNT = "count";
+
+ private final ExtendedJSONObject pingData = new ExtendedJSONObject();
+
+ @Override
+ public String getDocType() {
+ return "sync";
+ }
+
+ @Override
+ public String[] getMandatoryFields() {
+ return new String[0];
+ }
+
+ public TelemetrySyncPingBundleBuilder setReason(@NonNull String reason) {
+ pingData.put("why", reason);
+ return this;
+ }
+
+ @Override
+ public TelemetryOutgoingPing build() {
+ final DateFormat pingCreationDateFormat = new SimpleDateFormat(
+ "yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US);
+ pingCreationDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
+
+ payload.put("type", PING_TYPE);
+ payload.put("version", PING_VERSION);
+ payload.put("id", docID);
+ payload.put("creationDate", pingCreationDateFormat.format(new Date()));
+
+ final ExtendedJSONObject application = new ExtendedJSONObject();
+ application.put("architecture", Build.CPU_ABI);
+ application.put("buildID", AppConstants.MOZ_APP_BUILDID);
+ application.put("platformVersion", AppConstants.MOZ_APP_VERSION);
+ application.put("name", AppConstants.MOZ_APP_BASENAME);
+ application.put("version", AppConstants.MOZ_APP_VERSION);
+ application.put("displayVersion", AppConstants.MOZ_APP_VERSION);
+ application.put("vendor", AppConstants.MOZ_APP_VENDOR);
+ application.put("xpcomAbi", AppConstants.MOZ_APP_ABI);
+ application.put("channel", AppConstants.MOZ_UPDATE_CHANNEL);
+
+ payload.put("application", application);
+ payload.put("payload", pingData);
+ return super.build();
+ }
+
+ @SuppressWarnings("unchecked")
+ public TelemetrySyncPingBundleBuilder setSyncStore(TelemetryPingStore store) {
+ final JSONArray syncs = new JSONArray();
+ List<TelemetryPing> pings = store.getAllPings();
+
+ // Please note how we're not including constituent ping's docID in the final payload. This is
+ // unfortunate and causes some grief when managing local ping storage and uploads, but needs
+ // to be resolved beyond this individual client. See Bug 1369186.
+ for (TelemetryPing ping : pings) {
+ syncs.add(ping.getPayload());
+ }
+
+ pingData.put("syncs", syncs);
+ return this;
+ }
+
+ // Event telemetry will be implemented in Bug 1363924.
+ public TelemetrySyncPingBundleBuilder setSyncEventStore(TelemetryPingStore store) {
+ return this;
+ }
+}
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/schedulers/TelemetryUploadAllPingsImmediatelyScheduler.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/schedulers/TelemetryUploadAllPingsImmediatelyScheduler.java
@@ -12,17 +12,17 @@ import org.mozilla.gecko.telemetry.store
import org.mozilla.gecko.telemetry.TelemetryUploadService;
/**
* Schedules an upload with all pings to be sent immediately.
*/
public class TelemetryUploadAllPingsImmediatelyScheduler implements TelemetryUploadScheduler {
@Override
- public boolean isReadyToUpload(final TelemetryPingStore store) {
+ public boolean isReadyToUpload(final Context applicationContext, final TelemetryPingStore store) {
// We're ready since we don't have any conditions to wait on (e.g. on wifi, accumulated X pings).
return true;
}
@Override
public void scheduleUpload(final Context applicationContext, final TelemetryPingStore store) {
final Intent i = new Intent(TelemetryUploadService.ACTION_UPLOAD);
i.setClass(applicationContext, TelemetryUploadService.class);
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/schedulers/TelemetryUploadScheduler.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/schedulers/TelemetryUploadScheduler.java
@@ -16,11 +16,11 @@ import org.mozilla.gecko.telemetry.store
* scheduled by sending an {@link android.content.Intent} to the
* {@link org.mozilla.gecko.telemetry.TelemetryUploadService}, either immediately or
* via an external scheduler (e.g. {@link android.app.job.JobScheduler}).
*
* N.B.: If the Store is not ready to upload, an implementation *should not* try to reschedule
* the check to see if it's time to upload - this is expected to be handled by the caller.
*/
public interface TelemetryUploadScheduler {
- boolean isReadyToUpload(TelemetryPingStore store);
+ boolean isReadyToUpload(Context applicationContext, TelemetryPingStore store);
void scheduleUpload(Context applicationContext, TelemetryPingStore store);
}
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryJSONFilePingStore.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryJSONFilePingStore.java
@@ -3,23 +3,26 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.stores;
import android.os.Parcel;
import android.os.Parcelable;
+import android.support.annotation.Nullable;
import android.support.annotation.VisibleForTesting;
import android.support.annotation.WorkerThread;
import android.util.Log;
import org.json.JSONException;
import org.json.JSONObject;
import org.mozilla.gecko.sync.ExtendedJSONObject;
import org.mozilla.gecko.sync.NonObjectJSONException;
+import org.mozilla.gecko.telemetry.TelemetryLocalPing;
+import org.mozilla.gecko.telemetry.TelemetryOutgoingPing;
import org.mozilla.gecko.telemetry.TelemetryPing;
import org.mozilla.gecko.util.FileUtils;
import org.mozilla.gecko.util.FileUtils.FileLastModifiedComparator;
import org.mozilla.gecko.util.FileUtils.FilenameRegexFilter;
import org.mozilla.gecko.util.FileUtils.FilenameWhitelistFilter;
import org.mozilla.gecko.util.StringUtils;
import org.mozilla.gecko.util.UUIDUtil;
@@ -28,21 +31,20 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.channels.FileLock;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
-import java.util.SortedSet;
-import java.util.TreeSet;
/**
* An implementation of TelemetryPingStore that is backed by JSON files.
*
* This implementation seeks simplicity. Each ping to upload is stored in its own file with its doc ID
* as the filename. The doc ID is sent with a ping to be uploaded and is expected to be returned with
* {@link #onUploadAttemptComplete(Set)} so the associated file can be removed.
*
@@ -95,29 +97,34 @@ public class TelemetryJSONFilePingStore
}
@VisibleForTesting File getPingFile(final String docID) {
return new File(storeDir, docID);
}
@Override
public void storePing(final TelemetryPing ping) throws IOException {
- final String output;
+ storePing(ping.getPayload(), ping.getDocID(), ping.getURLPath());
+ }
+
+ public void storePing(final ExtendedJSONObject payload, String docID, @Nullable String urlPath) throws IOException {
+ final JSONObject output;
try {
- output = new JSONObject()
- .put(KEY_PAYLOAD, ping.getPayload())
- .put(KEY_URL_PATH, ping.getURLPath())
- .toString();
+ output = new JSONObject().put(KEY_PAYLOAD, payload);
+
+ if (urlPath != null) {
+ output.put(KEY_URL_PATH, urlPath);
+ }
} catch (final JSONException e) {
// Do not log the exception to avoid leaking personal data.
throw new IOException("Unable to create JSON to store to disk");
}
- final FileOutputStream outputStream = new FileOutputStream(getPingFile(ping.getDocID()), false);
- blockForLockAndWriteFileAndCloseStream(outputStream, output);
+ final FileOutputStream outputStream = new FileOutputStream(getPingFile(docID), false);
+ blockForLockAndWriteFileAndCloseStream(outputStream, output.toString());
}
@Override
public void maybePrunePings() {
final File[] files = storeDir.listFiles(uuidFilenameFilter);
if (files == null) {
return;
}
@@ -162,28 +169,59 @@ public class TelemetryJSONFilePingStore
final ArrayList<TelemetryPing> out = new ArrayList<>(files.size());
for (final File file : files) {
final JSONObject obj = lockAndReadJSONFromFile(file);
if (obj == null) {
// We log in the method to get the JSONObject if we return null.
continue;
}
+ final ExtendedJSONObject payload;
try {
- final String url = obj.getString(KEY_URL_PATH);
- final ExtendedJSONObject payload = new ExtendedJSONObject(obj.getString(KEY_PAYLOAD));
- out.add(new TelemetryPing(url, payload, file.getName()));
- } catch (final IOException | JSONException | NonObjectJSONException e) {
+ payload = new ExtendedJSONObject(obj.getString(KEY_PAYLOAD));
+ } catch (IOException | JSONException | NonObjectJSONException e) {
Log.w(LOGTAG, "Bad json in ping. Ignoring.");
continue;
}
+
+ try {
+ final String url = obj.getString(KEY_URL_PATH);
+ out.add(new TelemetryOutgoingPing(url, payload, file.getName()));
+ } catch (JSONException e) {
+ out.add(new TelemetryLocalPing(payload, file.getName()));
+ }
}
return out;
}
+ @Override
+ public int getCount() {
+ final File[] fileArray = storeDir.listFiles(uuidFilenameFilter);
+ if (fileArray == null) {
+ Log.w(LOGTAG, "listFiles unexpectedly returned null - unable to retrieve pings. Assuming 0. " +
+ "Debug: exists? " + storeDir.exists() + "; directory? " + storeDir.isDirectory());
+ return 0;
+ }
+ return fileArray.length;
+ }
+
+ @Override
+ public Set<String> getStoredIDs() {
+ final Set<String> ids = new HashSet<>();
+ final File[] fileArray = storeDir.listFiles(uuidFilenameFilter);
+ if (fileArray == null) {
+ return ids;
+ }
+ // Map list of files to a set of IDs.
+ for (File file : fileArray) {
+ ids.add(file.getName());
+ }
+ return ids;
+ }
+
/**
* Logs if there is an error.
*
* @return the JSON object from the given file or null if there is an error.
*/
private JSONObject lockAndReadJSONFromFile(final File file) {
// lockAndReadFileAndCloseStream doesn't handle file size of 0.
if (file.length() == 0) {
--- a/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryPingStore.java
+++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/stores/TelemetryPingStore.java
@@ -2,16 +2,20 @@
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, you can obtain one at http://mozilla.org/MPL/2.0/.
*/
package org.mozilla.gecko.telemetry.stores;
import android.os.Parcelable;
+import android.support.annotation.VisibleForTesting;
+
+import org.mozilla.gecko.telemetry.TelemetryLocalPing;
+import org.mozilla.gecko.telemetry.TelemetryOutgoingPing;
import org.mozilla.gecko.telemetry.TelemetryPing;
import java.io.IOException;
import java.util.List;
import java.util.Set;
/**
* Persistent storage for TelemetryPings that are queued for upload.
@@ -21,33 +25,40 @@ import java.util.Set;
* to synchronize state (or be stateless!).
*
* The pings in {@link #getAllPings()} and {@link #maybePrunePings()} are returned in the
* same order in order to guarantee consistent results.
*/
public abstract class TelemetryPingStore implements Parcelable {
private final String profileName;
+ @VisibleForTesting
public TelemetryPingStore(final String profileName) {
this.profileName = profileName;
}
/**
* @return the profile name associated with this store.
*/
public String getProfileName() {
return profileName;
}
/**
- * @return a list of all the telemetry pings in the store that are ready for upload, ascending oldest to newest.
+ * @return a list of all the telemetry pings in the store, ascending oldest to newest. Depending
+ * on the store.
*/
public abstract List<TelemetryPing> getAllPings();
/**
+ * @return a number of all currently stored pings.
+ */
+ public abstract int getCount();
+
+ /**
* Save a ping to the store.
*
* @param ping the ping to store
* @throws IOException for underlying store access errors
*/
public abstract void storePing(TelemetryPing ping) throws IOException;
/**
@@ -58,9 +69,14 @@ public abstract class TelemetryPingStore
/**
* Removes the successfully uploaded pings from the database and performs another other actions necessary
* for when upload is completed.
*
* @param successfulRemoveIDs doc ids of pings that were successfully uploaded
*/
public abstract void onUploadAttemptComplete(Set<String> successfulRemoveIDs);
+
+ /**
+ * Returns a set of currently stored IDs.
+ */
+ public abstract Set<String> getStoredIDs();
}
--- a/mobile/android/base/moz.build
+++ b/mobile/android/base/moz.build
@@ -894,24 +894,31 @@ gbjar.sources += ['java/org/mozilla/geck
'tabs/TabsPanel.java',
'tabs/TabsPanelThumbnailView.java',
'tabs/TabsTouchHelperCallback.java',
'Telemetry.java',
'telemetry/measurements/CampaignIdMeasurements.java',
'telemetry/measurements/SearchCountMeasurements.java',
'telemetry/measurements/SessionMeasurements.java',
'telemetry/pingbuilders/TelemetryCorePingBuilder.java',
+ 'telemetry/pingbuilders/TelemetryLocalPingBuilder.java',
'telemetry/pingbuilders/TelemetryPingBuilder.java',
+ 'telemetry/pingbuilders/TelemetrySyncEventPingBuilder.java',
+ 'telemetry/pingbuilders/TelemetrySyncPingBuilder.java',
+ 'telemetry/pingbuilders/TelemetrySyncPingBundleBuilder.java',
'telemetry/schedulers/TelemetryUploadAllPingsImmediatelyScheduler.java',
'telemetry/schedulers/TelemetryUploadScheduler.java',
'telemetry/stores/TelemetryJSONFilePingStore.java',
'telemetry/stores/TelemetryPingStore.java',
+ 'telemetry/TelemetryBackgroundReceiver.java',
'telemetry/TelemetryConstants.java',
'telemetry/TelemetryCorePingDelegate.java',
'telemetry/TelemetryDispatcher.java',
+ 'telemetry/TelemetryLocalPing.java',
+ 'telemetry/TelemetryOutgoingPing.java',
'telemetry/TelemetryPing.java',
'telemetry/TelemetryPreferences.java',
'telemetry/TelemetryUploadService.java',
'TelemetryContract.java',
'text/FloatingActionModeCallback.java',
'text/FloatingToolbarTextSelection.java',
'text/TextAction.java',
'text/TextSelection.java',
new file mode 100644
--- /dev/null
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBuilderTest.java
@@ -0,0 +1,93 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+package org.mozilla.gecko.telemetry.pingbuilders;
+
+import android.os.Bundle;
+import android.os.Parcelable;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mozilla.gecko.background.testhelpers.TestRunner;
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.telemetry.TelemetryLocalPing;
+
+import java.util.ArrayList;
+
+import static org.junit.Assert.*;
+
+@RunWith(TestRunner.class)
+public class TelemetrySyncPingBuilderTest {
+ private TelemetrySyncPingBundleBuilderTest.MockTelemetryPingStore pingStore;
+ private TelemetrySyncPingBuilder builder;
+
+ @Before
+ public void setUp() throws Exception {
+ pingStore = new TelemetrySyncPingBundleBuilderTest.MockTelemetryPingStore();
+ builder = new TelemetrySyncPingBuilder();
+ }
+
+ @Test
+ public void testGeneralShape() throws Exception {
+ TelemetryLocalPing localPing = builder
+ .setDeviceID("device-id")
+ .setUID("uid")
+ .setTook(123L)
+ .setRestarted(false)
+ .build();
+ ExtendedJSONObject payload = localPing.getPayload();
+ assertEquals(
+ "{\"uid\":\"uid\",\"took\":123,\"deviceID\":\"device-id\",\"version\":1}",
+ payload.toString()
+ );
+ }
+
+ @Test
+ public void testDevices() throws Exception {
+ ArrayList<Parcelable> devices = new ArrayList<>();
+
+ TelemetryLocalPing localPing = builder
+ .setDevices(devices)
+ .build();
+ ExtendedJSONObject payload = localPing.getPayload();
+
+ // Empty list isn't part of the JSON.
+ assertEquals(
+ "{\"version\":1}",
+ payload.toString()
+ );
+
+ Bundle device = new Bundle();
+ device.putString("os", "Android");
+ device.putString("version", "53.0a1");
+ device.putString("id", "80daf12dsadsa4236914cff2cc6e9d0f80a965380e2cf8e976e4004ead887521b5d9");
+ devices.add(device);
+
+ // Test with only one device
+ payload = builder
+ .setDevices(devices)
+ .build()
+ .getPayload();
+ assertEquals(
+ "{\"devices\":[{\"os\":\"Android\",\"id\":\"80daf12dsadsa4236914cff2cc6e9d0f80a965380e2cf8e976e4004ead887521b5d9\",\"version\":\"53.0a1\"}],\"version\":1}",
+ payload.toString()
+ );
+
+ device = new Bundle();
+ device.putString("os", "iOS");
+ device.putString("version", "8.0");
+ device.putString("id", "fa813452774b3cdc8f5f73290b5346df800f644b7b92a1ab94b6e2af748d261362");
+ devices.add(device);
+
+ // Test with more than one device
+ payload = builder
+ .setDevices(devices)
+ .build()
+ .getPayload();
+ assertEquals(
+ "{\"devices\":[{\"os\":\"Android\",\"id\":\"80daf12dsadsa4236914cff2cc6e9d0f80a965380e2cf8e976e4004ead887521b5d9\",\"version\":\"53.0a1\"},{\"os\":\"iOS\",\"id\":\"fa813452774b3cdc8f5f73290b5346df800f644b7b92a1ab94b6e2af748d261362\",\"version\":\"8.0\"}],\"version\":1}",
+ payload.toString()
+ );
+ }
+}
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/pingbuilders/TelemetrySyncPingBundleBuilderTest.java
@@ -0,0 +1,178 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+package org.mozilla.gecko.telemetry.pingbuilders;
+
+import android.os.Parcel;
+import android.os.Parcelable;
+
+import org.json.simple.JSONArray;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mozilla.gecko.background.testhelpers.TestRunner;
+import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.telemetry.TelemetryOutgoingPing;
+import org.mozilla.gecko.telemetry.TelemetryPing;
+import org.mozilla.gecko.telemetry.stores.TelemetryJSONFilePingStore;
+import org.mozilla.gecko.telemetry.stores.TelemetryPingStore;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.*;
+
+@RunWith(TestRunner.class)
+public class TelemetrySyncPingBundleBuilderTest {
+ public static class MockTelemetryPingStore extends TelemetryPingStore {
+ public MockTelemetryPingStore() {
+ super("default");
+ }
+
+ private HashMap<String, TelemetryPing> pings = new HashMap<>();
+
+ @Override
+ public List<TelemetryPing> getAllPings() {
+ return new ArrayList<>(pings.values());
+ }
+
+ @Override
+ public int getCount() {
+ return pings.size();
+ }
+
+ @Override
+ public void storePing(TelemetryPing ping) throws IOException {
+ pings.put(ping.getDocID(), ping);
+ }
+
+ @Override
+ public void maybePrunePings() {}
+
+ @Override
+ public void onUploadAttemptComplete(Set<String> successfulRemoveIDs) {
+ for (String id : successfulRemoveIDs) {
+ pings.remove(id);
+ }
+ }
+
+ @Override
+ public Set<String> getStoredIDs() {
+ return pings.keySet();
+ }
+
+ @Override
+ public int describeContents() {
+ return 0;
+ }
+
+ @Override
+ public void writeToParcel(Parcel dest, int flags) {
+
+ }
+
+ public final Parcelable.Creator<TelemetryJSONFilePingStore> CREATOR = new Parcelable.Creator<TelemetryJSONFilePingStore>() {
+ @Override
+ public TelemetryJSONFilePingStore createFromParcel(Parcel source) {
+ return null;
+ }
+
+ @Override
+ public TelemetryJSONFilePingStore[] newArray(int size) {
+ return new TelemetryJSONFilePingStore[0];
+ }
+ };
+ }
+
+ private TelemetrySyncPingBundleBuilder builder;
+ private MockTelemetryPingStore syncPings = new MockTelemetryPingStore();
+ private MockTelemetryPingStore eventPings = new MockTelemetryPingStore();
+
+ @Before
+ public void setUp() throws Exception {
+ builder = new TelemetrySyncPingBundleBuilder();
+ builder.setReason(TelemetrySyncPingBundleBuilder.UPLOAD_REASON_SCHEDULE);
+ syncPings = new MockTelemetryPingStore();
+ eventPings = new MockTelemetryPingStore();
+ }
+
+ @Test
+ public void testGeneralShape() throws Exception {
+ builder.setSyncStore(syncPings);
+ builder.setSyncEventStore(eventPings);
+
+ TelemetryOutgoingPing outgoingPing = builder.build();
+
+ assertTrue(outgoingPing.getPayload().containsKey("application"));
+ assertTrue(outgoingPing.getPayload().containsKey("payload"));
+ assertTrue(outgoingPing.getPayload().containsKey("id"));
+ assertEquals("sync", outgoingPing.getPayload().getString("type"));
+ assertEquals(Integer.valueOf(4), outgoingPing.getPayload().getIntegerSafely("version"));
+
+ // Test application key.
+ ExtendedJSONObject application = outgoingPing.getPayload().getObject("application");
+ assertEquals("Mozilla", application.getString("vendor"));
+ assertTrue(application.containsKey("architecture"));
+ assertTrue(application.containsKey("platformVersion"));
+ assertTrue(application.containsKey("displayVersion"));
+ assertTrue(application.containsKey("version"));
+ assertTrue(application.containsKey("name"));
+ assertTrue(application.containsKey("channel"));
+ assertTrue(application.containsKey("buildID"));
+ assertTrue(application.containsKey("xpcomAbi"));
+
+ // Test general shape of payload.
+ // NB that even though we set an empty sync event store, it's not in the json string.
+ // That's because sync events are not yet instrumented.
+ ExtendedJSONObject payload = outgoingPing.getPayload().getObject("payload");
+ assertEquals("{\"syncs\":[],\"why\":\"schedule\"}", payload.toJSONString());
+ }
+
+ @Test
+ public void testBundlingOfMultiplePings() throws Exception {
+ // Try just one ping first.
+ syncPings.storePing(new TelemetrySyncPingBuilder()
+ .setDeviceID("test-device-id")
+ .setRestarted(true)
+ .setTook(123L)
+ .setUID("test-uid")
+ .build()
+ );
+ builder.setSyncStore(syncPings);
+
+ TelemetryOutgoingPing outgoingPing = builder.build();
+
+ // Ensure we have that one ping.
+ ExtendedJSONObject payload = outgoingPing.getPayload().getObject("payload");
+ assertEquals(
+ "{\"syncs\":[{\"took\":123,\"uid\":\"test-uid\",\"restarted\":true,\"deviceID\":\"test-device-id\",\"version\":1}],\"why\":\"schedule\"}",
+ payload.toString()
+ );
+
+ // Add another ping.
+ syncPings.storePing(new TelemetrySyncPingBuilder()
+ .setDeviceID("test-device-id")
+ .setRestarted(true)
+ .setTook(321L)
+ .setUID("test-uid")
+ .build()
+ );
+ builder.setSyncStore(syncPings);
+
+ // We should have two pings now.
+ outgoingPing = builder.build();
+ JSONArray syncs = outgoingPing.getPayload()
+ .getObject("payload")
+ .getArray("syncs");
+ assertEquals(2, syncs.size());
+ assertTrue(
+ syncs.toString().contains("{\"took\":123,\"uid\":\"test-uid\",\"restarted\":true,\"deviceID\":\"test-device-id\",\"version\":1}")
+ );
+ assertTrue(
+ syncs.toString().contains("{\"took\":321,\"uid\":\"test-uid\",\"restarted\":true,\"deviceID\":\"test-device-id\",\"version\":1}")
+ );
+ }
+}
\ No newline at end of file
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/schedulers/TestTelemetryUploadAllPingsImmediatelyScheduler.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/schedulers/TestTelemetryUploadAllPingsImmediatelyScheduler.java
@@ -34,17 +34,18 @@ public class TestTelemetryUploadAllPings
@Before
public void setUp() {
testScheduler = new TelemetryUploadAllPingsImmediatelyScheduler();
testStore = mock(TelemetryPingStore.class);
}
@Test
public void testReadyToUpload() {
- assertTrue("Scheduler is always ready to upload", testScheduler.isReadyToUpload(testStore));
+ assertTrue("Scheduler is always ready to upload", testScheduler.isReadyToUpload(
+ mock(Context.class), testStore));
}
@Test
public void testScheduleUpload() {
final Context context = mock(Context.class);
testScheduler.scheduleUpload(context, testStore);
--- a/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/stores/TestTelemetryJSONFilePingStore.java
+++ b/mobile/android/tests/background/junit4/src/org/mozilla/gecko/telemetry/stores/TestTelemetryJSONFilePingStore.java
@@ -9,22 +9,22 @@ package org.mozilla.gecko.telemetry.stor
import org.json.JSONObject;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mozilla.gecko.background.testhelpers.TestRunner;
import org.mozilla.gecko.sync.ExtendedJSONObject;
+import org.mozilla.gecko.telemetry.TelemetryOutgoingPing;
import org.mozilla.gecko.telemetry.TelemetryPing;
import org.mozilla.gecko.util.FileUtils;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.FilenameFilter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import static org.junit.Assert.*;
@@ -98,40 +98,40 @@ public class TestTelemetryJSONFilePingSt
new TelemetryJSONFilePingStore(dir, "profileName"); // expected to throw.
}
@Test
public void testStorePingStoresCorrectData() throws Exception {
assertStoreFileCount(0);
final String expectedID = getDocID();
- final TelemetryPing expectedPing = new TelemetryPing("a/server/url", generateTelemetryPayload(), expectedID);
+ final TelemetryOutgoingPing expectedPing = new TelemetryOutgoingPing("a/server/url", generateTelemetryPayload(), expectedID);
testStore.storePing(expectedPing);
assertStoreFileCount(1);
final String filename = testDir.list()[0];
assertTrue("Filename contains expected ID", filename.equals(expectedID));
final JSONObject actual = FileUtils.readJSONObjectFromFile(new File(testDir, filename));
assertEquals("Ping url paths are equal", expectedPing.getURLPath(), actual.getString(TelemetryJSONFilePingStore.KEY_URL_PATH));
assertIsGeneratedPayload(new ExtendedJSONObject(actual.getString(TelemetryJSONFilePingStore.KEY_PAYLOAD)));
}
@Test
public void testStorePingMultiplePingsStoreSeparateFiles() throws Exception {
assertStoreFileCount(0);
for (int i = 1; i < 10; ++i) {
- testStore.storePing(new TelemetryPing("server", generateTelemetryPayload(), getDocID()));
+ testStore.storePing(new TelemetryOutgoingPing("server", generateTelemetryPayload(), getDocID()));
assertStoreFileCount(i);
}
}
@Test
public void testStorePingReleasesFileLock() throws Exception {
assertStoreFileCount(0);
- testStore.storePing(new TelemetryPing("server", generateTelemetryPayload(), getDocID()));
+ testStore.storePing(new TelemetryOutgoingPing("server", generateTelemetryPayload(), getDocID()));
assertStoreFileCount(1);
final File file = new File(testDir, testDir.list()[0]);
final FileOutputStream stream = new FileOutputStream(file);
try {
assertNotNull("File lock is released after store write", stream.getChannel().tryLock());
} finally {
stream.close(); // releases lock
}