Bug 1456209 - 3. Scale timeouts based on device; r?gbrown draft
authorJim Chen <nchen@mozilla.com>
Tue, 24 Apr 2018 10:13:35 -0400
changeset 787193 a2db8bdea7d7b64bd6cd926b84981f2c8ed732ad
parent 787192 ded6e2a1d06ffdce26bc2fef062973baa25da853
push id107676
push userbmo:nchen@mozilla.com
push dateTue, 24 Apr 2018 14:14:30 +0000
reviewersgbrown
bugs1456209
milestone61.0a1
Bug 1456209 - 3. Scale timeouts based on device; r?gbrown Scale the default test timeout based on the architecture, and whether we're running inside an emulator or on a real device. This is done to account for performance differences in the device under test. We have some tests that are expected to timeout. In order to maintain accuracy of these tests across devices, their custom timeout values are scaled as well. MozReview-Commit-ID: 7a65eAAPHOB
mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java
--- a/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java
+++ b/mobile/android/geckoview/src/androidTest/java/org/mozilla/geckoview/test/rule/GeckoSessionTestRule.java
@@ -1,16 +1,17 @@
 /* -*- Mode: Java; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: nil; -*-
  * This Source Code Form is subject to the terms of the Mozilla Public
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 package org.mozilla.geckoview.test.rule;
 
 import org.mozilla.gecko.gfx.GeckoDisplay;
+import org.mozilla.geckoview.BuildConfig;
 import org.mozilla.geckoview.GeckoRuntime;
 import org.mozilla.geckoview.GeckoRuntimeSettings;
 import org.mozilla.geckoview.GeckoSession;
 import org.mozilla.geckoview.GeckoSessionSettings;
 import org.mozilla.geckoview.test.util.Callbacks;
 
 import static org.hamcrest.Matchers.*;
 import static org.junit.Assert.assertThat;
@@ -19,16 +20,17 @@ import org.hamcrest.Matcher;
 
 import org.junit.rules.ErrorCollector;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
 
 import android.app.Instrumentation;
 import android.graphics.Point;
 import android.graphics.SurfaceTexture;
+import android.os.Build;
 import android.os.Bundle;
 import android.os.Debug;
 import android.os.Handler;
 import android.os.Looper;
 import android.os.Message;
 import android.os.MessageQueue;
 import android.os.SystemClock;
 import android.support.annotation.NonNull;
@@ -69,17 +71,22 @@ import kotlin.reflect.KClass;
  * and tears down the GeckoSession at the end of the test. The rule also provides methods
  * for waiting on particular callbacks to be called, and methods for asserting that
  * callbacks are called in the proper order.
  */
 public class GeckoSessionTestRule extends UiThreadTestRule {
     private static final String LOGTAG = "GeckoSessionTestRule";
 
     private static final long DEFAULT_TIMEOUT_MILLIS = 10000;
-    private static final long DEFAULT_DEBUG_TIMEOUT_MILLIS = 86400000;
+    private static final long DEFAULT_ARM_DEVICE_TIMEOUT_MILLIS = 30000;
+    private static final long DEFAULT_ARM_EMULATOR_TIMEOUT_MILLIS = 120000;
+    private static final long DEFAULT_X86_DEVICE_TIMEOUT_MILLIS = 30000;
+    private static final long DEFAULT_X86_EMULATOR_TIMEOUT_MILLIS = 5000;
+    private static final long DEFAULT_IDE_DEBUG_TIMEOUT_MILLIS = 86400000;
+
     public static final String APK_URI_PREFIX = "resource://android/";
 
     private static final Method sOnLocationChange;
     private static final Method sOnPageStop;
 
     static {
         try {
             sOnLocationChange = GeckoSession.NavigationDelegate.class.getMethod(
@@ -87,18 +94,20 @@ public class GeckoSessionTestRule extend
             sOnPageStop = GeckoSession.ProgressDelegate.class.getMethod(
                     "onPageStop", GeckoSession.class, boolean.class);
         } catch (final NoSuchMethodException e) {
             throw new RuntimeException(e);
         }
     }
 
     /**
-     * Specify the timeout for any of the wait methods, in milliseconds. Can be used
-     * on classes or methods.
+     * Specify the timeout for any of the wait methods, in milliseconds, relative to
+     * {@link #DEFAULT_TIMEOUT_MILLIS}. When the default timeout scales to account
+     * for differences in the device under test, the timeout value here will be
+     * scaled as well. Can be used on classes or methods.
      */
     @Target({ElementType.METHOD, ElementType.TYPE})
     @Retention(RetentionPolicy.RUNTIME)
     public @interface TimeoutMillis {
         long value();
     }
 
     /**
@@ -415,16 +424,28 @@ public class GeckoSessionTestRule extend
             return Boolean.valueOf(InstrumentationRegistry.getArguments()
                                                           .getString("use_multiprocess",
                                                                      "true"));
         }
 
         public boolean isDebugging() {
             return Debug.isDebuggerConnected();
         }
+
+        public boolean isEmulator() {
+            return "generic".equals(Build.DEVICE) || Build.DEVICE.startsWith("generic_");
+        }
+
+        public boolean isDebugBuild() {
+            return BuildConfig.DEBUG_BUILD;
+        }
+
+        public String getCPUArch() {
+            return BuildConfig.ANDROID_CPU_ARCH;
+        }
     }
 
     protected class CallbackDelegates {
         private final Map<Pair<GeckoSession, Method>, MethodCall> mDelegates = new HashMap<>();
         private int mOrder;
 
         public void delegate(final @Nullable GeckoSession session,
                              final @NonNull Object callback) {
@@ -642,17 +663,20 @@ public class GeckoSessionTestRule extend
             throws NoSuchMethodException {
         return GeckoSession.class.getMethod("get" + cls.getSimpleName());
     }
 
     protected void applyAnnotations(final Collection<Annotation> annotations,
                                     final GeckoSessionSettings settings) {
         for (final Annotation annotation : annotations) {
             if (TimeoutMillis.class.equals(annotation.annotationType())) {
-                mTimeoutMillis = Math.max(((TimeoutMillis) annotation).value(), 100);
+                // Scale timeout based on the default timeout to account for the device under test.
+                final long value = ((TimeoutMillis) annotation).value();
+                final long timeout = value * getDefaultTimeoutMillis() / DEFAULT_TIMEOUT_MILLIS;
+                mTimeoutMillis = Math.max(timeout, 1000);
             } else if (Setting.class.equals(annotation.annotationType())) {
                 ((Setting) annotation).key().set(settings, ((Setting) annotation).value());
             } else if (Setting.List.class.equals(annotation.annotationType())) {
                 for (final Setting setting : ((Setting.List) annotation).value()) {
                     setting.key().set(settings, setting.value());
                 }
             } else if (WithDisplay.class.equals(annotation.annotationType())) {
                 final WithDisplay displaySize = (WithDisplay)annotation;
@@ -669,20 +693,29 @@ public class GeckoSessionTestRule extend
             return (RuntimeException) cause;
         } else if (e instanceof RuntimeException) {
             return (RuntimeException) e;
         }
 
         return new RuntimeException(cause != null ? cause : e);
     }
 
+    private long getDefaultTimeoutMillis() {
+        if ("x86".equals(env.getCPUArch())) {
+            return env.isEmulator() ? DEFAULT_X86_EMULATOR_TIMEOUT_MILLIS
+                                    : DEFAULT_X86_DEVICE_TIMEOUT_MILLIS;
+        }
+        return env.isEmulator() ? DEFAULT_ARM_EMULATOR_TIMEOUT_MILLIS
+                                : DEFAULT_ARM_DEVICE_TIMEOUT_MILLIS;
+    }
+
     protected void prepareStatement(final Description description) throws Throwable {
         final GeckoSessionSettings settings = new GeckoSessionSettings(mDefaultSettings);
-        mTimeoutMillis = env.isDebugging() ? DEFAULT_DEBUG_TIMEOUT_MILLIS
-                                           : DEFAULT_TIMEOUT_MILLIS;
+        mTimeoutMillis = env.isDebugging() ? DEFAULT_IDE_DEBUG_TIMEOUT_MILLIS
+                                           : getDefaultTimeoutMillis();
         mClosedSession = false;
 
         applyAnnotations(Arrays.asList(description.getTestClass().getAnnotations()), settings);
         applyAnnotations(description.getAnnotations(), settings);
 
         final List<CallRecord> records = new ArrayList<>();
         final CallbackDelegates waitDelegates = new CallbackDelegates();
         final CallbackDelegates testDelegates = new CallbackDelegates();
@@ -804,17 +837,17 @@ public class GeckoSessionTestRule extend
                         }
                         return true;
                     }
                     return false;
                 }
             };
 
             do {
-                loopUntilIdle(DEFAULT_TIMEOUT_MILLIS);
+                loopUntilIdle(getDefaultTimeoutMillis());
             } while (mCallRecordHandler != null);
 
         } finally {
             mCallRecordHandler = null;
         }
     }
 
     /**