--- a/pylib/mozreview/mozreview/extension.py
+++ b/pylib/mozreview/mozreview/extension.py
@@ -99,49 +99,36 @@ class ParentJSExtension(JSExtension):
 
 
 class MozReviewExtension(Extension):
     metadata = {
         'Name': 'mozreview',
         'Summary': 'MozReview extension to Review Board',
     }
 
-    default_settings = {
-        'enabled': False,
-        'pulse_host': '',
-        'pulse_port': '',
-        'pulse_user': '',
-        'pulse_password': '',
-        'pulse_ssl': False,
-        'autoland_try_ui_enabled': False,
-        'autoland_url': '',
-        'autoland_user': '',
-        'autoland_password': '',
-        'autoland_testing': False,
-        'autoland_import_pullrequest_ui_enabled': False,
-        'ldap_url': '',
-        'ldap_user': '',
-        'ldap_password': '',
-    }
+    default_settings = {}
 
     is_configurable = True
 
     js_extensions = [ParentJSExtension]
 
     css_bundles = {
         'default': {
             'source_filenames': ['mozreview/css/common.less'],
         },
         'review': {
             'source_filenames': ['mozreview/css/review.less',
                                  'mozreview/css/commits.less'],
         },
         'viewdiff': {
             'source_filenames': ['mozreview/css/viewdiff.less'],
         },
+        'admin': {
+            'source_filenames': ['mozreview/css/admin.less'],
+        },
     }
     js_bundles = {
         'default': {
             'source_filenames': ['mozreview/js/logout.js'],
         },
         'filediffreviewer': {
             'source_filenames': ['mozreview/js/models/filediffreviewermodel.js',
                                  'mozreview/js/collections/filediffreviewercollection.js',
@@ -259,16 +246,18 @@ class MozReviewExtension(Extension):
 
         # All of our review request styling is injected via
         # review-stylings-css, which in turn loads the review.css static
         # bundle.
         TemplateHook(self, 'base-css', 'mozreview/review-stylings-css.html',
                      apply_to=review_request_url_names)
         TemplateHook(self, 'base-css', 'mozreview/viewdiff-stylings-css.html',
                      apply_to=diffviewer_url_names)
+        TemplateHook(self, 'base-css', 'mozreview/admin-stylings-css.html',
+                     apply_to=['reviewboard.extensions.views.configure_extension'])
         TemplateHook(self, 'base-scripts-post',
                      'mozreview/review-scripts-js.html',
                      apply_to=review_request_url_names)
         TemplateHook(self, 'base-extrahead',
                      'mozreview/base-extrahead-login-form.html',
                      apply_to=['login'])
         TemplateHook(self, 'before-login-form',
                      'mozreview/before-login-form.html', apply_to=['login'])

--- a/pylib/mozreview/mozreview/forms.py
+++ b/pylib/mozreview/mozreview/forms.py
@@ -1,52 +1,13 @@
 from django import forms
 from django.utils.translation import ugettext as _
 from djblets.extensions.forms import SettingsForm
 
+import mozreview.extension
+
 
 class MozReviewSettingsForm(SettingsForm):
-    enabled = forms.BooleanField(
-        initial=False,
-        required=False,
-        label=_('Enable Pulse'))
-    pulse_host = forms.CharField(
+    config = forms.CharField(
         required=False,
-        widget=forms.TextInput(attrs={
-            'size': 100,
-        }))
-    pulse_port = forms.IntegerField(required=False)
-    pulse_ssl = forms.BooleanField(required=False)
-    pulse_user = forms.CharField(required=False)
-    pulse_password = forms.CharField(required=False,
-                                     widget=forms.PasswordInput(
-                                        render_value=True))
-    autoland_import_pullrequest_ui_enabled = forms.BooleanField(
-        label=_('Enable Autoland Import Pullrequest UI'),
-        help_text=_('This allows access to the web endpoint for importing '
-                    'Github pull requests'),
-        initial=False,
-        required=False)
-    autoland_try_ui_enabled = forms.BooleanField(
-        label=_('Enable Autoland Try UI'),
-        help_text=_('This exposes the field in a push-based review request '
-                    'for scheduling Try Autoland jobs'),
-        initial=False,
-        required=False)
-    autoland_url = forms.CharField(
-        required=False,
-        widget=forms.TextInput(attrs={
-            'size': 100,
-        }))
-    autoland_user = forms.CharField(required=False)
-    autoland_password = forms.CharField(required=False,
-                                        widget=forms.PasswordInput(
-                                            render_value=True))
-    autoland_testing = forms.BooleanField(initial=False, required=False)
-    ldap_url = forms.CharField(
-        required=False,
-        widget=forms.TextInput(attrs={
-            'size': 100,
-        }))
-    ldap_user = forms.CharField(required=False)
-    ldap_password = forms.CharField(required=False,
-                                    widget=forms.PasswordInput(
-                                        render_value=True))
+        help_text=_('Configure this extension by editing ' +
+                    mozreview.extension.SETTINGS_PATH)
+    )

new file mode 100644
--- /dev/null
+++ b/pylib/mozreview/mozreview/static/mozreview/css/admin.less
@@ -0,0 +1,3 @@
+#id_config {
+  display: none;
+}

new file mode 100644
--- /dev/null
+++ b/pylib/mozreview/mozreview/templates/mozreview/admin-stylings-css.html
@@ -0,0 +1,2 @@
+{% load djblets_extensions %}
+{% ext_css_bundle extension "admin" %}