Bug 1244160 - Create json-schema for build telemetry data r?gps
This adds a simple schema for build telemetry data. We can make it more
restrictive once we have a better feeling for what kind of data we want
to submit.
This also moves more common data about the system to the telemetry handler.
We leave psutil derivied information in the resource usage data as not every
system will have psutil installed.
MozReview-Commit-ID: CFRq1Ow6AOf
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -249,18 +249,36 @@ def bootstrap(topsrcdir, mozilla_dir=Non
outgoing_dir = os.path.join(telemetry_dir, 'outgoing')
try:
os.mkdir(outgoing_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
# Add common metadata to help submit sorted data later on.
- # For now, we'll just record the mach command that was invoked.
data['argv'] = sys.argv
+ data.setdefault('system', {}).update(dict(
+ architecture=list(platform.architecture()),
+ machine=platform.machine(),
+ python_version=platform.python_version(),
+ release=platform.release(),
+ system=platform.system(),
+ version=platform.version(),
+ ))
+
+ if platform.system() == 'Linux':
+ dist = list(platform.linux_distribution())
+ data['system']['linux_distribution'] = dist
+ elif platform.system() == 'Windows':
+ win32_ver=list((platform.win32_ver())),
+ data['system']['win32_ver'] = win32_ver
+ elif platform.system() == 'Darwin':
+ # mac version is a special Cupertino snowflake
+ r, v, m = platform.mac_ver()
+ data['system']['mac_ver'] = [r, list(v), m]
with open(os.path.join(outgoing_dir, str(uuid.uuid4()) + '.json'),
'w') as f:
json.dump(data, f, sort_keys=True)
def should_skip_dispatch(context, handler):
# The user is performing a maintenance command.
if handler.name in ('bootstrap', 'doctor', 'mach-commands', 'mercurial-setup'):
new file mode 100644
--- /dev/null
+++ b/build/telemetry-schema.json
@@ -0,0 +1,24 @@
+{
+ "$schema": "http://json-schema.org/draft-04/schema#",
+ "type": "object",
+ "properties": {
+ "argv": {"type": "array"},
+ "system": {
+ "type": "object",
+ "properties": {
+ "architecture": {"type": "array"},
+ "linux_distribution": {"type": "array"},
+ "mac_ver": {"type": "array"},
+ "machine": {"type": "string"},
+ "python_version": {"type": "string"},
+ "release": {"type": "string"},
+ "system": {"type": "string"},
+ "version": {"type": "string"},
+ "win_ver": {"type": "array"}
+ },
+ "required": ["architecture", "machine", "python_version",
+ "release", "system", "version"]
+ }
+ },
+ "required": ["argv", "system"]
+}
--- a/python/mozbuild/mozbuild/controller/building.py
+++ b/python/mozbuild/mozbuild/controller/building.py
@@ -372,17 +372,17 @@ class BuildMonitor(MozbuildObject):
if excessive is not None and (sin or sout):
sin /= 1048576
sout /= 1048576
self.log(logging.WARNING, 'swap_activity',
{'sin': sin, 'sout': sout},
'Swap in/out (MB): {sin}/{sout}')
o = dict(
- version=2,
+ version=3,
argv=sys.argv,
start=self.start_time,
end=self.end_time,
duration=self.end_time - self.start_time,
resources=[],
cpu_percent=cpu_percent,
cpu_times=cpu_times,
io=io,
@@ -406,51 +406,33 @@ class BuildMonitor(MozbuildObject):
swap=list(usage.swap),
)
self.tiers.add_resources_to_dict(entry, start=usage.start,
end=usage.end)
o['resources'].append(entry)
- # TODO: it would be nice to collect data on the storage device as well
- o['system'] = dict(
- architecture=list(platform.architecture()),
- machine=platform.machine(),
- python_version=platform.python_version(),
- release=platform.release(),
- system=platform.system(),
- version=platform.version(),
- )
# If the imports for this file ran before the in-tree virtualenv
# was bootstrapped (for instance, for a clobber build in automation),
# psutil might not be available.
#
# Treat psutil as optional to avoid an outright failure to log resources
+ # TODO: it would be nice to collect data on the storage device as well
# in this case.
+ o['system'] = {}
if psutil:
o['system'].update(dict(
logical_cpu_count=psutil.cpu_count(),
physical_cpu_count=psutil.cpu_count(logical=False),
swap_total=psutil.swap_memory()[0],
vmem_total=psutil.virtual_memory()[0],
))
- if platform.system() == 'Linux':
- dist = list(platform.linux_distribution())
- o['system']['linux_distribution'] = dist
- elif platform.system() == 'Windows':
- win32_ver=list((platform.win32_ver())),
- o['system']['win32_ver'] = win32_ver
- elif platform.system() == 'Darwin':
- # mac version is a special Cupertino snowflake
- r, v, m = platform.mac_ver()
- o['system']['mac_ver'] = [r, list(v), m]
-
return o
def _log_resource_usage(self, prefix, m_type, duration, cpu_percent,
cpu_times, io, extra_params={}):
params = dict(
duration=duration,
cpu_percent=cpu_percent,
--- a/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
+++ b/python/mozbuild/mozbuild/resources/html-build-viewer/index.html
@@ -51,17 +51,17 @@ svg {
<body>
<script>
var currentResources;
/**
* Interface for a build resources JSON file.
*/
function BuildResources(data) {
- if (data.version != 1 && data.version != 2) {
+ if (data.version < 1 || data.version > 3) {
throw new Error("Unsupported version of the JSON format: " + data.version);
}
this.resources = [];
var cpu_fields = data.cpu_times_fields;
var io_fields = data.io_fields;
var virt_fields = data.virt_fields;