Bug 1111730 - (e10s) access to preliminary data from JS_TELEMETRY_ADDON_EXCEPTIONS probe
import numpy
import simplejson as json
import matplotlib.pyplot as plt
import pandas as pd
import IPython
import re
from moztelemetry.spark import get_pings
from __future__ import division
IPython.core.pylabtools.figsize(18, 7)
sc.defaultParallelism
16
Let's have a look at the past 7 days:
%%capture
pings = get_pings(sc, "Firefox", "nightly", "*", "*", ("20150218", "20150224"))
filtered = pings.filter(lambda x: "JS_TELEMETRY_ADDON_EXCEPTIONS" in x)
cached = filtered.map(lambda x: json.loads(x)["keyedHistograms"].get("JS_TELEMETRY_ADDON_EXCEPTIONS", None)).filter(lambda x: x).cache()
Number of pings with a non-empty JS_TELEMETRY_ADDON_EXCEPTIONS histogram to examine:
%%capture
n_pings = cached.count()
n_pings
177510
Group exceptions by key (addon, file, line) and count the number of submissions a key appears in:
%%capture
def clean(key): return re.sub("\?[0-9]+", "", key)
exceptions = cached.flatMap(lambda excs: [(clean(key), 1) for key in excs])
grouped = exceptions.countByKey()
exception_counts = pd.Series(grouped)
Show the top offenders by submission frequency:
exception_counts.sort(ascending=False)
for key, value in exception_counts[:100].iteritems():
print key, "{0:.2f}%".format(100*value/n_pings)
{d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} contentPolicy.js 404 17.72% {DDC359D1-844A-42a7-9AA1-88A850A938A8} preferences.jsm 90 15.29% abs@avira.com DNTContentPolicy.js 369 5.99% firefox@ghostery.com utils.js 234 5.40% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 302 3.69% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} FILE_NOT_FOUND 0 2.95% {c45c406e-ab73-11d8-be73-000a95be3b12} overlay.js 7333 2.63% jid1-F9UJ2thwoAm5gQ@jetpack policy.js 94 2.46% {19503e42-ca3c-4c27-b1e2-9cdb2170ee34} Youtube.js 1040 2.39% {b9db16a4-6edc-47ec-a1f4-b86292ed211d} util-service.jsm 322 2.16% {d40f5e7b-d2cf-4856-b441-cc613eeffbe3} bprivacy.js 93 2.08% {b9db16a4-6edc-47ec-a1f4-b86292ed211d} FILE_NOT_FOUND 0 1.98% jid1-4P0kohSJxU1qGg@jetpack FILE_NOT_FOUND 0 1.83% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 364 1.77% jid1-ZAdIEUB7XOzOJw@jetpack context-menu.js 689 1.58% {fe272bd1-5f76-4ea4-8501-a05d35d823fc} contentPolicy.js 390 1.50% nimbusscreencaptureff@everhelper.me context-menu.js 689 1.24% {7b1bf0b6-a1b9-42b0-b75d-252036438bdc} bootstrap.js 1124 1.22% {635abd67-4fe9-1b23-4f01-e679fa7484c1} FILE_NOT_FOUND 0 1.19% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} contentPolicy.js 625 1.19% {73a6fe31-595d-460b-a920-fcc0f8843232} FILE_NOT_FOUND 0 1.13% autopager@mozilla.org AutopagerCOMP.js 572 1.13% support@lastpass.com FILE_NOT_FOUND 0 1.11% jid1-dgnIBwQga0SIBw@jetpack context-menu.js 689 1.06% {7f57cf46-4467-4c2d-adfa-0cba7c507e54} mafContentPolicy.js 87 1.02% jid0-k75TfRGfOXPHfEZmJ9cKu5eCgLc@jetpack context-menu.js 689 0.99% youtubeunblocker@unblocker.yt proxy.js 83 0.91% {f3bd3dd2-2888-44c5-91a2-2caeb33fb898} bootstrap.js 1077 0.86% printPages2Pdf@reinhold.ripper FILE_NOT_FOUND 0 0.82% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 438 0.80% autopager@mozilla.org autoConfig.js 289 0.79% {F003DA68-8256-4b37-A6C4-350FA04494DF} overlay.js 398 0.79% https-everywhere@eff.org FILE_NOT_FOUND 0 0.71% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} FILE_NOT_FOUND 0 0.69% wrc@avast.com Utils.jsm 53 0.66% {3e9bb2a7-62ca-4efa-a4e6-f6f6168a652d} FILE_NOT_FOUND 0 0.66% jid1-d7uz7G3OvoZTJQ@jetpack context-menu.js 689 0.65% thumbnailZoom@dadler.github.com overlay.js 1666 0.64% foxyproxy@eric.h.jung FILE_NOT_FOUND 0 0.63% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} vapi-background.js 641 0.61% thumbnailZoom@dadler.github.com browser.xml 416 0.61% keefox@chris.tomlinson session.js 127 0.60% abs@avira.com FILE_NOT_FOUND 0 0.60% adblockpopups@jessehakanen.net FILE_NOT_FOUND 0 0.59% {9AA46F4F-4DC7-4c06-97AF-5035170634FE} imtranslatorOverlay.js?v 0.59% tmbepff@trendmicro.com TmBpFfUtil.js 122 0.59% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 333 0.55% safesearch@avira.com background.js 7412 0.54% https-everywhere@eff.org toolbar_button.js 64 0.52% abs@avira.com background.js 6 0.49% nicofox@littlebtc DownloadManager.jsm 513 0.49% jid1-BYcQOfYfmBMd9A@jetpack context-menu.js 689 0.48% privateTab@infocatcher FILE_NOT_FOUND 0 0.48% https-everywhere@eff.org ssl-observatory.js 621 0.48% {b9acf540-acba-11e1-8ccb-001fd0e08bd4} content.js 585 0.46% adguardadblocker@adguard.com context-menu.js 689 0.46% jid1-sNL73VCI4UB0Fw@jetpack context-menu.js 689 0.45% https-everywhere@eff.org tabbrowser.xml 430 0.44% mozilla_cc@internetdownloadmanager.com RemoteAddonsParent.jsm 5 0.44% fx-devtools FILE_NOT_FOUND 0 0.43% {73a6fe31-595d-460b-a920-fcc0f8843232} RemoteAddonsParent.jsm 8 0.40% firefox@zenmate.com FILE_NOT_FOUND 0 0.39% custombuttons@xsms.org FILE_NOT_FOUND 0 0.39% cliqz@cliqz.com FILE_NOT_FOUND 0 0.38% https-everywhere@eff.org https-everywhere.js 291 0.37% translator@dontfollowme.net context-menu.js 689 0.37% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} frameModule.js 140 0.37% smarterwiki@wikiatic.com smarterwiki.js 33 0.37% foxmarks@kei.com FILE_NOT_FOUND 0 0.37% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} frameModule.js 148 0.36% jid1-NIfFY2CA8fy1tg@jetpack FILE_NOT_FOUND 0 0.36% jid1-93WyvpgvxzGATw@jetpack context-menu.js 689 0.36% jid1-xKH0EoS44u1a2w@jetpack context-menu.js 689 0.35% jid1-MnnxcxisBPnSXQ@jetpack FILE_NOT_FOUND 0 0.35% privateTab@infocatcher PrivateBrowsingUtils.jsm 49 0.34% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} downloader.js 278 0.33% {DDC359D1-844A-42a7-9AA1-88A850A938A8} tree.js 907 0.31% jid0-raWjElI57dRa4jx9CCiYm5qZUQU@jetpack context-menu.js 689 0.31% firefox@ghostery.com FILE_NOT_FOUND 0 0.31% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} frameModule.js 150 0.31% jid1-4P0kohSJxU1qGg@jetpack tab-firefox.js 98 0.30% donottrackplus@abine.com FILE_NOT_FOUND 0 0.29% r2d2b2g@mozilla.org context-menu.js 689 0.29% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} vapi-background.js 378 0.29% {f36c6cd1-da73-491d-b290-8fc9115bfa55} RemoteAddonsParent.jsm 8 0.29% {73a6fe31-595d-460b-a920-fcc0f8843232} RemoteAddonsParent.jsm 5 0.28% nosquint@urandom.ca interfaces.js 95 0.28% wrc@avast.com bal.js 931 0.28% heartbeat-telemetry-experiment-1@mozilla.com FILE_NOT_FOUND 0 0.28% {a0d7ccb3-214d-498b-b4aa-0e8fda9a7bf7} FILE_NOT_FOUND 0 0.27% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} vapi-background.js 1091 0.26% extension@one-tab.com context-menu.js 689 0.25% firefox@ghostery.com utils.js 153 0.25% vb@yandex.ru WindowListener.jsm 19 0.25% 2.0@disconnect.me overlay.js 1314 0.25% tmbepff@trendmicro.com TmBpFfUtil.js 135 0.25% {dc572301-7619-498c-a57d-39143191b318} browser.js 1128 0.25% heartbeat-telemetry-experiment-1@mozilla.com tab-firefox.js 98 0.24% artur.dubovoy@gmail.com FILE_NOT_FOUND 0 0.24% jid0-RwTySlpoKU14fw7yw2AflOAihhA@jetpack context-menu.js 689 0.24%
Group exceptions by key (addon, file, line) and count the number of exception for each key:
%%capture
exceptions = cached.flatMap(lambda excs: [(clean(key), value["sum"]) for key, value in excs.items()])
exception_counts = pd.Series(exceptions.groupByKey().map(lambda x: (x[0], np.sum(list(x[1])))).collectAsMap())
Show the top offenders by count:
exception_counts.sort(ascending=False)
total = exception_counts.sum()
for key, value in exception_counts[:100].iteritems():
print key, "{0:.2f}%".format(100*value/total)
{d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} contentPolicy.js 404 10.31% {DDC359D1-844A-42a7-9AA1-88A850A938A8} preferences.jsm 90 8.89% abs@avira.com DNTContentPolicy.js 369 3.49% firefox@ghostery.com utils.js 234 3.14% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 302 2.14% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} FILE_NOT_FOUND 0 1.72% {c45c406e-ab73-11d8-be73-000a95be3b12} overlay.js 7333 1.53% jid1-F9UJ2thwoAm5gQ@jetpack policy.js 94 1.43% {19503e42-ca3c-4c27-b1e2-9cdb2170ee34} Youtube.js 1040 1.39% {b9db16a4-6edc-47ec-a1f4-b86292ed211d} util-service.jsm 322 1.25% {d40f5e7b-d2cf-4856-b441-cc613eeffbe3} bprivacy.js 93 1.21% {b9db16a4-6edc-47ec-a1f4-b86292ed211d} FILE_NOT_FOUND 0 1.15% jid1-4P0kohSJxU1qGg@jetpack FILE_NOT_FOUND 0 1.06% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 364 1.03% jid1-ZAdIEUB7XOzOJw@jetpack context-menu.js 689 0.92% {fe272bd1-5f76-4ea4-8501-a05d35d823fc} contentPolicy.js 390 0.87% nimbusscreencaptureff@everhelper.me context-menu.js 689 0.72% {7b1bf0b6-a1b9-42b0-b75d-252036438bdc} bootstrap.js 1124 0.71% {635abd67-4fe9-1b23-4f01-e679fa7484c1} FILE_NOT_FOUND 0 0.69% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} contentPolicy.js 625 0.69% {73a6fe31-595d-460b-a920-fcc0f8843232} FILE_NOT_FOUND 0 0.66% autopager@mozilla.org AutopagerCOMP.js 572 0.66% support@lastpass.com FILE_NOT_FOUND 0 0.64% jid1-dgnIBwQga0SIBw@jetpack context-menu.js 689 0.62% {7f57cf46-4467-4c2d-adfa-0cba7c507e54} mafContentPolicy.js 87 0.59% jid0-k75TfRGfOXPHfEZmJ9cKu5eCgLc@jetpack context-menu.js 689 0.57% youtubeunblocker@unblocker.yt proxy.js 83 0.53% {f3bd3dd2-2888-44c5-91a2-2caeb33fb898} bootstrap.js 1077 0.50% printPages2Pdf@reinhold.ripper FILE_NOT_FOUND 0 0.48% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 438 0.47% autopager@mozilla.org autoConfig.js 289 0.46% {F003DA68-8256-4b37-A6C4-350FA04494DF} overlay.js 398 0.46% https-everywhere@eff.org FILE_NOT_FOUND 0 0.42% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} FILE_NOT_FOUND 0 0.40% wrc@avast.com Utils.jsm 53 0.39% {3e9bb2a7-62ca-4efa-a4e6-f6f6168a652d} FILE_NOT_FOUND 0 0.38% jid1-d7uz7G3OvoZTJQ@jetpack context-menu.js 689 0.38% thumbnailZoom@dadler.github.com overlay.js 1666 0.37% foxyproxy@eric.h.jung FILE_NOT_FOUND 0 0.37% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} vapi-background.js 641 0.36% thumbnailZoom@dadler.github.com browser.xml 416 0.36% keefox@chris.tomlinson session.js 127 0.35% abs@avira.com FILE_NOT_FOUND 0 0.35% adblockpopups@jessehakanen.net FILE_NOT_FOUND 0 0.34% {9AA46F4F-4DC7-4c06-97AF-5035170634FE} imtranslatorOverlay.js?v 0.34% tmbepff@trendmicro.com TmBpFfUtil.js 122 0.34% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} objectTabs.js 333 0.32% safesearch@avira.com background.js 7412 0.31% https-everywhere@eff.org toolbar_button.js 64 0.30% abs@avira.com background.js 6 0.28% nicofox@littlebtc DownloadManager.jsm 513 0.28% jid1-BYcQOfYfmBMd9A@jetpack context-menu.js 689 0.28% privateTab@infocatcher FILE_NOT_FOUND 0 0.28% https-everywhere@eff.org ssl-observatory.js 621 0.28% {b9acf540-acba-11e1-8ccb-001fd0e08bd4} content.js 585 0.27% adguardadblocker@adguard.com context-menu.js 689 0.27% jid1-sNL73VCI4UB0Fw@jetpack context-menu.js 689 0.26% https-everywhere@eff.org tabbrowser.xml 430 0.26% mozilla_cc@internetdownloadmanager.com RemoteAddonsParent.jsm 5 0.25% fx-devtools FILE_NOT_FOUND 0 0.25% {73a6fe31-595d-460b-a920-fcc0f8843232} RemoteAddonsParent.jsm 8 0.23% firefox@zenmate.com FILE_NOT_FOUND 0 0.23% custombuttons@xsms.org FILE_NOT_FOUND 0 0.23% cliqz@cliqz.com FILE_NOT_FOUND 0 0.22% https-everywhere@eff.org https-everywhere.js 291 0.22% translator@dontfollowme.net context-menu.js 689 0.22% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} frameModule.js 140 0.22% smarterwiki@wikiatic.com smarterwiki.js 33 0.21% foxmarks@kei.com FILE_NOT_FOUND 0 0.21% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} frameModule.js 148 0.21% jid1-NIfFY2CA8fy1tg@jetpack FILE_NOT_FOUND 0 0.21% jid1-93WyvpgvxzGATw@jetpack context-menu.js 689 0.21% jid1-xKH0EoS44u1a2w@jetpack context-menu.js 689 0.20% jid1-MnnxcxisBPnSXQ@jetpack FILE_NOT_FOUND 0 0.20% privateTab@infocatcher PrivateBrowsingUtils.jsm 49 0.20% {d10d0bf8-f5b5-c8b4-a8b2-2b9879e08c5d} downloader.js 278 0.19% {DDC359D1-844A-42a7-9AA1-88A850A938A8} tree.js 907 0.18% jid0-raWjElI57dRa4jx9CCiYm5qZUQU@jetpack context-menu.js 689 0.18% firefox@ghostery.com FILE_NOT_FOUND 0 0.18% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} frameModule.js 150 0.18% jid1-4P0kohSJxU1qGg@jetpack tab-firefox.js 98 0.17% donottrackplus@abine.com FILE_NOT_FOUND 0 0.17% r2d2b2g@mozilla.org context-menu.js 689 0.17% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} vapi-background.js 378 0.17% {f36c6cd1-da73-491d-b290-8fc9115bfa55} RemoteAddonsParent.jsm 8 0.17% {73a6fe31-595d-460b-a920-fcc0f8843232} RemoteAddonsParent.jsm 5 0.17% nosquint@urandom.ca interfaces.js 95 0.17% wrc@avast.com bal.js 931 0.16% heartbeat-telemetry-experiment-1@mozilla.com FILE_NOT_FOUND 0 0.16% {a0d7ccb3-214d-498b-b4aa-0e8fda9a7bf7} FILE_NOT_FOUND 0 0.16% {2b10c1c8-a11f-4bad-fe9c-1c11e82cac42} vapi-background.js 1091 0.15% extension@one-tab.com context-menu.js 689 0.15% vb@yandex.ru WindowListener.jsm 19 0.15% firefox@ghostery.com utils.js 153 0.15% 2.0@disconnect.me overlay.js 1314 0.15% tmbepff@trendmicro.com TmBpFfUtil.js 135 0.14% {dc572301-7619-498c-a57d-39143191b318} browser.js 1128 0.14% heartbeat-telemetry-experiment-1@mozilla.com tab-firefox.js 98 0.14% artur.dubovoy@gmail.com FILE_NOT_FOUND 0 0.14% jid0-RwTySlpoKU14fw7yw2AflOAihhA@jetpack context-menu.js 689 0.14%