1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
3 /* This Source Code Form is subject to the terms of the Mozilla Public
4 * License, v. 2.0. If a copy of the MPL was not distributed with this
5 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
6
7 #include "MemoryTelemetry.h"
8 #include "nsMemoryReporterManager.h"
9
10 #include "mozilla/ClearOnShutdown.h"
11 #include "mozilla/Result.h"
12 #include "mozilla/ResultExtensions.h"
13 #include "mozilla/Services.h"
14 #include "mozilla/ScopeExit.h"
15 #include "mozilla/SimpleEnumerator.h"
16 #include "mozilla/Telemetry.h"
17 #include "mozilla/TimeStamp.h"
18 #include "mozilla/dom/ContentParent.h"
19 #include "mozilla/dom/ScriptSettings.h"
20 #include "nsContentUtils.h"
21 #include "nsIBrowserDOMWindow.h"
22 #include "nsIDOMChromeWindow.h"
23 #include "nsIMemoryReporter.h"
24 #include "nsIWindowMediator.h"
25 #include "nsImportModule.h"
26 #include "nsITelemetry.h"
27 #include "nsNetCID.h"
28 #include "nsObserverService.h"
29 #include "nsReadableUtils.h"
30 #include "nsThreadUtils.h"
31 #include "nsXULAppAPI.h"
32 #include "xpcpublic.h"
33
34 #include <cstdlib>
35
36 using namespace mozilla;
37
38 using mozilla::dom::AutoJSAPI;
39 using mozilla::dom::ContentParent;
40
41 // Do not gather data more than once a minute (ms)
42 static constexpr uint32_t kTelemetryInterval = 60 * 1000;
43
44 static constexpr const char* kTopicCycleCollectorBegin =
45 "cycle-collector-begin";
46
47 // How long to wait in millis for all the child memory reports to come in
48 static constexpr uint32_t kTotalMemoryCollectorTimeout = 200;
49
50 namespace {
51
52 enum class PrevValue : uint32_t {
53 #ifdef XP_WIN
54 LOW_MEMORY_EVENTS_VIRTUAL,
55 LOW_MEMORY_EVENTS_COMMIT_SPACE,
56 LOW_MEMORY_EVENTS_PHYSICAL,
57 #endif
58 #if defined(XP_LINUX) && !defined(ANDROID)
59 PAGE_FAULTS_HARD,
60 #endif
61 SIZE_,
62 };
63
64 } // anonymous namespace
65
66 constexpr uint32_t kUninitialized = ~0;
67
68 static uint32_t gPrevValues[uint32_t(PrevValue::SIZE_)];
69
PrevValueIndex(Telemetry::HistogramID aId)70 static uint32_t PrevValueIndex(Telemetry::HistogramID aId) {
71 switch (aId) {
72 #ifdef XP_WIN
73 case Telemetry::LOW_MEMORY_EVENTS_VIRTUAL:
74 return uint32_t(PrevValue::LOW_MEMORY_EVENTS_VIRTUAL);
75 case Telemetry::LOW_MEMORY_EVENTS_COMMIT_SPACE:
76 return uint32_t(PrevValue::LOW_MEMORY_EVENTS_COMMIT_SPACE);
77 case Telemetry::LOW_MEMORY_EVENTS_PHYSICAL:
78 return uint32_t(PrevValue::LOW_MEMORY_EVENTS_PHYSICAL);
79 #endif
80 #if defined(XP_LINUX) && !defined(ANDROID)
81 case Telemetry::PAGE_FAULTS_HARD:
82 return uint32_t(PrevValue::PAGE_FAULTS_HARD);
83 #endif
84 default:
85 MOZ_ASSERT_UNREACHABLE("Unexpected histogram ID");
86 return 0;
87 }
88 }
89
NS_IMPL_ISUPPORTS(MemoryTelemetry,nsIObserver,nsISupportsWeakReference)90 NS_IMPL_ISUPPORTS(MemoryTelemetry, nsIObserver, nsISupportsWeakReference)
91
92 MemoryTelemetry::MemoryTelemetry()
93 : mThreadPool(do_GetService(NS_STREAMTRANSPORTSERVICE_CONTRACTID)) {}
94
Init()95 void MemoryTelemetry::Init() {
96 for (auto& val : gPrevValues) {
97 val = kUninitialized;
98 }
99
100 if (XRE_IsContentProcess()) {
101 nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
102 MOZ_RELEASE_ASSERT(obs);
103
104 obs->AddObserver(this, "content-child-shutdown", true);
105 }
106 }
107
Get()108 /* static */ MemoryTelemetry& MemoryTelemetry::Get() {
109 static RefPtr<MemoryTelemetry> sInstance;
110
111 MOZ_ASSERT(NS_IsMainThread());
112
113 if (!sInstance) {
114 sInstance = new MemoryTelemetry();
115 sInstance->Init();
116 ClearOnShutdown(&sInstance);
117 }
118 return *sInstance;
119 }
120
DelayedInit()121 nsresult MemoryTelemetry::DelayedInit() {
122 if (Telemetry::CanRecordExtended()) {
123 nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
124 MOZ_RELEASE_ASSERT(obs);
125
126 obs->AddObserver(this, kTopicCycleCollectorBegin, true);
127 }
128
129 GatherReports();
130
131 return NS_OK;
132 }
133
Shutdown()134 nsresult MemoryTelemetry::Shutdown() {
135 nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
136 MOZ_RELEASE_ASSERT(obs);
137
138 obs->RemoveObserver(this, kTopicCycleCollectorBegin);
139
140 return NS_OK;
141 }
142
HandleMemoryReport(Telemetry::HistogramID aId,int32_t aUnits,uint64_t aAmount,const nsCString & aKey=VoidCString ())143 static inline void HandleMemoryReport(Telemetry::HistogramID aId,
144 int32_t aUnits, uint64_t aAmount,
145 const nsCString& aKey = VoidCString()) {
146 uint32_t val;
147 switch (aUnits) {
148 case nsIMemoryReporter::UNITS_BYTES:
149 val = uint32_t(aAmount / 1024);
150 break;
151
152 case nsIMemoryReporter::UNITS_PERCENTAGE:
153 // UNITS_PERCENTAGE amounts are 100x greater than their raw value.
154 val = uint32_t(aAmount / 100);
155 break;
156
157 case nsIMemoryReporter::UNITS_COUNT:
158 val = uint32_t(aAmount);
159 break;
160
161 case nsIMemoryReporter::UNITS_COUNT_CUMULATIVE: {
162 // If the reporter gives us a cumulative count, we'll report the
163 // difference in its value between now and our previous ping.
164
165 uint32_t idx = PrevValueIndex(aId);
166 uint32_t prev = gPrevValues[idx];
167 gPrevValues[idx] = aAmount;
168
169 if (prev == kUninitialized) {
170 // If this is the first time we're reading this reporter, store its
171 // current value but don't report it in the telemetry ping, so we
172 // ignore the effect startup had on the reporter.
173 return;
174 }
175 val = aAmount - prev;
176 break;
177 }
178
179 default:
180 MOZ_ASSERT_UNREACHABLE("Unexpected aUnits value");
181 return;
182 }
183
184 // Note: The reference equality check here should allow the compiler to
185 // optimize this case out at compile time when we weren't given a key,
186 // while IsEmpty() or IsVoid() most likely will not.
187 if (&aKey == &VoidCString()) {
188 Telemetry::Accumulate(aId, val);
189 } else {
190 Telemetry::Accumulate(aId, aKey, val);
191 }
192 }
193
GatherReports(const std::function<void ()> & aCompletionCallback)194 nsresult MemoryTelemetry::GatherReports(
195 const std::function<void()>& aCompletionCallback) {
196 auto cleanup = MakeScopeExit([&]() {
197 if (aCompletionCallback) {
198 aCompletionCallback();
199 }
200 });
201
202 RefPtr<nsMemoryReporterManager> mgr = nsMemoryReporterManager::GetOrCreate();
203 MOZ_DIAGNOSTIC_ASSERT(mgr);
204 NS_ENSURE_TRUE(mgr, NS_ERROR_FAILURE);
205
206 #define RECORD(id, metric, units) \
207 do { \
208 int64_t amt; \
209 nsresult rv = mgr->Get##metric(&amt); \
210 if (NS_SUCCEEDED(rv)) { \
211 HandleMemoryReport(Telemetry::id, nsIMemoryReporter::units, amt); \
212 } else if (rv != NS_ERROR_NOT_AVAILABLE) { \
213 NS_WARNING("Failed to retrieve memory telemetry for " #metric); \
214 } \
215 } while (0)
216
217 // GHOST_WINDOWS is opt-out as of Firefox 55
218 RECORD(GHOST_WINDOWS, GhostWindows, UNITS_COUNT);
219
220 // If we're running in the parent process, collect data from all processes for
221 // the MEMORY_TOTAL histogram.
222 if (XRE_IsParentProcess() && !mGatheringTotalMemory) {
223 GatherTotalMemory();
224 }
225
226 if (!Telemetry::CanRecordReleaseData()) {
227 return NS_OK;
228 }
229
230 // Get memory measurements from distinguished amount attributes. We used
231 // to measure "explicit" too, but it could cause hangs, and the data was
232 // always really noisy anyway. See bug 859657.
233 //
234 // test_TelemetrySession.js relies on some of these histograms being
235 // here. If you remove any of the following histograms from here, you'll
236 // have to modify test_TelemetrySession.js:
237 //
238 // * MEMORY_TOTAL,
239 // * MEMORY_JS_GC_HEAP, and
240 // * MEMORY_JS_COMPARTMENTS_SYSTEM.
241 //
242 // The distinguished amount attribute names don't match the telemetry id
243 // names in some cases due to a combination of (a) historical reasons, and
244 // (b) the fact that we can't change telemetry id names without breaking
245 // data continuity.
246
247 // Collect cheap or main-thread only metrics synchronously, on the main
248 // thread.
249 RECORD(MEMORY_JS_GC_HEAP, JSMainRuntimeGCHeap, UNITS_BYTES);
250 RECORD(MEMORY_JS_COMPARTMENTS_SYSTEM, JSMainRuntimeCompartmentsSystem,
251 UNITS_COUNT);
252 RECORD(MEMORY_JS_COMPARTMENTS_USER, JSMainRuntimeCompartmentsUser,
253 UNITS_COUNT);
254 RECORD(MEMORY_JS_REALMS_SYSTEM, JSMainRuntimeRealmsSystem, UNITS_COUNT);
255 RECORD(MEMORY_JS_REALMS_USER, JSMainRuntimeRealmsUser, UNITS_COUNT);
256 RECORD(MEMORY_IMAGES_CONTENT_USED_UNCOMPRESSED, ImagesContentUsedUncompressed,
257 UNITS_BYTES);
258 RECORD(MEMORY_STORAGE_SQLITE, StorageSQLite, UNITS_BYTES);
259 #ifdef XP_WIN
260 RECORD(LOW_MEMORY_EVENTS_PHYSICAL, LowMemoryEventsPhysical,
261 UNITS_COUNT_CUMULATIVE);
262 #endif
263 #if defined(XP_LINUX) && !defined(ANDROID)
264 RECORD(PAGE_FAULTS_HARD, PageFaultsHard, UNITS_COUNT_CUMULATIVE);
265 #endif
266
267 RefPtr<Runnable> completionRunnable;
268 if (aCompletionCallback) {
269 completionRunnable = NS_NewRunnableFunction(__func__, aCompletionCallback);
270 }
271
272 // Collect expensive metrics that can be calculated off-main-thread
273 // asynchronously, on a background thread.
274 RefPtr<Runnable> runnable = NS_NewRunnableFunction(
275 "MemoryTelemetry::GatherReports", [mgr, completionRunnable]() mutable {
276 RECORD(MEMORY_VSIZE, Vsize, UNITS_BYTES);
277 #if !defined(HAVE_64BIT_BUILD) || !defined(XP_WIN)
278 RECORD(MEMORY_VSIZE_MAX_CONTIGUOUS, VsizeMaxContiguous, UNITS_BYTES);
279 #endif
280 RECORD(MEMORY_RESIDENT_FAST, ResidentFast, UNITS_BYTES);
281 RECORD(MEMORY_RESIDENT_PEAK, ResidentPeak, UNITS_BYTES);
282 RECORD(MEMORY_UNIQUE, ResidentUnique, UNITS_BYTES);
283 RECORD(MEMORY_HEAP_ALLOCATED, HeapAllocated, UNITS_BYTES);
284 RECORD(MEMORY_HEAP_OVERHEAD_FRACTION, HeapOverheadFraction,
285 UNITS_PERCENTAGE);
286
287 if (completionRunnable) {
288 NS_DispatchToMainThread(completionRunnable.forget(),
289 NS_DISPATCH_NORMAL);
290 }
291 });
292
293 #undef RECORD
294
295 nsresult rv = mThreadPool->Dispatch(runnable.forget(), NS_DISPATCH_NORMAL);
296 if (!NS_WARN_IF(NS_FAILED(rv))) {
297 cleanup.release();
298 }
299
300 return NS_OK;
301 }
302
303 namespace {
304 struct ChildProcessInfo {
305 GeckoProcessType mType;
306 #if defined(XP_WIN)
307 HANDLE mHandle;
308 #elif defined(XP_MACOSX)
309 task_t mHandle;
310 #else
311 pid_t mHandle;
312 #endif
313 };
314 } // namespace
315
316 /**
317 * Runs a task on the background thread pool to fetch the memory usage of all
318 * processes.
319 */
GatherTotalMemory()320 void MemoryTelemetry::GatherTotalMemory() {
321 MOZ_ASSERT(!mGatheringTotalMemory);
322 mGatheringTotalMemory = true;
323
324 nsTArray<ChildProcessInfo> infos;
325 mozilla::ipc::GeckoChildProcessHost::GetAll(
326 [&](mozilla::ipc::GeckoChildProcessHost* aGeckoProcess) {
327 if (!aGeckoProcess->GetChildProcessHandle()) {
328 return;
329 }
330
331 ChildProcessInfo info{};
332 info.mType = aGeckoProcess->GetProcessType();
333
334 // NOTE: For now we ignore non-content processes here for compatibility
335 // with the existing probe. We may want to introduce a new probe in the
336 // future which also collects data for non-content processes.
337 if (info.mType != GeckoProcessType_Content) {
338 return;
339 }
340
341 #if defined(XP_WIN)
342 if (!::DuplicateHandle(::GetCurrentProcess(),
343 aGeckoProcess->GetChildProcessHandle(),
344 ::GetCurrentProcess(), &info.mHandle, 0, false,
345 DUPLICATE_SAME_ACCESS)) {
346 return;
347 }
348 #elif defined(XP_MACOSX)
349 info.mHandle = aGeckoProcess->GetChildTask();
350 if (mach_port_mod_refs(mach_task_self(), info.mHandle,
351 MACH_PORT_RIGHT_SEND, 1) != KERN_SUCCESS) {
352 return;
353 }
354 #else
355 info.mHandle = base::GetProcId(aGeckoProcess->GetChildProcessHandle());
356 #endif
357
358 infos.AppendElement(info);
359 });
360
361 mThreadPool->Dispatch(NS_NewRunnableFunction(
362 "MemoryTelemetry::GatherTotalMemory", [infos = std::move(infos)] {
363 RefPtr<nsMemoryReporterManager> mgr =
364 nsMemoryReporterManager::GetOrCreate();
365 MOZ_RELEASE_ASSERT(mgr);
366
367 int64_t totalMemory = mgr->ResidentFast();
368 nsTArray<int64_t> childSizes(infos.Length());
369
370 // Use our handle for the remote process to collect resident unique set
371 // size information for that process.
372 for (const auto& info : infos) {
373 int64_t memory =
374 nsMemoryReporterManager::ResidentUnique(info.mHandle);
375 if (memory > 0) {
376 childSizes.AppendElement(memory);
377 totalMemory += memory;
378 }
379
380 #if defined(XP_WIN)
381 ::CloseHandle(info.mHandle);
382 #elif defined(XP_MACOSX)
383 mach_port_deallocate(mach_task_self(), info.mHandle);
384 #endif
385 }
386
387 NS_DispatchToMainThread(NS_NewRunnableFunction(
388 "MemoryTelemetry::FinishGatheringTotalMemory",
389 [totalMemory, childSizes = std::move(childSizes)] {
390 MemoryTelemetry::Get().FinishGatheringTotalMemory(totalMemory,
391 childSizes);
392 }));
393 }));
394 }
395
FinishGatheringTotalMemory(int64_t aTotalMemory,const nsTArray<int64_t> & aChildSizes)396 nsresult MemoryTelemetry::FinishGatheringTotalMemory(
397 int64_t aTotalMemory, const nsTArray<int64_t>& aChildSizes) {
398 mGatheringTotalMemory = false;
399
400 HandleMemoryReport(Telemetry::MEMORY_TOTAL, nsIMemoryReporter::UNITS_BYTES,
401 aTotalMemory);
402
403 if (aChildSizes.Length() > 1) {
404 int32_t tabsCount;
405 MOZ_TRY_VAR(tabsCount, GetOpenTabsCount());
406
407 nsCString key;
408 if (tabsCount <= 10) {
409 key = "0 - 10 tabs";
410 } else if (tabsCount <= 500) {
411 key = "11 - 500 tabs";
412 } else {
413 key = "more tabs";
414 }
415
416 // Mean of the USS of all the content processes.
417 int64_t mean = 0;
418 for (auto size : aChildSizes) {
419 mean += size;
420 }
421 mean /= aChildSizes.Length();
422
423 // For some users, for unknown reasons (though most likely because they're
424 // in a sandbox without procfs mounted), we wind up with 0 here, which
425 // triggers a floating point exception if we try to calculate values using
426 // it.
427 if (!mean) {
428 return NS_ERROR_UNEXPECTED;
429 }
430
431 // Absolute error of USS for each content process, normalized by the mean
432 // (*100 to get it in percentage). 20% means for a content process that it
433 // is using 20% more or 20% less than the mean.
434 for (auto size : aChildSizes) {
435 int64_t diff = llabs(size - mean) * 100 / mean;
436
437 HandleMemoryReport(Telemetry::MEMORY_DISTRIBUTION_AMONG_CONTENT,
438 nsIMemoryReporter::UNITS_COUNT, diff, key);
439 }
440 }
441
442 // This notification is for testing only.
443 if (nsCOMPtr<nsIObserverService> obs = services::GetObserverService()) {
444 obs->NotifyObservers(nullptr, "gather-memory-telemetry-finished", nullptr);
445 }
446
447 return NS_OK;
448 }
449
GetOpenTabsCount()450 /* static */ Result<uint32_t, nsresult> MemoryTelemetry::GetOpenTabsCount() {
451 nsresult rv;
452
453 nsCOMPtr<nsIWindowMediator> windowMediator(
454 do_GetService(NS_WINDOWMEDIATOR_CONTRACTID, &rv));
455 MOZ_TRY(rv);
456
457 nsCOMPtr<nsISimpleEnumerator> enumerator;
458 MOZ_TRY(windowMediator->GetEnumerator(u"navigator:browser",
459 getter_AddRefs(enumerator)));
460
461 uint32_t total = 0;
462 for (auto& window : SimpleEnumerator<nsIDOMChromeWindow>(enumerator)) {
463 nsCOMPtr<nsIBrowserDOMWindow> browserWin;
464 MOZ_TRY(window->GetBrowserDOMWindow(getter_AddRefs(browserWin)));
465
466 NS_ENSURE_TRUE(browserWin, Err(NS_ERROR_UNEXPECTED));
467
468 uint32_t tabCount;
469 MOZ_TRY(browserWin->GetTabCount(&tabCount));
470 total += tabCount;
471 }
472
473 return total;
474 }
475
GetUniqueSetSize(std::function<void (const int64_t &)> && aCallback)476 void MemoryTelemetry::GetUniqueSetSize(
477 std::function<void(const int64_t&)>&& aCallback) {
478 mThreadPool->Dispatch(
479 NS_NewRunnableFunction(
480 "MemoryTelemetry::GetUniqueSetSize",
481 [callback = std::move(aCallback)]() mutable {
482 RefPtr<nsMemoryReporterManager> mgr =
483 nsMemoryReporterManager::GetOrCreate();
484 MOZ_RELEASE_ASSERT(mgr);
485
486 int64_t uss = mgr->ResidentUnique();
487
488 NS_DispatchToMainThread(NS_NewRunnableFunction(
489 "MemoryTelemetry::GetUniqueSetSizeResult",
490 [uss, callback = std::move(callback)]() { callback(uss); }));
491 }),
492 NS_DISPATCH_NORMAL);
493 }
494
Observe(nsISupports * aSubject,const char * aTopic,const char16_t * aData)495 nsresult MemoryTelemetry::Observe(nsISupports* aSubject, const char* aTopic,
496 const char16_t* aData) {
497 if (strcmp(aTopic, kTopicCycleCollectorBegin) == 0) {
498 auto now = TimeStamp::Now();
499 if (!mLastPoll.IsNull() &&
500 (now - mLastPoll).ToMilliseconds() < kTelemetryInterval) {
501 return NS_OK;
502 }
503
504 mLastPoll = now;
505
506 NS_DispatchToCurrentThreadQueue(
507 NewRunnableMethod<std::function<void()>>(
508 "MemoryTelemetry::GatherReports", this,
509 &MemoryTelemetry::GatherReports, nullptr),
510 EventQueuePriority::Idle);
511 } else if (strcmp(aTopic, "content-child-shutdown") == 0) {
512 if (nsCOMPtr<nsITelemetry> telemetry =
513 do_GetService("@mozilla.org/base/telemetry;1")) {
514 telemetry->FlushBatchedChildTelemetry();
515 }
516 }
517 return NS_OK;
518 }
519