1 //===-- xray_profile_collector.cpp -----------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of XRay, a dynamic runtime instrumentation system.
10 //
11 // This implements the interface for the profileCollectorService.
12 //
13 //===----------------------------------------------------------------------===//
14 #include "xray_profile_collector.h"
15 #include "sanitizer_common/sanitizer_common.h"
16 #include "xray_allocator.h"
17 #include "xray_defs.h"
18 #include "xray_profiling_flags.h"
19 #include "xray_segmented_array.h"
20 #include <memory>
21 #include <pthread.h>
22 #include <utility>
23 
24 namespace __xray {
25 namespace profileCollectorService {
26 
27 namespace {
28 
29 SpinMutex GlobalMutex;
30 struct ThreadTrie {
31   tid_t TId;
32   typename std::aligned_storage<sizeof(FunctionCallTrie)>::type TrieStorage;
33 };
34 
35 struct ProfileBuffer {
36   void *Data;
37   size_t Size;
38 };
39 
40 // Current version of the profile format.
41 constexpr u64 XRayProfilingVersion = 0x20180424;
42 
43 // Identifier for XRay profiling files 'xrayprof' in hex.
44 constexpr u64 XRayMagicBytes = 0x7872617970726f66;
45 
46 struct XRayProfilingFileHeader {
47   const u64 MagicBytes = XRayMagicBytes;
48   const u64 Version = XRayProfilingVersion;
49   u64 Timestamp = 0; // System time in nanoseconds.
50   u64 PID = 0;       // Process ID.
51 };
52 
53 struct BlockHeader {
54   u32 BlockSize;
55   u32 BlockNum;
56   u64 ThreadId;
57 };
58 
59 struct ThreadData {
60   BufferQueue *BQ;
61   FunctionCallTrie::Allocators::Buffers Buffers;
62   FunctionCallTrie::Allocators Allocators;
63   FunctionCallTrie FCT;
64   tid_t TId;
65 };
66 
67 using ThreadDataArray = Array<ThreadData>;
68 using ThreadDataAllocator = ThreadDataArray::AllocatorType;
69 
70 // We use a separate buffer queue for the backing store for the allocator used
71 // by the ThreadData array. This lets us host the buffers, allocators, and tries
72 // associated with a thread by moving the data into the array instead of
73 // attempting to copy the data to a separately backed set of tries.
74 static typename std::aligned_storage<
75     sizeof(BufferQueue), alignof(BufferQueue)>::type BufferQueueStorage;
76 static BufferQueue *BQ = nullptr;
77 static BufferQueue::Buffer Buffer;
78 static typename std::aligned_storage<sizeof(ThreadDataAllocator),
79                                      alignof(ThreadDataAllocator)>::type
80     ThreadDataAllocatorStorage;
81 static typename std::aligned_storage<sizeof(ThreadDataArray),
82                                      alignof(ThreadDataArray)>::type
83     ThreadDataArrayStorage;
84 
85 static ThreadDataAllocator *TDAllocator = nullptr;
86 static ThreadDataArray *TDArray = nullptr;
87 
88 using ProfileBufferArray = Array<ProfileBuffer>;
89 using ProfileBufferArrayAllocator = typename ProfileBufferArray::AllocatorType;
90 
91 // These need to be global aligned storage to avoid dynamic initialization. We
92 // need these to be aligned to allow us to placement new objects into the
93 // storage, and have pointers to those objects be appropriately aligned.
94 static typename std::aligned_storage<sizeof(ProfileBufferArray)>::type
95     ProfileBuffersStorage;
96 static typename std::aligned_storage<sizeof(ProfileBufferArrayAllocator)>::type
97     ProfileBufferArrayAllocatorStorage;
98 
99 static ProfileBufferArrayAllocator *ProfileBuffersAllocator = nullptr;
100 static ProfileBufferArray *ProfileBuffers = nullptr;
101 
102 // Use a global flag to determine whether the collector implementation has been
103 // initialized.
104 static atomic_uint8_t CollectorInitialized{0};
105 
106 } // namespace
107 
108 void post(BufferQueue *Q, FunctionCallTrie &&T,
109           FunctionCallTrie::Allocators &&A,
110           FunctionCallTrie::Allocators::Buffers &&B,
111           tid_t TId) XRAY_NEVER_INSTRUMENT {
112   DCHECK_NE(Q, nullptr);
113 
114   // Bail out early if the collector has not been initialized.
115   if (!atomic_load(&CollectorInitialized, memory_order_acquire)) {
116     T.~FunctionCallTrie();
117     A.~Allocators();
118     Q->releaseBuffer(B.NodeBuffer);
119     Q->releaseBuffer(B.RootsBuffer);
120     Q->releaseBuffer(B.ShadowStackBuffer);
121     Q->releaseBuffer(B.NodeIdPairBuffer);
122     B.~Buffers();
123     return;
124   }
125 
126   {
127     SpinMutexLock Lock(&GlobalMutex);
128     DCHECK_NE(TDAllocator, nullptr);
129     DCHECK_NE(TDArray, nullptr);
130 
131     if (TDArray->AppendEmplace(Q, std::move(B), std::move(A), std::move(T),
132                                TId) == nullptr) {
133       // If we fail to add the data to the array, we should destroy the objects
134       // handed us.
135       T.~FunctionCallTrie();
136       A.~Allocators();
137       Q->releaseBuffer(B.NodeBuffer);
138       Q->releaseBuffer(B.RootsBuffer);
139       Q->releaseBuffer(B.ShadowStackBuffer);
140       Q->releaseBuffer(B.NodeIdPairBuffer);
141       B.~Buffers();
142     }
143   }
144 }
145 
146 // A PathArray represents the function id's representing a stack trace. In this
147 // context a path is almost always represented from the leaf function in a call
148 // stack to a root of the call trie.
149 using PathArray = Array<int32_t>;
150 
151 struct ProfileRecord {
152   using PathAllocator = typename PathArray::AllocatorType;
153 
154   // The Path in this record is the function id's from the leaf to the root of
155   // the function call stack as represented from a FunctionCallTrie.
156   PathArray Path;
157   const FunctionCallTrie::Node *Node;
158 };
159 
160 namespace {
161 
162 using ProfileRecordArray = Array<ProfileRecord>;
163 
164 // Walk a depth-first traversal of each root of the FunctionCallTrie to generate
165 // the path(s) and the data associated with the path.
166 static void
167 populateRecords(ProfileRecordArray &PRs, ProfileRecord::PathAllocator &PA,
168                 const FunctionCallTrie &Trie) XRAY_NEVER_INSTRUMENT {
169   using StackArray = Array<const FunctionCallTrie::Node *>;
170   using StackAllocator = typename StackArray::AllocatorType;
171   StackAllocator StackAlloc(profilingFlags()->stack_allocator_max);
172   StackArray DFSStack(StackAlloc);
173   for (const auto *R : Trie.getRoots()) {
174     DFSStack.Append(R);
175     while (!DFSStack.empty()) {
176       auto *Node = DFSStack.back();
177       DFSStack.trim(1);
178       if (Node == nullptr)
179         continue;
180       auto Record = PRs.AppendEmplace(PathArray{PA}, Node);
181       if (Record == nullptr)
182         return;
183       DCHECK_NE(Record, nullptr);
184 
185       // Traverse the Node's parents and as we're doing so, get the FIds in
186       // the order they appear.
187       for (auto N = Node; N != nullptr; N = N->Parent)
188         Record->Path.Append(N->FId);
189       DCHECK(!Record->Path.empty());
190 
191       for (const auto C : Node->Callees)
192         DFSStack.Append(C.NodePtr);
193     }
194   }
195 }
196 
197 static void serializeRecords(ProfileBuffer *Buffer, const BlockHeader &Header,
198                              const ProfileRecordArray &ProfileRecords)
199     XRAY_NEVER_INSTRUMENT {
200   auto NextPtr = static_cast<uint8_t *>(
201                      internal_memcpy(Buffer->Data, &Header, sizeof(Header))) +
202                  sizeof(Header);
203   for (const auto &Record : ProfileRecords) {
204     // List of IDs follow:
205     for (const auto FId : Record.Path)
206       NextPtr =
207           static_cast<uint8_t *>(internal_memcpy(NextPtr, &FId, sizeof(FId))) +
208           sizeof(FId);
209 
210     // Add the sentinel here.
211     constexpr int32_t SentinelFId = 0;
212     NextPtr = static_cast<uint8_t *>(
213                   internal_memset(NextPtr, SentinelFId, sizeof(SentinelFId))) +
214               sizeof(SentinelFId);
215 
216     // Add the node data here.
217     NextPtr =
218         static_cast<uint8_t *>(internal_memcpy(
219             NextPtr, &Record.Node->CallCount, sizeof(Record.Node->CallCount))) +
220         sizeof(Record.Node->CallCount);
221     NextPtr = static_cast<uint8_t *>(
222                   internal_memcpy(NextPtr, &Record.Node->CumulativeLocalTime,
223                                   sizeof(Record.Node->CumulativeLocalTime))) +
224               sizeof(Record.Node->CumulativeLocalTime);
225   }
226 
227   DCHECK_EQ(NextPtr - static_cast<uint8_t *>(Buffer->Data), Buffer->Size);
228 }
229 
230 } // namespace
231 
232 void serialize() XRAY_NEVER_INSTRUMENT {
233   if (!atomic_load(&CollectorInitialized, memory_order_acquire))
234     return;
235 
236   SpinMutexLock Lock(&GlobalMutex);
237 
238   // Clear out the global ProfileBuffers, if it's not empty.
239   for (auto &B : *ProfileBuffers)
240     deallocateBuffer(reinterpret_cast<unsigned char *>(B.Data), B.Size);
241   ProfileBuffers->trim(ProfileBuffers->size());
242 
243   DCHECK_NE(TDArray, nullptr);
244   if (TDArray->empty())
245     return;
246 
247   // Then repopulate the global ProfileBuffers.
248   u32 I = 0;
249   auto MaxSize = profilingFlags()->global_allocator_max;
250   auto ProfileArena = allocateBuffer(MaxSize);
251   if (ProfileArena == nullptr)
252     return;
253 
254   auto ProfileArenaCleanup = at_scope_exit(
255       [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(ProfileArena, MaxSize); });
256 
257   auto PathArena = allocateBuffer(profilingFlags()->global_allocator_max);
258   if (PathArena == nullptr)
259     return;
260 
261   auto PathArenaCleanup = at_scope_exit(
262       [&]() XRAY_NEVER_INSTRUMENT { deallocateBuffer(PathArena, MaxSize); });
263 
264   for (const auto &ThreadTrie : *TDArray) {
265     using ProfileRecordAllocator = typename ProfileRecordArray::AllocatorType;
266     ProfileRecordAllocator PRAlloc(ProfileArena,
267                                    profilingFlags()->global_allocator_max);
268     ProfileRecord::PathAllocator PathAlloc(
269         PathArena, profilingFlags()->global_allocator_max);
270     ProfileRecordArray ProfileRecords(PRAlloc);
271 
272     // First, we want to compute the amount of space we're going to need. We'll
273     // use a local allocator and an __xray::Array<...> to store the intermediary
274     // data, then compute the size as we're going along. Then we'll allocate the
275     // contiguous space to contain the thread buffer data.
276     if (ThreadTrie.FCT.getRoots().empty())
277       continue;
278 
279     populateRecords(ProfileRecords, PathAlloc, ThreadTrie.FCT);
280     DCHECK(!ThreadTrie.FCT.getRoots().empty());
281     DCHECK(!ProfileRecords.empty());
282 
283     // Go through each record, to compute the sizes.
284     //
285     // header size = block size (4 bytes)
286     //   + block number (4 bytes)
287     //   + thread id (8 bytes)
288     // record size = path ids (4 bytes * number of ids + sentinel 4 bytes)
289     //   + call count (8 bytes)
290     //   + local time (8 bytes)
291     //   + end of record (8 bytes)
292     u32 CumulativeSizes = 0;
293     for (const auto &Record : ProfileRecords)
294       CumulativeSizes += 20 + (4 * Record.Path.size());
295 
296     BlockHeader Header{16 + CumulativeSizes, I++, ThreadTrie.TId};
297     auto B = ProfileBuffers->Append({});
298     B->Size = sizeof(Header) + CumulativeSizes;
299     B->Data = allocateBuffer(B->Size);
300     DCHECK_NE(B->Data, nullptr);
301     serializeRecords(B, Header, ProfileRecords);
302   }
303 }
304 
305 void reset() XRAY_NEVER_INSTRUMENT {
306   atomic_store(&CollectorInitialized, 0, memory_order_release);
307   SpinMutexLock Lock(&GlobalMutex);
308 
309   if (ProfileBuffers != nullptr) {
310     // Clear out the profile buffers that have been serialized.
311     for (auto &B : *ProfileBuffers)
312       deallocateBuffer(reinterpret_cast<uint8_t *>(B.Data), B.Size);
313     ProfileBuffers->trim(ProfileBuffers->size());
314     ProfileBuffers = nullptr;
315   }
316 
317   if (TDArray != nullptr) {
318     // Release the resources as required.
319     for (auto &TD : *TDArray) {
320       TD.BQ->releaseBuffer(TD.Buffers.NodeBuffer);
321       TD.BQ->releaseBuffer(TD.Buffers.RootsBuffer);
322       TD.BQ->releaseBuffer(TD.Buffers.ShadowStackBuffer);
323       TD.BQ->releaseBuffer(TD.Buffers.NodeIdPairBuffer);
324     }
325     // We don't bother destroying the array here because we've already
326     // potentially freed the backing store for the array. Instead we're going to
327     // reset the pointer to nullptr, and re-use the storage later instead
328     // (placement-new'ing into the storage as-is).
329     TDArray = nullptr;
330   }
331 
332   if (TDAllocator != nullptr) {
333     TDAllocator->~Allocator();
334     TDAllocator = nullptr;
335   }
336 
337   if (Buffer.Data != nullptr) {
338     BQ->releaseBuffer(Buffer);
339   }
340 
341   if (BQ == nullptr) {
342     bool Success = false;
343     new (&BufferQueueStorage)
344         BufferQueue(profilingFlags()->global_allocator_max, 1, Success);
345     if (!Success)
346       return;
347     BQ = reinterpret_cast<BufferQueue *>(&BufferQueueStorage);
348   } else {
349     BQ->finalize();
350 
351     if (BQ->init(profilingFlags()->global_allocator_max, 1) !=
352         BufferQueue::ErrorCode::Ok)
353       return;
354   }
355 
356   if (BQ->getBuffer(Buffer) != BufferQueue::ErrorCode::Ok)
357     return;
358 
359   new (&ProfileBufferArrayAllocatorStorage)
360       ProfileBufferArrayAllocator(profilingFlags()->global_allocator_max);
361   ProfileBuffersAllocator = reinterpret_cast<ProfileBufferArrayAllocator *>(
362       &ProfileBufferArrayAllocatorStorage);
363 
364   new (&ProfileBuffersStorage) ProfileBufferArray(*ProfileBuffersAllocator);
365   ProfileBuffers =
366       reinterpret_cast<ProfileBufferArray *>(&ProfileBuffersStorage);
367 
368   new (&ThreadDataAllocatorStorage)
369       ThreadDataAllocator(Buffer.Data, Buffer.Size);
370   TDAllocator =
371       reinterpret_cast<ThreadDataAllocator *>(&ThreadDataAllocatorStorage);
372   new (&ThreadDataArrayStorage) ThreadDataArray(*TDAllocator);
373   TDArray = reinterpret_cast<ThreadDataArray *>(&ThreadDataArrayStorage);
374 
375   atomic_store(&CollectorInitialized, 1, memory_order_release);
376 }
377 
378 XRayBuffer nextBuffer(XRayBuffer B) XRAY_NEVER_INSTRUMENT {
379   SpinMutexLock Lock(&GlobalMutex);
380 
381   if (ProfileBuffers == nullptr || ProfileBuffers->size() == 0)
382     return {nullptr, 0};
383 
384   static pthread_once_t Once = PTHREAD_ONCE_INIT;
385   static typename std::aligned_storage<sizeof(XRayProfilingFileHeader)>::type
386       FileHeaderStorage;
387   pthread_once(
388       &Once, +[]() XRAY_NEVER_INSTRUMENT {
389         new (&FileHeaderStorage) XRayProfilingFileHeader{};
390       });
391 
392   if (UNLIKELY(B.Data == nullptr)) {
393     // The first buffer should always contain the file header information.
394     auto &FileHeader =
395         *reinterpret_cast<XRayProfilingFileHeader *>(&FileHeaderStorage);
396     FileHeader.Timestamp = NanoTime();
397     FileHeader.PID = internal_getpid();
398     return {&FileHeaderStorage, sizeof(XRayProfilingFileHeader)};
399   }
400 
401   if (UNLIKELY(B.Data == &FileHeaderStorage))
402     return {(*ProfileBuffers)[0].Data, (*ProfileBuffers)[0].Size};
403 
404   BlockHeader Header;
405   internal_memcpy(&Header, B.Data, sizeof(BlockHeader));
406   auto NextBlock = Header.BlockNum + 1;
407   if (NextBlock < ProfileBuffers->size())
408     return {(*ProfileBuffers)[NextBlock].Data,
409             (*ProfileBuffers)[NextBlock].Size};
410   return {nullptr, 0};
411 }
412 
413 } // namespace profileCollectorService
414 } // namespace __xray
415