1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8
9 // Note: This is linked into the Darwin kernel, and must remain compatible
10 // with freestanding compilation. See `darwin_add_builtin_libraries`.
11
12 #ifdef _MSC_VER
13 /* For _alloca */
14 #include <malloc.h>
15 #endif
16 #include <string.h>
17
18 #include "InstrProfiling.h"
19 #include "InstrProfilingInternal.h"
20 #include "InstrProfilingPort.h"
21
22 #define INSTR_PROF_VALUE_PROF_DATA
23 #include "profile/InstrProfData.inc"
24
25 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
26 static ProfBufferIO TheBufferIO;
27 #define VP_BUFFER_SIZE 8 * 1024
28 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
29 static InstrProfValueData VPDataArray[16];
30 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
31
32 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
33 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
34
35 /* The buffer writer is responsible in keeping writer state
36 * across the call.
37 */
lprofBufferWriter(ProfDataWriter * This,ProfDataIOVec * IOVecs,uint32_t NumIOVecs)38 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
39 ProfDataIOVec *IOVecs,
40 uint32_t NumIOVecs) {
41 uint32_t I;
42 char **Buffer = (char **)&This->WriterCtx;
43 for (I = 0; I < NumIOVecs; I++) {
44 size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
45 if (IOVecs[I].Data)
46 memcpy(*Buffer, IOVecs[I].Data, Length);
47 else if (IOVecs[I].UseZeroPadding) {
48 /* Allocating the buffer should zero fill. */
49 }
50 *Buffer += Length;
51 }
52 return 0;
53 }
54
llvmInitBufferIO(ProfBufferIO * BufferIO,ProfDataWriter * FileWriter,uint8_t * Buffer,uint32_t BufferSz)55 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
56 uint8_t *Buffer, uint32_t BufferSz) {
57 BufferIO->FileWriter = FileWriter;
58 BufferIO->OwnFileWriter = 0;
59 BufferIO->BufferStart = Buffer;
60 BufferIO->BufferSz = BufferSz;
61 BufferIO->CurOffset = 0;
62 }
63
64 COMPILER_RT_VISIBILITY ProfBufferIO *
lprofCreateBufferIO(ProfDataWriter * FileWriter)65 lprofCreateBufferIO(ProfDataWriter *FileWriter) {
66 uint8_t *Buffer = DynamicBufferIOBuffer;
67 uint32_t BufferSize = VPBufferSize;
68 if (!Buffer) {
69 Buffer = &BufferIOBuffer[0];
70 BufferSize = sizeof(BufferIOBuffer);
71 }
72 llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize);
73 return &TheBufferIO;
74 }
75
lprofDeleteBufferIO(ProfBufferIO * BufferIO)76 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
77 if (BufferIO->OwnFileWriter)
78 FreeHook(BufferIO->FileWriter);
79 if (DynamicBufferIOBuffer) {
80 FreeHook(DynamicBufferIOBuffer);
81 DynamicBufferIOBuffer = 0;
82 VPBufferSize = 0;
83 }
84 }
85
86 COMPILER_RT_VISIBILITY int
lprofBufferIOWrite(ProfBufferIO * BufferIO,const uint8_t * Data,uint32_t Size)87 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
88 /* Buffer is not large enough, it is time to flush. */
89 if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
90 if (lprofBufferIOFlush(BufferIO) != 0)
91 return -1;
92 }
93 /* Special case, bypass the buffer completely. */
94 ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
95 if (Size > BufferIO->BufferSz) {
96 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
97 return -1;
98 } else {
99 /* Write the data to buffer */
100 uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
101 ProfDataWriter BufferWriter;
102 initBufferWriter(&BufferWriter, (char *)Buffer);
103 lprofBufferWriter(&BufferWriter, IO, 1);
104 BufferIO->CurOffset =
105 (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
106 }
107 return 0;
108 }
109
lprofBufferIOFlush(ProfBufferIO * BufferIO)110 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
111 if (BufferIO->CurOffset) {
112 ProfDataIOVec IO[] = {
113 {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
114 if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
115 return -1;
116 BufferIO->CurOffset = 0;
117 }
118 return 0;
119 }
120
121 /* Write out value profile data for function specified with \c Data.
122 * The implementation does not use the method \c serializeValueProfData
123 * which depends on dynamic memory allocation. In this implementation,
124 * value profile data is written out to \c BufferIO piecemeal.
125 */
writeOneValueProfData(ProfBufferIO * BufferIO,VPDataReaderType * VPDataReader,const __llvm_profile_data * Data)126 static int writeOneValueProfData(ProfBufferIO *BufferIO,
127 VPDataReaderType *VPDataReader,
128 const __llvm_profile_data *Data) {
129 unsigned I, NumValueKinds = 0;
130 ValueProfData VPHeader;
131 uint8_t *SiteCountArray[IPVK_Last + 1];
132
133 for (I = 0; I <= IPVK_Last; I++) {
134 if (!Data->NumValueSites[I])
135 SiteCountArray[I] = 0;
136 else {
137 uint32_t Sz =
138 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
139 offsetof(ValueProfRecord, SiteCountArray);
140 /* Only use alloca for this small byte array to avoid excessive
141 * stack growth. */
142 SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
143 memset(SiteCountArray[I], 0, Sz);
144 }
145 }
146
147 /* If NumValueKinds returned is 0, there is nothing to write, report
148 success and return. This should match the raw profile reader's behavior. */
149 if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
150 return 0;
151
152 /* First write the header structure. */
153 VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
154 VPHeader.NumValueKinds = NumValueKinds;
155 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,
156 sizeof(ValueProfData)))
157 return -1;
158
159 /* Make sure nothing else needs to be written before value profile
160 * records. */
161 if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
162 (void *)(&VPHeader + 1))
163 return -1;
164
165 /* Write out the value profile record for each value kind
166 * one by one. */
167 for (I = 0; I <= IPVK_Last; I++) {
168 uint32_t J;
169 ValueProfRecord RecordHeader;
170 /* The size of the value prof record header without counting the
171 * site count array .*/
172 uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
173 uint32_t SiteCountArraySize;
174
175 if (!Data->NumValueSites[I])
176 continue;
177
178 /* Write out the record header. */
179 RecordHeader.Kind = I;
180 RecordHeader.NumValueSites = Data->NumValueSites[I];
181 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,
182 RecordHeaderSize))
183 return -1;
184
185 /* Write out the site value count array including padding space. */
186 SiteCountArraySize =
187 VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
188 RecordHeaderSize;
189 if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))
190 return -1;
191
192 /* Write out the value profile data for each value site. */
193 for (J = 0; J < Data->NumValueSites[I]; J++) {
194 uint32_t NRead, NRemain;
195 ValueProfNode *NextStartNode = 0;
196 NRemain = VPDataReader->GetNumValueDataForSite(I, J);
197 if (!NRemain)
198 continue;
199 /* Read and write out value data in small chunks till it is done. */
200 do {
201 NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
202 NextStartNode =
203 VPDataReader->GetValueData(I, /* ValueKind */
204 J, /* Site */
205 &VPDataArray[0], NextStartNode, NRead);
206 if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],
207 NRead * sizeof(InstrProfValueData)))
208 return -1;
209 NRemain -= NRead;
210 } while (NRemain != 0);
211 }
212 }
213 /* All done report success. */
214 return 0;
215 }
216
writeValueProfData(ProfDataWriter * Writer,VPDataReaderType * VPDataReader,const __llvm_profile_data * DataBegin,const __llvm_profile_data * DataEnd)217 static int writeValueProfData(ProfDataWriter *Writer,
218 VPDataReaderType *VPDataReader,
219 const __llvm_profile_data *DataBegin,
220 const __llvm_profile_data *DataEnd) {
221 ProfBufferIO *BufferIO;
222 const __llvm_profile_data *DI = 0;
223
224 if (!VPDataReader)
225 return 0;
226
227 BufferIO = lprofCreateBufferIO(Writer);
228
229 for (DI = DataBegin; DI < DataEnd; DI++) {
230 if (writeOneValueProfData(BufferIO, VPDataReader, DI))
231 return -1;
232 }
233
234 if (lprofBufferIOFlush(BufferIO) != 0)
235 return -1;
236 lprofDeleteBufferIO(BufferIO);
237
238 return 0;
239 }
240
lprofWriteData(ProfDataWriter * Writer,VPDataReaderType * VPDataReader,int SkipNameDataWrite)241 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
242 VPDataReaderType *VPDataReader,
243 int SkipNameDataWrite) {
244 /* Match logic in __llvm_profile_write_buffer(). */
245 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
246 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
247 const char *CountersBegin = __llvm_profile_begin_counters();
248 const char *CountersEnd = __llvm_profile_end_counters();
249 const char *BitmapBegin = __llvm_profile_begin_bitmap();
250 const char *BitmapEnd = __llvm_profile_end_bitmap();
251 const char *NamesBegin = __llvm_profile_begin_names();
252 const char *NamesEnd = __llvm_profile_end_names();
253 return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
254 CountersEnd, BitmapBegin, BitmapEnd, VPDataReader,
255 NamesBegin, NamesEnd, SkipNameDataWrite);
256 }
257
258 COMPILER_RT_VISIBILITY int
lprofWriteDataImpl(ProfDataWriter * Writer,const __llvm_profile_data * DataBegin,const __llvm_profile_data * DataEnd,const char * CountersBegin,const char * CountersEnd,const char * BitmapBegin,const char * BitmapEnd,VPDataReaderType * VPDataReader,const char * NamesBegin,const char * NamesEnd,int SkipNameDataWrite)259 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
260 const __llvm_profile_data *DataEnd,
261 const char *CountersBegin, const char *CountersEnd,
262 const char *BitmapBegin, const char *BitmapEnd,
263 VPDataReaderType *VPDataReader, const char *NamesBegin,
264 const char *NamesEnd, int SkipNameDataWrite) {
265 /* Calculate size of sections. */
266 const uint64_t DataSectionSize =
267 __llvm_profile_get_data_size(DataBegin, DataEnd);
268 const uint64_t NumData = __llvm_profile_get_num_data(DataBegin, DataEnd);
269 const uint64_t CountersSectionSize =
270 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
271 const uint64_t NumCounters =
272 __llvm_profile_get_num_counters(CountersBegin, CountersEnd);
273 const uint64_t NumBitmapBytes =
274 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
275 const uint64_t NamesSize = __llvm_profile_get_name_size(NamesBegin, NamesEnd);
276
277 /* Create the header. */
278 __llvm_profile_header Header;
279
280 /* Determine how much padding is needed before/after the counters and after
281 * the names. */
282 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
283 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes;
284 __llvm_profile_get_padding_sizes_for_counters(
285 DataSectionSize, CountersSectionSize, NumBitmapBytes, NamesSize,
286 &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
287 &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames);
288
289 {
290 /* Initialize header structure. */
291 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
292 #include "profile/InstrProfData.inc"
293 }
294
295 /* On WIN64, label differences are truncated 32-bit values. Truncate
296 * CountersDelta to match. */
297 #ifdef _WIN64
298 Header.CountersDelta = (uint32_t)Header.CountersDelta;
299 Header.BitmapDelta = (uint32_t)Header.BitmapDelta;
300 #endif
301
302 /* The data and names sections are omitted in lightweight mode. */
303 if (NumData == 0 && NamesSize == 0) {
304 Header.CountersDelta = 0;
305 Header.NamesDelta = 0;
306 }
307
308 /* Write the profile header. */
309 ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
310 if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
311 return -1;
312
313 /* Write the binary id lengths and data. */
314 if (__llvm_write_binary_ids(Writer) == -1)
315 return -1;
316
317 /* Write the profile data. */
318 ProfDataIOVec IOVecData[] = {
319 {DataBegin, sizeof(uint8_t), DataSectionSize, 0},
320 {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
321 {CountersBegin, sizeof(uint8_t), CountersSectionSize, 0},
322 {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
323 {BitmapBegin, sizeof(uint8_t), NumBitmapBytes, 0},
324 {NULL, sizeof(uint8_t), PaddingBytesAfterBitmapBytes, 1},
325 {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
326 {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
327 if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
328 return -1;
329
330 /* Value profiling is not yet supported in continuous mode and profile
331 * correlation mode. */
332 if (__llvm_profile_is_continuous_mode_enabled() ||
333 (NumData == 0 && NamesSize == 0))
334 return 0;
335
336 return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
337 }
338
339 /*
340 * Write binary id length and then its data, because binary id does not
341 * have a fixed length.
342 */
343 COMPILER_RT_VISIBILITY
lprofWriteOneBinaryId(ProfDataWriter * Writer,uint64_t BinaryIdLen,const uint8_t * BinaryIdData,uint64_t BinaryIdPadding)344 int lprofWriteOneBinaryId(ProfDataWriter *Writer, uint64_t BinaryIdLen,
345 const uint8_t *BinaryIdData,
346 uint64_t BinaryIdPadding) {
347 ProfDataIOVec BinaryIdIOVec[] = {
348 {&BinaryIdLen, sizeof(uint64_t), 1, 0},
349 {BinaryIdData, sizeof(uint8_t), BinaryIdLen, 0},
350 {NULL, sizeof(uint8_t), BinaryIdPadding, 1},
351 };
352 if (Writer->Write(Writer, BinaryIdIOVec,
353 sizeof(BinaryIdIOVec) / sizeof(*BinaryIdIOVec)))
354 return -1;
355
356 /* Successfully wrote binary id, report success. */
357 return 0;
358 }
359