1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8 
9 // Note: This is linked into the Darwin kernel, and must remain compatible
10 // with freestanding compilation. See `darwin_add_builtin_libraries`.
11 
12 #ifdef _MSC_VER
13 /* For _alloca */
14 #include <malloc.h>
15 #endif
16 #include <string.h>
17 
18 #include "InstrProfiling.h"
19 #include "InstrProfilingInternal.h"
20 #include "InstrProfilingPort.h"
21 
22 #define INSTR_PROF_VALUE_PROF_DATA
23 #include "profile/InstrProfData.inc"
24 
25 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
26 static ProfBufferIO TheBufferIO;
27 #define VP_BUFFER_SIZE 8 * 1024
28 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
29 static InstrProfValueData VPDataArray[16];
30 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
31 
32 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
33 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
34 
35 /* The buffer writer is responsible in keeping writer state
36  * across the call.
37  */
38 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
39                                                   ProfDataIOVec *IOVecs,
40                                                   uint32_t NumIOVecs) {
41   uint32_t I;
42   char **Buffer = (char **)&This->WriterCtx;
43   for (I = 0; I < NumIOVecs; I++) {
44     size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
45     if (IOVecs[I].Data)
46       memcpy(*Buffer, IOVecs[I].Data, Length);
47     else if (IOVecs[I].UseZeroPadding) {
48       /* Allocating the buffer should zero fill. */
49     }
50     *Buffer += Length;
51   }
52   return 0;
53 }
54 
55 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
56                              uint8_t *Buffer, uint32_t BufferSz) {
57   BufferIO->FileWriter = FileWriter;
58   BufferIO->OwnFileWriter = 0;
59   BufferIO->BufferStart = Buffer;
60   BufferIO->BufferSz = BufferSz;
61   BufferIO->CurOffset = 0;
62 }
63 
64 COMPILER_RT_VISIBILITY ProfBufferIO *
65 lprofCreateBufferIO(ProfDataWriter *FileWriter) {
66   uint8_t *Buffer = DynamicBufferIOBuffer;
67   uint32_t BufferSize = VPBufferSize;
68   if (!Buffer) {
69     Buffer = &BufferIOBuffer[0];
70     BufferSize = sizeof(BufferIOBuffer);
71   }
72   llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize);
73   return &TheBufferIO;
74 }
75 
76 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
77   if (BufferIO->OwnFileWriter)
78     FreeHook(BufferIO->FileWriter);
79   if (DynamicBufferIOBuffer) {
80     FreeHook(DynamicBufferIOBuffer);
81     DynamicBufferIOBuffer = 0;
82     VPBufferSize = 0;
83   }
84 }
85 
86 COMPILER_RT_VISIBILITY int
87 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
88   /* Buffer is not large enough, it is time to flush.  */
89   if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
90     if (lprofBufferIOFlush(BufferIO) != 0)
91       return -1;
92   }
93   /* Special case, bypass the buffer completely. */
94   ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
95   if (Size > BufferIO->BufferSz) {
96     if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
97       return -1;
98   } else {
99     /* Write the data to buffer */
100     uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
101     ProfDataWriter BufferWriter;
102     initBufferWriter(&BufferWriter, (char *)Buffer);
103     lprofBufferWriter(&BufferWriter, IO, 1);
104     BufferIO->CurOffset =
105         (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
106   }
107   return 0;
108 }
109 
110 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
111   if (BufferIO->CurOffset) {
112     ProfDataIOVec IO[] = {
113         {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
114     if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
115       return -1;
116     BufferIO->CurOffset = 0;
117   }
118   return 0;
119 }
120 
121 /* Write out value profile data for function specified with \c Data.
122  * The implementation does not use the method \c serializeValueProfData
123  * which depends on dynamic memory allocation. In this implementation,
124  * value profile data is written out to \c BufferIO piecemeal.
125  */
126 static int writeOneValueProfData(ProfBufferIO *BufferIO,
127                                  VPDataReaderType *VPDataReader,
128                                  const __llvm_profile_data *Data) {
129   unsigned I, NumValueKinds = 0;
130   ValueProfData VPHeader;
131   uint8_t *SiteCountArray[IPVK_Last + 1];
132 
133   for (I = 0; I <= IPVK_Last; I++) {
134     if (!Data->NumValueSites[I])
135       SiteCountArray[I] = 0;
136     else {
137       uint32_t Sz =
138           VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
139           offsetof(ValueProfRecord, SiteCountArray);
140       /* Only use alloca for this small byte array to avoid excessive
141        * stack growth.  */
142       SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
143       memset(SiteCountArray[I], 0, Sz);
144     }
145   }
146 
147   /* If NumValueKinds returned is 0, there is nothing to write, report
148      success and return. This should match the raw profile reader's behavior. */
149   if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
150     return 0;
151 
152   /* First write the header structure. */
153   VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
154   VPHeader.NumValueKinds = NumValueKinds;
155   if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,
156                          sizeof(ValueProfData)))
157     return -1;
158 
159   /* Make sure nothing else needs to be written before value profile
160    * records. */
161   if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
162       (void *)(&VPHeader + 1))
163     return -1;
164 
165   /* Write out the value profile record for each value kind
166    * one by one. */
167   for (I = 0; I <= IPVK_Last; I++) {
168     uint32_t J;
169     ValueProfRecord RecordHeader;
170     /* The size of the value prof record header without counting the
171      * site count array .*/
172     uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
173     uint32_t SiteCountArraySize;
174 
175     if (!Data->NumValueSites[I])
176       continue;
177 
178     /* Write out the record header.  */
179     RecordHeader.Kind = I;
180     RecordHeader.NumValueSites = Data->NumValueSites[I];
181     if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,
182                            RecordHeaderSize))
183       return -1;
184 
185     /* Write out the site value count array including padding space. */
186     SiteCountArraySize =
187         VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
188         RecordHeaderSize;
189     if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))
190       return -1;
191 
192     /* Write out the value profile data for each value site.  */
193     for (J = 0; J < Data->NumValueSites[I]; J++) {
194       uint32_t NRead, NRemain;
195       ValueProfNode *NextStartNode = 0;
196       NRemain = VPDataReader->GetNumValueDataForSite(I, J);
197       if (!NRemain)
198         continue;
199       /* Read and write out value data in small chunks till it is done. */
200       do {
201         NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
202         NextStartNode =
203             VPDataReader->GetValueData(I, /* ValueKind */
204                                        J, /* Site */
205                                        &VPDataArray[0], NextStartNode, NRead);
206         if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],
207                                NRead * sizeof(InstrProfValueData)))
208           return -1;
209         NRemain -= NRead;
210       } while (NRemain != 0);
211     }
212   }
213   /* All done report success.  */
214   return 0;
215 }
216 
217 static int writeValueProfData(ProfDataWriter *Writer,
218                               VPDataReaderType *VPDataReader,
219                               const __llvm_profile_data *DataBegin,
220                               const __llvm_profile_data *DataEnd) {
221   ProfBufferIO *BufferIO;
222   const __llvm_profile_data *DI = 0;
223 
224   if (!VPDataReader)
225     return 0;
226 
227   BufferIO = lprofCreateBufferIO(Writer);
228 
229   for (DI = DataBegin; DI < DataEnd; DI++) {
230     if (writeOneValueProfData(BufferIO, VPDataReader, DI))
231       return -1;
232   }
233 
234   if (lprofBufferIOFlush(BufferIO) != 0)
235     return -1;
236   lprofDeleteBufferIO(BufferIO);
237 
238   return 0;
239 }
240 
241 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
242                                           VPDataReaderType *VPDataReader,
243                                           int SkipNameDataWrite) {
244   /* Match logic in __llvm_profile_write_buffer(). */
245   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
246   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
247   const char *CountersBegin = __llvm_profile_begin_counters();
248   const char *CountersEnd = __llvm_profile_end_counters();
249   const char *NamesBegin = __llvm_profile_begin_names();
250   const char *NamesEnd = __llvm_profile_end_names();
251   return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
252                             CountersEnd, VPDataReader, NamesBegin, NamesEnd,
253                             SkipNameDataWrite);
254 }
255 
256 COMPILER_RT_VISIBILITY int
257 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
258                    const __llvm_profile_data *DataEnd,
259                    const char *CountersBegin, const char *CountersEnd,
260                    VPDataReaderType *VPDataReader, const char *NamesBegin,
261                    const char *NamesEnd, int SkipNameDataWrite) {
262   int DebugInfoCorrelate =
263       (__llvm_profile_get_version() & VARIANT_MASK_DBG_CORRELATE) != 0ULL;
264 
265   /* Calculate size of sections. */
266   const uint64_t DataSize =
267       DebugInfoCorrelate ? 0 : __llvm_profile_get_data_size(DataBegin, DataEnd);
268   const uint64_t NumData =
269       DebugInfoCorrelate ? 0 : __llvm_profile_get_num_data(DataBegin, DataEnd);
270   const uint64_t CountersSize =
271       __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
272   const uint64_t NumCounters =
273       __llvm_profile_get_num_counters(CountersBegin, CountersEnd);
274   const uint64_t NamesSize = DebugInfoCorrelate ? 0 : NamesEnd - NamesBegin;
275 
276   /* Create the header. */
277   __llvm_profile_header Header;
278 
279   if (!NumData && (!DebugInfoCorrelate || !NumCounters))
280     return 0;
281 
282   /* Determine how much padding is needed before/after the counters and after
283    * the names. */
284   uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
285       PaddingBytesAfterNames;
286   __llvm_profile_get_padding_sizes_for_counters(
287       DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
288       &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
289 
290   {
291     // TODO: Unfortunately the header's fields are named DataSize and
292     // CountersSize when they should be named NumData and NumCounters,
293     // respectively.
294     const uint64_t CountersSize = NumCounters;
295     const uint64_t DataSize = NumData;
296 /* Initialize header structure.  */
297 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
298 #include "profile/InstrProfData.inc"
299   }
300 
301   /* On WIN64, label differences are truncated 32-bit values. Truncate
302    * CountersDelta to match. */
303 #ifdef _WIN64
304   Header.CountersDelta = (uint32_t)Header.CountersDelta;
305 #endif
306 
307   /* The data and names sections are omitted in lightweight mode. */
308   if (DebugInfoCorrelate) {
309     Header.CountersDelta = 0;
310     Header.NamesDelta = 0;
311   }
312 
313   /* Write the profile header. */
314   ProfDataIOVec IOVec[] = {{&Header, sizeof(__llvm_profile_header), 1, 0}};
315   if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
316     return -1;
317 
318   /* Write the binary id lengths and data. */
319   if (__llvm_write_binary_ids(Writer) == -1)
320     return -1;
321 
322   /* Write the profile data. */
323   ProfDataIOVec IOVecData[] = {
324       {DebugInfoCorrelate ? NULL : DataBegin, sizeof(uint8_t), DataSize, 0},
325       {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
326       {CountersBegin, sizeof(uint8_t), CountersSize, 0},
327       {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
328       {(SkipNameDataWrite || DebugInfoCorrelate) ? NULL : NamesBegin,
329        sizeof(uint8_t), NamesSize, 0},
330       {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
331   if (Writer->Write(Writer, IOVecData, sizeof(IOVecData) / sizeof(*IOVecData)))
332     return -1;
333 
334   /* Value profiling is not yet supported in continuous mode. */
335   if (__llvm_profile_is_continuous_mode_enabled())
336     return 0;
337 
338   return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
339 }
340