1 /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8 
9 #ifdef _MSC_VER
10 /* For _alloca */
11 #include <malloc.h>
12 #endif
13 #include <string.h>
14 
15 #include "InstrProfiling.h"
16 #include "InstrProfilingInternal.h"
17 #include "InstrProfilingPort.h"
18 
19 #define INSTR_PROF_VALUE_PROF_DATA
20 #include "profile/InstrProfData.inc"
21 
22 COMPILER_RT_VISIBILITY void (*FreeHook)(void *) = NULL;
23 static ProfBufferIO TheBufferIO;
24 #define VP_BUFFER_SIZE 8 * 1024
25 static uint8_t BufferIOBuffer[VP_BUFFER_SIZE];
26 static InstrProfValueData VPDataArray[16];
27 static uint32_t VPDataArraySize = sizeof(VPDataArray) / sizeof(*VPDataArray);
28 
29 COMPILER_RT_VISIBILITY uint8_t *DynamicBufferIOBuffer = 0;
30 COMPILER_RT_VISIBILITY uint32_t VPBufferSize = 0;
31 
32 /* The buffer writer is reponsponsible in keeping writer state
33  * across the call.
34  */
35 COMPILER_RT_VISIBILITY uint32_t lprofBufferWriter(ProfDataWriter *This,
36                                                   ProfDataIOVec *IOVecs,
37                                                   uint32_t NumIOVecs) {
38   uint32_t I;
39   char **Buffer = (char **)&This->WriterCtx;
40   for (I = 0; I < NumIOVecs; I++) {
41     size_t Length = IOVecs[I].ElmSize * IOVecs[I].NumElm;
42     if (IOVecs[I].Data)
43       memcpy(*Buffer, IOVecs[I].Data, Length);
44     else if (IOVecs[I].UseZeroPadding) {
45       /* Allocating the buffer should zero fill. */
46     }
47     *Buffer += Length;
48   }
49   return 0;
50 }
51 
52 static void llvmInitBufferIO(ProfBufferIO *BufferIO, ProfDataWriter *FileWriter,
53                              uint8_t *Buffer, uint32_t BufferSz) {
54   BufferIO->FileWriter = FileWriter;
55   BufferIO->OwnFileWriter = 0;
56   BufferIO->BufferStart = Buffer;
57   BufferIO->BufferSz = BufferSz;
58   BufferIO->CurOffset = 0;
59 }
60 
61 COMPILER_RT_VISIBILITY ProfBufferIO *
62 lprofCreateBufferIO(ProfDataWriter *FileWriter) {
63   uint8_t *Buffer = DynamicBufferIOBuffer;
64   uint32_t BufferSize = VPBufferSize;
65   if (!Buffer) {
66     Buffer = &BufferIOBuffer[0];
67     BufferSize = sizeof(BufferIOBuffer);
68   }
69   llvmInitBufferIO(&TheBufferIO, FileWriter, Buffer, BufferSize);
70   return &TheBufferIO;
71 }
72 
73 COMPILER_RT_VISIBILITY void lprofDeleteBufferIO(ProfBufferIO *BufferIO) {
74   if (BufferIO->OwnFileWriter)
75     FreeHook(BufferIO->FileWriter);
76   if (DynamicBufferIOBuffer) {
77     FreeHook(DynamicBufferIOBuffer);
78     DynamicBufferIOBuffer = 0;
79     VPBufferSize = 0;
80   }
81 }
82 
83 COMPILER_RT_VISIBILITY int
84 lprofBufferIOWrite(ProfBufferIO *BufferIO, const uint8_t *Data, uint32_t Size) {
85   /* Buffer is not large enough, it is time to flush.  */
86   if (Size + BufferIO->CurOffset > BufferIO->BufferSz) {
87     if (lprofBufferIOFlush(BufferIO) != 0)
88       return -1;
89   }
90   /* Special case, bypass the buffer completely. */
91   ProfDataIOVec IO[] = {{Data, sizeof(uint8_t), Size, 0}};
92   if (Size > BufferIO->BufferSz) {
93     if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
94       return -1;
95   } else {
96     /* Write the data to buffer */
97     uint8_t *Buffer = BufferIO->BufferStart + BufferIO->CurOffset;
98     ProfDataWriter BufferWriter;
99     initBufferWriter(&BufferWriter, (char *)Buffer);
100     lprofBufferWriter(&BufferWriter, IO, 1);
101     BufferIO->CurOffset =
102         (uint8_t *)BufferWriter.WriterCtx - BufferIO->BufferStart;
103   }
104   return 0;
105 }
106 
107 COMPILER_RT_VISIBILITY int lprofBufferIOFlush(ProfBufferIO *BufferIO) {
108   if (BufferIO->CurOffset) {
109     ProfDataIOVec IO[] = {
110         {BufferIO->BufferStart, sizeof(uint8_t), BufferIO->CurOffset, 0}};
111     if (BufferIO->FileWriter->Write(BufferIO->FileWriter, IO, 1))
112       return -1;
113     BufferIO->CurOffset = 0;
114   }
115   return 0;
116 }
117 
118 /* Write out value profile data for function specified with \c Data.
119  * The implementation does not use the method \c serializeValueProfData
120  * which depends on dynamic memory allocation. In this implementation,
121  * value profile data is written out to \c BufferIO piecemeal.
122  */
123 static int writeOneValueProfData(ProfBufferIO *BufferIO,
124                                  VPDataReaderType *VPDataReader,
125                                  const __llvm_profile_data *Data) {
126   unsigned I, NumValueKinds = 0;
127   ValueProfData VPHeader;
128   uint8_t *SiteCountArray[IPVK_Last + 1];
129 
130   for (I = 0; I <= IPVK_Last; I++) {
131     if (!Data->NumValueSites[I])
132       SiteCountArray[I] = 0;
133     else {
134       uint32_t Sz =
135           VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
136           offsetof(ValueProfRecord, SiteCountArray);
137       /* Only use alloca for this small byte array to avoid excessive
138        * stack growth.  */
139       SiteCountArray[I] = (uint8_t *)COMPILER_RT_ALLOCA(Sz);
140       memset(SiteCountArray[I], 0, Sz);
141     }
142   }
143 
144   /* If NumValueKinds returned is 0, there is nothing to write, report
145      success and return. This should match the raw profile reader's behavior. */
146   if (!(NumValueKinds = VPDataReader->InitRTRecord(Data, SiteCountArray)))
147     return 0;
148 
149   /* First write the header structure. */
150   VPHeader.TotalSize = VPDataReader->GetValueProfDataSize();
151   VPHeader.NumValueKinds = NumValueKinds;
152   if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPHeader,
153                          sizeof(ValueProfData)))
154     return -1;
155 
156   /* Make sure nothing else needs to be written before value profile
157    * records. */
158   if ((void *)VPDataReader->GetFirstValueProfRecord(&VPHeader) !=
159       (void *)(&VPHeader + 1))
160     return -1;
161 
162   /* Write out the value profile record for each value kind
163    * one by one. */
164   for (I = 0; I <= IPVK_Last; I++) {
165     uint32_t J;
166     ValueProfRecord RecordHeader;
167     /* The size of the value prof record header without counting the
168      * site count array .*/
169     uint32_t RecordHeaderSize = offsetof(ValueProfRecord, SiteCountArray);
170     uint32_t SiteCountArraySize;
171 
172     if (!Data->NumValueSites[I])
173       continue;
174 
175     /* Write out the record header.  */
176     RecordHeader.Kind = I;
177     RecordHeader.NumValueSites = Data->NumValueSites[I];
178     if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&RecordHeader,
179                            RecordHeaderSize))
180       return -1;
181 
182     /* Write out the site value count array including padding space. */
183     SiteCountArraySize =
184         VPDataReader->GetValueProfRecordHeaderSize(Data->NumValueSites[I]) -
185         RecordHeaderSize;
186     if (lprofBufferIOWrite(BufferIO, SiteCountArray[I], SiteCountArraySize))
187       return -1;
188 
189     /* Write out the value profile data for each value site.  */
190     for (J = 0; J < Data->NumValueSites[I]; J++) {
191       uint32_t NRead, NRemain;
192       ValueProfNode *NextStartNode = 0;
193       NRemain = VPDataReader->GetNumValueDataForSite(I, J);
194       if (!NRemain)
195         continue;
196       /* Read and write out value data in small chunks till it is done. */
197       do {
198         NRead = (NRemain > VPDataArraySize ? VPDataArraySize : NRemain);
199         NextStartNode =
200             VPDataReader->GetValueData(I, /* ValueKind */
201                                        J, /* Site */
202                                        &VPDataArray[0], NextStartNode, NRead);
203         if (lprofBufferIOWrite(BufferIO, (const uint8_t *)&VPDataArray[0],
204                                NRead * sizeof(InstrProfValueData)))
205           return -1;
206         NRemain -= NRead;
207       } while (NRemain != 0);
208     }
209   }
210   /* All done report success.  */
211   return 0;
212 }
213 
214 static int writeValueProfData(ProfDataWriter *Writer,
215                               VPDataReaderType *VPDataReader,
216                               const __llvm_profile_data *DataBegin,
217                               const __llvm_profile_data *DataEnd) {
218   ProfBufferIO *BufferIO;
219   const __llvm_profile_data *DI = 0;
220 
221   if (!VPDataReader)
222     return 0;
223 
224   BufferIO = lprofCreateBufferIO(Writer);
225 
226   for (DI = DataBegin; DI < DataEnd; DI++) {
227     if (writeOneValueProfData(BufferIO, VPDataReader, DI))
228       return -1;
229   }
230 
231   if (lprofBufferIOFlush(BufferIO) != 0)
232     return -1;
233   lprofDeleteBufferIO(BufferIO);
234 
235   return 0;
236 }
237 
238 COMPILER_RT_VISIBILITY int lprofWriteData(ProfDataWriter *Writer,
239                                           VPDataReaderType *VPDataReader,
240                                           int SkipNameDataWrite) {
241   /* Match logic in __llvm_profile_write_buffer(). */
242   const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
243   const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
244   const uint64_t *CountersBegin = __llvm_profile_begin_counters();
245   const uint64_t *CountersEnd = __llvm_profile_end_counters();
246   const char *NamesBegin = __llvm_profile_begin_names();
247   const char *NamesEnd = __llvm_profile_end_names();
248   return lprofWriteDataImpl(Writer, DataBegin, DataEnd, CountersBegin,
249                             CountersEnd, VPDataReader, NamesBegin, NamesEnd,
250                             SkipNameDataWrite);
251 }
252 
253 COMPILER_RT_VISIBILITY int
254 lprofWriteDataImpl(ProfDataWriter *Writer, const __llvm_profile_data *DataBegin,
255                    const __llvm_profile_data *DataEnd,
256                    const uint64_t *CountersBegin, const uint64_t *CountersEnd,
257                    VPDataReaderType *VPDataReader, const char *NamesBegin,
258                    const char *NamesEnd, int SkipNameDataWrite) {
259 
260   /* Calculate size of sections. */
261   const uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
262   const uint64_t CountersSize = CountersEnd - CountersBegin;
263   const uint64_t NamesSize = NamesEnd - NamesBegin;
264 
265   /* Create the header. */
266   __llvm_profile_header Header;
267 
268   if (!DataSize)
269     return 0;
270 
271   /* Determine how much padding is needed before/after the counters and after
272    * the names. */
273   uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
274       PaddingBytesAfterNames;
275   __llvm_profile_get_padding_sizes_for_counters(
276       DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
277       &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
278 
279 /* Initialize header structure.  */
280 #define INSTR_PROF_RAW_HEADER(Type, Name, Init) Header.Name = Init;
281 #include "profile/InstrProfData.inc"
282 
283   /* Write the data. */
284   ProfDataIOVec IOVec[] = {
285       {&Header, sizeof(__llvm_profile_header), 1, 0},
286       {DataBegin, sizeof(__llvm_profile_data), DataSize, 0},
287       {NULL, sizeof(uint8_t), PaddingBytesBeforeCounters, 1},
288       {CountersBegin, sizeof(uint64_t), CountersSize, 0},
289       {NULL, sizeof(uint8_t), PaddingBytesAfterCounters, 1},
290       {SkipNameDataWrite ? NULL : NamesBegin, sizeof(uint8_t), NamesSize, 0},
291       {NULL, sizeof(uint8_t), PaddingBytesAfterNames, 1}};
292   if (Writer->Write(Writer, IOVec, sizeof(IOVec) / sizeof(*IOVec)))
293     return -1;
294 
295   /* Value profiling is not yet supported in continuous mode. */
296   if (__llvm_profile_is_continuous_mode_enabled())
297     return 0;
298 
299   return writeValueProfData(Writer, VPDataReader, DataBegin, DataEnd);
300 }
301