1 /*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 |*===----------------------------------------------------------------------===*
8 |* This file defines the API needed for in-process merging of profile data
9 |* stored in memory buffer.
10 \*===---------------------------------------------------------------------===*/
11
12 #include "InstrProfiling.h"
13 #include "InstrProfilingInternal.h"
14 #include "InstrProfilingUtil.h"
15
16 #define INSTR_PROF_VALUE_PROF_DATA
17 #include "profile/InstrProfData.inc"
18
19 COMPILER_RT_VISIBILITY
20 void (*VPMergeHook)(ValueProfData *, __llvm_profile_data *);
21
22 COMPILER_RT_VISIBILITY
lprofGetLoadModuleSignature(void)23 uint64_t lprofGetLoadModuleSignature(void) {
24 /* A very fast way to compute a module signature. */
25 uint64_t Version = __llvm_profile_get_version();
26 uint64_t NumCounters = __llvm_profile_get_num_counters(
27 __llvm_profile_begin_counters(), __llvm_profile_end_counters());
28 uint64_t NumData = __llvm_profile_get_num_data(__llvm_profile_begin_data(),
29 __llvm_profile_end_data());
30 uint64_t NamesSize =
31 (uint64_t)(__llvm_profile_end_names() - __llvm_profile_begin_names());
32 uint64_t NumVnodes =
33 (uint64_t)(__llvm_profile_end_vnodes() - __llvm_profile_begin_vnodes());
34 const __llvm_profile_data *FirstD = __llvm_profile_begin_data();
35
36 return (NamesSize << 40) + (NumCounters << 30) + (NumData << 20) +
37 (NumVnodes << 10) + (NumData > 0 ? FirstD->NameRef : 0) + Version +
38 __llvm_profile_get_magic();
39 }
40
41 #ifdef __GNUC__
42 #pragma GCC diagnostic push
43 #pragma GCC diagnostic ignored "-Wcast-qual"
44 #elif defined(__clang__)
45 #pragma clang diagnostic push
46 #pragma clang diagnostic ignored "-Wcast-qual"
47 #endif
48
49 /* Returns 1 if profile is not structurally compatible. */
50 COMPILER_RT_VISIBILITY
__llvm_profile_check_compatibility(const char * ProfileData,uint64_t ProfileSize)51 int __llvm_profile_check_compatibility(const char *ProfileData,
52 uint64_t ProfileSize) {
53 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
54 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
55 SrcDataStart =
56 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
57 Header->BinaryIdsSize);
58 SrcDataEnd = SrcDataStart + Header->NumData;
59
60 if (ProfileSize < sizeof(__llvm_profile_header))
61 return 1;
62
63 /* Check the header first. */
64 if (Header->Magic != __llvm_profile_get_magic() ||
65 Header->Version != __llvm_profile_get_version() ||
66 Header->NumData !=
67 __llvm_profile_get_num_data(__llvm_profile_begin_data(),
68 __llvm_profile_end_data()) ||
69 Header->NumCounters !=
70 __llvm_profile_get_num_counters(__llvm_profile_begin_counters(),
71 __llvm_profile_end_counters()) ||
72 Header->NumBitmapBytes !=
73 __llvm_profile_get_num_bitmap_bytes(__llvm_profile_begin_bitmap(),
74 __llvm_profile_end_bitmap()) ||
75 Header->NamesSize !=
76 __llvm_profile_get_name_size(__llvm_profile_begin_names(),
77 __llvm_profile_end_names()) ||
78 Header->ValueKindLast != IPVK_Last)
79 return 1;
80
81 if (ProfileSize <
82 sizeof(__llvm_profile_header) + Header->BinaryIdsSize +
83 Header->NumData * sizeof(__llvm_profile_data) + Header->NamesSize +
84 Header->NumCounters * __llvm_profile_counter_entry_size() +
85 Header->NumBitmapBytes)
86 return 1;
87
88 for (SrcData = SrcDataStart,
89 DstData = (__llvm_profile_data *)__llvm_profile_begin_data();
90 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
91 if (SrcData->NameRef != DstData->NameRef ||
92 SrcData->FuncHash != DstData->FuncHash ||
93 SrcData->NumCounters != DstData->NumCounters ||
94 SrcData->NumBitmapBytes != DstData->NumBitmapBytes)
95 return 1;
96 }
97
98 /* Matched! */
99 return 0;
100 }
101
signextIfWin64(void * V)102 static uintptr_t signextIfWin64(void *V) {
103 #ifdef _WIN64
104 return (uintptr_t)(int32_t)(uintptr_t)V;
105 #else
106 return (uintptr_t)V;
107 #endif
108 }
109
110 COMPILER_RT_VISIBILITY
__llvm_profile_merge_from_buffer(const char * ProfileData,uint64_t ProfileSize)111 int __llvm_profile_merge_from_buffer(const char *ProfileData,
112 uint64_t ProfileSize) {
113 if (__llvm_profile_get_version() & VARIANT_MASK_TEMPORAL_PROF) {
114 PROF_ERR("%s\n",
115 "Temporal profiles do not support profile merging at runtime. "
116 "Instead, merge raw profiles using the llvm-profdata tool.");
117 return 1;
118 }
119
120 __llvm_profile_data *SrcDataStart, *SrcDataEnd, *SrcData, *DstData;
121 __llvm_profile_header *Header = (__llvm_profile_header *)ProfileData;
122 char *SrcCountersStart, *DstCounter;
123 const char *SrcCountersEnd, *SrcCounter;
124 const char *SrcBitmapStart;
125 const char *SrcNameStart;
126 const char *SrcValueProfDataStart, *SrcValueProfData;
127 uintptr_t CountersDelta = Header->CountersDelta;
128 uintptr_t BitmapDelta = Header->BitmapDelta;
129
130 SrcDataStart =
131 (__llvm_profile_data *)(ProfileData + sizeof(__llvm_profile_header) +
132 Header->BinaryIdsSize);
133 SrcDataEnd = SrcDataStart + Header->NumData;
134 SrcCountersStart = (char *)SrcDataEnd;
135 SrcCountersEnd = SrcCountersStart +
136 Header->NumCounters * __llvm_profile_counter_entry_size();
137 SrcBitmapStart = SrcCountersEnd;
138 SrcNameStart = SrcBitmapStart + Header->NumBitmapBytes;
139 SrcValueProfDataStart =
140 SrcNameStart + Header->NamesSize +
141 __llvm_profile_get_num_padding_bytes(Header->NamesSize);
142 if (SrcNameStart < SrcCountersStart || SrcNameStart < SrcBitmapStart)
143 return 1;
144
145 // Merge counters by iterating the entire counter section when data section is
146 // empty due to correlation.
147 if (Header->NumData == 0) {
148 for (SrcCounter = SrcCountersStart,
149 DstCounter = __llvm_profile_begin_counters();
150 SrcCounter < SrcCountersEnd;) {
151 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
152 *DstCounter &= *SrcCounter;
153 } else {
154 *(uint64_t *)DstCounter += *(uint64_t *)SrcCounter;
155 }
156 SrcCounter += __llvm_profile_counter_entry_size();
157 DstCounter += __llvm_profile_counter_entry_size();
158 }
159 return 0;
160 }
161
162 for (SrcData = SrcDataStart,
163 DstData = (__llvm_profile_data *)__llvm_profile_begin_data(),
164 SrcValueProfData = SrcValueProfDataStart;
165 SrcData < SrcDataEnd; ++SrcData, ++DstData) {
166 // For the in-memory destination, CounterPtr is the distance from the start
167 // address of the data to the start address of the counter. On WIN64,
168 // CounterPtr is a truncated 32-bit value due to COFF limitation. Sign
169 // extend CounterPtr to get the original value.
170 char *DstCounters =
171 (char *)((uintptr_t)DstData + signextIfWin64(DstData->CounterPtr));
172 char *DstBitmap =
173 (char *)((uintptr_t)DstData + signextIfWin64(DstData->BitmapPtr));
174 unsigned NVK = 0;
175
176 // SrcData is a serialized representation of the memory image. We need to
177 // compute the in-buffer counter offset from the in-memory address distance.
178 // The initial CountersDelta is the in-memory address difference
179 // start(__llvm_prf_cnts)-start(__llvm_prf_data), so SrcData->CounterPtr -
180 // CountersDelta computes the offset into the in-buffer counter section.
181 //
182 // On WIN64, CountersDelta is truncated as well, so no need for signext.
183 char *SrcCounters =
184 SrcCountersStart + ((uintptr_t)SrcData->CounterPtr - CountersDelta);
185 // CountersDelta needs to be decreased as we advance to the next data
186 // record.
187 CountersDelta -= sizeof(*SrcData);
188 unsigned NC = SrcData->NumCounters;
189 if (NC == 0)
190 return 1;
191 if (SrcCounters < SrcCountersStart || SrcCounters >= SrcNameStart ||
192 (SrcCounters + __llvm_profile_counter_entry_size() * NC) > SrcNameStart)
193 return 1;
194 for (unsigned I = 0; I < NC; I++) {
195 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE) {
196 // A value of zero signifies the function is covered.
197 DstCounters[I] &= SrcCounters[I];
198 } else {
199 ((uint64_t *)DstCounters)[I] += ((uint64_t *)SrcCounters)[I];
200 }
201 }
202
203 const char *SrcBitmap =
204 SrcBitmapStart + ((uintptr_t)SrcData->BitmapPtr - BitmapDelta);
205 // BitmapDelta also needs to be decreased as we advance to the next data
206 // record.
207 BitmapDelta -= sizeof(*SrcData);
208 unsigned NB = SrcData->NumBitmapBytes;
209 // NumBitmapBytes may legitimately be 0. Just keep going.
210 if (NB != 0) {
211 if (SrcBitmap < SrcBitmapStart || (SrcBitmap + NB) > SrcNameStart)
212 return 1;
213 // Merge Src and Dst Bitmap bytes by simply ORing them together.
214 for (unsigned I = 0; I < NB; I++)
215 DstBitmap[I] |= SrcBitmap[I];
216 }
217
218 /* Now merge value profile data. */
219 if (!VPMergeHook)
220 continue;
221
222 for (unsigned I = 0; I <= IPVK_Last; I++)
223 NVK += (SrcData->NumValueSites[I] != 0);
224
225 if (!NVK)
226 continue;
227
228 if (SrcValueProfData >= ProfileData + ProfileSize)
229 return 1;
230 VPMergeHook((ValueProfData *)SrcValueProfData, DstData);
231 SrcValueProfData =
232 SrcValueProfData + ((ValueProfData *)SrcValueProfData)->TotalSize;
233 }
234
235 return 0;
236 }
237
238 #ifdef __GNUC__
239 #pragma GCC diagnostic pop
240 #elif defined(__clang__)
241 #pragma clang diagnostic pop
242 #endif
243