1 /*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8
9 // Note: This is linked into the Darwin kernel, and must remain compatible
10 // with freestanding compilation. See `darwin_add_builtin_libraries`.
11
12 #include "InstrProfiling.h"
13 #include "InstrProfilingInternal.h"
14 #include "InstrProfilingPort.h"
15
16 /* When continuous mode is enabled (%c), this parameter is set to 1.
17 *
18 * This parameter is defined here in InstrProfilingBuffer.o, instead of in
19 * InstrProfilingFile.o, to sequester all libc-dependent code in
20 * InstrProfilingFile.o. The test `instrprof-without-libc` will break if this
21 * layering is violated. */
22 static int ContinuouslySyncProfile = 0;
23
24 /* The system page size. Only valid when non-zero. If 0, the page size is
25 * unavailable. */
26 static unsigned PageSize = 0;
27
__llvm_profile_is_continuous_mode_enabled(void)28 COMPILER_RT_VISIBILITY int __llvm_profile_is_continuous_mode_enabled(void) {
29 return ContinuouslySyncProfile && PageSize;
30 }
31
__llvm_profile_enable_continuous_mode(void)32 COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) {
33 ContinuouslySyncProfile = 1;
34 }
35
__llvm_profile_disable_continuous_mode(void)36 COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) {
37 ContinuouslySyncProfile = 0;
38 }
39
__llvm_profile_set_page_size(unsigned PS)40 COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) {
41 PageSize = PS;
42 }
43
44 COMPILER_RT_VISIBILITY
__llvm_profile_get_size_for_buffer(void)45 uint64_t __llvm_profile_get_size_for_buffer(void) {
46 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
47 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
48 const char *CountersBegin = __llvm_profile_begin_counters();
49 const char *CountersEnd = __llvm_profile_end_counters();
50 const char *BitmapBegin = __llvm_profile_begin_bitmap();
51 const char *BitmapEnd = __llvm_profile_end_bitmap();
52 const char *NamesBegin = __llvm_profile_begin_names();
53 const char *NamesEnd = __llvm_profile_end_names();
54
55 return __llvm_profile_get_size_for_buffer_internal(
56 DataBegin, DataEnd, CountersBegin, CountersEnd, BitmapBegin, BitmapEnd,
57 NamesBegin, NamesEnd);
58 }
59
60 COMPILER_RT_VISIBILITY
__llvm_profile_get_num_data(const __llvm_profile_data * Begin,const __llvm_profile_data * End)61 uint64_t __llvm_profile_get_num_data(const __llvm_profile_data *Begin,
62 const __llvm_profile_data *End) {
63 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
64 return ((EndI + sizeof(__llvm_profile_data) - 1) - BeginI) /
65 sizeof(__llvm_profile_data);
66 }
67
68 COMPILER_RT_VISIBILITY
__llvm_profile_get_data_size(const __llvm_profile_data * Begin,const __llvm_profile_data * End)69 uint64_t __llvm_profile_get_data_size(const __llvm_profile_data *Begin,
70 const __llvm_profile_data *End) {
71 return __llvm_profile_get_num_data(Begin, End) * sizeof(__llvm_profile_data);
72 }
73
__llvm_profile_counter_entry_size(void)74 COMPILER_RT_VISIBILITY size_t __llvm_profile_counter_entry_size(void) {
75 if (__llvm_profile_get_version() & VARIANT_MASK_BYTE_COVERAGE)
76 return sizeof(uint8_t);
77 return sizeof(uint64_t);
78 }
79
80 COMPILER_RT_VISIBILITY
__llvm_profile_get_num_counters(const char * Begin,const char * End)81 uint64_t __llvm_profile_get_num_counters(const char *Begin, const char *End) {
82 intptr_t BeginI = (intptr_t)Begin, EndI = (intptr_t)End;
83 return ((EndI + __llvm_profile_counter_entry_size() - 1) - BeginI) /
84 __llvm_profile_counter_entry_size();
85 }
86
87 COMPILER_RT_VISIBILITY
__llvm_profile_get_counters_size(const char * Begin,const char * End)88 uint64_t __llvm_profile_get_counters_size(const char *Begin, const char *End) {
89 return __llvm_profile_get_num_counters(Begin, End) *
90 __llvm_profile_counter_entry_size();
91 }
92
93 COMPILER_RT_VISIBILITY
__llvm_profile_get_num_bitmap_bytes(const char * Begin,const char * End)94 uint64_t __llvm_profile_get_num_bitmap_bytes(const char *Begin,
95 const char *End) {
96 return (End - Begin);
97 }
98
99 COMPILER_RT_VISIBILITY
__llvm_profile_get_name_size(const char * Begin,const char * End)100 uint64_t __llvm_profile_get_name_size(const char *Begin, const char *End) {
101 return End - Begin;
102 }
103
104 /// Calculate the number of padding bytes needed to add to \p Offset in order
105 /// for (\p Offset + Padding) to be page-aligned.
calculateBytesNeededToPageAlign(uint64_t Offset)106 static uint64_t calculateBytesNeededToPageAlign(uint64_t Offset) {
107 uint64_t OffsetModPage = Offset % PageSize;
108 if (OffsetModPage > 0)
109 return PageSize - OffsetModPage;
110 return 0;
111 }
112
needsCounterPadding(void)113 static int needsCounterPadding(void) {
114 #if defined(__APPLE__)
115 return __llvm_profile_is_continuous_mode_enabled();
116 #else
117 return 0;
118 #endif
119 }
120
121 COMPILER_RT_VISIBILITY
__llvm_profile_get_padding_sizes_for_counters(uint64_t DataSize,uint64_t CountersSize,uint64_t NumBitmapBytes,uint64_t NamesSize,uint64_t * PaddingBytesBeforeCounters,uint64_t * PaddingBytesAfterCounters,uint64_t * PaddingBytesAfterBitmapBytes,uint64_t * PaddingBytesAfterNames)122 void __llvm_profile_get_padding_sizes_for_counters(
123 uint64_t DataSize, uint64_t CountersSize, uint64_t NumBitmapBytes,
124 uint64_t NamesSize, uint64_t *PaddingBytesBeforeCounters,
125 uint64_t *PaddingBytesAfterCounters, uint64_t *PaddingBytesAfterBitmapBytes,
126 uint64_t *PaddingBytesAfterNames) {
127 if (!needsCounterPadding()) {
128 *PaddingBytesBeforeCounters = 0;
129 *PaddingBytesAfterCounters =
130 __llvm_profile_get_num_padding_bytes(CountersSize);
131 *PaddingBytesAfterBitmapBytes =
132 __llvm_profile_get_num_padding_bytes(NumBitmapBytes);
133 *PaddingBytesAfterNames = __llvm_profile_get_num_padding_bytes(NamesSize);
134 return;
135 }
136
137 // In continuous mode, the file offsets for headers and for the start of
138 // counter sections need to be page-aligned.
139 *PaddingBytesBeforeCounters =
140 calculateBytesNeededToPageAlign(sizeof(__llvm_profile_header) + DataSize);
141 *PaddingBytesAfterCounters = calculateBytesNeededToPageAlign(CountersSize);
142 *PaddingBytesAfterBitmapBytes =
143 calculateBytesNeededToPageAlign(NumBitmapBytes);
144 *PaddingBytesAfterNames = calculateBytesNeededToPageAlign(NamesSize);
145 }
146
147 COMPILER_RT_VISIBILITY
__llvm_profile_get_size_for_buffer_internal(const __llvm_profile_data * DataBegin,const __llvm_profile_data * DataEnd,const char * CountersBegin,const char * CountersEnd,const char * BitmapBegin,const char * BitmapEnd,const char * NamesBegin,const char * NamesEnd)148 uint64_t __llvm_profile_get_size_for_buffer_internal(
149 const __llvm_profile_data *DataBegin, const __llvm_profile_data *DataEnd,
150 const char *CountersBegin, const char *CountersEnd, const char *BitmapBegin,
151 const char *BitmapEnd, const char *NamesBegin, const char *NamesEnd) {
152 /* Match logic in __llvm_profile_write_buffer(). */
153 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
154 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
155 uint64_t CountersSize =
156 __llvm_profile_get_counters_size(CountersBegin, CountersEnd);
157 const uint64_t NumBitmapBytes =
158 __llvm_profile_get_num_bitmap_bytes(BitmapBegin, BitmapEnd);
159
160 /* Determine how much padding is needed before/after the counters and after
161 * the names. */
162 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
163 PaddingBytesAfterNames, PaddingBytesAfterBitmapBytes;
164 __llvm_profile_get_padding_sizes_for_counters(
165 DataSize, CountersSize, NumBitmapBytes, NamesSize,
166 &PaddingBytesBeforeCounters, &PaddingBytesAfterCounters,
167 &PaddingBytesAfterBitmapBytes, &PaddingBytesAfterNames);
168
169 return sizeof(__llvm_profile_header) + __llvm_write_binary_ids(NULL) +
170 DataSize + PaddingBytesBeforeCounters + CountersSize +
171 PaddingBytesAfterCounters + NumBitmapBytes +
172 PaddingBytesAfterBitmapBytes + NamesSize + PaddingBytesAfterNames;
173 }
174
175 COMPILER_RT_VISIBILITY
initBufferWriter(ProfDataWriter * BufferWriter,char * Buffer)176 void initBufferWriter(ProfDataWriter *BufferWriter, char *Buffer) {
177 BufferWriter->Write = lprofBufferWriter;
178 BufferWriter->WriterCtx = Buffer;
179 }
180
__llvm_profile_write_buffer(char * Buffer)181 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer(char *Buffer) {
182 ProfDataWriter BufferWriter;
183 initBufferWriter(&BufferWriter, Buffer);
184 return lprofWriteData(&BufferWriter, 0, 0);
185 }
186
__llvm_profile_write_buffer_internal(char * Buffer,const __llvm_profile_data * DataBegin,const __llvm_profile_data * DataEnd,const char * CountersBegin,const char * CountersEnd,const char * BitmapBegin,const char * BitmapEnd,const char * NamesBegin,const char * NamesEnd)187 COMPILER_RT_VISIBILITY int __llvm_profile_write_buffer_internal(
188 char *Buffer, const __llvm_profile_data *DataBegin,
189 const __llvm_profile_data *DataEnd, const char *CountersBegin,
190 const char *CountersEnd, const char *BitmapBegin, const char *BitmapEnd,
191 const char *NamesBegin, const char *NamesEnd) {
192 ProfDataWriter BufferWriter;
193 initBufferWriter(&BufferWriter, Buffer);
194 return lprofWriteDataImpl(&BufferWriter, DataBegin, DataEnd, CountersBegin,
195 CountersEnd, BitmapBegin, BitmapEnd, 0, NamesBegin,
196 NamesEnd, 0);
197 }
198