1 /*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\
2 |*
3 |* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 |* See https://llvm.org/LICENSE.txt for license information.
5 |* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 |*
7 \*===----------------------------------------------------------------------===*/
8
9 #if !defined(__Fuchsia__)
10
11 #include <errno.h>
12 #include <stdio.h>
13 #include <stdlib.h>
14 #include <string.h>
15 #ifdef _MSC_VER
16 /* For _alloca. */
17 #include <malloc.h>
18 #endif
19 #if defined(_WIN32)
20 #include "WindowsMMap.h"
21 /* For _chsize_s */
22 #include <io.h>
23 #include <process.h>
24 #else
25 #include <sys/file.h>
26 #include <sys/mman.h>
27 #include <unistd.h>
28 #if defined(__linux__)
29 #include <sys/types.h>
30 #endif
31 #endif
32
33 #include "InstrProfiling.h"
34 #include "InstrProfilingInternal.h"
35 #include "InstrProfilingPort.h"
36 #include "InstrProfilingUtil.h"
37
38 /* From where is profile name specified.
39 * The order the enumerators define their
40 * precedence. Re-order them may lead to
41 * runtime behavior change. */
42 typedef enum ProfileNameSpecifier {
43 PNS_unknown = 0,
44 PNS_default,
45 PNS_command_line,
46 PNS_environment,
47 PNS_runtime_api
48 } ProfileNameSpecifier;
49
getPNSStr(ProfileNameSpecifier PNS)50 static const char *getPNSStr(ProfileNameSpecifier PNS) {
51 switch (PNS) {
52 case PNS_default:
53 return "default setting";
54 case PNS_command_line:
55 return "command line";
56 case PNS_environment:
57 return "environment variable";
58 case PNS_runtime_api:
59 return "runtime API";
60 default:
61 return "Unknown";
62 }
63 }
64
65 #define MAX_PID_SIZE 16
66 /* Data structure holding the result of parsed filename pattern. */
67 typedef struct lprofFilename {
68 /* File name string possibly with %p or %h specifiers. */
69 const char *FilenamePat;
70 /* A flag indicating if FilenamePat's memory is allocated
71 * by runtime. */
72 unsigned OwnsFilenamePat;
73 const char *ProfilePathPrefix;
74 char PidChars[MAX_PID_SIZE];
75 char Hostname[COMPILER_RT_MAX_HOSTLEN];
76 unsigned NumPids;
77 unsigned NumHosts;
78 /* When in-process merging is enabled, this parameter specifies
79 * the total number of profile data files shared by all the processes
80 * spawned from the same binary. By default the value is 1. If merging
81 * is not enabled, its value should be 0. This parameter is specified
82 * by the %[0-9]m specifier. For instance %2m enables merging using
83 * 2 profile data files. %1m is equivalent to %m. Also %m specifier
84 * can only appear once at the end of the name pattern. */
85 unsigned MergePoolSize;
86 ProfileNameSpecifier PNS;
87 } lprofFilename;
88
89 static lprofFilename lprofCurFilename = {0, 0, 0, {0}, {0},
90 0, 0, 0, PNS_unknown};
91
92 static int ProfileMergeRequested = 0;
isProfileMergeRequested()93 static int isProfileMergeRequested() { return ProfileMergeRequested; }
setProfileMergeRequested(int EnableMerge)94 static void setProfileMergeRequested(int EnableMerge) {
95 ProfileMergeRequested = EnableMerge;
96 }
97
98 static FILE *ProfileFile = NULL;
getProfileFile()99 static FILE *getProfileFile() { return ProfileFile; }
setProfileFile(FILE * File)100 static void setProfileFile(FILE *File) { ProfileFile = File; }
101
__llvm_profile_set_file_object(FILE * File,int EnableMerge)102 COMPILER_RT_VISIBILITY void __llvm_profile_set_file_object(FILE *File,
103 int EnableMerge) {
104 if (__llvm_profile_is_continuous_mode_enabled()) {
105 PROF_WARN("__llvm_profile_set_file_object(fd=%d) not supported, because "
106 "continuous sync mode (%%c) is enabled",
107 fileno(File));
108 return;
109 }
110 setProfileFile(File);
111 setProfileMergeRequested(EnableMerge);
112 }
113
114 static int getCurFilenameLength();
115 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf);
doMerging()116 static unsigned doMerging() {
117 return lprofCurFilename.MergePoolSize || isProfileMergeRequested();
118 }
119
120 /* Return 1 if there is an error, otherwise return 0. */
fileWriter(ProfDataWriter * This,ProfDataIOVec * IOVecs,uint32_t NumIOVecs)121 static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs,
122 uint32_t NumIOVecs) {
123 uint32_t I;
124 FILE *File = (FILE *)This->WriterCtx;
125 char Zeroes[sizeof(uint64_t)] = {0};
126 for (I = 0; I < NumIOVecs; I++) {
127 if (IOVecs[I].Data) {
128 if (fwrite(IOVecs[I].Data, IOVecs[I].ElmSize, IOVecs[I].NumElm, File) !=
129 IOVecs[I].NumElm)
130 return 1;
131 } else if (IOVecs[I].UseZeroPadding) {
132 size_t BytesToWrite = IOVecs[I].ElmSize * IOVecs[I].NumElm;
133 while (BytesToWrite > 0) {
134 size_t PartialWriteLen =
135 (sizeof(uint64_t) > BytesToWrite) ? BytesToWrite : sizeof(uint64_t);
136 if (fwrite(Zeroes, sizeof(uint8_t), PartialWriteLen, File) !=
137 PartialWriteLen) {
138 return 1;
139 }
140 BytesToWrite -= PartialWriteLen;
141 }
142 } else {
143 if (fseek(File, IOVecs[I].ElmSize * IOVecs[I].NumElm, SEEK_CUR) == -1)
144 return 1;
145 }
146 }
147 return 0;
148 }
149
150 /* TODO: make buffer size controllable by an internal option, and compiler can pass the size
151 to runtime via a variable. */
orderFileWriter(FILE * File,const uint32_t * DataStart)152 static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) {
153 if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) !=
154 INSTR_ORDER_FILE_BUFFER_SIZE)
155 return 1;
156 return 0;
157 }
158
initFileWriter(ProfDataWriter * This,FILE * File)159 static void initFileWriter(ProfDataWriter *This, FILE *File) {
160 This->Write = fileWriter;
161 This->WriterCtx = File;
162 }
163
164 COMPILER_RT_VISIBILITY ProfBufferIO *
lprofCreateBufferIOInternal(void * File,uint32_t BufferSz)165 lprofCreateBufferIOInternal(void *File, uint32_t BufferSz) {
166 FreeHook = &free;
167 DynamicBufferIOBuffer = (uint8_t *)calloc(BufferSz, 1);
168 VPBufferSize = BufferSz;
169 ProfDataWriter *fileWriter =
170 (ProfDataWriter *)calloc(sizeof(ProfDataWriter), 1);
171 initFileWriter(fileWriter, File);
172 ProfBufferIO *IO = lprofCreateBufferIO(fileWriter);
173 IO->OwnFileWriter = 1;
174 return IO;
175 }
176
setupIOBuffer()177 static void setupIOBuffer() {
178 const char *BufferSzStr = 0;
179 BufferSzStr = getenv("LLVM_VP_BUFFER_SIZE");
180 if (BufferSzStr && BufferSzStr[0]) {
181 VPBufferSize = atoi(BufferSzStr);
182 DynamicBufferIOBuffer = (uint8_t *)calloc(VPBufferSize, 1);
183 }
184 }
185
186 /* Get the size of the profile file. If there are any errors, print the
187 * message under the assumption that the profile is being read for merging
188 * purposes, and return -1. Otherwise return the file size in the inout param
189 * \p ProfileFileSize. */
getProfileFileSizeForMerging(FILE * ProfileFile,uint64_t * ProfileFileSize)190 static int getProfileFileSizeForMerging(FILE *ProfileFile,
191 uint64_t *ProfileFileSize) {
192 if (fseek(ProfileFile, 0L, SEEK_END) == -1) {
193 PROF_ERR("Unable to merge profile data, unable to get size: %s\n",
194 strerror(errno));
195 return -1;
196 }
197 *ProfileFileSize = ftell(ProfileFile);
198
199 /* Restore file offset. */
200 if (fseek(ProfileFile, 0L, SEEK_SET) == -1) {
201 PROF_ERR("Unable to merge profile data, unable to rewind: %s\n",
202 strerror(errno));
203 return -1;
204 }
205
206 if (*ProfileFileSize > 0 &&
207 *ProfileFileSize < sizeof(__llvm_profile_header)) {
208 PROF_WARN("Unable to merge profile data: %s\n",
209 "source profile file is too small.");
210 return -1;
211 }
212 return 0;
213 }
214
215 /* mmap() \p ProfileFile for profile merging purposes, assuming that an
216 * exclusive lock is held on the file and that \p ProfileFileSize is the
217 * length of the file. Return the mmap'd buffer in the inout variable
218 * \p ProfileBuffer. Returns -1 on failure. On success, the caller is
219 * responsible for unmapping the mmap'd buffer in \p ProfileBuffer. */
mmapProfileForMerging(FILE * ProfileFile,uint64_t ProfileFileSize,char ** ProfileBuffer)220 static int mmapProfileForMerging(FILE *ProfileFile, uint64_t ProfileFileSize,
221 char **ProfileBuffer) {
222 *ProfileBuffer = mmap(NULL, ProfileFileSize, PROT_READ, MAP_SHARED | MAP_FILE,
223 fileno(ProfileFile), 0);
224 if (*ProfileBuffer == MAP_FAILED) {
225 PROF_ERR("Unable to merge profile data, mmap failed: %s\n",
226 strerror(errno));
227 return -1;
228 }
229
230 if (__llvm_profile_check_compatibility(*ProfileBuffer, ProfileFileSize)) {
231 (void)munmap(*ProfileBuffer, ProfileFileSize);
232 PROF_WARN("Unable to merge profile data: %s\n",
233 "source profile file is not compatible.");
234 return -1;
235 }
236 return 0;
237 }
238
239 /* Read profile data in \c ProfileFile and merge with in-memory
240 profile counters. Returns -1 if there is fatal error, otheriwse
241 0 is returned. Returning 0 does not mean merge is actually
242 performed. If merge is actually done, *MergeDone is set to 1.
243 */
doProfileMerging(FILE * ProfileFile,int * MergeDone)244 static int doProfileMerging(FILE *ProfileFile, int *MergeDone) {
245 uint64_t ProfileFileSize;
246 char *ProfileBuffer;
247
248 /* Get the size of the profile on disk. */
249 if (getProfileFileSizeForMerging(ProfileFile, &ProfileFileSize) == -1)
250 return -1;
251
252 /* Nothing to merge. */
253 if (!ProfileFileSize)
254 return 0;
255
256 /* mmap() the profile and check that it is compatible with the data in
257 * the current image. */
258 if (mmapProfileForMerging(ProfileFile, ProfileFileSize, &ProfileBuffer) == -1)
259 return -1;
260
261 /* Now start merging */
262 __llvm_profile_merge_from_buffer(ProfileBuffer, ProfileFileSize);
263
264 // Truncate the file in case merging of value profile did not happend to
265 // prevent from leaving garbage data at the end of the profile file.
266 COMPILER_RT_FTRUNCATE(ProfileFile, __llvm_profile_get_size_for_buffer());
267
268 (void)munmap(ProfileBuffer, ProfileFileSize);
269 *MergeDone = 1;
270
271 return 0;
272 }
273
274 /* Create the directory holding the file, if needed. */
createProfileDir(const char * Filename)275 static void createProfileDir(const char *Filename) {
276 size_t Length = strlen(Filename);
277 if (lprofFindFirstDirSeparator(Filename)) {
278 char *Copy = (char *)COMPILER_RT_ALLOCA(Length + 1);
279 strncpy(Copy, Filename, Length + 1);
280 __llvm_profile_recursive_mkdir(Copy);
281 }
282 }
283
284 /* Open the profile data for merging. It opens the file in r+b mode with
285 * file locking. If the file has content which is compatible with the
286 * current process, it also reads in the profile data in the file and merge
287 * it with in-memory counters. After the profile data is merged in memory,
288 * the original profile data is truncated and gets ready for the profile
289 * dumper. With profile merging enabled, each executable as well as any of
290 * its instrumented shared libraries dump profile data into their own data file.
291 */
openFileForMerging(const char * ProfileFileName,int * MergeDone)292 static FILE *openFileForMerging(const char *ProfileFileName, int *MergeDone) {
293 FILE *ProfileFile = NULL;
294 int rc;
295
296 ProfileFile = getProfileFile();
297 if (ProfileFile) {
298 lprofLockFileHandle(ProfileFile);
299 } else {
300 createProfileDir(ProfileFileName);
301 ProfileFile = lprofOpenFileEx(ProfileFileName);
302 }
303 if (!ProfileFile)
304 return NULL;
305
306 rc = doProfileMerging(ProfileFile, MergeDone);
307 if (rc || (!*MergeDone && COMPILER_RT_FTRUNCATE(ProfileFile, 0L)) ||
308 fseek(ProfileFile, 0L, SEEK_SET) == -1) {
309 PROF_ERR("Profile Merging of file %s failed: %s\n", ProfileFileName,
310 strerror(errno));
311 fclose(ProfileFile);
312 return NULL;
313 }
314 return ProfileFile;
315 }
316
getFileObject(const char * OutputName)317 static FILE *getFileObject(const char *OutputName) {
318 FILE *File;
319 File = getProfileFile();
320 if (File != NULL) {
321 return File;
322 }
323
324 return fopen(OutputName, "ab");
325 }
326
327 /* Write profile data to file \c OutputName. */
writeFile(const char * OutputName)328 static int writeFile(const char *OutputName) {
329 int RetVal;
330 FILE *OutputFile;
331
332 int MergeDone = 0;
333 VPMergeHook = &lprofMergeValueProfData;
334 if (doMerging())
335 OutputFile = openFileForMerging(OutputName, &MergeDone);
336 else
337 OutputFile = getFileObject(OutputName);
338
339 if (!OutputFile)
340 return -1;
341
342 FreeHook = &free;
343 setupIOBuffer();
344 ProfDataWriter fileWriter;
345 initFileWriter(&fileWriter, OutputFile);
346 RetVal = lprofWriteData(&fileWriter, lprofGetVPDataReader(), MergeDone);
347
348 if (OutputFile == getProfileFile()) {
349 fflush(OutputFile);
350 if (doMerging()) {
351 lprofUnlockFileHandle(OutputFile);
352 }
353 } else {
354 fclose(OutputFile);
355 }
356
357 return RetVal;
358 }
359
360 /* Write order data to file \c OutputName. */
writeOrderFile(const char * OutputName)361 static int writeOrderFile(const char *OutputName) {
362 int RetVal;
363 FILE *OutputFile;
364
365 OutputFile = fopen(OutputName, "w");
366
367 if (!OutputFile) {
368 PROF_WARN("can't open file with mode ab: %s\n", OutputName);
369 return -1;
370 }
371
372 FreeHook = &free;
373 setupIOBuffer();
374 const uint32_t *DataBegin = __llvm_profile_begin_orderfile();
375 RetVal = orderFileWriter(OutputFile, DataBegin);
376
377 fclose(OutputFile);
378 return RetVal;
379 }
380
381 #define LPROF_INIT_ONCE_ENV "__LLVM_PROFILE_RT_INIT_ONCE"
382
truncateCurrentFile(void)383 static void truncateCurrentFile(void) {
384 const char *Filename;
385 char *FilenameBuf;
386 FILE *File;
387 int Length;
388
389 Length = getCurFilenameLength();
390 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
391 Filename = getCurFilename(FilenameBuf, 0);
392 if (!Filename)
393 return;
394
395 /* Only create the profile directory and truncate an existing profile once.
396 * In continuous mode, this is necessary, as the profile is written-to by the
397 * runtime initializer. */
398 int initialized = getenv(LPROF_INIT_ONCE_ENV) != NULL;
399 if (initialized)
400 return;
401 #if defined(_WIN32)
402 _putenv(LPROF_INIT_ONCE_ENV "=" LPROF_INIT_ONCE_ENV);
403 #else
404 setenv(LPROF_INIT_ONCE_ENV, LPROF_INIT_ONCE_ENV, 1);
405 #endif
406
407 /* Create the profile dir (even if online merging is enabled), so that
408 * the profile file can be set up if continuous mode is enabled. */
409 createProfileDir(Filename);
410
411 /* By pass file truncation to allow online raw profile merging. */
412 if (lprofCurFilename.MergePoolSize)
413 return;
414
415 /* Truncate the file. Later we'll reopen and append. */
416 File = fopen(Filename, "w");
417 if (!File)
418 return;
419 fclose(File);
420 }
421
422 #ifndef _MSC_VER
assertIsZero(int * i)423 static void assertIsZero(int *i) {
424 if (*i)
425 PROF_WARN("Expected flag to be 0, but got: %d\n", *i);
426 }
427 #endif
428
429 #if !defined(__Fuchsia__) && !defined(_WIN32)
430 /* Write a partial profile to \p Filename, which is required to be backed by
431 * the open file object \p File. */
writeProfileWithFileObject(const char * Filename,FILE * File)432 static int writeProfileWithFileObject(const char *Filename, FILE *File) {
433 setProfileFile(File);
434 int rc = writeFile(Filename);
435 if (rc)
436 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
437 setProfileFile(NULL);
438 return rc;
439 }
440
441 /* Unlock the profile \p File and clear the unlock flag. */
unlockProfile(int * ProfileRequiresUnlock,FILE * File)442 static void unlockProfile(int *ProfileRequiresUnlock, FILE *File) {
443 if (!*ProfileRequiresUnlock) {
444 PROF_WARN("%s", "Expected to require profile unlock\n");
445 }
446 lprofUnlockFileHandle(File);
447 *ProfileRequiresUnlock = 0;
448 }
449 #endif // !defined(__Fuchsia__) && !defined(_WIN32)
450
writeMMappedFile(FILE * OutputFile,char ** Profile)451 static int writeMMappedFile(FILE *OutputFile, char **Profile) {
452 if (!OutputFile)
453 return -1;
454
455 /* Write the data into a file. */
456 setupIOBuffer();
457 ProfDataWriter fileWriter;
458 initFileWriter(&fileWriter, OutputFile);
459 if (lprofWriteData(&fileWriter, NULL, 0)) {
460 PROF_ERR("Failed to write profile: %s\n", strerror(errno));
461 return -1;
462 }
463 fflush(OutputFile);
464
465 /* Get the file size. */
466 uint64_t FileSize = ftell(OutputFile);
467
468 /* Map the profile. */
469 *Profile = (char *)mmap(
470 NULL, FileSize, PROT_READ | PROT_WRITE, MAP_SHARED, fileno(OutputFile), 0);
471 if (*Profile == MAP_FAILED) {
472 PROF_ERR("Unable to mmap profile: %s\n", strerror(errno));
473 return -1;
474 }
475
476 return 0;
477 }
478
relocateCounters(void)479 static void relocateCounters(void) {
480 if (!__llvm_profile_is_continuous_mode_enabled() ||
481 !lprofRuntimeCounterRelocation())
482 return;
483
484 /* Get the sizes of various profile data sections. Taken from
485 * __llvm_profile_get_size_for_buffer(). */
486 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
487 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
488 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
489 const uint64_t CountersOffset = sizeof(__llvm_profile_header) +
490 (DataSize * sizeof(__llvm_profile_data));
491
492 int Length = getCurFilenameLength();
493 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
494 const char *Filename = getCurFilename(FilenameBuf, 0);
495 if (!Filename)
496 return;
497
498 FILE *File = NULL;
499 char *Profile = NULL;
500
501 if (!doMerging()) {
502 File = fopen(Filename, "w+b");
503 if (!File)
504 return;
505
506 if (writeMMappedFile(File, &Profile) == -1) {
507 fclose(File);
508 return;
509 }
510 } else {
511 File = lprofOpenFileEx(Filename);
512 if (!File)
513 return;
514
515 uint64_t ProfileFileSize = 0;
516 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1) {
517 lprofUnlockFileHandle(File);
518 fclose(File);
519 return;
520 }
521
522 if (!ProfileFileSize) {
523 if (writeMMappedFile(File, &Profile) == -1) {
524 fclose(File);
525 return;
526 }
527 } else {
528 /* The merged profile has a non-zero length. Check that it is compatible
529 * with the data in this process. */
530 if (mmapProfileForMerging(File, ProfileFileSize, &Profile) == -1) {
531 fclose(File);
532 return;
533 }
534 }
535
536 lprofUnlockFileHandle(File);
537 }
538
539 /* Update the profile fields based on the current mapping. */
540 __llvm_profile_counter_bias = (intptr_t)Profile -
541 (uintptr_t)__llvm_profile_begin_counters() + CountersOffset;
542 }
543
initializeProfileForContinuousMode(void)544 static void initializeProfileForContinuousMode(void) {
545 if (!__llvm_profile_is_continuous_mode_enabled())
546 return;
547
548 #if defined(__Fuchsia__) || defined(_WIN32)
549 PROF_ERR("%s\n", "Continuous mode not yet supported on Fuchsia or Windows.");
550 #else // defined(__Fuchsia__) || defined(_WIN32)
551 /* Get the sizes of various profile data sections. Taken from
552 * __llvm_profile_get_size_for_buffer(). */
553 const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
554 const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
555 const uint64_t *CountersBegin = __llvm_profile_begin_counters();
556 const uint64_t *CountersEnd = __llvm_profile_end_counters();
557 const char *NamesBegin = __llvm_profile_begin_names();
558 const char *NamesEnd = __llvm_profile_end_names();
559 const uint64_t NamesSize = (NamesEnd - NamesBegin) * sizeof(char);
560 uint64_t DataSize = __llvm_profile_get_data_size(DataBegin, DataEnd);
561 uint64_t CountersSize = CountersEnd - CountersBegin;
562
563 /* Check that the counter and data sections in this image are page-aligned. */
564 unsigned PageSize = getpagesize();
565 if ((intptr_t)CountersBegin % PageSize != 0) {
566 PROF_ERR("Counters section not page-aligned (start = %p, pagesz = %u).\n",
567 CountersBegin, PageSize);
568 return;
569 }
570 if ((intptr_t)DataBegin % PageSize != 0) {
571 PROF_ERR("Data section not page-aligned (start = %p, pagesz = %u).\n",
572 DataBegin, PageSize);
573 return;
574 }
575
576 int Length = getCurFilenameLength();
577 char *FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
578 const char *Filename = getCurFilename(FilenameBuf, 0);
579 if (!Filename)
580 return;
581
582 FILE *File = NULL;
583 off_t CurrentFileOffset = 0;
584 off_t OffsetModPage = 0;
585
586 /* Whether an exclusive lock on the profile must be dropped after init.
587 * Use a cleanup to warn if the unlock does not occur. */
588 COMPILER_RT_CLEANUP(assertIsZero) int ProfileRequiresUnlock = 0;
589
590 if (!doMerging()) {
591 /* We are not merging profiles, so open the raw profile in append mode. */
592 File = fopen(Filename, "a+b");
593 if (!File)
594 return;
595
596 /* Check that the offset within the file is page-aligned. */
597 CurrentFileOffset = ftello(File);
598 OffsetModPage = CurrentFileOffset % PageSize;
599 if (OffsetModPage != 0) {
600 PROF_ERR("Continuous counter sync mode is enabled, but raw profile is not"
601 "page-aligned. CurrentFileOffset = %" PRIu64 ", pagesz = %u.\n",
602 (uint64_t)CurrentFileOffset, PageSize);
603 return;
604 }
605
606 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
607 * the file should stay open. */
608 if (writeProfileWithFileObject(Filename, File) != 0)
609 return;
610 } else {
611 /* We are merging profiles. Map the counter section as shared memory into
612 * the profile, i.e. into each participating process. An increment in one
613 * process should be visible to every other process with the same counter
614 * section mapped. */
615 File = lprofOpenFileEx(Filename);
616 if (!File)
617 return;
618
619 ProfileRequiresUnlock = 1;
620
621 uint64_t ProfileFileSize;
622 if (getProfileFileSizeForMerging(File, &ProfileFileSize) == -1)
623 return unlockProfile(&ProfileRequiresUnlock, File);
624
625 if (ProfileFileSize == 0) {
626 /* Grow the profile so that mmap() can succeed. Leak the file handle, as
627 * the file should stay open. */
628 if (writeProfileWithFileObject(Filename, File) != 0)
629 return unlockProfile(&ProfileRequiresUnlock, File);
630 } else {
631 /* The merged profile has a non-zero length. Check that it is compatible
632 * with the data in this process. */
633 char *ProfileBuffer;
634 if (mmapProfileForMerging(File, ProfileFileSize, &ProfileBuffer) == -1 ||
635 munmap(ProfileBuffer, ProfileFileSize) == -1)
636 return unlockProfile(&ProfileRequiresUnlock, File);
637 }
638 }
639
640 int Fileno = fileno(File);
641
642 /* Determine how much padding is needed before/after the counters and after
643 * the names. */
644 uint64_t PaddingBytesBeforeCounters, PaddingBytesAfterCounters,
645 PaddingBytesAfterNames;
646 __llvm_profile_get_padding_sizes_for_counters(
647 DataSize, CountersSize, NamesSize, &PaddingBytesBeforeCounters,
648 &PaddingBytesAfterCounters, &PaddingBytesAfterNames);
649
650 uint64_t PageAlignedCountersLength =
651 (CountersSize * sizeof(uint64_t)) + PaddingBytesAfterCounters;
652 uint64_t FileOffsetToCounters =
653 CurrentFileOffset + sizeof(__llvm_profile_header) +
654 (DataSize * sizeof(__llvm_profile_data)) + PaddingBytesBeforeCounters;
655
656 uint64_t *CounterMmap = (uint64_t *)mmap(
657 (void *)CountersBegin, PageAlignedCountersLength, PROT_READ | PROT_WRITE,
658 MAP_FIXED | MAP_SHARED, Fileno, FileOffsetToCounters);
659 if (CounterMmap != CountersBegin) {
660 PROF_ERR(
661 "Continuous counter sync mode is enabled, but mmap() failed (%s).\n"
662 " - CountersBegin: %p\n"
663 " - PageAlignedCountersLength: %" PRIu64 "\n"
664 " - Fileno: %d\n"
665 " - FileOffsetToCounters: %" PRIu64 "\n",
666 strerror(errno), CountersBegin, PageAlignedCountersLength, Fileno,
667 FileOffsetToCounters);
668 }
669
670 unlockProfile(&ProfileRequiresUnlock, File);
671 #endif // defined(__Fuchsia__) || defined(_WIN32)
672 }
673
674 static const char *DefaultProfileName = "default.profraw";
resetFilenameToDefault(void)675 static void resetFilenameToDefault(void) {
676 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
677 free((void *)lprofCurFilename.FilenamePat);
678 }
679 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
680 lprofCurFilename.FilenamePat = DefaultProfileName;
681 lprofCurFilename.PNS = PNS_default;
682 }
683
getMergePoolSize(const char * FilenamePat,int * I)684 static unsigned getMergePoolSize(const char *FilenamePat, int *I) {
685 unsigned J = 0, Num = 0;
686 for (;; ++J) {
687 char C = FilenamePat[*I + J];
688 if (C == 'm') {
689 *I += J;
690 return Num ? Num : 1;
691 }
692 if (C < '0' || C > '9')
693 break;
694 Num = Num * 10 + C - '0';
695
696 /* If FilenamePat[*I+J] is between '0' and '9', the next byte is guaranteed
697 * to be in-bound as the string is null terminated. */
698 }
699 return 0;
700 }
701
702 /* Parses the pattern string \p FilenamePat and stores the result to
703 * lprofcurFilename structure. */
parseFilenamePattern(const char * FilenamePat,unsigned CopyFilenamePat)704 static int parseFilenamePattern(const char *FilenamePat,
705 unsigned CopyFilenamePat) {
706 int NumPids = 0, NumHosts = 0, I;
707 char *PidChars = &lprofCurFilename.PidChars[0];
708 char *Hostname = &lprofCurFilename.Hostname[0];
709 int MergingEnabled = 0;
710
711 /* Clean up cached prefix and filename. */
712 if (lprofCurFilename.ProfilePathPrefix)
713 free((void *)lprofCurFilename.ProfilePathPrefix);
714
715 if (lprofCurFilename.FilenamePat && lprofCurFilename.OwnsFilenamePat) {
716 free((void *)lprofCurFilename.FilenamePat);
717 }
718
719 memset(&lprofCurFilename, 0, sizeof(lprofCurFilename));
720
721 if (!CopyFilenamePat)
722 lprofCurFilename.FilenamePat = FilenamePat;
723 else {
724 lprofCurFilename.FilenamePat = strdup(FilenamePat);
725 lprofCurFilename.OwnsFilenamePat = 1;
726 }
727 /* Check the filename for "%p", which indicates a pid-substitution. */
728 for (I = 0; FilenamePat[I]; ++I)
729 if (FilenamePat[I] == '%') {
730 if (FilenamePat[++I] == 'p') {
731 if (!NumPids++) {
732 if (snprintf(PidChars, MAX_PID_SIZE, "%ld", (long)getpid()) <= 0) {
733 PROF_WARN("Unable to get pid for filename pattern %s. Using the "
734 "default name.",
735 FilenamePat);
736 return -1;
737 }
738 }
739 } else if (FilenamePat[I] == 'h') {
740 if (!NumHosts++)
741 if (COMPILER_RT_GETHOSTNAME(Hostname, COMPILER_RT_MAX_HOSTLEN)) {
742 PROF_WARN("Unable to get hostname for filename pattern %s. Using "
743 "the default name.",
744 FilenamePat);
745 return -1;
746 }
747 } else if (FilenamePat[I] == 'c') {
748 if (__llvm_profile_is_continuous_mode_enabled()) {
749 PROF_WARN("%%c specifier can only be specified once in %s.\n",
750 FilenamePat);
751 return -1;
752 }
753
754 __llvm_profile_enable_continuous_mode();
755 I++; /* advance to 'c' */
756 } else {
757 unsigned MergePoolSize = getMergePoolSize(FilenamePat, &I);
758 if (!MergePoolSize)
759 continue;
760 if (MergingEnabled) {
761 PROF_WARN("%%m specifier can only be specified once in %s.\n",
762 FilenamePat);
763 return -1;
764 }
765 MergingEnabled = 1;
766 lprofCurFilename.MergePoolSize = MergePoolSize;
767 }
768 }
769
770 lprofCurFilename.NumPids = NumPids;
771 lprofCurFilename.NumHosts = NumHosts;
772 return 0;
773 }
774
parseAndSetFilename(const char * FilenamePat,ProfileNameSpecifier PNS,unsigned CopyFilenamePat)775 static void parseAndSetFilename(const char *FilenamePat,
776 ProfileNameSpecifier PNS,
777 unsigned CopyFilenamePat) {
778
779 const char *OldFilenamePat = lprofCurFilename.FilenamePat;
780 ProfileNameSpecifier OldPNS = lprofCurFilename.PNS;
781
782 /* The old profile name specifier takes precedence over the old one. */
783 if (PNS < OldPNS)
784 return;
785
786 if (!FilenamePat)
787 FilenamePat = DefaultProfileName;
788
789 if (OldFilenamePat && !strcmp(OldFilenamePat, FilenamePat)) {
790 lprofCurFilename.PNS = PNS;
791 return;
792 }
793
794 /* When PNS >= OldPNS, the last one wins. */
795 if (!FilenamePat || parseFilenamePattern(FilenamePat, CopyFilenamePat))
796 resetFilenameToDefault();
797 lprofCurFilename.PNS = PNS;
798
799 if (!OldFilenamePat) {
800 if (getenv("LLVM_PROFILE_VERBOSE"))
801 PROF_NOTE("Set profile file path to \"%s\" via %s.\n",
802 lprofCurFilename.FilenamePat, getPNSStr(PNS));
803 } else {
804 if (getenv("LLVM_PROFILE_VERBOSE"))
805 PROF_NOTE("Override old profile path \"%s\" via %s to \"%s\" via %s.\n",
806 OldFilenamePat, getPNSStr(OldPNS), lprofCurFilename.FilenamePat,
807 getPNSStr(PNS));
808 }
809
810 truncateCurrentFile();
811 if (__llvm_profile_is_continuous_mode_enabled()) {
812 if (lprofRuntimeCounterRelocation())
813 relocateCounters();
814 else
815 initializeProfileForContinuousMode();
816 }
817 }
818
819 /* Return buffer length that is required to store the current profile
820 * filename with PID and hostname substitutions. */
821 /* The length to hold uint64_t followed by 3 digits pool id including '_' */
822 #define SIGLEN 24
getCurFilenameLength()823 static int getCurFilenameLength() {
824 int Len;
825 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
826 return 0;
827
828 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
829 lprofCurFilename.MergePoolSize))
830 return strlen(lprofCurFilename.FilenamePat);
831
832 Len = strlen(lprofCurFilename.FilenamePat) +
833 lprofCurFilename.NumPids * (strlen(lprofCurFilename.PidChars) - 2) +
834 lprofCurFilename.NumHosts * (strlen(lprofCurFilename.Hostname) - 2);
835 if (lprofCurFilename.MergePoolSize)
836 Len += SIGLEN;
837 return Len;
838 }
839
840 /* Return the pointer to the current profile file name (after substituting
841 * PIDs and Hostnames in filename pattern. \p FilenameBuf is the buffer
842 * to store the resulting filename. If no substitution is needed, the
843 * current filename pattern string is directly returned, unless ForceUseBuf
844 * is enabled. */
getCurFilename(char * FilenameBuf,int ForceUseBuf)845 static const char *getCurFilename(char *FilenameBuf, int ForceUseBuf) {
846 int I, J, PidLength, HostNameLength, FilenamePatLength;
847 const char *FilenamePat = lprofCurFilename.FilenamePat;
848
849 if (!lprofCurFilename.FilenamePat || !lprofCurFilename.FilenamePat[0])
850 return 0;
851
852 if (!(lprofCurFilename.NumPids || lprofCurFilename.NumHosts ||
853 lprofCurFilename.MergePoolSize ||
854 __llvm_profile_is_continuous_mode_enabled())) {
855 if (!ForceUseBuf)
856 return lprofCurFilename.FilenamePat;
857
858 FilenamePatLength = strlen(lprofCurFilename.FilenamePat);
859 memcpy(FilenameBuf, lprofCurFilename.FilenamePat, FilenamePatLength);
860 FilenameBuf[FilenamePatLength] = '\0';
861 return FilenameBuf;
862 }
863
864 PidLength = strlen(lprofCurFilename.PidChars);
865 HostNameLength = strlen(lprofCurFilename.Hostname);
866 /* Construct the new filename. */
867 for (I = 0, J = 0; FilenamePat[I]; ++I)
868 if (FilenamePat[I] == '%') {
869 if (FilenamePat[++I] == 'p') {
870 memcpy(FilenameBuf + J, lprofCurFilename.PidChars, PidLength);
871 J += PidLength;
872 } else if (FilenamePat[I] == 'h') {
873 memcpy(FilenameBuf + J, lprofCurFilename.Hostname, HostNameLength);
874 J += HostNameLength;
875 } else {
876 if (!getMergePoolSize(FilenamePat, &I))
877 continue;
878 char LoadModuleSignature[SIGLEN + 1];
879 int S;
880 int ProfilePoolId = getpid() % lprofCurFilename.MergePoolSize;
881 S = snprintf(LoadModuleSignature, SIGLEN + 1, "%" PRIu64 "_%d",
882 lprofGetLoadModuleSignature(), ProfilePoolId);
883 if (S == -1 || S > SIGLEN)
884 S = SIGLEN;
885 memcpy(FilenameBuf + J, LoadModuleSignature, S);
886 J += S;
887 }
888 /* Drop any unknown substitutions. */
889 } else
890 FilenameBuf[J++] = FilenamePat[I];
891 FilenameBuf[J] = 0;
892
893 return FilenameBuf;
894 }
895
896 /* Returns the pointer to the environment variable
897 * string. Returns null if the env var is not set. */
getFilenamePatFromEnv(void)898 static const char *getFilenamePatFromEnv(void) {
899 const char *Filename = getenv("LLVM_PROFILE_FILE");
900 if (!Filename || !Filename[0])
901 return 0;
902 return Filename;
903 }
904
905 COMPILER_RT_VISIBILITY
__llvm_profile_get_path_prefix(void)906 const char *__llvm_profile_get_path_prefix(void) {
907 int Length;
908 char *FilenameBuf, *Prefix;
909 const char *Filename, *PrefixEnd;
910
911 if (lprofCurFilename.ProfilePathPrefix)
912 return lprofCurFilename.ProfilePathPrefix;
913
914 Length = getCurFilenameLength();
915 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
916 Filename = getCurFilename(FilenameBuf, 0);
917 if (!Filename)
918 return "\0";
919
920 PrefixEnd = lprofFindLastDirSeparator(Filename);
921 if (!PrefixEnd)
922 return "\0";
923
924 Length = PrefixEnd - Filename + 1;
925 Prefix = (char *)malloc(Length + 1);
926 if (!Prefix) {
927 PROF_ERR("Failed to %s\n", "allocate memory.");
928 return "\0";
929 }
930 memcpy(Prefix, Filename, Length);
931 Prefix[Length] = '\0';
932 lprofCurFilename.ProfilePathPrefix = Prefix;
933 return Prefix;
934 }
935
936 COMPILER_RT_VISIBILITY
__llvm_profile_get_filename(void)937 const char *__llvm_profile_get_filename(void) {
938 int Length;
939 char *FilenameBuf;
940 const char *Filename;
941
942 Length = getCurFilenameLength();
943 FilenameBuf = (char *)malloc(Length + 1);
944 if (!FilenameBuf) {
945 PROF_ERR("Failed to %s\n", "allocate memory.");
946 return "\0";
947 }
948 Filename = getCurFilename(FilenameBuf, 1);
949 if (!Filename)
950 return "\0";
951
952 return FilenameBuf;
953 }
954
955 /* This API initializes the file handling, both user specified
956 * profile path via -fprofile-instr-generate= and LLVM_PROFILE_FILE
957 * environment variable can override this default value.
958 */
959 COMPILER_RT_VISIBILITY
__llvm_profile_initialize_file(void)960 void __llvm_profile_initialize_file(void) {
961 const char *EnvFilenamePat;
962 const char *SelectedPat = NULL;
963 ProfileNameSpecifier PNS = PNS_unknown;
964 int hasCommandLineOverrider = (INSTR_PROF_PROFILE_NAME_VAR[0] != 0);
965
966 if (__llvm_profile_counter_bias != -1)
967 lprofSetRuntimeCounterRelocation(1);
968
969 EnvFilenamePat = getFilenamePatFromEnv();
970 if (EnvFilenamePat) {
971 /* Pass CopyFilenamePat = 1, to ensure that the filename would be valid
972 at the moment when __llvm_profile_write_file() gets executed. */
973 parseAndSetFilename(EnvFilenamePat, PNS_environment, 1);
974 return;
975 } else if (hasCommandLineOverrider) {
976 SelectedPat = INSTR_PROF_PROFILE_NAME_VAR;
977 PNS = PNS_command_line;
978 } else {
979 SelectedPat = NULL;
980 PNS = PNS_default;
981 }
982
983 parseAndSetFilename(SelectedPat, PNS, 0);
984 }
985
986 /* This method is invoked by the runtime initialization hook
987 * InstrProfilingRuntime.o if it is linked in.
988 */
989 COMPILER_RT_VISIBILITY
__llvm_profile_initialize(void)990 void __llvm_profile_initialize(void) {
991 __llvm_profile_initialize_file();
992 if (!__llvm_profile_is_continuous_mode_enabled())
993 __llvm_profile_register_write_file_atexit();
994 }
995
996 /* This API is directly called by the user application code. It has the
997 * highest precedence compared with LLVM_PROFILE_FILE environment variable
998 * and command line option -fprofile-instr-generate=<profile_name>.
999 */
1000 COMPILER_RT_VISIBILITY
__llvm_profile_set_filename(const char * FilenamePat)1001 void __llvm_profile_set_filename(const char *FilenamePat) {
1002 if (__llvm_profile_is_continuous_mode_enabled())
1003 return;
1004 parseAndSetFilename(FilenamePat, PNS_runtime_api, 1);
1005 }
1006
1007 /* The public API for writing profile data into the file with name
1008 * set by previous calls to __llvm_profile_set_filename or
1009 * __llvm_profile_override_default_filename or
1010 * __llvm_profile_initialize_file. */
1011 COMPILER_RT_VISIBILITY
__llvm_profile_write_file(void)1012 int __llvm_profile_write_file(void) {
1013 int rc, Length;
1014 const char *Filename;
1015 char *FilenameBuf;
1016 int PDeathSig = 0;
1017
1018 if (lprofProfileDumped() || __llvm_profile_is_continuous_mode_enabled()) {
1019 PROF_NOTE("Profile data not written to file: %s.\n", "already written");
1020 return 0;
1021 }
1022
1023 Length = getCurFilenameLength();
1024 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1025 Filename = getCurFilename(FilenameBuf, 0);
1026
1027 /* Check the filename. */
1028 if (!Filename) {
1029 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1030 return -1;
1031 }
1032
1033 /* Check if there is llvm/runtime version mismatch. */
1034 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1035 PROF_ERR("Runtime and instrumentation version mismatch : "
1036 "expected %d, but get %d\n",
1037 INSTR_PROF_RAW_VERSION,
1038 (int)GET_VERSION(__llvm_profile_get_version()));
1039 return -1;
1040 }
1041
1042 // Temporarily suspend getting SIGKILL when the parent exits.
1043 PDeathSig = lprofSuspendSigKill();
1044
1045 /* Write profile data to the file. */
1046 rc = writeFile(Filename);
1047 if (rc)
1048 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1049
1050 // Restore SIGKILL.
1051 if (PDeathSig == 1)
1052 lprofRestoreSigKill();
1053
1054 return rc;
1055 }
1056
1057 COMPILER_RT_VISIBILITY
__llvm_profile_dump(void)1058 int __llvm_profile_dump(void) {
1059 if (!doMerging())
1060 PROF_WARN("Later invocation of __llvm_profile_dump can lead to clobbering "
1061 " of previously dumped profile data : %s. Either use %%m "
1062 "in profile name or change profile name before dumping.\n",
1063 "online profile merging is not on");
1064 int rc = __llvm_profile_write_file();
1065 lprofSetProfileDumped(1);
1066 return rc;
1067 }
1068
1069 /* Order file data will be saved in a file with suffx .order. */
1070 static const char *OrderFileSuffix = ".order";
1071
1072 COMPILER_RT_VISIBILITY
__llvm_orderfile_write_file(void)1073 int __llvm_orderfile_write_file(void) {
1074 int rc, Length, LengthBeforeAppend, SuffixLength;
1075 const char *Filename;
1076 char *FilenameBuf;
1077 int PDeathSig = 0;
1078
1079 SuffixLength = strlen(OrderFileSuffix);
1080 Length = getCurFilenameLength() + SuffixLength;
1081 FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1);
1082 Filename = getCurFilename(FilenameBuf, 1);
1083
1084 /* Check the filename. */
1085 if (!Filename) {
1086 PROF_ERR("Failed to write file : %s\n", "Filename not set");
1087 return -1;
1088 }
1089
1090 /* Append order file suffix */
1091 LengthBeforeAppend = strlen(Filename);
1092 memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength);
1093 FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0';
1094
1095 /* Check if there is llvm/runtime version mismatch. */
1096 if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) {
1097 PROF_ERR("Runtime and instrumentation version mismatch : "
1098 "expected %d, but get %d\n",
1099 INSTR_PROF_RAW_VERSION,
1100 (int)GET_VERSION(__llvm_profile_get_version()));
1101 return -1;
1102 }
1103
1104 // Temporarily suspend getting SIGKILL when the parent exits.
1105 PDeathSig = lprofSuspendSigKill();
1106
1107 /* Write order data to the file. */
1108 rc = writeOrderFile(Filename);
1109 if (rc)
1110 PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno));
1111
1112 // Restore SIGKILL.
1113 if (PDeathSig == 1)
1114 lprofRestoreSigKill();
1115
1116 return rc;
1117 }
1118
1119 COMPILER_RT_VISIBILITY
__llvm_orderfile_dump(void)1120 int __llvm_orderfile_dump(void) {
1121 int rc = __llvm_orderfile_write_file();
1122 return rc;
1123 }
1124
writeFileWithoutReturn(void)1125 static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); }
1126
1127 COMPILER_RT_VISIBILITY
__llvm_profile_register_write_file_atexit(void)1128 int __llvm_profile_register_write_file_atexit(void) {
1129 static int HasBeenRegistered = 0;
1130
1131 if (HasBeenRegistered)
1132 return 0;
1133
1134 lprofSetupValueProfiler();
1135
1136 HasBeenRegistered = 1;
1137 return atexit(writeFileWithoutReturn);
1138 }
1139
1140 #endif
1141