1 //===- OffloadBundler.cpp - File Bundling and Unbundling ------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements an offload bundling API that bundles different files
11 /// that relate with the same source code but different targets into a single
12 /// one. Also the implements the opposite functionality, i.e. unbundle files
13 /// previous created by this API.
14 ///
15 //===----------------------------------------------------------------------===//
16 
17 #include "clang/Driver/OffloadBundler.h"
18 #include "clang/Basic/Cuda.h"
19 #include "clang/Basic/TargetID.h"
20 #include "clang/Basic/Version.h"
21 #include "llvm/ADT/ArrayRef.h"
22 #include "llvm/ADT/SmallString.h"
23 #include "llvm/ADT/SmallVector.h"
24 #include "llvm/ADT/StringMap.h"
25 #include "llvm/ADT/StringRef.h"
26 #include "llvm/Object/Archive.h"
27 #include "llvm/Object/ArchiveWriter.h"
28 #include "llvm/Object/Binary.h"
29 #include "llvm/Object/ObjectFile.h"
30 #include "llvm/Support/Casting.h"
31 #include "llvm/Support/Debug.h"
32 #include "llvm/Support/EndianStream.h"
33 #include "llvm/Support/Errc.h"
34 #include "llvm/Support/Error.h"
35 #include "llvm/Support/ErrorOr.h"
36 #include "llvm/Support/FileSystem.h"
37 #include "llvm/Support/MemoryBuffer.h"
38 #include "llvm/Support/Path.h"
39 #include "llvm/Support/Program.h"
40 #include "llvm/Support/Signals.h"
41 #include "llvm/Support/StringSaver.h"
42 #include "llvm/Support/WithColor.h"
43 #include "llvm/Support/raw_ostream.h"
44 #include "llvm/TargetParser/Host.h"
45 #include "llvm/TargetParser/Triple.h"
46 #include <algorithm>
47 #include <cassert>
48 #include <cstddef>
49 #include <cstdint>
50 #include <forward_list>
51 #include <memory>
52 #include <set>
53 #include <string>
54 #include <system_error>
55 #include <utility>
56 
57 using namespace llvm;
58 using namespace llvm::object;
59 using namespace clang;
60 
61 /// Magic string that marks the existence of offloading data.
62 #define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
63 
64 OffloadTargetInfo::OffloadTargetInfo(const StringRef Target,
65                                      const OffloadBundlerConfig &BC)
66     : BundlerConfig(BC) {
67 
68   // TODO: Add error checking from ClangOffloadBundler.cpp
69   auto TargetFeatures = Target.split(':');
70   auto TripleOrGPU = TargetFeatures.first.rsplit('-');
71 
72   if (clang::StringToCudaArch(TripleOrGPU.second) != clang::CudaArch::UNKNOWN) {
73     auto KindTriple = TripleOrGPU.first.split('-');
74     this->OffloadKind = KindTriple.first;
75 
76     // Enforce optional env field to standardize bundles
77     llvm::Triple t = llvm::Triple(KindTriple.second);
78     this->Triple = llvm::Triple(t.getArchName(), t.getVendorName(),
79                                 t.getOSName(), t.getEnvironmentName());
80 
81     this->TargetID = Target.substr(Target.find(TripleOrGPU.second));
82   } else {
83     auto KindTriple = TargetFeatures.first.split('-');
84     this->OffloadKind = KindTriple.first;
85 
86     // Enforce optional env field to standardize bundles
87     llvm::Triple t = llvm::Triple(KindTriple.second);
88     this->Triple = llvm::Triple(t.getArchName(), t.getVendorName(),
89                                 t.getOSName(), t.getEnvironmentName());
90 
91     this->TargetID = "";
92   }
93 }
94 
95 bool OffloadTargetInfo::hasHostKind() const {
96   return this->OffloadKind == "host";
97 }
98 
99 bool OffloadTargetInfo::isOffloadKindValid() const {
100   return OffloadKind == "host" || OffloadKind == "openmp" ||
101          OffloadKind == "hip" || OffloadKind == "hipv4";
102 }
103 
104 bool OffloadTargetInfo::isOffloadKindCompatible(
105     const StringRef TargetOffloadKind) const {
106   if (OffloadKind == TargetOffloadKind)
107     return true;
108   if (BundlerConfig.HipOpenmpCompatible) {
109     bool HIPCompatibleWithOpenMP = OffloadKind.starts_with_insensitive("hip") &&
110                                    TargetOffloadKind == "openmp";
111     bool OpenMPCompatibleWithHIP =
112         OffloadKind == "openmp" &&
113         TargetOffloadKind.starts_with_insensitive("hip");
114     return HIPCompatibleWithOpenMP || OpenMPCompatibleWithHIP;
115   }
116   return false;
117 }
118 
119 bool OffloadTargetInfo::isTripleValid() const {
120   return !Triple.str().empty() && Triple.getArch() != Triple::UnknownArch;
121 }
122 
123 bool OffloadTargetInfo::operator==(const OffloadTargetInfo &Target) const {
124   return OffloadKind == Target.OffloadKind &&
125          Triple.isCompatibleWith(Target.Triple) && TargetID == Target.TargetID;
126 }
127 
128 std::string OffloadTargetInfo::str() const {
129   return Twine(OffloadKind + "-" + Triple.str() + "-" + TargetID).str();
130 }
131 
132 static StringRef getDeviceFileExtension(StringRef Device,
133                                         StringRef BundleFileName) {
134   if (Device.contains("gfx"))
135     return ".bc";
136   if (Device.contains("sm_"))
137     return ".cubin";
138   return sys::path::extension(BundleFileName);
139 }
140 
141 static std::string getDeviceLibraryFileName(StringRef BundleFileName,
142                                             StringRef Device) {
143   StringRef LibName = sys::path::stem(BundleFileName);
144   StringRef Extension = getDeviceFileExtension(Device, BundleFileName);
145 
146   std::string Result;
147   Result += LibName;
148   Result += Extension;
149   return Result;
150 }
151 
152 /// @brief Checks if a code object \p CodeObjectInfo is compatible with a given
153 /// target \p TargetInfo.
154 /// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id
155 bool isCodeObjectCompatible(const OffloadTargetInfo &CodeObjectInfo,
156                             const OffloadTargetInfo &TargetInfo) {
157 
158   // Compatible in case of exact match.
159   if (CodeObjectInfo == TargetInfo) {
160     DEBUG_WITH_TYPE("CodeObjectCompatibility",
161                     dbgs() << "Compatible: Exact match: \t[CodeObject: "
162                            << CodeObjectInfo.str()
163                            << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
164     return true;
165   }
166 
167   // Incompatible if Kinds or Triples mismatch.
168   if (!CodeObjectInfo.isOffloadKindCompatible(TargetInfo.OffloadKind) ||
169       !CodeObjectInfo.Triple.isCompatibleWith(TargetInfo.Triple)) {
170     DEBUG_WITH_TYPE(
171         "CodeObjectCompatibility",
172         dbgs() << "Incompatible: Kind/Triple mismatch \t[CodeObject: "
173                << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
174                << "]\n");
175     return false;
176   }
177 
178   // Incompatible if target IDs are incompatible.
179   if (!clang::isCompatibleTargetID(CodeObjectInfo.TargetID,
180                                    TargetInfo.TargetID)) {
181     DEBUG_WITH_TYPE(
182         "CodeObjectCompatibility",
183         dbgs() << "Incompatible: target IDs are incompatible \t[CodeObject: "
184                << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
185                << "]\n");
186     return false;
187   }
188 
189   DEBUG_WITH_TYPE(
190       "CodeObjectCompatibility",
191       dbgs() << "Compatible: Code Objects are compatible \t[CodeObject: "
192              << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
193              << "]\n");
194   return true;
195 }
196 
197 namespace {
198 /// Generic file handler interface.
199 class FileHandler {
200 public:
201   struct BundleInfo {
202     StringRef BundleID;
203   };
204 
205   FileHandler() {}
206 
207   virtual ~FileHandler() {}
208 
209   /// Update the file handler with information from the header of the bundled
210   /// file.
211   virtual Error ReadHeader(MemoryBuffer &Input) = 0;
212 
213   /// Read the marker of the next bundled to be read in the file. The bundle
214   /// name is returned if there is one in the file, or `std::nullopt` if there
215   /// are no more bundles to be read.
216   virtual Expected<std::optional<StringRef>>
217   ReadBundleStart(MemoryBuffer &Input) = 0;
218 
219   /// Read the marker that closes the current bundle.
220   virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0;
221 
222   /// Read the current bundle and write the result into the stream \a OS.
223   virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
224 
225   /// Write the header of the bundled file to \a OS based on the information
226   /// gathered from \a Inputs.
227   virtual Error WriteHeader(raw_fd_ostream &OS,
228                             ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) = 0;
229 
230   /// Write the marker that initiates a bundle for the triple \a TargetTriple to
231   /// \a OS.
232   virtual Error WriteBundleStart(raw_fd_ostream &OS,
233                                  StringRef TargetTriple) = 0;
234 
235   /// Write the marker that closes a bundle for the triple \a TargetTriple to \a
236   /// OS.
237   virtual Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
238 
239   /// Write the bundle from \a Input into \a OS.
240   virtual Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
241 
242   /// List bundle IDs in \a Input.
243   virtual Error listBundleIDs(MemoryBuffer &Input) {
244     if (Error Err = ReadHeader(Input))
245       return Err;
246     return forEachBundle(Input, [&](const BundleInfo &Info) -> Error {
247       llvm::outs() << Info.BundleID << '\n';
248       Error Err = listBundleIDsCallback(Input, Info);
249       if (Err)
250         return Err;
251       return Error::success();
252     });
253   }
254 
255   /// For each bundle in \a Input, do \a Func.
256   Error forEachBundle(MemoryBuffer &Input,
257                       std::function<Error(const BundleInfo &)> Func) {
258     while (true) {
259       Expected<std::optional<StringRef>> CurTripleOrErr =
260           ReadBundleStart(Input);
261       if (!CurTripleOrErr)
262         return CurTripleOrErr.takeError();
263 
264       // No more bundles.
265       if (!*CurTripleOrErr)
266         break;
267 
268       StringRef CurTriple = **CurTripleOrErr;
269       assert(!CurTriple.empty());
270 
271       BundleInfo Info{CurTriple};
272       if (Error Err = Func(Info))
273         return Err;
274     }
275     return Error::success();
276   }
277 
278 protected:
279   virtual Error listBundleIDsCallback(MemoryBuffer &Input,
280                                       const BundleInfo &Info) {
281     return Error::success();
282   }
283 };
284 
285 /// Handler for binary files. The bundled file will have the following format
286 /// (all integers are stored in little-endian format):
287 ///
288 /// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
289 ///
290 /// NumberOfOffloadBundles (8-byte integer)
291 ///
292 /// OffsetOfBundle1 (8-byte integer)
293 /// SizeOfBundle1 (8-byte integer)
294 /// NumberOfBytesInTripleOfBundle1 (8-byte integer)
295 /// TripleOfBundle1 (byte length defined before)
296 ///
297 /// ...
298 ///
299 /// OffsetOfBundleN (8-byte integer)
300 /// SizeOfBundleN (8-byte integer)
301 /// NumberOfBytesInTripleOfBundleN (8-byte integer)
302 /// TripleOfBundleN (byte length defined before)
303 ///
304 /// Bundle1
305 /// ...
306 /// BundleN
307 
308 /// Read 8-byte integers from a buffer in little-endian format.
309 static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
310   return llvm::support::endian::read64le(Buffer.data() + pos);
311 }
312 
313 /// Write 8-byte integers to a buffer in little-endian format.
314 static void Write8byteIntegerToBuffer(raw_fd_ostream &OS, uint64_t Val) {
315   llvm::support::endian::write(OS, Val, llvm::support::little);
316 }
317 
318 class BinaryFileHandler final : public FileHandler {
319   /// Information about the bundles extracted from the header.
320   struct BinaryBundleInfo final : public BundleInfo {
321     /// Size of the bundle.
322     uint64_t Size = 0u;
323     /// Offset at which the bundle starts in the bundled file.
324     uint64_t Offset = 0u;
325 
326     BinaryBundleInfo() {}
327     BinaryBundleInfo(uint64_t Size, uint64_t Offset)
328         : Size(Size), Offset(Offset) {}
329   };
330 
331   /// Map between a triple and the corresponding bundle information.
332   StringMap<BinaryBundleInfo> BundlesInfo;
333 
334   /// Iterator for the bundle information that is being read.
335   StringMap<BinaryBundleInfo>::iterator CurBundleInfo;
336   StringMap<BinaryBundleInfo>::iterator NextBundleInfo;
337 
338   /// Current bundle target to be written.
339   std::string CurWriteBundleTarget;
340 
341   /// Configuration options and arrays for this bundler job
342   const OffloadBundlerConfig &BundlerConfig;
343 
344 public:
345   // TODO: Add error checking from ClangOffloadBundler.cpp
346   BinaryFileHandler(const OffloadBundlerConfig &BC) : BundlerConfig(BC) {}
347 
348   ~BinaryFileHandler() final {}
349 
350   Error ReadHeader(MemoryBuffer &Input) final {
351     StringRef FC = Input.getBuffer();
352 
353     // Initialize the current bundle with the end of the container.
354     CurBundleInfo = BundlesInfo.end();
355 
356     // Check if buffer is smaller than magic string.
357     size_t ReadChars = sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
358     if (ReadChars > FC.size())
359       return Error::success();
360 
361     // Check if no magic was found.
362     StringRef Magic(FC.data(), sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
363     if (!Magic.equals(OFFLOAD_BUNDLER_MAGIC_STR))
364       return Error::success();
365 
366     // Read number of bundles.
367     if (ReadChars + 8 > FC.size())
368       return Error::success();
369 
370     uint64_t NumberOfBundles = Read8byteIntegerFromBuffer(FC, ReadChars);
371     ReadChars += 8;
372 
373     // Read bundle offsets, sizes and triples.
374     for (uint64_t i = 0; i < NumberOfBundles; ++i) {
375 
376       // Read offset.
377       if (ReadChars + 8 > FC.size())
378         return Error::success();
379 
380       uint64_t Offset = Read8byteIntegerFromBuffer(FC, ReadChars);
381       ReadChars += 8;
382 
383       // Read size.
384       if (ReadChars + 8 > FC.size())
385         return Error::success();
386 
387       uint64_t Size = Read8byteIntegerFromBuffer(FC, ReadChars);
388       ReadChars += 8;
389 
390       // Read triple size.
391       if (ReadChars + 8 > FC.size())
392         return Error::success();
393 
394       uint64_t TripleSize = Read8byteIntegerFromBuffer(FC, ReadChars);
395       ReadChars += 8;
396 
397       // Read triple.
398       if (ReadChars + TripleSize > FC.size())
399         return Error::success();
400 
401       StringRef Triple(&FC.data()[ReadChars], TripleSize);
402       ReadChars += TripleSize;
403 
404       // Check if the offset and size make sense.
405       if (!Offset || Offset + Size > FC.size())
406         return Error::success();
407 
408       assert(!BundlesInfo.contains(Triple) && "Triple is duplicated??");
409       BundlesInfo[Triple] = BinaryBundleInfo(Size, Offset);
410     }
411     // Set the iterator to where we will start to read.
412     CurBundleInfo = BundlesInfo.end();
413     NextBundleInfo = BundlesInfo.begin();
414     return Error::success();
415   }
416 
417   Expected<std::optional<StringRef>>
418   ReadBundleStart(MemoryBuffer &Input) final {
419     if (NextBundleInfo == BundlesInfo.end())
420       return std::nullopt;
421     CurBundleInfo = NextBundleInfo++;
422     return CurBundleInfo->first();
423   }
424 
425   Error ReadBundleEnd(MemoryBuffer &Input) final {
426     assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
427     return Error::success();
428   }
429 
430   Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
431     assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
432     StringRef FC = Input.getBuffer();
433     OS.write(FC.data() + CurBundleInfo->second.Offset,
434              CurBundleInfo->second.Size);
435     return Error::success();
436   }
437 
438   Error WriteHeader(raw_fd_ostream &OS,
439                     ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
440 
441     // Compute size of the header.
442     uint64_t HeaderSize = 0;
443 
444     HeaderSize += sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1;
445     HeaderSize += 8; // Number of Bundles
446 
447     for (auto &T : BundlerConfig.TargetNames) {
448       HeaderSize += 3 * 8; // Bundle offset, Size of bundle and size of triple.
449       HeaderSize += T.size(); // The triple.
450     }
451 
452     // Write to the buffer the header.
453     OS << OFFLOAD_BUNDLER_MAGIC_STR;
454 
455     Write8byteIntegerToBuffer(OS, BundlerConfig.TargetNames.size());
456 
457     unsigned Idx = 0;
458     for (auto &T : BundlerConfig.TargetNames) {
459       MemoryBuffer &MB = *Inputs[Idx++];
460       HeaderSize = alignTo(HeaderSize, BundlerConfig.BundleAlignment);
461       // Bundle offset.
462       Write8byteIntegerToBuffer(OS, HeaderSize);
463       // Size of the bundle (adds to the next bundle's offset)
464       Write8byteIntegerToBuffer(OS, MB.getBufferSize());
465       BundlesInfo[T] = BinaryBundleInfo(MB.getBufferSize(), HeaderSize);
466       HeaderSize += MB.getBufferSize();
467       // Size of the triple
468       Write8byteIntegerToBuffer(OS, T.size());
469       // Triple
470       OS << T;
471     }
472     return Error::success();
473   }
474 
475   Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
476     CurWriteBundleTarget = TargetTriple.str();
477     return Error::success();
478   }
479 
480   Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
481     return Error::success();
482   }
483 
484   Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
485     auto BI = BundlesInfo[CurWriteBundleTarget];
486     OS.seek(BI.Offset);
487     OS.write(Input.getBufferStart(), Input.getBufferSize());
488     return Error::success();
489   }
490 };
491 
492 // This class implements a list of temporary files that are removed upon
493 // object destruction.
494 class TempFileHandlerRAII {
495 public:
496   ~TempFileHandlerRAII() {
497     for (const auto &File : Files)
498       sys::fs::remove(File);
499   }
500 
501   // Creates temporary file with given contents.
502   Expected<StringRef> Create(std::optional<ArrayRef<char>> Contents) {
503     SmallString<128u> File;
504     if (std::error_code EC =
505             sys::fs::createTemporaryFile("clang-offload-bundler", "tmp", File))
506       return createFileError(File, EC);
507     Files.push_front(File);
508 
509     if (Contents) {
510       std::error_code EC;
511       raw_fd_ostream OS(File, EC);
512       if (EC)
513         return createFileError(File, EC);
514       OS.write(Contents->data(), Contents->size());
515     }
516     return Files.front().str();
517   }
518 
519 private:
520   std::forward_list<SmallString<128u>> Files;
521 };
522 
523 /// Handler for object files. The bundles are organized by sections with a
524 /// designated name.
525 ///
526 /// To unbundle, we just copy the contents of the designated section.
527 class ObjectFileHandler final : public FileHandler {
528 
529   /// The object file we are currently dealing with.
530   std::unique_ptr<ObjectFile> Obj;
531 
532   /// Return the input file contents.
533   StringRef getInputFileContents() const { return Obj->getData(); }
534 
535   /// Return bundle name (<kind>-<triple>) if the provided section is an offload
536   /// section.
537   static Expected<std::optional<StringRef>>
538   IsOffloadSection(SectionRef CurSection) {
539     Expected<StringRef> NameOrErr = CurSection.getName();
540     if (!NameOrErr)
541       return NameOrErr.takeError();
542 
543     // If it does not start with the reserved suffix, just skip this section.
544     if (!NameOrErr->startswith(OFFLOAD_BUNDLER_MAGIC_STR))
545       return std::nullopt;
546 
547     // Return the triple that is right after the reserved prefix.
548     return NameOrErr->substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
549   }
550 
551   /// Total number of inputs.
552   unsigned NumberOfInputs = 0;
553 
554   /// Total number of processed inputs, i.e, inputs that were already
555   /// read from the buffers.
556   unsigned NumberOfProcessedInputs = 0;
557 
558   /// Iterator of the current and next section.
559   section_iterator CurrentSection;
560   section_iterator NextSection;
561 
562   /// Configuration options and arrays for this bundler job
563   const OffloadBundlerConfig &BundlerConfig;
564 
565 public:
566   // TODO: Add error checking from ClangOffloadBundler.cpp
567   ObjectFileHandler(std::unique_ptr<ObjectFile> ObjIn,
568                     const OffloadBundlerConfig &BC)
569       : Obj(std::move(ObjIn)), CurrentSection(Obj->section_begin()),
570         NextSection(Obj->section_begin()), BundlerConfig(BC) {}
571 
572   ~ObjectFileHandler() final {}
573 
574   Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
575 
576   Expected<std::optional<StringRef>>
577   ReadBundleStart(MemoryBuffer &Input) final {
578     while (NextSection != Obj->section_end()) {
579       CurrentSection = NextSection;
580       ++NextSection;
581 
582       // Check if the current section name starts with the reserved prefix. If
583       // so, return the triple.
584       Expected<std::optional<StringRef>> TripleOrErr =
585           IsOffloadSection(*CurrentSection);
586       if (!TripleOrErr)
587         return TripleOrErr.takeError();
588       if (*TripleOrErr)
589         return **TripleOrErr;
590     }
591     return std::nullopt;
592   }
593 
594   Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
595 
596   Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
597     Expected<StringRef> ContentOrErr = CurrentSection->getContents();
598     if (!ContentOrErr)
599       return ContentOrErr.takeError();
600     StringRef Content = *ContentOrErr;
601 
602     // Copy fat object contents to the output when extracting host bundle.
603     if (Content.size() == 1u && Content.front() == 0)
604       Content = StringRef(Input.getBufferStart(), Input.getBufferSize());
605 
606     OS.write(Content.data(), Content.size());
607     return Error::success();
608   }
609 
610   Error WriteHeader(raw_fd_ostream &OS,
611                     ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
612     assert(BundlerConfig.HostInputIndex != ~0u &&
613            "Host input index not defined.");
614 
615     // Record number of inputs.
616     NumberOfInputs = Inputs.size();
617     return Error::success();
618   }
619 
620   Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
621     ++NumberOfProcessedInputs;
622     return Error::success();
623   }
624 
625   Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
626     assert(NumberOfProcessedInputs <= NumberOfInputs &&
627            "Processing more inputs that actually exist!");
628     assert(BundlerConfig.HostInputIndex != ~0u &&
629            "Host input index not defined.");
630 
631     // If this is not the last output, we don't have to do anything.
632     if (NumberOfProcessedInputs != NumberOfInputs)
633       return Error::success();
634 
635     // We will use llvm-objcopy to add target objects sections to the output
636     // fat object. These sections should have 'exclude' flag set which tells
637     // link editor to remove them from linker inputs when linking executable or
638     // shared library.
639 
640     assert(BundlerConfig.ObjcopyPath != "" &&
641            "llvm-objcopy path not specified");
642 
643     // We write to the output file directly. So, we close it and use the name
644     // to pass down to llvm-objcopy.
645     OS.close();
646 
647     // Temporary files that need to be removed.
648     TempFileHandlerRAII TempFiles;
649 
650     // Compose llvm-objcopy command line for add target objects' sections with
651     // appropriate flags.
652     BumpPtrAllocator Alloc;
653     StringSaver SS{Alloc};
654     SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"};
655 
656     for (unsigned I = 0; I < NumberOfInputs; ++I) {
657       StringRef InputFile = BundlerConfig.InputFileNames[I];
658       if (I == BundlerConfig.HostInputIndex) {
659         // Special handling for the host bundle. We do not need to add a
660         // standard bundle for the host object since we are going to use fat
661         // object as a host object. Therefore use dummy contents (one zero byte)
662         // when creating section for the host bundle.
663         Expected<StringRef> TempFileOrErr = TempFiles.Create(ArrayRef<char>(0));
664         if (!TempFileOrErr)
665           return TempFileOrErr.takeError();
666         InputFile = *TempFileOrErr;
667       }
668 
669       ObjcopyArgs.push_back(
670           SS.save(Twine("--add-section=") + OFFLOAD_BUNDLER_MAGIC_STR +
671                   BundlerConfig.TargetNames[I] + "=" + InputFile));
672       ObjcopyArgs.push_back(
673           SS.save(Twine("--set-section-flags=") + OFFLOAD_BUNDLER_MAGIC_STR +
674                   BundlerConfig.TargetNames[I] + "=readonly,exclude"));
675     }
676     ObjcopyArgs.push_back("--");
677     ObjcopyArgs.push_back(
678         BundlerConfig.InputFileNames[BundlerConfig.HostInputIndex]);
679     ObjcopyArgs.push_back(BundlerConfig.OutputFileNames.front());
680 
681     if (Error Err = executeObjcopy(BundlerConfig.ObjcopyPath, ObjcopyArgs))
682       return Err;
683 
684     return Error::success();
685   }
686 
687   Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
688     return Error::success();
689   }
690 
691 private:
692   Error executeObjcopy(StringRef Objcopy, ArrayRef<StringRef> Args) {
693     // If the user asked for the commands to be printed out, we do that
694     // instead of executing it.
695     if (BundlerConfig.PrintExternalCommands) {
696       errs() << "\"" << Objcopy << "\"";
697       for (StringRef Arg : drop_begin(Args, 1))
698         errs() << " \"" << Arg << "\"";
699       errs() << "\n";
700     } else {
701       if (sys::ExecuteAndWait(Objcopy, Args))
702         return createStringError(inconvertibleErrorCode(),
703                                  "'llvm-objcopy' tool failed");
704     }
705     return Error::success();
706   }
707 };
708 
709 /// Handler for text files. The bundled file will have the following format.
710 ///
711 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
712 /// Bundle 1
713 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
714 /// ...
715 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
716 /// Bundle N
717 /// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
718 class TextFileHandler final : public FileHandler {
719   /// String that begins a line comment.
720   StringRef Comment;
721 
722   /// String that initiates a bundle.
723   std::string BundleStartString;
724 
725   /// String that closes a bundle.
726   std::string BundleEndString;
727 
728   /// Number of chars read from input.
729   size_t ReadChars = 0u;
730 
731 protected:
732   Error ReadHeader(MemoryBuffer &Input) final { return Error::success(); }
733 
734   Expected<std::optional<StringRef>>
735   ReadBundleStart(MemoryBuffer &Input) final {
736     StringRef FC = Input.getBuffer();
737 
738     // Find start of the bundle.
739     ReadChars = FC.find(BundleStartString, ReadChars);
740     if (ReadChars == FC.npos)
741       return std::nullopt;
742 
743     // Get position of the triple.
744     size_t TripleStart = ReadChars = ReadChars + BundleStartString.size();
745 
746     // Get position that closes the triple.
747     size_t TripleEnd = ReadChars = FC.find("\n", ReadChars);
748     if (TripleEnd == FC.npos)
749       return std::nullopt;
750 
751     // Next time we read after the new line.
752     ++ReadChars;
753 
754     return StringRef(&FC.data()[TripleStart], TripleEnd - TripleStart);
755   }
756 
757   Error ReadBundleEnd(MemoryBuffer &Input) final {
758     StringRef FC = Input.getBuffer();
759 
760     // Read up to the next new line.
761     assert(FC[ReadChars] == '\n' && "The bundle should end with a new line.");
762 
763     size_t TripleEnd = ReadChars = FC.find("\n", ReadChars + 1);
764     if (TripleEnd != FC.npos)
765       // Next time we read after the new line.
766       ++ReadChars;
767 
768     return Error::success();
769   }
770 
771   Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
772     StringRef FC = Input.getBuffer();
773     size_t BundleStart = ReadChars;
774 
775     // Find end of the bundle.
776     size_t BundleEnd = ReadChars = FC.find(BundleEndString, ReadChars);
777 
778     StringRef Bundle(&FC.data()[BundleStart], BundleEnd - BundleStart);
779     OS << Bundle;
780 
781     return Error::success();
782   }
783 
784   Error WriteHeader(raw_fd_ostream &OS,
785                     ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) final {
786     return Error::success();
787   }
788 
789   Error WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) final {
790     OS << BundleStartString << TargetTriple << "\n";
791     return Error::success();
792   }
793 
794   Error WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) final {
795     OS << BundleEndString << TargetTriple << "\n";
796     return Error::success();
797   }
798 
799   Error WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
800     OS << Input.getBuffer();
801     return Error::success();
802   }
803 
804 public:
805   TextFileHandler(StringRef Comment) : Comment(Comment), ReadChars(0) {
806     BundleStartString =
807         "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__START__ ";
808     BundleEndString =
809         "\n" + Comment.str() + " " OFFLOAD_BUNDLER_MAGIC_STR "__END__ ";
810   }
811 
812   Error listBundleIDsCallback(MemoryBuffer &Input,
813                               const BundleInfo &Info) final {
814     // TODO: To list bundle IDs in a bundled text file we need to go through
815     // all bundles. The format of bundled text file may need to include a
816     // header if the performance of listing bundle IDs of bundled text file is
817     // important.
818     ReadChars = Input.getBuffer().find(BundleEndString, ReadChars);
819     if (Error Err = ReadBundleEnd(Input))
820       return Err;
821     return Error::success();
822   }
823 };
824 } // namespace
825 
826 /// Return an appropriate object file handler. We use the specific object
827 /// handler if we know how to deal with that format, otherwise we use a default
828 /// binary file handler.
829 static std::unique_ptr<FileHandler>
830 CreateObjectFileHandler(MemoryBuffer &FirstInput,
831                         const OffloadBundlerConfig &BundlerConfig) {
832   // Check if the input file format is one that we know how to deal with.
833   Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
834 
835   // We only support regular object files. If failed to open the input as a
836   // known binary or this is not an object file use the default binary handler.
837   if (errorToBool(BinaryOrErr.takeError()) || !isa<ObjectFile>(*BinaryOrErr))
838     return std::make_unique<BinaryFileHandler>(BundlerConfig);
839 
840   // Otherwise create an object file handler. The handler will be owned by the
841   // client of this function.
842   return std::make_unique<ObjectFileHandler>(
843       std::unique_ptr<ObjectFile>(cast<ObjectFile>(BinaryOrErr->release())),
844       BundlerConfig);
845 }
846 
847 /// Return an appropriate handler given the input files and options.
848 static Expected<std::unique_ptr<FileHandler>>
849 CreateFileHandler(MemoryBuffer &FirstInput,
850                   const OffloadBundlerConfig &BundlerConfig) {
851   std::string FilesType = BundlerConfig.FilesType;
852 
853   if (FilesType == "i")
854     return std::make_unique<TextFileHandler>(/*Comment=*/"//");
855   if (FilesType == "ii")
856     return std::make_unique<TextFileHandler>(/*Comment=*/"//");
857   if (FilesType == "cui")
858     return std::make_unique<TextFileHandler>(/*Comment=*/"//");
859   if (FilesType == "hipi")
860     return std::make_unique<TextFileHandler>(/*Comment=*/"//");
861   // TODO: `.d` should be eventually removed once `-M` and its variants are
862   // handled properly in offload compilation.
863   if (FilesType == "d")
864     return std::make_unique<TextFileHandler>(/*Comment=*/"#");
865   if (FilesType == "ll")
866     return std::make_unique<TextFileHandler>(/*Comment=*/";");
867   if (FilesType == "bc")
868     return std::make_unique<BinaryFileHandler>(BundlerConfig);
869   if (FilesType == "s")
870     return std::make_unique<TextFileHandler>(/*Comment=*/"#");
871   if (FilesType == "o")
872     return CreateObjectFileHandler(FirstInput, BundlerConfig);
873   if (FilesType == "a")
874     return CreateObjectFileHandler(FirstInput, BundlerConfig);
875   if (FilesType == "gch")
876     return std::make_unique<BinaryFileHandler>(BundlerConfig);
877   if (FilesType == "ast")
878     return std::make_unique<BinaryFileHandler>(BundlerConfig);
879 
880   return createStringError(errc::invalid_argument,
881                            "'" + FilesType + "': invalid file type specified");
882 }
883 
884 // List bundle IDs. Return true if an error was found.
885 Error OffloadBundler::ListBundleIDsInFile(
886     StringRef InputFileName, const OffloadBundlerConfig &BundlerConfig) {
887   // Open Input file.
888   ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
889       MemoryBuffer::getFileOrSTDIN(InputFileName);
890   if (std::error_code EC = CodeOrErr.getError())
891     return createFileError(InputFileName, EC);
892 
893   MemoryBuffer &Input = **CodeOrErr;
894 
895   // Select the right files handler.
896   Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
897       CreateFileHandler(Input, BundlerConfig);
898   if (!FileHandlerOrErr)
899     return FileHandlerOrErr.takeError();
900 
901   std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
902   assert(FH);
903   return FH->listBundleIDs(Input);
904 }
905 
906 /// Bundle the files. Return true if an error was found.
907 Error OffloadBundler::BundleFiles() {
908   std::error_code EC;
909 
910   // Create output file.
911   raw_fd_ostream OutputFile(BundlerConfig.OutputFileNames.front(), EC,
912                             sys::fs::OF_None);
913   if (EC)
914     return createFileError(BundlerConfig.OutputFileNames.front(), EC);
915 
916   // Open input files.
917   SmallVector<std::unique_ptr<MemoryBuffer>, 8u> InputBuffers;
918   InputBuffers.reserve(BundlerConfig.InputFileNames.size());
919   for (auto &I : BundlerConfig.InputFileNames) {
920     ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
921         MemoryBuffer::getFileOrSTDIN(I);
922     if (std::error_code EC = CodeOrErr.getError())
923       return createFileError(I, EC);
924     InputBuffers.emplace_back(std::move(*CodeOrErr));
925   }
926 
927   // Get the file handler. We use the host buffer as reference.
928   assert((BundlerConfig.HostInputIndex != ~0u || BundlerConfig.AllowNoHost) &&
929          "Host input index undefined??");
930   Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr = CreateFileHandler(
931       *InputBuffers[BundlerConfig.AllowNoHost ? 0
932                                               : BundlerConfig.HostInputIndex],
933       BundlerConfig);
934   if (!FileHandlerOrErr)
935     return FileHandlerOrErr.takeError();
936 
937   std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
938   assert(FH);
939 
940   // Write header.
941   if (Error Err = FH->WriteHeader(OutputFile, InputBuffers))
942     return Err;
943 
944   // Write all bundles along with the start/end markers. If an error was found
945   // writing the end of the bundle component, abort the bundle writing.
946   auto Input = InputBuffers.begin();
947   for (auto &Triple : BundlerConfig.TargetNames) {
948     if (Error Err = FH->WriteBundleStart(OutputFile, Triple))
949       return Err;
950     if (Error Err = FH->WriteBundle(OutputFile, **Input))
951       return Err;
952     if (Error Err = FH->WriteBundleEnd(OutputFile, Triple))
953       return Err;
954     ++Input;
955   }
956   return Error::success();
957 }
958 
959 // Unbundle the files. Return true if an error was found.
960 Error OffloadBundler::UnbundleFiles() {
961   // Open Input file.
962   ErrorOr<std::unique_ptr<MemoryBuffer>> CodeOrErr =
963       MemoryBuffer::getFileOrSTDIN(BundlerConfig.InputFileNames.front());
964   if (std::error_code EC = CodeOrErr.getError())
965     return createFileError(BundlerConfig.InputFileNames.front(), EC);
966 
967   MemoryBuffer &Input = **CodeOrErr;
968 
969   // Select the right files handler.
970   Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
971       CreateFileHandler(Input, BundlerConfig);
972   if (!FileHandlerOrErr)
973     return FileHandlerOrErr.takeError();
974 
975   std::unique_ptr<FileHandler> &FH = *FileHandlerOrErr;
976   assert(FH);
977 
978   // Read the header of the bundled file.
979   if (Error Err = FH->ReadHeader(Input))
980     return Err;
981 
982   // Create a work list that consist of the map triple/output file.
983   StringMap<StringRef> Worklist;
984   auto Output = BundlerConfig.OutputFileNames.begin();
985   for (auto &Triple : BundlerConfig.TargetNames) {
986     Worklist[Triple] = *Output;
987     ++Output;
988   }
989 
990   // Read all the bundles that are in the work list. If we find no bundles we
991   // assume the file is meant for the host target.
992   bool FoundHostBundle = false;
993   while (!Worklist.empty()) {
994     Expected<std::optional<StringRef>> CurTripleOrErr =
995         FH->ReadBundleStart(Input);
996     if (!CurTripleOrErr)
997       return CurTripleOrErr.takeError();
998 
999     // We don't have more bundles.
1000     if (!*CurTripleOrErr)
1001       break;
1002 
1003     StringRef CurTriple = **CurTripleOrErr;
1004     assert(!CurTriple.empty());
1005 
1006     auto Output = Worklist.begin();
1007     for (auto E = Worklist.end(); Output != E; Output++) {
1008       if (isCodeObjectCompatible(
1009               OffloadTargetInfo(CurTriple, BundlerConfig),
1010               OffloadTargetInfo((*Output).first(), BundlerConfig))) {
1011         break;
1012       }
1013     }
1014 
1015     if (Output == Worklist.end())
1016       continue;
1017     // Check if the output file can be opened and copy the bundle to it.
1018     std::error_code EC;
1019     raw_fd_ostream OutputFile((*Output).second, EC, sys::fs::OF_None);
1020     if (EC)
1021       return createFileError((*Output).second, EC);
1022     if (Error Err = FH->ReadBundle(OutputFile, Input))
1023       return Err;
1024     if (Error Err = FH->ReadBundleEnd(Input))
1025       return Err;
1026     Worklist.erase(Output);
1027 
1028     // Record if we found the host bundle.
1029     auto OffloadInfo = OffloadTargetInfo(CurTriple, BundlerConfig);
1030     if (OffloadInfo.hasHostKind())
1031       FoundHostBundle = true;
1032   }
1033 
1034   if (!BundlerConfig.AllowMissingBundles && !Worklist.empty()) {
1035     std::string ErrMsg = "Can't find bundles for";
1036     std::set<StringRef> Sorted;
1037     for (auto &E : Worklist)
1038       Sorted.insert(E.first());
1039     unsigned I = 0;
1040     unsigned Last = Sorted.size() - 1;
1041     for (auto &E : Sorted) {
1042       if (I != 0 && Last > 1)
1043         ErrMsg += ",";
1044       ErrMsg += " ";
1045       if (I == Last && I != 0)
1046         ErrMsg += "and ";
1047       ErrMsg += E.str();
1048       ++I;
1049     }
1050     return createStringError(inconvertibleErrorCode(), ErrMsg);
1051   }
1052 
1053   // If no bundles were found, assume the input file is the host bundle and
1054   // create empty files for the remaining targets.
1055   if (Worklist.size() == BundlerConfig.TargetNames.size()) {
1056     for (auto &E : Worklist) {
1057       std::error_code EC;
1058       raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1059       if (EC)
1060         return createFileError(E.second, EC);
1061 
1062       // If this entry has a host kind, copy the input file to the output file.
1063       auto OffloadInfo = OffloadTargetInfo(E.getKey(), BundlerConfig);
1064       if (OffloadInfo.hasHostKind())
1065         OutputFile.write(Input.getBufferStart(), Input.getBufferSize());
1066     }
1067     return Error::success();
1068   }
1069 
1070   // If we found elements, we emit an error if none of those were for the host
1071   // in case host bundle name was provided in command line.
1072   if (!(FoundHostBundle || BundlerConfig.HostInputIndex == ~0u ||
1073         BundlerConfig.AllowMissingBundles))
1074     return createStringError(inconvertibleErrorCode(),
1075                              "Can't find bundle for the host target");
1076 
1077   // If we still have any elements in the worklist, create empty files for them.
1078   for (auto &E : Worklist) {
1079     std::error_code EC;
1080     raw_fd_ostream OutputFile(E.second, EC, sys::fs::OF_None);
1081     if (EC)
1082       return createFileError(E.second, EC);
1083   }
1084 
1085   return Error::success();
1086 }
1087 
1088 static Archive::Kind getDefaultArchiveKindForHost() {
1089   return Triple(sys::getDefaultTargetTriple()).isOSDarwin() ? Archive::K_DARWIN
1090                                                             : Archive::K_GNU;
1091 }
1092 
1093 /// @brief Computes a list of targets among all given targets which are
1094 /// compatible with this code object
1095 /// @param [in] CodeObjectInfo Code Object
1096 /// @param [out] CompatibleTargets List of all compatible targets among all
1097 /// given targets
1098 /// @return false, if no compatible target is found.
1099 static bool
1100 getCompatibleOffloadTargets(OffloadTargetInfo &CodeObjectInfo,
1101                             SmallVectorImpl<StringRef> &CompatibleTargets,
1102                             const OffloadBundlerConfig &BundlerConfig) {
1103   if (!CompatibleTargets.empty()) {
1104     DEBUG_WITH_TYPE("CodeObjectCompatibility",
1105                     dbgs() << "CompatibleTargets list should be empty\n");
1106     return false;
1107   }
1108   for (auto &Target : BundlerConfig.TargetNames) {
1109     auto TargetInfo = OffloadTargetInfo(Target, BundlerConfig);
1110     if (isCodeObjectCompatible(CodeObjectInfo, TargetInfo))
1111       CompatibleTargets.push_back(Target);
1112   }
1113   return !CompatibleTargets.empty();
1114 }
1115 
1116 /// UnbundleArchive takes an archive file (".a") as input containing bundled
1117 /// code object files, and a list of offload targets (not host), and extracts
1118 /// the code objects into a new archive file for each offload target. Each
1119 /// resulting archive file contains all code object files corresponding to that
1120 /// particular offload target. The created archive file does not
1121 /// contain an index of the symbols and code object files are named as
1122 /// <<Parent Bundle Name>-<CodeObject's GPUArch>>, with ':' replaced with '_'.
1123 Error OffloadBundler::UnbundleArchive() {
1124   std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
1125 
1126   /// Map of target names with list of object files that will form the device
1127   /// specific archive for that target
1128   StringMap<std::vector<NewArchiveMember>> OutputArchivesMap;
1129 
1130   // Map of target names and output archive filenames
1131   StringMap<StringRef> TargetOutputFileNameMap;
1132 
1133   auto Output = BundlerConfig.OutputFileNames.begin();
1134   for (auto &Target : BundlerConfig.TargetNames) {
1135     TargetOutputFileNameMap[Target] = *Output;
1136     ++Output;
1137   }
1138 
1139   StringRef IFName = BundlerConfig.InputFileNames.front();
1140 
1141   ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
1142       MemoryBuffer::getFileOrSTDIN(IFName, true, false);
1143   if (std::error_code EC = BufOrErr.getError())
1144     return createFileError(BundlerConfig.InputFileNames.front(), EC);
1145 
1146   ArchiveBuffers.push_back(std::move(*BufOrErr));
1147   Expected<std::unique_ptr<llvm::object::Archive>> LibOrErr =
1148       Archive::create(ArchiveBuffers.back()->getMemBufferRef());
1149   if (!LibOrErr)
1150     return LibOrErr.takeError();
1151 
1152   auto Archive = std::move(*LibOrErr);
1153 
1154   Error ArchiveErr = Error::success();
1155   auto ChildEnd = Archive->child_end();
1156 
1157   /// Iterate over all bundled code object files in the input archive.
1158   for (auto ArchiveIter = Archive->child_begin(ArchiveErr);
1159        ArchiveIter != ChildEnd; ++ArchiveIter) {
1160     if (ArchiveErr)
1161       return ArchiveErr;
1162     auto ArchiveChildNameOrErr = (*ArchiveIter).getName();
1163     if (!ArchiveChildNameOrErr)
1164       return ArchiveChildNameOrErr.takeError();
1165 
1166     StringRef BundledObjectFile = sys::path::filename(*ArchiveChildNameOrErr);
1167 
1168     auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef();
1169     if (!CodeObjectBufferRefOrErr)
1170       return CodeObjectBufferRefOrErr.takeError();
1171 
1172     auto CodeObjectBuffer =
1173         MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
1174 
1175     Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
1176         CreateFileHandler(*CodeObjectBuffer, BundlerConfig);
1177     if (!FileHandlerOrErr)
1178       return FileHandlerOrErr.takeError();
1179 
1180     std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
1181     assert(FileHandler &&
1182            "FileHandle creation failed for file in the archive!");
1183 
1184     if (Error ReadErr = FileHandler->ReadHeader(*CodeObjectBuffer))
1185       return ReadErr;
1186 
1187     Expected<std::optional<StringRef>> CurBundleIDOrErr =
1188         FileHandler->ReadBundleStart(*CodeObjectBuffer);
1189     if (!CurBundleIDOrErr)
1190       return CurBundleIDOrErr.takeError();
1191 
1192     std::optional<StringRef> OptionalCurBundleID = *CurBundleIDOrErr;
1193     // No device code in this child, skip.
1194     if (!OptionalCurBundleID)
1195       continue;
1196     StringRef CodeObject = *OptionalCurBundleID;
1197 
1198     // Process all bundle entries (CodeObjects) found in this child of input
1199     // archive.
1200     while (!CodeObject.empty()) {
1201       SmallVector<StringRef> CompatibleTargets;
1202       auto CodeObjectInfo = OffloadTargetInfo(CodeObject, BundlerConfig);
1203       if (CodeObjectInfo.hasHostKind()) {
1204         // Do nothing, we don't extract host code yet.
1205       } else if (getCompatibleOffloadTargets(CodeObjectInfo, CompatibleTargets,
1206                                              BundlerConfig)) {
1207         std::string BundleData;
1208         raw_string_ostream DataStream(BundleData);
1209         if (Error Err = FileHandler->ReadBundle(DataStream, *CodeObjectBuffer))
1210           return Err;
1211 
1212         for (auto &CompatibleTarget : CompatibleTargets) {
1213           SmallString<128> BundledObjectFileName;
1214           BundledObjectFileName.assign(BundledObjectFile);
1215           auto OutputBundleName =
1216               Twine(llvm::sys::path::stem(BundledObjectFileName) + "-" +
1217                     CodeObject +
1218                     getDeviceLibraryFileName(BundledObjectFileName,
1219                                              CodeObjectInfo.TargetID))
1220                   .str();
1221           // Replace ':' in optional target feature list with '_' to ensure
1222           // cross-platform validity.
1223           std::replace(OutputBundleName.begin(), OutputBundleName.end(), ':',
1224                        '_');
1225 
1226           std::unique_ptr<MemoryBuffer> MemBuf = MemoryBuffer::getMemBufferCopy(
1227               DataStream.str(), OutputBundleName);
1228           ArchiveBuffers.push_back(std::move(MemBuf));
1229           llvm::MemoryBufferRef MemBufRef =
1230               MemoryBufferRef(*(ArchiveBuffers.back()));
1231 
1232           // For inserting <CompatibleTarget, list<CodeObject>> entry in
1233           // OutputArchivesMap.
1234           if (!OutputArchivesMap.contains(CompatibleTarget)) {
1235 
1236             std::vector<NewArchiveMember> ArchiveMembers;
1237             ArchiveMembers.push_back(NewArchiveMember(MemBufRef));
1238             OutputArchivesMap.insert_or_assign(CompatibleTarget,
1239                                                std::move(ArchiveMembers));
1240           } else {
1241             OutputArchivesMap[CompatibleTarget].push_back(
1242                 NewArchiveMember(MemBufRef));
1243           }
1244         }
1245       }
1246 
1247       if (Error Err = FileHandler->ReadBundleEnd(*CodeObjectBuffer))
1248         return Err;
1249 
1250       Expected<std::optional<StringRef>> NextTripleOrErr =
1251           FileHandler->ReadBundleStart(*CodeObjectBuffer);
1252       if (!NextTripleOrErr)
1253         return NextTripleOrErr.takeError();
1254 
1255       CodeObject = ((*NextTripleOrErr).has_value()) ? **NextTripleOrErr : "";
1256     } // End of processing of all bundle entries of this child of input archive.
1257   }   // End of while over children of input archive.
1258 
1259   assert(!ArchiveErr && "Error occurred while reading archive!");
1260 
1261   /// Write out an archive for each target
1262   for (auto &Target : BundlerConfig.TargetNames) {
1263     StringRef FileName = TargetOutputFileNameMap[Target];
1264     StringMapIterator<std::vector<llvm::NewArchiveMember>> CurArchiveMembers =
1265         OutputArchivesMap.find(Target);
1266     if (CurArchiveMembers != OutputArchivesMap.end()) {
1267       if (Error WriteErr = writeArchive(FileName, CurArchiveMembers->getValue(),
1268                                         true, getDefaultArchiveKindForHost(),
1269                                         true, false, nullptr))
1270         return WriteErr;
1271     } else if (!BundlerConfig.AllowMissingBundles) {
1272       std::string ErrMsg =
1273           Twine("no compatible code object found for the target '" + Target +
1274                 "' in heterogeneous archive library: " + IFName)
1275               .str();
1276       return createStringError(inconvertibleErrorCode(), ErrMsg);
1277     } else { // Create an empty archive file if no compatible code object is
1278              // found and "allow-missing-bundles" is enabled. It ensures that
1279              // the linker using output of this step doesn't complain about
1280              // the missing input file.
1281       std::vector<llvm::NewArchiveMember> EmptyArchive;
1282       EmptyArchive.clear();
1283       if (Error WriteErr = writeArchive(FileName, EmptyArchive, true,
1284                                         getDefaultArchiveKindForHost(), true,
1285                                         false, nullptr))
1286         return WriteErr;
1287     }
1288   }
1289 
1290   return Error::success();
1291 }
1292