1 /*========================== begin_copyright_notice ============================
2
3 Copyright (C) 2020-2021 Intel Corporation
4
5 SPDX-License-Identifier: MIT
6
7 ============================= end_copyright_notice ===========================*/
8
9 #include "SPIRVWrapper.h"
10 #include "vc/Support/PassManager.h"
11
12 #include "vc/Driver/Driver.h"
13
14 #include "igc/Options/Options.h"
15 #include "vc/GenXCodeGen/GenXOCLRuntimeInfo.h"
16 #include "vc/GenXCodeGen/GenXTarget.h"
17 #include "vc/GenXOpts/GenXOpts.h"
18 #include "vc/GenXOpts/Utils/KernelInfo.h"
19 #include "vc/Support/BackendConfig.h"
20 #include "vc/Support/Status.h"
21 #include "llvm/GenXIntrinsics/GenXIntrOpts.h"
22 #include "llvm/GenXIntrinsics/GenXIntrinsics.h"
23 #include "llvm/GenXIntrinsics/GenXSPIRVReaderAdaptor.h"
24
25 #include "llvm/ADT/ScopeExit.h"
26 #include "llvm/ADT/SmallString.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/Statistic.h"
29 #include "llvm/ADT/StringExtras.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/Analysis/TargetLibraryInfo.h"
32 #include "llvm/Analysis/TargetTransformInfo.h"
33 #include "llvm/Bitcode/BitcodeReader.h"
34 #include "llvm/CodeGen/MachineModuleInfo.h"
35 #include "llvm/IR/LLVMContext.h"
36 #include "llvm/IR/Verifier.h"
37 #include "llvm/IRReader/IRReader.h"
38 #include "llvm/InitializePasses.h"
39 #include "llvm/Option/ArgList.h"
40 #include "llvm/Support/Allocator.h"
41 #include "llvm/Support/CommandLine.h"
42 #include "llvm/Support/Error.h"
43 #include "llvm/Support/FileSystem.h"
44 #include "llvm/Support/MemoryBuffer.h"
45 #include "llvm/Support/SourceMgr.h"
46 #include "llvm/Support/StringSaver.h"
47 #include "llvm/Support/TargetRegistry.h"
48 #include "llvm/Support/TargetSelect.h"
49 #include "llvm/Support/Timer.h"
50 #include "llvm/Support/raw_ostream.h"
51 #include "llvm/Target/TargetOptions.h"
52 #include "llvm/Transforms/IPO.h"
53 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
54 #include "llvm/Transforms/Scalar.h"
55
56 #include "llvmWrapper/Option/OptTable.h"
57 #include "llvmWrapper/Target/TargetMachine.h"
58
59 #include "Probe/Assertion.h"
60
61 #include <cctype>
62 #include <memory>
63 #include <new>
64 #include <set>
65 #include <sstream>
66 #include <string>
67 #include <vector>
68
69 using namespace llvm;
70
71 static Expected<std::unique_ptr<llvm::Module>>
getModuleFromLLVMText(ArrayRef<char> Input,LLVMContext & C)72 getModuleFromLLVMText(ArrayRef<char> Input, LLVMContext &C) {
73 SMDiagnostic Err;
74 llvm::MemoryBufferRef BufferRef(llvm::StringRef(Input.data(), Input.size()),
75 "LLVM IR Module");
76 Expected<std::unique_ptr<llvm::Module>> ExpModule =
77 llvm::parseIR(BufferRef, Err, C);
78
79 if (!ExpModule)
80 Err.print("getModuleLL", errs());
81
82 if (verifyModule(*ExpModule.get()))
83 return make_error<vc::InvalidModuleError>();
84
85 return ExpModule;
86 }
87
88 static Expected<std::unique_ptr<llvm::Module>>
getModuleFromLLVMBinary(ArrayRef<char> Input,LLVMContext & C)89 getModuleFromLLVMBinary(ArrayRef<char> Input, LLVMContext& C) {
90
91 llvm::MemoryBufferRef BufferRef(llvm::StringRef(Input.data(), Input.size()),
92 "Deserialized LLVM Module");
93 auto ExpModule = llvm::parseBitcodeFile(BufferRef, C);
94
95 if (!ExpModule)
96 return llvm::handleExpected(
97 std::move(ExpModule),
98 []() -> llvm::Error {
99 IGC_ASSERT_EXIT_MESSAGE(0, "Should create new error");
100 },
101 [](const llvm::ErrorInfoBase& E) {
102 return make_error<vc::BadBitcodeError>(E.message());
103 });
104
105 if (verifyModule(*ExpModule.get()))
106 return make_error<vc::InvalidModuleError>();
107
108 return ExpModule;
109 }
110
111 static Expected<std::unique_ptr<llvm::Module>>
getModuleFromSPIRV(ArrayRef<char> Input,ArrayRef<uint32_t> SpecConstIds,ArrayRef<uint64_t> SpecConstValues,LLVMContext & Ctx)112 getModuleFromSPIRV(ArrayRef<char> Input, ArrayRef<uint32_t> SpecConstIds,
113 ArrayRef<uint64_t> SpecConstValues, LLVMContext &Ctx) {
114 auto ExpIR = vc::translateSPIRVToIR(Input, SpecConstIds, SpecConstValues);
115 if (!ExpIR)
116 return ExpIR.takeError();
117
118 return getModuleFromLLVMBinary(ExpIR.get(), Ctx);
119 }
120
121 static Expected<std::unique_ptr<llvm::Module>>
getModule(ArrayRef<char> Input,vc::FileType FType,ArrayRef<uint32_t> SpecConstIds,ArrayRef<uint64_t> SpecConstValues,LLVMContext & Ctx)122 getModule(ArrayRef<char> Input, vc::FileType FType,
123 ArrayRef<uint32_t> SpecConstIds, ArrayRef<uint64_t> SpecConstValues,
124 LLVMContext &Ctx) {
125 switch (FType) {
126 case vc::FileType::SPIRV:
127 return getModuleFromSPIRV(Input, SpecConstIds, SpecConstValues, Ctx);
128 case vc::FileType::LLVM_TEXT:
129 return getModuleFromLLVMText(Input, Ctx);
130 case vc::FileType::LLVM_BINARY:
131 return getModuleFromLLVMBinary(Input, Ctx);
132 }
133 IGC_ASSERT_EXIT_MESSAGE(0, "Unknown input kind");
134 }
135
overrideTripleWithVC(StringRef TripleStr)136 static Triple overrideTripleWithVC(StringRef TripleStr) {
137 Triple T{TripleStr};
138 // Normalize triple.
139 bool Is32Bit = T.isArch32Bit();
140 if (TripleStr.startswith("genx32"))
141 Is32Bit = true;
142 return Triple{Is32Bit ? "genx32-unknown-unknown" : "genx64-unknown-unknown"};
143 }
144
getSubtargetFeatureString(const vc::CompileOptions & Opts)145 static std::string getSubtargetFeatureString(const vc::CompileOptions &Opts) {
146
147 SubtargetFeatures Features;
148
149 if (!Opts.FeaturesString.empty()) {
150 SmallVector<StringRef, 8> AuxFeatures;
151 StringRef(Opts.FeaturesString).split(AuxFeatures, ",", -1, false);
152 for (const auto& F: AuxFeatures) {
153 auto Feature = F.trim();
154 bool Enabled = Feature.consume_front("+");
155 if (!Enabled) {
156 bool Disabled = Feature.consume_front("-");
157 IGC_ASSERT_MESSAGE(Disabled, "unexpected feature format");
158 }
159 Features.AddFeature(Feature.str(), Enabled);
160 }
161 }
162 if (Opts.HasL1ReadOnlyCache)
163 Features.AddFeature("has_l1_read_only_cache");
164 if (Opts.HasLocalMemFenceSupress)
165 Features.AddFeature("supress_local_mem_fence");
166 if (Opts.NoVecDecomp)
167 Features.AddFeature("disable_vec_decomp");
168 if (Opts.NoJumpTables)
169 Features.AddFeature("disable_jump_tables");
170 if (Opts.TranslateLegacyMemoryIntrinsics)
171 Features.AddFeature("translate_legacy_message");
172 if (Opts.Binary == vc::BinaryKind::OpenCL ||
173 Opts.Binary == vc::BinaryKind::ZE)
174 Features.AddFeature("ocl_runtime");
175
176 return Features.getString();
177 }
178
getCodeGenOptLevel(const vc::CompileOptions & Opts)179 static CodeGenOpt::Level getCodeGenOptLevel(const vc::CompileOptions &Opts) {
180 if (Opts.OptLevel == vc::OptimizerLevel::None)
181 return CodeGenOpt::None;
182 return CodeGenOpt::Default;
183 }
184
getTargetOptions(const vc::CompileOptions & Opts)185 static TargetOptions getTargetOptions(const vc::CompileOptions &Opts) {
186 TargetOptions Options;
187 Options.AllowFPOpFusion = Opts.AllowFPOpFusion;
188 return Options;
189 }
190
191 static Expected<std::unique_ptr<TargetMachine>>
createTargetMachine(const vc::CompileOptions & Opts,Triple & TheTriple)192 createTargetMachine(const vc::CompileOptions &Opts, Triple &TheTriple) {
193 std::string Error;
194 const Target *TheTarget = TargetRegistry::lookupTarget(
195 TheTriple.getArchName().str(), TheTriple, Error);
196 IGC_ASSERT_MESSAGE(TheTarget, "vc target was not registered");
197
198 const std::string FeaturesStr = getSubtargetFeatureString(Opts);
199
200 const TargetOptions Options = getTargetOptions(Opts);
201
202 CodeGenOpt::Level OptLevel = getCodeGenOptLevel(Opts);
203 std::unique_ptr<TargetMachine> TM{
204 TheTarget->createTargetMachine(TheTriple.getTriple(), Opts.CPUStr,
205 FeaturesStr, Options, /*RelocModel=*/None,
206 /*CodeModel=*/None, OptLevel)};
207 if (!TM)
208 return make_error<vc::TargetMachineError>();
209 return {std::move(TM)};
210 }
211
getDefaultOverridableFlag(T OptFlag,bool Default)212 template <typename T> bool getDefaultOverridableFlag(T OptFlag, bool Default) {
213 switch (OptFlag) {
214 default:
215 return Default;
216 case T::Enable:
217 return true;
218 case T::Disable:
219 return false;
220 }
221 }
222
223 // Create backend options for immutable config pass. Override default
224 // values with provided ones.
createBackendOptions(const vc::CompileOptions & Opts)225 static GenXBackendOptions createBackendOptions(const vc::CompileOptions &Opts) {
226 GenXBackendOptions BackendOpts;
227 if (Opts.StackMemSize) {
228 BackendOpts.StackSurfaceMaxSize = Opts.StackMemSize.getValue();
229 BackendOpts.StatelessPrivateMemSize = Opts.StackMemSize.getValue();
230 }
231 BackendOpts.DisableFinalizerMsg = Opts.DisableFinalizerMsg;
232 BackendOpts.EmitDebuggableKernels = Opts.EmitDebuggableKernels;
233 BackendOpts.DebugInfoForZeBin = (Opts.Binary == vc::BinaryKind::ZE);
234 BackendOpts.DebugInfoValidationEnable = Opts.ForceDebugInfoValidation;
235 BackendOpts.EnableAsmDumps = Opts.DumpAsm;
236 BackendOpts.EnableDebugInfoDumps = Opts.DumpDebugInfo;
237 BackendOpts.Dumper = Opts.Dumper.get();
238 BackendOpts.ShaderOverrider = Opts.ShaderOverrider.get();
239 BackendOpts.DisableStructSplitting = Opts.DisableStructSplitting;
240 BackendOpts.ForceArrayPromotion = (Opts.Binary == vc::BinaryKind::CM);
241 BackendOpts.ReserveBTIZero = (Opts.Binary != vc::BinaryKind::CM) &&
242 Opts.EmitDebuggableKernels;
243 if (Opts.ForceLiveRangesLocalizationForAccUsage)
244 BackendOpts.LocalizeLRsForAccUsage = true;
245 if (Opts.ForceDisableNonOverlappingRegionOpt)
246 BackendOpts.DisableNonOverlappingRegionOpt = true;
247 BackendOpts.FCtrl = Opts.FCtrl;
248 BackendOpts.WATable = Opts.WATable;
249 BackendOpts.IsLargeGRFMode = Opts.IsLargeGRFMode;
250 BackendOpts.UseBindlessBuffers = Opts.UseBindlessBuffers;
251 if (Opts.SaveStackCallLinkage)
252 BackendOpts.SaveStackCallLinkage = true;
253 BackendOpts.UsePlain2DImages = Opts.UsePlain2DImages;
254 BackendOpts.EnablePreemption = Opts.EnablePreemption;
255
256 bool IsOptLevel_O0 =
257 Opts.OptLevel == vc::OptimizerLevel::None && Opts.EmitDebuggableKernels;
258 BackendOpts.PassDebugToFinalizer =
259 getDefaultOverridableFlag(Opts.NoOptFinalizerMode, IsOptLevel_O0);
260 BackendOpts.DisableLiveRangesCoalescing =
261 getDefaultOverridableFlag(Opts.DisableLRCoalescingMode, false);
262 return BackendOpts;
263 }
264
createBackendData(const vc::ExternalData & Data,int PointerSizeInBits)265 static GenXBackendData createBackendData(const vc::ExternalData &Data,
266 int PointerSizeInBits) {
267 IGC_ASSERT_MESSAGE(PointerSizeInBits == 32 || PointerSizeInBits == 64,
268 "only 32 and 64 bit pointers are expected");
269 GenXBackendData BackendData;
270 BackendData.BiFModule[BiFKind::OCLGeneric] =
271 IGCLLVM::makeMemoryBufferRef(*Data.OCLGenericBIFModule);
272 BackendData.BiFModule[BiFKind::VCEmulation] =
273 IGCLLVM::makeMemoryBufferRef(*Data.VCEmulationBIFModule);
274 BackendData.BiFModule[BiFKind::VCSPIRVBuiltins] =
275 IGCLLVM::makeMemoryBufferRef(*Data.VCSPIRVBuiltinsBIFModule);
276 if (PointerSizeInBits == 64)
277 BackendData.BiFModule[BiFKind::VCPrintf] =
278 IGCLLVM::makeMemoryBufferRef(*Data.VCPrintf64BIFModule);
279 else
280 BackendData.BiFModule[BiFKind::VCPrintf] =
281 IGCLLVM::makeMemoryBufferRef(*Data.VCPrintf32BIFModule);
282 return std::move(BackendData);
283 }
284
optimizeIR(const vc::CompileOptions & Opts,const vc::ExternalData & ExtData,TargetMachine & TM,Module & M)285 static void optimizeIR(const vc::CompileOptions &Opts,
286 const vc::ExternalData &ExtData, TargetMachine &TM,
287 Module &M) {
288 vc::PassManager PerModulePasses;
289 legacy::FunctionPassManager PerFunctionPasses(&M);
290
291 PerModulePasses.add(
292 createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
293 PerModulePasses.add(new GenXBackendConfig{createBackendOptions(Opts),
294 createBackendData(ExtData,
295 TM.getPointerSizeInBits(0))});
296 PerFunctionPasses.add(
297 createTargetTransformInfoWrapperPass(TM.getTargetIRAnalysis()));
298
299 unsigned OptLevel;
300 if (Opts.OptLevel == vc::OptimizerLevel::None)
301 OptLevel = 0;
302 else
303 OptLevel = 2;
304
305 PassManagerBuilder PMBuilder;
306 PMBuilder.Inliner = createFunctionInliningPass(2, 2, false);
307 PMBuilder.OptLevel = OptLevel;
308 PMBuilder.SizeLevel = OptLevel;
309 PMBuilder.SLPVectorize = false;
310 PMBuilder.LoopVectorize = false;
311 PMBuilder.DisableUnrollLoops = false;
312 PMBuilder.MergeFunctions = false;
313 PMBuilder.PrepareForThinLTO = false;
314 PMBuilder.PrepareForLTO = false;
315 PMBuilder.RerollLoops = true;
316
317 TM.adjustPassManager(PMBuilder);
318
319 PMBuilder.populateFunctionPassManager(PerFunctionPasses);
320 PMBuilder.populateModulePassManager(PerModulePasses);
321
322 // Do we need per function passes at all?
323 PerFunctionPasses.doInitialization();
324 for (Function &F : M) {
325 if (!F.isDeclaration())
326 PerFunctionPasses.run(F);
327 }
328 PerFunctionPasses.doFinalization();
329
330 PerModulePasses.run(M);
331 }
332
dumpFinalOutput(const vc::CompileOptions & Opts,const Module & M,StringRef IsaBinary)333 static void dumpFinalOutput(const vc::CompileOptions &Opts, const Module &M,
334 StringRef IsaBinary) {
335 if (Opts.DumpIR && Opts.Dumper)
336 Opts.Dumper->dumpModule(M, "final.ll");
337 if (Opts.DumpIsa && Opts.Dumper)
338 Opts.Dumper->dumpBinary({IsaBinary.data(), IsaBinary.size()}, "final.isa");
339 }
340
populateCodeGenPassManager(const vc::CompileOptions & Opts,const vc::ExternalData & ExtData,TargetMachine & TM,raw_pwrite_stream & OS,legacy::PassManager & PM)341 static void populateCodeGenPassManager(const vc::CompileOptions &Opts,
342 const vc::ExternalData &ExtData,
343 TargetMachine &TM, raw_pwrite_stream &OS,
344 legacy::PassManager &PM) {
345 TargetLibraryInfoImpl TLII{TM.getTargetTriple()};
346 PM.add(new TargetLibraryInfoWrapperPass(TLII));
347 PM.add(new GenXBackendConfig{createBackendOptions(Opts),
348 createBackendData(ExtData,
349 TM.getPointerSizeInBits(0))});
350
351 #ifndef NDEBUG
352 // Do not enforce IR verification at an arbitrary moments in release builds
353 constexpr bool DisableIrVerifier = false;
354 #else
355 constexpr bool DisableIrVerifier = true;
356 #endif
357
358 auto FileType = IGCLLVM::TargetMachine::CodeGenFileType::CGFT_AssemblyFile;
359
360 bool AddPasses =
361 TM.addPassesToEmitFile(PM, OS, nullptr, FileType, DisableIrVerifier);
362 IGC_ASSERT_MESSAGE(!AddPasses, "Bad filetype for vc-codegen");
363 }
364
runOclCodeGen(const vc::CompileOptions & Opts,const vc::ExternalData & ExtData,TargetMachine & TM,Module & M)365 static vc::ocl::CompileOutput runOclCodeGen(const vc::CompileOptions &Opts,
366 const vc::ExternalData &ExtData,
367 TargetMachine &TM, Module &M) {
368 vc::PassManager PM;
369
370 SmallString<32> IsaBinary;
371 raw_svector_ostream OS(IsaBinary);
372 raw_null_ostream NullOS;
373 if (Opts.DumpIsa)
374 populateCodeGenPassManager(Opts, ExtData, TM, OS, PM);
375 else
376 populateCodeGenPassManager(Opts, ExtData, TM, NullOS, PM);
377
378 GenXOCLRuntimeInfo::CompiledModuleT CompiledModule;
379 PM.add(createGenXOCLInfoExtractorPass(CompiledModule));
380
381 PM.run(M);
382 dumpFinalOutput(Opts, M, IsaBinary);
383
384 return CompiledModule;
385 }
386
runCmCodeGen(const vc::CompileOptions & Opts,const vc::ExternalData & ExtData,TargetMachine & TM,Module & M)387 static vc::cm::CompileOutput runCmCodeGen(const vc::CompileOptions &Opts,
388 const vc::ExternalData &ExtData,
389 TargetMachine &TM, Module &M) {
390 vc::PassManager PM;
391
392 SmallString<32> IsaBinary;
393 raw_svector_ostream OS(IsaBinary);
394 populateCodeGenPassManager(Opts, ExtData, TM, OS, PM);
395 PM.run(M);
396 dumpFinalOutput(Opts, M, IsaBinary);
397 vc::cm::CompileOutput Output;
398 Output.IsaBinary.assign(IsaBinary.begin(), IsaBinary.end());
399 return Output;
400 }
401
runCodeGen(const vc::CompileOptions & Opts,const vc::ExternalData & ExtData,TargetMachine & TM,Module & M)402 static vc::CompileOutput runCodeGen(const vc::CompileOptions &Opts,
403 const vc::ExternalData &ExtData,
404 TargetMachine &TM, Module &M) {
405 switch (Opts.Binary) {
406 case vc::BinaryKind::CM:
407 return runCmCodeGen(Opts, ExtData, TM, M);
408 case vc::BinaryKind::OpenCL:
409 case vc::BinaryKind::ZE:
410 return runOclCodeGen(Opts, ExtData, TM, M);
411 }
412 IGC_ASSERT_EXIT_MESSAGE(0, "Unknown runtime kind");
413 }
414
415 // Parse global llvm cl options.
416 // Parsing of cl options should not fail under any circumstances.
parseLLVMOptions(const std::string & Args)417 static void parseLLVMOptions(const std::string &Args) {
418 BumpPtrAllocator Alloc;
419 StringSaver Saver{Alloc};
420 SmallVector<const char *, 8> Argv{"vc-codegen"};
421 cl::TokenizeGNUCommandLine(Args, Saver, Argv);
422
423 // Reset all options to ensure that scalar part does not affect
424 // vector compilation.
425 cl::ResetAllOptionOccurrences();
426 cl::ParseCommandLineOptions(Argv.size(), Argv.data());
427 }
428
Compile(ArrayRef<char> Input,const vc::CompileOptions & Opts,const vc::ExternalData & ExtData,ArrayRef<uint32_t> SpecConstIds,ArrayRef<uint64_t> SpecConstValues)429 Expected<vc::CompileOutput> vc::Compile(ArrayRef<char> Input,
430 const vc::CompileOptions &Opts,
431 const vc::ExternalData &ExtData,
432 ArrayRef<uint32_t> SpecConstIds,
433 ArrayRef<uint64_t> SpecConstValues) {
434 parseLLVMOptions(Opts.LLVMOptions);
435 // Reset options when everything is done here. This is needed to not
436 // interfere with subsequent translations (including scalar part).
437 const auto ClOptGuard =
438 llvm::make_scope_exit([]() { cl::ResetAllOptionOccurrences(); });
439
440 LLVMContext Context;
441 LLVMInitializeGenXTarget();
442 LLVMInitializeGenXTargetInfo();
443 llvm::PassRegistry &Registry = *llvm::PassRegistry::getPassRegistry();
444 llvm::initializeTarget(Registry);
445
446 Expected<std::unique_ptr<llvm::Module>> ExpModule =
447 getModule(Input, Opts.FType, SpecConstIds, SpecConstValues, Context);
448 if (!ExpModule)
449 return ExpModule.takeError();
450 Module &M = *ExpModule.get();
451
452 if (Opts.DumpIR && Opts.Dumper)
453 Opts.Dumper->dumpModule(M, "after_spirv_reader.ll");
454
455 vc::PassManager PerModulePasses;
456 PerModulePasses.add(createGenXSPIRVReaderAdaptorPass());
457 PerModulePasses.add(createGenXRestoreIntrAttrPass());
458 PerModulePasses.run(M);
459
460 Triple TheTriple = overrideTripleWithVC(M.getTargetTriple());
461 M.setTargetTriple(TheTriple.getTriple());
462
463 auto ExpTargetMachine = createTargetMachine(Opts, TheTriple);
464 if (!ExpTargetMachine)
465 return ExpTargetMachine.takeError();
466 TargetMachine &TM = *ExpTargetMachine.get();
467 M.setDataLayout(TM.createDataLayout());
468
469 // Save old value and restore at the end.
470 bool TimePassesIsEnabledLocal = TimePassesIsEnabled;
471 if (Opts.TimePasses)
472 TimePassesIsEnabled = true;
473
474 // Enable LLVM statistics recording if required.
475 if (Opts.ShowStats || !Opts.StatsFile.empty())
476 llvm::EnableStatistics(false);
477
478 if (Opts.DumpIR && Opts.Dumper)
479 Opts.Dumper->dumpModule(M, "after_ir_adaptors.ll");
480
481 optimizeIR(Opts, ExtData, TM, M);
482
483 if (Opts.DumpIR && Opts.Dumper)
484 Opts.Dumper->dumpModule(M, "optimized.ll");
485
486 vc::CompileOutput Output = runCodeGen(Opts, ExtData, TM, M);
487
488 // Print timers if any and restore old TimePassesIsEnabled value.
489 TimerGroup::printAll(llvm::errs());
490 TimePassesIsEnabled = TimePassesIsEnabledLocal;
491
492 // Print LLVM statistics if required.
493 if (Opts.ShowStats)
494 llvm::PrintStatistics(llvm::errs());
495 if (!Opts.StatsFile.empty()) {
496 std::error_code EC;
497 auto StatS = std::make_unique<llvm::raw_fd_ostream>(
498 Opts.StatsFile, EC, llvm::sys::fs::OF_Text);
499 if (EC)
500 llvm::errs() << Opts.StatsFile << ": " << EC.message();
501 else
502 llvm::PrintStatisticsJSON(*StatS);
503 }
504 return Output;
505 }
506
507 template <typename ID, ID... UnknownIDs>
508 static Expected<opt::InputArgList>
parseOptions(const SmallVectorImpl<const char * > & Argv,unsigned FlagsToInclude,const opt::OptTable & Options,bool IsStrictMode)509 parseOptions(const SmallVectorImpl<const char *> &Argv, unsigned FlagsToInclude,
510 const opt::OptTable &Options, bool IsStrictMode) {
511 const bool IsInternal = FlagsToInclude & IGC::options::VCInternalOption;
512
513 unsigned MissingArgIndex = 0;
514 unsigned MissingArgCount = 0;
515 opt::InputArgList InputArgs =
516 Options.ParseArgs(Argv, MissingArgIndex, MissingArgCount, FlagsToInclude);
517 if (MissingArgCount)
518 return make_error<vc::OptionError>(Argv[MissingArgIndex], IsInternal);
519
520 // ocloc uncoditionally passes opencl options to internal options.
521 // Skip checking of internal options for now.
522 if (IsStrictMode) {
523 if (opt::Arg *A = InputArgs.getLastArg(UnknownIDs...)) {
524 std::string BadOpt = A->getAsString(InputArgs);
525 return make_error<vc::OptionError>(BadOpt, IsInternal);
526 }
527 }
528
529 return {std::move(InputArgs)};
530 }
531
532 static Expected<opt::InputArgList>
parseApiOptions(StringSaver & Saver,StringRef ApiOptions,bool IsStrictMode)533 parseApiOptions(StringSaver &Saver, StringRef ApiOptions, bool IsStrictMode) {
534 using namespace IGC::options::api;
535
536 SmallVector<const char *, 8> Argv;
537 cl::TokenizeGNUCommandLine(ApiOptions, Saver, Argv);
538
539 const opt::OptTable &Options = IGC::getApiOptTable();
540 // This can be rewritten to parse options and then check for
541 // OPT_vc_codegen, but it would be better to manually check for
542 // this option before any real parsing. If it is missing,
543 // then no parsing should be done at all.
544 auto HasOption = [&Argv](const std::string &Opt) {
545 return std::any_of(Argv.begin(), Argv.end(),
546 [&Opt](const char *ArgStr) { return Opt == ArgStr; });
547 };
548 const std::string VCCodeGenOptName =
549 Options.getOption(OPT_vc_codegen).getPrefixedName();
550 if (HasOption(VCCodeGenOptName)) {
551 const unsigned FlagsToInclude =
552 IGC::options::VCApiOption | IGC::options::IGCApiOption;
553 return parseOptions<ID, OPT_UNKNOWN, OPT_INPUT>(Argv, FlagsToInclude,
554 Options, IsStrictMode);
555 }
556 // Deprecated -cmc parsing just for compatibility.
557 const std::string IgcmcOptName =
558 Options.getOption(OPT_igcmc).getPrefixedName();
559 if (HasOption(IgcmcOptName)) {
560 llvm::errs()
561 << "'" << IgcmcOptName
562 << "' option is deprecated and will be removed in the future release. "
563 "Use -vc-codegen instead for compiling from SPIRV.\n";
564 const unsigned FlagsToInclude =
565 IGC::options::IgcmcApiOption | IGC::options::IGCApiOption;
566 return parseOptions<ID, OPT_UNKNOWN, OPT_INPUT>(Argv, FlagsToInclude,
567 Options, IsStrictMode);
568 }
569
570 return make_error<vc::NotVCError>();
571 }
572
573 static Expected<opt::InputArgList>
parseInternalOptions(StringSaver & Saver,StringRef InternalOptions)574 parseInternalOptions(StringSaver &Saver, StringRef InternalOptions) {
575 using namespace IGC::options::internal;
576
577 SmallVector<const char *, 8> Argv;
578 cl::TokenizeGNUCommandLine(InternalOptions, Saver, Argv);
579 // Internal options are always unchecked.
580 constexpr bool IsStrictMode = false;
581 const opt::OptTable &Options = IGC::getInternalOptTable();
582 const unsigned FlagsToInclude =
583 IGC::options::VCInternalOption | IGC::options::IGCInternalOption;
584 return parseOptions<ID, OPT_UNKNOWN, OPT_INPUT>(Argv, FlagsToInclude, Options,
585 IsStrictMode);
586 }
587
makeOptionError(const opt::Arg & A,const opt::ArgList & Opts,bool IsInternal)588 static Error makeOptionError(const opt::Arg &A, const opt::ArgList &Opts,
589 bool IsInternal) {
590 const std::string BadOpt = A.getAsString(Opts);
591 return make_error<vc::OptionError>(BadOpt, IsInternal);
592 }
593
fillApiOptions(const opt::ArgList & ApiOptions,vc::CompileOptions & Opts)594 static Error fillApiOptions(const opt::ArgList &ApiOptions,
595 vc::CompileOptions &Opts) {
596 using namespace IGC::options::api;
597
598 if (ApiOptions.hasArg(OPT_no_vector_decomposition))
599 Opts.NoVecDecomp = true;
600 if (ApiOptions.hasArg(OPT_emit_debug))
601 Opts.EmitDebuggableKernels = true;
602 if (ApiOptions.hasArg(OPT_vc_fno_struct_splitting))
603 Opts.DisableStructSplitting = true;
604 if (ApiOptions.hasArg(OPT_vc_fno_jump_tables))
605 Opts.NoJumpTables = true;
606 if (ApiOptions.hasArg(OPT_vc_ftranslate_legacy_memory_intrinsics))
607 Opts.TranslateLegacyMemoryIntrinsics = true;
608 if (ApiOptions.hasArg(OPT_vc_disable_finalizer_msg))
609 Opts.DisableFinalizerMsg = true;
610 if (ApiOptions.hasArg(OPT_large_GRF))
611 Opts.IsLargeGRFMode = true;
612 if (ApiOptions.hasArg(OPT_vc_use_plain_2d_images))
613 Opts.UsePlain2DImages = true;
614 if (ApiOptions.hasArg(OPT_vc_enable_preemption))
615 Opts.EnablePreemption = true;
616
617 if (opt::Arg *A = ApiOptions.getLastArg(OPT_fp_contract)) {
618 StringRef Val = A->getValue();
619 auto MayBeAllowFPOPFusion =
620 StringSwitch<Optional<FPOpFusion::FPOpFusionMode>>(Val)
621 .Case("on", FPOpFusion::Standard)
622 .Case("fast", FPOpFusion::Fast)
623 .Case("off", FPOpFusion::Strict)
624 .Default(None);
625 if (!MayBeAllowFPOPFusion)
626 return makeOptionError(*A, ApiOptions, /*IsInternal=*/false);
627 Opts.AllowFPOpFusion = MayBeAllowFPOPFusion.getValue();
628 }
629
630 if (opt::Arg *A =
631 ApiOptions.getLastArg(OPT_vc_optimize, OPT_opt_disable_ze)) {
632 if (A->getOption().matches(OPT_vc_optimize)) {
633 StringRef Val = A->getValue();
634 auto MaybeLevel = StringSwitch<Optional<vc::OptimizerLevel>>(Val)
635 .Case("none", vc::OptimizerLevel::None)
636 .Case("full", vc::OptimizerLevel::Full)
637 .Default(None);
638 if (!MaybeLevel)
639 return makeOptionError(*A, ApiOptions, /*IsInternal=*/false);
640 Opts.OptLevel = MaybeLevel.getValue();
641 } else {
642 IGC_ASSERT(A->getOption().matches(OPT_opt_disable_ze));
643 Opts.OptLevel = vc::OptimizerLevel::None;
644 }
645 }
646
647 if (opt::Arg *A = ApiOptions.getLastArg(OPT_vc_stateless_private_size)) {
648 StringRef Val = A->getValue();
649 unsigned Result;
650 if (Val.getAsInteger(/*Radix=*/0, Result))
651 return makeOptionError(*A, ApiOptions, /*IsInternal=*/false);
652 Opts.StackMemSize = Result;
653 }
654
655 return Error::success();
656 }
657
fillInternalOptions(const opt::ArgList & InternalOptions,vc::CompileOptions & Opts)658 static Error fillInternalOptions(const opt::ArgList &InternalOptions,
659 vc::CompileOptions &Opts) {
660 using namespace IGC::options::internal;
661
662 if (InternalOptions.hasArg(OPT_dump_isa_binary))
663 Opts.DumpIsa = true;
664 if (InternalOptions.hasArg(OPT_dump_llvm_ir))
665 Opts.DumpIR = true;
666 if (InternalOptions.hasArg(OPT_dump_asm))
667 Opts.DumpAsm = true;
668 if (InternalOptions.hasArg(OPT_ftime_report))
669 Opts.TimePasses = true;
670 if (InternalOptions.hasArg(OPT_print_stats))
671 Opts.ShowStats = true;
672 Opts.StatsFile = InternalOptions.getLastArgValue(OPT_stats_file).str();
673 if (InternalOptions.hasArg(OPT_intel_use_bindless_buffers_ze))
674 Opts.UseBindlessBuffers = true;
675
676 if (opt::Arg *A = InternalOptions.getLastArg(OPT_binary_format)) {
677 StringRef Val = A->getValue();
678 auto MaybeBinary = StringSwitch<Optional<vc::BinaryKind>>(Val)
679 .Case("cm", vc::BinaryKind::CM)
680 .Case("ocl", vc::BinaryKind::OpenCL)
681 .Case("ze", vc::BinaryKind::ZE)
682 .Default(None);
683 if (!MaybeBinary)
684 return makeOptionError(*A, InternalOptions, /*IsInternal=*/true);
685 Opts.Binary = MaybeBinary.getValue();
686 }
687
688 Opts.FeaturesString =
689 llvm::join(InternalOptions.getAllArgValues(OPT_target_features), ",");
690
691 if (InternalOptions.hasArg(OPT_help)) {
692 constexpr const char *Usage = "-options \"-vc-codegen [options]\"";
693 constexpr const char *Title = "Vector compiler options";
694 constexpr unsigned FlagsToInclude = IGC::options::VCApiOption;
695 constexpr unsigned FlagsToExclude = 0;
696 constexpr bool ShowAllAliases = false;
697 IGCLLVM::printHelp(IGC::getApiOptTable(), llvm::errs(), Usage, Title,
698 FlagsToInclude, FlagsToExclude, ShowAllAliases);
699 }
700 if (InternalOptions.hasArg(OPT_help_internal)) {
701 constexpr const char *Usage =
702 "-options \"-vc-codegen\" -internal_options \"[options]\"";
703 constexpr const char *Title = "Vector compiler internal options";
704 constexpr unsigned FlagsToInclude = IGC::options::VCInternalOption;
705 constexpr unsigned FlagsToExclude = 0;
706 constexpr bool ShowAllAliases = false;
707 IGCLLVM::printHelp(IGC::getInternalOptTable(), llvm::errs(), Usage, Title,
708 FlagsToInclude, FlagsToExclude, ShowAllAliases);
709 }
710
711 return Error::success();
712 }
713
714 // Prepare llvm options string using different API and internal options.
composeLLVMArgs(const opt::ArgList & ApiArgs,const opt::ArgList & InternalArgs)715 static std::string composeLLVMArgs(const opt::ArgList &ApiArgs,
716 const opt::ArgList &InternalArgs) {
717 std::string Result;
718
719 // Handle input llvm options.
720 if (InternalArgs.hasArg(IGC::options::internal::OPT_llvm_options))
721 Result += join(
722 InternalArgs.getAllArgValues(IGC::options::internal::OPT_llvm_options),
723 " ");
724
725 // Add visaopts if any.
726 for (auto OptID : {IGC::options::api::OPT_igcmc_visaopts,
727 IGC::options::api::OPT_Xfinalizer}) {
728 if (!ApiArgs.hasArg(OptID))
729 continue;
730 Result += " -finalizer-opts='";
731 Result += join(ApiArgs.getAllArgValues(OptID), " ");
732 Result += "'";
733 }
734
735 // Add gtpin options if any.
736 if (ApiArgs.hasArg(IGC::options::api::OPT_gtpin_rera))
737 Result += " -finalizer-opts='-GTPinReRA'";
738 if (ApiArgs.hasArg(IGC::options::api::OPT_gtpin_grf_info))
739 Result += " -finalizer-opts='-getfreegrfinfo -rerapostschedule'";
740 if (opt::Arg *A =
741 ApiArgs.getLastArg(IGC::options::api::OPT_gtpin_scratch_area_size)) {
742 Result += " -finalizer-opts='-GTPinScratchAreaSize ";
743 Result += A->getValue();
744 Result += "'";
745 }
746
747 return Result;
748 }
749
750 static Expected<vc::CompileOptions>
fillOptions(const opt::ArgList & ApiOptions,const opt::ArgList & InternalOptions)751 fillOptions(const opt::ArgList &ApiOptions,
752 const opt::ArgList &InternalOptions) {
753 vc::CompileOptions Opts;
754 Error Status = fillApiOptions(ApiOptions, Opts);
755 if (Status)
756 return {std::move(Status)};
757
758 Status = fillInternalOptions(InternalOptions, Opts);
759 if (Status)
760 return {std::move(Status)};
761
762 // Prepare additional llvm options (like finalizer args).
763 Opts.LLVMOptions = composeLLVMArgs(ApiOptions, InternalOptions);
764
765 return {std::move(Opts)};
766 }
767
768 // Filter input argument list to derive options that will contribute
769 // to subsequent translation.
770 // InputArgs -- argument list to filter, should outlive resulting
771 // derived option list.
772 // IncludeFlag -- options with that flag will be included in result.
filterUsedOptions(opt::InputArgList & InputArgs,IGC::options::Flags IncludeFlag)773 static opt::DerivedArgList filterUsedOptions(opt::InputArgList &InputArgs,
774 IGC::options::Flags IncludeFlag) {
775 opt::DerivedArgList FilteredArgs(InputArgs);
776
777 // InputArg is not a constant. This is required to pass it to append
778 // function of derived argument list. Derived argument list will not
779 // own added argument so it will not try to free this memory.
780 // Additionally note that InputArgs are used in derived arg list as
781 // a constant so added arguments should not be modified through
782 // derived list to avoid unexpected results.
783 for (opt::Arg *InputArg : InputArgs) {
784 const opt::Arg *Arg = InputArg;
785 // Get alias as unaliased form can belong to used flags
786 // (see cl intel gtpin options).
787 if (const opt::Arg *AliasArg = InputArg->getAlias())
788 Arg = AliasArg;
789 // Ignore options without required flag.
790 if (!Arg->getOption().hasFlag(IncludeFlag))
791 continue;
792 FilteredArgs.append(InputArg);
793 }
794
795 return FilteredArgs;
796 }
797
filterApiOptions(opt::InputArgList & InputArgs)798 opt::DerivedArgList filterApiOptions(opt::InputArgList &InputArgs) {
799 if (InputArgs.hasArg(IGC::options::api::OPT_igcmc))
800 return filterUsedOptions(InputArgs, IGC::options::IgcmcApiOption);
801
802 return filterUsedOptions(InputArgs, IGC::options::VCApiOption);
803 }
804
805 llvm::Expected<vc::CompileOptions>
ParseOptions(llvm::StringRef ApiOptions,llvm::StringRef InternalOptions,bool IsStrictMode)806 vc::ParseOptions(llvm::StringRef ApiOptions, llvm::StringRef InternalOptions,
807 bool IsStrictMode) {
808 llvm::BumpPtrAllocator Alloc;
809 llvm::StringSaver Saver{Alloc};
810 auto ExpApiArgList = parseApiOptions(Saver, ApiOptions, IsStrictMode);
811 if (!ExpApiArgList)
812 return ExpApiArgList.takeError();
813 opt::InputArgList &ApiArgs = ExpApiArgList.get();
814 const opt::DerivedArgList VCApiArgs = filterApiOptions(ApiArgs);
815
816 auto ExpInternalArgList = parseInternalOptions(Saver, InternalOptions);
817 if (!ExpInternalArgList)
818 return ExpInternalArgList.takeError();
819 opt::InputArgList &InternalArgs = ExpInternalArgList.get();
820 const opt::DerivedArgList VCInternalArgs =
821 filterUsedOptions(InternalArgs, IGC::options::VCInternalOption);
822
823 return fillOptions(VCApiArgs, VCInternalArgs);
824 }
825