1 /* 2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 16 #ifndef GLOW_NNPI_IMPORTER_H 17 #define GLOW_NNPI_IMPORTER_H 18 19 #include "NNPIOptions.h" 20 #include "glow/Backends/BackendOptions.h" 21 #include "glow/Graph/Nodes.h" 22 #include "glow/Support/Compiler.h" 23 #include "nnpi_network_builder.h" 24 #include "nnpi_network_builder_EXPERIMENTAL.h" 25 #include "llvm/ADT/ArrayRef.h" 26 #include "llvm/Support/raw_ostream.h" 27 #include <string> 28 #include <unordered_map> 29 #include <unordered_set> 30 31 namespace glow { 32 class Function; 33 class Placeholder; 34 class Value; 35 class Node; 36 class Tensor; 37 class Storage; 38 struct Type; 39 class INNPINodeImporter; 40 41 /// This class imports Glow IR to the NNPI backend. 42 class NNPIImporter { 43 public: 44 /// Constructor. 45 NNPIImporter(const NNPICompilationOptions &compileOptions); 46 47 /// Destructor. 48 ~NNPIImporter(); 49 50 /// The main entry point for the importer functionality 51 /// Imports a Function \p F using options taken from \p opts 52 /// \return true iff the import succeeded. 53 NNPINetwork importFunction(Function *F, const BackendOptions &opts); 54 55 /// Get a new name for an internal object. getInternalName()56 std::string getInternalName() { 57 return internalName_ + std::to_string(internalNameCounter_++); 58 } 59 /// Get the network handle. getNetwork()60 NNPINetwork getNetwork() const { return network_; } 61 /// Add a value to the network by Glow::Value. 62 NNPIErrorCode addValueIfTensor(Value *v); 63 /// Add a value to the network by parameters. 64 NNPIErrorCode addValue(std::string name, const glow::Type *vType, 65 bool alternativeLayout = false, bool input = false, 66 bool output = false, 67 const std::string &scaleTensor = {}, 68 const std::string &offsetTensor = {}, 69 bool forceSymlowp = false); 70 /// Set given tensor names as inputs/outputs. 71 void 72 setUsedTensors(const std::unordered_set<std::string> &readTensors = {}, 73 const std::unordered_set<std::string> &writeTensors = {}) { 74 readTensors_.insert(readTensors.begin(), readTensors.end()); 75 writeTensors_.insert(writeTensors.begin(), writeTensors.end()); 76 } 77 /// Add an external tensor to the network (by name). 78 NNPIErrorCode addTensor(std::string name, bool alternativeLayout = false, 79 const std::string &scaleTensor = {}, 80 const std::string &offsetTensor = {}, 81 bool forceSymlowp = false); 82 /// Add an external tensor to the network (by parameters). 83 NNPIErrorCode addTensor(std::string name, const NNPITensorDesc &desc, 84 const void *pData); 85 /// Update the NNPITensorDesc \p desc by the dimensions array \p glowDims. 86 static void updateDescDimsFromGlow(const llvm::ArrayRef<size_t> glowDims, 87 NNPITensorDesc &desc, 88 bool alternativeLayout = false); 89 /// Update the NNPITensorDesc \p desc quantization params by \p vType. 90 void updateDescQuantFromGlow(const glow::Type &t, NNPITensorDesc &desc, 91 const std::string &scaleTensor = {}, 92 const std::string &offsetTensor = {}, 93 bool forceSymlowp = false); 94 95 static bool isVariableUsingAlternativeLayout(Storage *v); 96 bool zeroes(const std::string &name) const; 97 /// Internal name header used for variables. 98 static const std::string internalName_; 99 100 /// Check whether a variable name is in the channelwiseConverters_ set. hasChannelWiseConverter(const std::string & s)101 bool hasChannelWiseConverter(const std::string &s) const { 102 return channelwiseConverters_.count(s); 103 } 104 105 /// Add a new (gemmlowp) variable name to the channelwiseConverters_ set. addChannelWiseConverter(const std::string & s)106 void addChannelWiseConverter(const std::string &s) { 107 channelwiseConverters_.emplace(s); 108 } 109 110 /// Add a path to AI extension (that will be loaded by the inference API). 111 /// Will fail if a file does not exist at this path, validity of the file is 112 /// checked only when the extension is loaded. 113 NNPIErrorCode addIAExtentionPath(const std::string &extPath); 114 115 /// Get AI extension paths. getIAExtensionPaths()116 const std::vector<std::string> &getIAExtensionPaths() const { 117 return iaExtensionPaths_; 118 } 119 120 /// Convert from Glow lengths mode enum to NNPI length type enum. 121 static NNPIErrorCode 122 convertLengthsModeToLengthType(glow::LengthsMode mode, 123 NNPI_LENGTH_TYPE &lengthType); 124 125 private: 126 /// Map of named external tensors (inputs, outputs, weights, etc...). 127 std::unordered_map<std::string, const Tensor *> constants_; 128 /// Set of tensors written to by the function. 129 std::unordered_set<std::string> writeTensors_; 130 /// Set of tensors read from by the function. 131 std::unordered_set<std::string> readTensors_; 132 /// Set of tensors already defined. 133 std::unordered_set<std::string> definedTensors_; 134 /// Number of internal names created for variables. 135 size_t internalNameCounter_; 136 137 /// NNPI network handle. 138 NNPINetwork network_; 139 140 /// Map of Glow node specific importers. 141 static const std::unordered_map<std::string, 142 std::unique_ptr<INNPINodeImporter>> 143 nodeImporters_; 144 /// NNPI Device configuration. 145 const NNPICompilationOptions &compileOptions_; 146 147 /// This set records all the gemmlowp variables that we created for 148 /// channelwise FC/Conv operator inputs/ouputs. When an variable name appears 149 /// in this set, it means that a (gemmlowp -> symlowp) converter has already 150 /// created. Querying of this can help us create duplicated converters when 1. 151 /// an input is feeding into more than one channelwise ops. 2. an output of 152 /// one channelwise op is consumed by another channelwise op. 153 std::unordered_set<std::string> channelwiseConverters_; 154 155 /// A list of IA extensions that need to be loaded by the device. 156 std::vector<std::string> iaExtensionPaths_; 157 }; 158 159 /// Interface class for all node specific importers. 160 class INNPINodeImporter { 161 public: 162 /// Import a single node \p n and add it to \p importer 163 /// \return the create NNPI layer or nullptr if no layer was created. 164 virtual NNPIErrorCode importNode(Node *n, NNPIImporter &importer) = 0; 165 166 /// Destructor. 167 virtual ~INNPINodeImporter() = default; 168 }; 169 170 } // namespace glow 171 #endif // GLOW_NNPI_IMPORTER_H 172