1 /** 2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 #ifndef GLOW_BACKENDS_INTERPRETER_INTERPRETERFUNCTION_H 17 #define GLOW_BACKENDS_INTERPRETER_INTERPRETERFUNCTION_H 18 19 #include "glow/Backend/Backend.h" 20 #include "glow/Backend/BackendUtils.h" 21 #include "glow/Backend/CompiledFunction.h" 22 #include "glow/Base/Tensor.h" 23 #include "glow/ExecutionContext/ExecutionContext.h" 24 #include "glow/Quantization/Base/Base.h" 25 26 #include "llvm/ADT/ArrayRef.h" 27 28 #include <memory> 29 #include <unordered_map> 30 31 namespace glow { 32 33 class IRFunction; 34 class Value; 35 class Tensor; 36 class Constant; 37 38 // Forward declare all of the classes. 39 #define DEF_VALUE(CLASS, NAME) class CLASS; 40 #define DEF_INSTR(CLASS, NAME) class CLASS; 41 #define DEF_BACKEND_SPECIFIC_INSTR(CLASS, NAME) 42 #include "glow/AutoGenInstr.def" 43 44 /// Function "compiled" for execution by the interpreter. 45 class InterpreterFunction final : public CompiledFunction, 46 public IRInstructionProcessingHandler { 47 /// The IR to be executed. 48 std::unique_ptr<IRFunction> F_; 49 50 /// Maps Value.name to tensors for constants. 51 std::unordered_map<std::string, Tensor *> constants_; 52 53 public: 54 InterpreterFunction(std::unique_ptr<IRFunction> F, 55 runtime::RuntimeBundle &&bundle); 56 57 /// \name CompiledFunction interface 58 ///@{ 59 ~InterpreterFunction() override; 60 61 Error execute(ExecutionContext *context) override; 62 63 /// Collects constants for runtime. 64 void collectConstants(const Module *module) override; 65 66 /// Add a constant to the function, this is used for loading static 67 /// placeholders. 68 void addConstant(std::string name, Tensor *T); 69 70 /// Get reference to IR function. getIR()71 IRFunction *getIR() { return F_.get(); } 72 73 /// Read trace events out of this func and write them into /p context 74 void translateTraceEvents(ExecutionContext *context) const override; 75 76 /// \returns the backend used to compile this function. getCompileBackendName()77 virtual std::string getCompileBackendName() const override { 78 return "Interpreter"; 79 } 80 ///@} 81 }; 82 83 /// An InterpreterFunction bound to a specific invocation. 84 class BoundInterpreterFunction : public IRInstructionProcessingHandler { 85 /// Maps values to Tensors, that are owned by this class. 86 std::unordered_map<const Value *, Tensor *> tensors_; 87 88 /// Maps values to Tensors, that are *not* owned by this class. 89 std::unordered_map<const Value *, Tensor *> externalTensors_; 90 91 /// A reference to the constant map from the owning InterpreterFunction. 92 const std::unordered_map<std::string, Tensor *> &constants_; 93 94 public: BoundInterpreterFunction(const std::unordered_map<std::string,Tensor * > & constants)95 explicit BoundInterpreterFunction( 96 const std::unordered_map<std::string, Tensor *> &constants) 97 : constants_(constants) {} 98 99 ~BoundInterpreterFunction(); 100 101 Error execute(IRFunction *F, ExecutionContext *context); 102 103 private: 104 /// \returns a pointer to the tensor that is saved under \p v. 105 Tensor *getTensor(const Value *v) const; 106 107 /// Allocate a tensor to back the value \p v. Do not allocate anything if a 108 /// tensor is already allocated for \p v. 109 /// \returns a tensor for \p v. 110 Tensor *getOrCreateTensor(const Value *v); 111 112 /// Allocate an unowned tensor to back the value \p v. The source tensor of 113 /// the unowned tensor is provided by \p src. 114 /// \returns a tensor for \p v. 115 Tensor *getOrCreateUnownedTensor(const Value *v, const Value *src, 116 llvm::ArrayRef<dim_t> offsets); 117 118 /// If a tensor is allocated for \p v then delete it. 119 void deleteTensor(const Value *v); 120 121 /// \returns a typed handle to the tensor that is stored at \p v. 122 template <class ElemTy = float> getWeightHandle(Value * v)123 Handle<ElemTy> getWeightHandle(Value *v) const { 124 return getTensor(v)->getHandle<ElemTy>(); 125 } 126 127 /// @name BoundInterpreterFunction methods. This is a list of method 128 /// declerations that are used by the interpreter to dispatch different 129 /// instructions. 130 ///@{ 131 132 #define DEF_VALUE(CLASS, NAME) 133 #define DEF_INSTR(CLASS, NAME) void fwd##CLASS(const CLASS *I); 134 #define DEF_BACKEND_SPECIFIC_INSTR(CLASS, NAME) 135 #include "glow/AutoGenInstr.def" 136 137 template <typename ElemTy, typename AccumulatorTy, 138 typename BiasElemTy = int32_t> 139 void fwdConvolutionInstQuantizedImpl(Value *inV, Value *outV, Value *filterV, 140 Value *biasV, 141 llvm::ArrayRef<unsigned_t> kernelSizes, 142 llvm::ArrayRef<unsigned_t> strides, 143 llvm::ArrayRef<unsigned_t> pads, 144 size_t group, size_t dilation); 145 146 template <typename ElemTy = float> 147 void fwdConvolutionInstFloatImpl(Value *inV, Value *outV, Value *filterV, 148 Value *biasV, 149 llvm::ArrayRef<unsigned_t> kernelSizes, 150 llvm::ArrayRef<unsigned_t> strides, 151 llvm::ArrayRef<unsigned_t> pads, 152 size_t group, size_t dilation); 153 154 template <typename ElemTy, typename AccumulatorTy, 155 typename BiasElemTy = int32_t> 156 void fwdConvolution3DInstQuantizedImpl(Value *inV, Value *outV, 157 Value *filterV, Value *biasV, 158 llvm::ArrayRef<unsigned_t> kernelSizes, 159 llvm::ArrayRef<unsigned_t> strides, 160 llvm::ArrayRef<unsigned_t> pads, 161 size_t group); 162 163 template <typename ElemTy = float> 164 void fwdConvolution3DInstFloatImpl(Value *inV, Value *outV, Value *filterV, 165 Value *biasV, 166 llvm::ArrayRef<unsigned_t> kernelSizes, 167 llvm::ArrayRef<unsigned_t> strides, 168 llvm::ArrayRef<unsigned_t> pads, 169 size_t group); 170 171 template <typename ElemTy = float> 172 void fwdConvTransposeInstFloatImpl(Value *inV, Value *outV, Value *filterV, 173 Value *biasV, 174 llvm::ArrayRef<unsigned_t> kernelSizes, 175 llvm::ArrayRef<unsigned_t> strides, 176 llvm::ArrayRef<unsigned_t> pads, 177 size_t group, size_t dilation); 178 179 void fwdAvgPoolInstI8Impl(const AvgPoolInst *I); 180 template <typename ElemTy> void fwdAvgPoolInstFloatImpl(const AvgPoolInst *I); 181 182 void fwdAvgPool3DInstI8Impl(const AvgPoolInst *I); 183 template <typename ElemTy> 184 void fwdAvgPool3DInstFloatImpl(const AvgPoolInst *I); 185 186 void fwdAdaptiveAvgPoolInstI8Impl(const AdaptiveAvgPoolInst *I); 187 template <typename ElemTy> 188 void fwdAdaptiveAvgPoolInstFloatImpl(const AdaptiveAvgPoolInst *I); 189 190 template <typename ElemTy> void fwdSoftMaxInstImpl(const SoftMaxInst *I); 191 192 template <typename ElemTy, typename AccumulatorTy> 193 void fwdMatMulInstQuantizedImpl(const MatMulInst *I); 194 template <typename ElemTy> void fwdMatMulInstFloatImpl(const MatMulInst *I); 195 196 template <typename ElemTy, typename AccumulatorTy, 197 typename BiasElemTy = int32_t> 198 void fwdFullyConnectedInstQuantizedImpl(const FullyConnectedInst *I); 199 template <typename ElemTy> 200 void fwdFullyConnectedInstFloatImpl(const FullyConnectedInst *I); 201 202 template <typename ElemTy, typename AccumulatorTy, 203 typename BiasElemTy = int32_t> 204 void fwdRowwiseQuantizedFullyConnectedInstImpl(Value *inV, Value *outV, 205 Value *weightsV, Value *biasV, 206 Value *scalesV, 207 Value *offsetsV); 208 209 template <typename ElemTy, typename AccumulatorTy, 210 typename BiasElemTy = int32_t> 211 void fwdChannelwiseQuantizedConv2DInstImpl( 212 const ChannelwiseQuantizedConvolutionInst *I); 213 214 template <typename ElemTy, typename AccumulatorTy, 215 typename BiasElemTy = int32_t> 216 void fwdChannelwiseQuantizedConv3DInstImpl( 217 const ChannelwiseQuantizedConvolutionInst *I); 218 219 void fwdElementAddInstI8Impl(const ElementAddInst *I); 220 template <typename ElemTy> 221 void fwdElementAddInstArithmeticImpl(const ElementAddInst *I); 222 223 void fwdElementMaxInstI8Impl(const ElementMaxInst *I); 224 template <typename ElemTy> 225 void fwdElementMaxInstArithmeticImpl(const ElementMaxInst *I); 226 227 template <typename ElemTy> 228 void fwdBatchedAddInstFloatImpl(const BatchedAddInst *I); 229 230 template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy, 231 typename CmpTy = ElemTy> 232 void fwdElementCmpEQInstImpl(const ElementCmpEQInst *I); 233 234 template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy, 235 typename CmpTy = ElemTy> 236 void fwdElementCmpNEQInstImpl(const ElementCmpNEQInst *I); 237 238 template <typename ElemTy> 239 void fwdBatchOneHotImpl(const glow::BatchOneHotInst *I); 240 241 template <typename ElemTy> 242 void fwdSpaceToDepthInstImpl(const glow::SpaceToDepthInst *I); 243 244 template <typename ElemTy> 245 void fwdResizeNearestInstImpl(const ResizeNearestInst *I); 246 247 template <typename ElemTy> 248 void fwdResizeBilinearInstImpl(const ResizeBilinearInst *I); 249 250 template <typename ElemTy> void fwdSigmoidInstFloatImpl(const SigmoidInst *I); 251 252 template <typename ElemTy> void fwdTanhInstFloatImpl(const TanhInst *I); 253 254 template <typename ElemTy> 255 void fwdCrossEntropyLossInstFloatImpl(const CrossEntropyLossInst *I); 256 257 template <typename ElemTy> 258 void fwdLocalResponseNormalizationInstFloatImpl( 259 const glow::LocalResponseNormalizationInst *I); 260 261 template <typename ElemTy> 262 void fwdElementSubInstArithmeticImpl(const ElementSubInst *I); 263 264 template <typename ElemTy> 265 void fwdElementMulInstArithmeticImpl(const ElementMulInst *I); 266 267 template <typename ElemTy> 268 void fwdElementMinInstArithmeticImpl(const ElementMinInst *I); 269 270 template <typename ElemTy, typename InstKind> 271 void fwdUnaryArithmeticImpl(const InstKind *I, 272 std::function<float(float)> func); 273 274 template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy, 275 typename CmpTy = ElemTy> 276 void fwdElementCmpLTEInstImpl(const ElementCmpLTEInst *I); 277 278 template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy, 279 typename CmpTy = ElemTy> 280 void fwdElementCmpLTInstImpl(const ElementCmpLTInst *I); 281 282 template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy, 283 typename CmpTy, typename InstCmpKind> 284 void 285 fwdElementCmpHelperImpl(const InstCmpKind *I, 286 std::function<bool(CmpTy LHS, CmpTy RHS)> cmpHelper); 287 288 template <typename ElemTy> 289 void fwdElementPowInstFloatImpl(const ElementPowInst *I); 290 291 template <typename ElemTy> 292 void fwdElementIsNaNInstFloatImpl(const ElementIsNaNInst *I); 293 294 template <typename ElemTy> 295 void fwdElementLogInstFloatImpl(const ElementLogInst *I); 296 297 template <typename ElemTy> 298 void fwdElementExpInstFloatImpl(const ElementExpInst *I); 299 300 template <typename ElemTy> 301 void fwdElementSelectInstFloatImpl(const ElementSelectInst *I); 302 303 template <typename ElemTy> 304 void fwdBatchedReduceAddInstFloatImpl(Value *batch, Value *dest, 305 unsigned_t axis, 306 const ShapeVector &eBatchDims, 307 const ShapeVector &eDestDims); 308 309 template <typename ElemTy> 310 void fwdBatchedReduceMinInstImpl(Value *batch, Value *dest, 311 const ShapeVector &eBatchDims, 312 const ShapeVector &eDestDims, ElemTy max); 313 314 template <typename ElemTy> 315 void fwdCumSumInstImpl(Value *input, Value *dest, bool exclusive, 316 bool reverse); 317 318 template <typename ElemTy> 319 void fwdLengthsSumInstFloatImpl(const LengthsSumInst *I); 320 321 template <typename ElemTy> void fwdGatherInstImpl(const GatherInst *I); 322 template <typename ElemTy> 323 void fwdGatherRangesInstImpl(const GatherRangesInst *I); 324 template <typename ElemTy> 325 void fwdScatterDataInstCopyImpl(const ScatterDataInst *I); 326 template <typename ElemTy> 327 void fwdScatterDataInstAddFloatImpl(const ScatterDataInst *I); 328 template <typename ElemTy> 329 void fwdScatterDataInstAddQuantizedImpl(const ScatterDataInst *I); 330 331 template <typename ElemTy> 332 void fwdSparseLengthsSumInstI8Impl(const SparseLengthsSumInst *I); 333 template <typename ElemTy, typename TI> 334 void fwdSparseLengthsSumInstFloatImpl(const SparseLengthsSumInst *I); 335 336 template <typename ElemTy> 337 void 338 fwdSparseLengthsWeightedSumInstI8Impl(const SparseLengthsWeightedSumInst *I); 339 template <typename ElemTy, typename TI> 340 void fwdSparseLengthsWeightedSumInstFloatImpl( 341 const SparseLengthsWeightedSumInst *I); 342 343 template <typename ElemTy> 344 void fwdEmbeddingBagInstFloatImpl(const EmbeddingBagInst *I); 345 346 template <typename ElemTy> 347 void fwdSparseToDenseInstImpl(const SparseToDenseInst *I); 348 349 template <class eTy> 350 void fwdRescaleQuantizedInstImpl(Value *src, Value *dest, 351 TensorQuantizationParams &srcQ, 352 TensorQuantizationParams &destQ); 353 354 template <typename ElemTy> void fwdModuloInstImpl(glow::ModuloInst const *I); 355 356 template <typename T, typename AccumT, typename TI> 357 void fwdRowwiseQuantizedSparseLengthsWeightedSumImpl( 358 const RowwiseQuantizedSparseLengthsWeightedSumInst *I); 359 360 template <typename T, typename AccumT, typename TI> 361 void fwdFusedRowwiseQuantizedSparseLengthsWeightedSumImpl( 362 const FusedRowwiseQuantizedSparseLengthsWeightedSumInst *I); 363 364 template <typename T> 365 void fwdNonMaxSuppressionInstImpl(glow::NonMaxSuppressionInst const *I); 366 367 void fwdAudioSpectrogramInstFloatImpl(glow::AudioSpectrogramInst const *I); 368 369 void fwdMFCCInstFloatImpl(glow::MFCCInst const *I); 370 371 template <typename T, typename AccumT> 372 void fwdEmbeddingBagByteRowwiseOffsetsImpl( 373 const EmbeddingBagByteRowwiseOffsetsInst *I); 374 375 template <typename ElemTy> void fwdFlipInstImpl(const FlipInst *I); 376 377 template <typename ElemTy> 378 void fwdBatchedPairwiseDotProductInstImpl( 379 const glow::BatchedPairwiseDotProductInst *I); 380 381 template <typename ElemTy> 382 void fwdBatchedPairwiseDotProductGradInstImpl( 383 const glow::BatchedPairwiseDotProductGradInst *I); 384 void fwdAvgPool2DGradInst(const AvgPoolGradInst *I); 385 void fwdAvgPool3DGradInst(const AvgPoolGradInst *I); 386 387 ///@} 388 }; 389 390 } // end namespace glow 391 392 #endif // GLOW_BACKENDS_INTERPRETER_INTERPRETERFUNCTION_H 393