1 /**
2  * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *     http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 #ifndef GLOW_BACKENDS_INTERPRETER_INTERPRETERFUNCTION_H
17 #define GLOW_BACKENDS_INTERPRETER_INTERPRETERFUNCTION_H
18 
19 #include "glow/Backend/Backend.h"
20 #include "glow/Backend/BackendUtils.h"
21 #include "glow/Backend/CompiledFunction.h"
22 #include "glow/Base/Tensor.h"
23 #include "glow/ExecutionContext/ExecutionContext.h"
24 #include "glow/Quantization/Base/Base.h"
25 
26 #include "llvm/ADT/ArrayRef.h"
27 
28 #include <memory>
29 #include <unordered_map>
30 
31 namespace glow {
32 
33 class IRFunction;
34 class Value;
35 class Tensor;
36 class Constant;
37 
38 // Forward declare all of the classes.
39 #define DEF_VALUE(CLASS, NAME) class CLASS;
40 #define DEF_INSTR(CLASS, NAME) class CLASS;
41 #define DEF_BACKEND_SPECIFIC_INSTR(CLASS, NAME)
42 #include "glow/AutoGenInstr.def"
43 
44 /// Function "compiled" for execution by the interpreter.
45 class InterpreterFunction final : public CompiledFunction,
46                                   public IRInstructionProcessingHandler {
47   /// The IR to be executed.
48   std::unique_ptr<IRFunction> F_;
49 
50   /// Maps Value.name to tensors for constants.
51   std::unordered_map<std::string, Tensor *> constants_;
52 
53 public:
54   InterpreterFunction(std::unique_ptr<IRFunction> F,
55                       runtime::RuntimeBundle &&bundle);
56 
57   /// \name CompiledFunction interface
58   ///@{
59   ~InterpreterFunction() override;
60 
61   Error execute(ExecutionContext *context) override;
62 
63   /// Collects constants for runtime.
64   void collectConstants(const Module *module) override;
65 
66   /// Add a constant to the function, this is used for loading static
67   /// placeholders.
68   void addConstant(std::string name, Tensor *T);
69 
70   /// Get reference to IR function.
getIR()71   IRFunction *getIR() { return F_.get(); }
72 
73   /// Read trace events out of this func and write them into /p context
74   void translateTraceEvents(ExecutionContext *context) const override;
75 
76   /// \returns the backend used to compile this function.
getCompileBackendName()77   virtual std::string getCompileBackendName() const override {
78     return "Interpreter";
79   }
80   ///@}
81 };
82 
83 /// An InterpreterFunction bound to a specific invocation.
84 class BoundInterpreterFunction : public IRInstructionProcessingHandler {
85   /// Maps values to Tensors, that are owned by this class.
86   std::unordered_map<const Value *, Tensor *> tensors_;
87 
88   /// Maps values to Tensors, that are *not* owned by this class.
89   std::unordered_map<const Value *, Tensor *> externalTensors_;
90 
91   /// A reference to the constant map from the owning InterpreterFunction.
92   const std::unordered_map<std::string, Tensor *> &constants_;
93 
94 public:
BoundInterpreterFunction(const std::unordered_map<std::string,Tensor * > & constants)95   explicit BoundInterpreterFunction(
96       const std::unordered_map<std::string, Tensor *> &constants)
97       : constants_(constants) {}
98 
99   ~BoundInterpreterFunction();
100 
101   Error execute(IRFunction *F, ExecutionContext *context);
102 
103 private:
104   /// \returns a pointer to the tensor that is saved under \p v.
105   Tensor *getTensor(const Value *v) const;
106 
107   /// Allocate a tensor to back the value \p v. Do not allocate anything if a
108   /// tensor is already allocated for \p v.
109   /// \returns a tensor for \p v.
110   Tensor *getOrCreateTensor(const Value *v);
111 
112   /// Allocate an unowned tensor to back the value \p v. The source tensor of
113   /// the unowned tensor is provided by \p src.
114   /// \returns a tensor for \p v.
115   Tensor *getOrCreateUnownedTensor(const Value *v, const Value *src,
116                                    llvm::ArrayRef<dim_t> offsets);
117 
118   /// If a tensor is allocated for \p v then delete it.
119   void deleteTensor(const Value *v);
120 
121   /// \returns a typed handle to the tensor that is stored at \p v.
122   template <class ElemTy = float>
getWeightHandle(Value * v)123   Handle<ElemTy> getWeightHandle(Value *v) const {
124     return getTensor(v)->getHandle<ElemTy>();
125   }
126 
127   /// @name BoundInterpreterFunction methods. This is a list of method
128   /// declerations that are used by the interpreter to dispatch different
129   /// instructions.
130   ///@{
131 
132 #define DEF_VALUE(CLASS, NAME)
133 #define DEF_INSTR(CLASS, NAME) void fwd##CLASS(const CLASS *I);
134 #define DEF_BACKEND_SPECIFIC_INSTR(CLASS, NAME)
135 #include "glow/AutoGenInstr.def"
136 
137   template <typename ElemTy, typename AccumulatorTy,
138             typename BiasElemTy = int32_t>
139   void fwdConvolutionInstQuantizedImpl(Value *inV, Value *outV, Value *filterV,
140                                        Value *biasV,
141                                        llvm::ArrayRef<unsigned_t> kernelSizes,
142                                        llvm::ArrayRef<unsigned_t> strides,
143                                        llvm::ArrayRef<unsigned_t> pads,
144                                        size_t group, size_t dilation);
145 
146   template <typename ElemTy = float>
147   void fwdConvolutionInstFloatImpl(Value *inV, Value *outV, Value *filterV,
148                                    Value *biasV,
149                                    llvm::ArrayRef<unsigned_t> kernelSizes,
150                                    llvm::ArrayRef<unsigned_t> strides,
151                                    llvm::ArrayRef<unsigned_t> pads,
152                                    size_t group, size_t dilation);
153 
154   template <typename ElemTy, typename AccumulatorTy,
155             typename BiasElemTy = int32_t>
156   void fwdConvolution3DInstQuantizedImpl(Value *inV, Value *outV,
157                                          Value *filterV, Value *biasV,
158                                          llvm::ArrayRef<unsigned_t> kernelSizes,
159                                          llvm::ArrayRef<unsigned_t> strides,
160                                          llvm::ArrayRef<unsigned_t> pads,
161                                          size_t group);
162 
163   template <typename ElemTy = float>
164   void fwdConvolution3DInstFloatImpl(Value *inV, Value *outV, Value *filterV,
165                                      Value *biasV,
166                                      llvm::ArrayRef<unsigned_t> kernelSizes,
167                                      llvm::ArrayRef<unsigned_t> strides,
168                                      llvm::ArrayRef<unsigned_t> pads,
169                                      size_t group);
170 
171   template <typename ElemTy = float>
172   void fwdConvTransposeInstFloatImpl(Value *inV, Value *outV, Value *filterV,
173                                      Value *biasV,
174                                      llvm::ArrayRef<unsigned_t> kernelSizes,
175                                      llvm::ArrayRef<unsigned_t> strides,
176                                      llvm::ArrayRef<unsigned_t> pads,
177                                      size_t group, size_t dilation);
178 
179   void fwdAvgPoolInstI8Impl(const AvgPoolInst *I);
180   template <typename ElemTy> void fwdAvgPoolInstFloatImpl(const AvgPoolInst *I);
181 
182   void fwdAvgPool3DInstI8Impl(const AvgPoolInst *I);
183   template <typename ElemTy>
184   void fwdAvgPool3DInstFloatImpl(const AvgPoolInst *I);
185 
186   void fwdAdaptiveAvgPoolInstI8Impl(const AdaptiveAvgPoolInst *I);
187   template <typename ElemTy>
188   void fwdAdaptiveAvgPoolInstFloatImpl(const AdaptiveAvgPoolInst *I);
189 
190   template <typename ElemTy> void fwdSoftMaxInstImpl(const SoftMaxInst *I);
191 
192   template <typename ElemTy, typename AccumulatorTy>
193   void fwdMatMulInstQuantizedImpl(const MatMulInst *I);
194   template <typename ElemTy> void fwdMatMulInstFloatImpl(const MatMulInst *I);
195 
196   template <typename ElemTy, typename AccumulatorTy,
197             typename BiasElemTy = int32_t>
198   void fwdFullyConnectedInstQuantizedImpl(const FullyConnectedInst *I);
199   template <typename ElemTy>
200   void fwdFullyConnectedInstFloatImpl(const FullyConnectedInst *I);
201 
202   template <typename ElemTy, typename AccumulatorTy,
203             typename BiasElemTy = int32_t>
204   void fwdRowwiseQuantizedFullyConnectedInstImpl(Value *inV, Value *outV,
205                                                  Value *weightsV, Value *biasV,
206                                                  Value *scalesV,
207                                                  Value *offsetsV);
208 
209   template <typename ElemTy, typename AccumulatorTy,
210             typename BiasElemTy = int32_t>
211   void fwdChannelwiseQuantizedConv2DInstImpl(
212       const ChannelwiseQuantizedConvolutionInst *I);
213 
214   template <typename ElemTy, typename AccumulatorTy,
215             typename BiasElemTy = int32_t>
216   void fwdChannelwiseQuantizedConv3DInstImpl(
217       const ChannelwiseQuantizedConvolutionInst *I);
218 
219   void fwdElementAddInstI8Impl(const ElementAddInst *I);
220   template <typename ElemTy>
221   void fwdElementAddInstArithmeticImpl(const ElementAddInst *I);
222 
223   void fwdElementMaxInstI8Impl(const ElementMaxInst *I);
224   template <typename ElemTy>
225   void fwdElementMaxInstArithmeticImpl(const ElementMaxInst *I);
226 
227   template <typename ElemTy>
228   void fwdBatchedAddInstFloatImpl(const BatchedAddInst *I);
229 
230   template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy,
231             typename CmpTy = ElemTy>
232   void fwdElementCmpEQInstImpl(const ElementCmpEQInst *I);
233 
234   template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy,
235             typename CmpTy = ElemTy>
236   void fwdElementCmpNEQInstImpl(const ElementCmpNEQInst *I);
237 
238   template <typename ElemTy>
239   void fwdBatchOneHotImpl(const glow::BatchOneHotInst *I);
240 
241   template <typename ElemTy>
242   void fwdSpaceToDepthInstImpl(const glow::SpaceToDepthInst *I);
243 
244   template <typename ElemTy>
245   void fwdResizeNearestInstImpl(const ResizeNearestInst *I);
246 
247   template <typename ElemTy>
248   void fwdResizeBilinearInstImpl(const ResizeBilinearInst *I);
249 
250   template <typename ElemTy> void fwdSigmoidInstFloatImpl(const SigmoidInst *I);
251 
252   template <typename ElemTy> void fwdTanhInstFloatImpl(const TanhInst *I);
253 
254   template <typename ElemTy>
255   void fwdCrossEntropyLossInstFloatImpl(const CrossEntropyLossInst *I);
256 
257   template <typename ElemTy>
258   void fwdLocalResponseNormalizationInstFloatImpl(
259       const glow::LocalResponseNormalizationInst *I);
260 
261   template <typename ElemTy>
262   void fwdElementSubInstArithmeticImpl(const ElementSubInst *I);
263 
264   template <typename ElemTy>
265   void fwdElementMulInstArithmeticImpl(const ElementMulInst *I);
266 
267   template <typename ElemTy>
268   void fwdElementMinInstArithmeticImpl(const ElementMinInst *I);
269 
270   template <typename ElemTy, typename InstKind>
271   void fwdUnaryArithmeticImpl(const InstKind *I,
272                               std::function<float(float)> func);
273 
274   template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy,
275             typename CmpTy = ElemTy>
276   void fwdElementCmpLTEInstImpl(const ElementCmpLTEInst *I);
277 
278   template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy,
279             typename CmpTy = ElemTy>
280   void fwdElementCmpLTInstImpl(const ElementCmpLTInst *I);
281 
282   template <typename ElemTy, typename ElemOffsetTy, typename ElemScaleTy,
283             typename CmpTy, typename InstCmpKind>
284   void
285   fwdElementCmpHelperImpl(const InstCmpKind *I,
286                           std::function<bool(CmpTy LHS, CmpTy RHS)> cmpHelper);
287 
288   template <typename ElemTy>
289   void fwdElementPowInstFloatImpl(const ElementPowInst *I);
290 
291   template <typename ElemTy>
292   void fwdElementIsNaNInstFloatImpl(const ElementIsNaNInst *I);
293 
294   template <typename ElemTy>
295   void fwdElementLogInstFloatImpl(const ElementLogInst *I);
296 
297   template <typename ElemTy>
298   void fwdElementExpInstFloatImpl(const ElementExpInst *I);
299 
300   template <typename ElemTy>
301   void fwdElementSelectInstFloatImpl(const ElementSelectInst *I);
302 
303   template <typename ElemTy>
304   void fwdBatchedReduceAddInstFloatImpl(Value *batch, Value *dest,
305                                         unsigned_t axis,
306                                         const ShapeVector &eBatchDims,
307                                         const ShapeVector &eDestDims);
308 
309   template <typename ElemTy>
310   void fwdBatchedReduceMinInstImpl(Value *batch, Value *dest,
311                                    const ShapeVector &eBatchDims,
312                                    const ShapeVector &eDestDims, ElemTy max);
313 
314   template <typename ElemTy>
315   void fwdCumSumInstImpl(Value *input, Value *dest, bool exclusive,
316                          bool reverse);
317 
318   template <typename ElemTy>
319   void fwdLengthsSumInstFloatImpl(const LengthsSumInst *I);
320 
321   template <typename ElemTy> void fwdGatherInstImpl(const GatherInst *I);
322   template <typename ElemTy>
323   void fwdGatherRangesInstImpl(const GatherRangesInst *I);
324   template <typename ElemTy>
325   void fwdScatterDataInstCopyImpl(const ScatterDataInst *I);
326   template <typename ElemTy>
327   void fwdScatterDataInstAddFloatImpl(const ScatterDataInst *I);
328   template <typename ElemTy>
329   void fwdScatterDataInstAddQuantizedImpl(const ScatterDataInst *I);
330 
331   template <typename ElemTy>
332   void fwdSparseLengthsSumInstI8Impl(const SparseLengthsSumInst *I);
333   template <typename ElemTy, typename TI>
334   void fwdSparseLengthsSumInstFloatImpl(const SparseLengthsSumInst *I);
335 
336   template <typename ElemTy>
337   void
338   fwdSparseLengthsWeightedSumInstI8Impl(const SparseLengthsWeightedSumInst *I);
339   template <typename ElemTy, typename TI>
340   void fwdSparseLengthsWeightedSumInstFloatImpl(
341       const SparseLengthsWeightedSumInst *I);
342 
343   template <typename ElemTy>
344   void fwdEmbeddingBagInstFloatImpl(const EmbeddingBagInst *I);
345 
346   template <typename ElemTy>
347   void fwdSparseToDenseInstImpl(const SparseToDenseInst *I);
348 
349   template <class eTy>
350   void fwdRescaleQuantizedInstImpl(Value *src, Value *dest,
351                                    TensorQuantizationParams &srcQ,
352                                    TensorQuantizationParams &destQ);
353 
354   template <typename ElemTy> void fwdModuloInstImpl(glow::ModuloInst const *I);
355 
356   template <typename T, typename AccumT, typename TI>
357   void fwdRowwiseQuantizedSparseLengthsWeightedSumImpl(
358       const RowwiseQuantizedSparseLengthsWeightedSumInst *I);
359 
360   template <typename T, typename AccumT, typename TI>
361   void fwdFusedRowwiseQuantizedSparseLengthsWeightedSumImpl(
362       const FusedRowwiseQuantizedSparseLengthsWeightedSumInst *I);
363 
364   template <typename T>
365   void fwdNonMaxSuppressionInstImpl(glow::NonMaxSuppressionInst const *I);
366 
367   void fwdAudioSpectrogramInstFloatImpl(glow::AudioSpectrogramInst const *I);
368 
369   void fwdMFCCInstFloatImpl(glow::MFCCInst const *I);
370 
371   template <typename T, typename AccumT>
372   void fwdEmbeddingBagByteRowwiseOffsetsImpl(
373       const EmbeddingBagByteRowwiseOffsetsInst *I);
374 
375   template <typename ElemTy> void fwdFlipInstImpl(const FlipInst *I);
376 
377   template <typename ElemTy>
378   void fwdBatchedPairwiseDotProductInstImpl(
379       const glow::BatchedPairwiseDotProductInst *I);
380 
381   template <typename ElemTy>
382   void fwdBatchedPairwiseDotProductGradInstImpl(
383       const glow::BatchedPairwiseDotProductGradInst *I);
384   void fwdAvgPool2DGradInst(const AvgPoolGradInst *I);
385   void fwdAvgPool3DGradInst(const AvgPoolGradInst *I);
386 
387   ///@}
388 };
389 
390 } // end namespace glow
391 
392 #endif // GLOW_BACKENDS_INTERPRETER_INTERPRETERFUNCTION_H
393