1 /*
2 * Copyright (c) Glow Contributors. See CONTRIBUTORS file.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "Importer.h"
17 #include "DebugMacros.h"
18 #include "NNPI.h"
19 #include "glow/IR/IR.h"
20 #include "glow/IR/Instrs.h"
21 #include "glow/Quantization/Base/Base.h"
22 #include "glow/Quantization/Quantization.h"
23 #include "nnpi_transformer.h"
24 #include <cmath>
25 #include <cstdio>
26 #include <fstream>
27 #include <limits>
28
29 #include "llvm/Support/CommandLine.h"
30
31 using namespace glow;
32
33 namespace glow {
34 llvm::cl::OptionCategory optionsForNNPIImporter("NNPI Importer Options");
35
36 bool GlowNNPISpecializeAllOneSLS = false;
37 static llvm::cl::opt<bool, /* ExternalStorage */ true>
38 GlowNNPISpecializeAllOneSLSOpt(
39 "glow_nnpi_specialize_all_one_sls",
40 llvm::cl::desc(
41 "Whether to import SLS ops with AllOne attribute to NNPI."),
42 llvm::cl::location(GlowNNPISpecializeAllOneSLS), llvm::cl::Optional,
43 llvm::cl::init(false), llvm::cl::cat(optionsForNNPIImporter));
44
45 } // namespace glow
46
47 const std::string NNPIImporter::internalName_("_NNPI_");
48
nodeValueName(const glow::NodeValue & nv)49 static std::string nodeValueName(const glow::NodeValue &nv) {
50 if (nv.getNode()->getKind() == glow::Kinded::Kind::PlaceholderKind) {
51 return nv.getNode()->getName();
52 } else if (nv.getNode()->getKind() == glow::Kinded::Kind::ConstantKind) {
53 return std::string(nv.getNode()->getName()) + std::string("__const");
54 }
55 return std::string(nv.getNode()->getName()) + std::string("__res_") +
56 std::to_string(nv.getResNo());
57 }
58
convertLengthsModeToLengthType(glow::LengthsMode mode,NNPI_LENGTH_TYPE & lengthType)59 NNPIErrorCode glow::NNPIImporter::convertLengthsModeToLengthType(
60 glow::LengthsMode mode, NNPI_LENGTH_TYPE &lengthType) {
61 if (!GlowNNPISpecializeAllOneSLS) {
62 mode = LengthsMode::Variable;
63 }
64 switch (mode) {
65 case LengthsMode::Variable:
66 lengthType = NNPI_LENGTH_VARIABLE;
67 break;
68 case LengthsMode::AllOne:
69 lengthType = NNPI_LENGTH_ALL_ONE;
70 break;
71 default:
72 return NNPI_INVALID_PARAM;
73 }
74 return NNPI_NO_ERROR;
75 }
76
NNPIImporter(const NNPICompilationOptions & compileOptions)77 glow::NNPIImporter::NNPIImporter(const NNPICompilationOptions &compileOptions)
78 : internalNameCounter_(0), network_(NNPI_INVALID_NNPIHANDLE),
79 compileOptions_(compileOptions) {
80 ASSERT_LOG_NNPI_ERROR(nnpiNetworkCreate(&network_),
81 "Failed to create NNPI network");
82 }
83
84 /// Destructor.
~NNPIImporter()85 glow::NNPIImporter::~NNPIImporter() {
86 if (network_ != NNPI_INVALID_NNPIHANDLE) {
87 LOG_NNPI_IF_ERROR(nnpiNetworkDestroy(network_),
88 "Failed to destroy NNPI network");
89 }
90 }
91
addTensor(std::string name,bool alternativeLayout,const std::string & scaleTensor,const std::string & offsetTensor,bool forceSymlowp)92 NNPIErrorCode glow::NNPIImporter::addTensor(std::string name,
93 bool alternativeLayout,
94 const std::string &scaleTensor,
95 const std::string &offsetTensor,
96 bool forceSymlowp) {
97 LOG_AND_RETURN_IF_NOT(
98 ERROR, constants_.count(name),
99 strFormat("Could not find Constants for tensor %s", name.c_str()),
100 NNPI_INVALID_PARAM);
101 const Tensor *t = constants_.at(name);
102
103 NNPITensorDesc desc;
104 desc.attributes.value = 0;
105 desc.attributes.constant = 1;
106 const auto &dims = t->dims();
107 desc.numDims = dims.size();
108 updateDescQuantFromGlow(t->getType(), desc, scaleTensor, offsetTensor,
109 forceSymlowp || compileOptions_.useSymlowp);
110 updateDescDimsFromGlow(dims, desc, alternativeLayout);
111
112 const void *pRawData(nullptr);
113 int32_t *pDataInt32(nullptr); // Used for converting int64_t to int32_t
114 switch (t->getType().getElementType()) {
115 case glow::ElemKind::FloatTy:
116 case glow::ElemKind::Float16Ty:
117 case glow::ElemKind::Int8QTy:
118 case glow::ElemKind::UInt8QTy:
119 case glow::ElemKind::UInt8FusedQTy:
120 case glow::ElemKind::UInt8FusedFP16QTy:
121 case glow::ElemKind::UInt4FusedFP16QTy:
122 case glow::ElemKind::Int32ITy:
123 case glow::ElemKind::Int32QTy:
124 case glow::ElemKind::BoolTy:
125 pRawData = t->getUnsafePtr();
126 break;
127 case glow::ElemKind::Int64ITy: {
128 auto *pDataInt64 = &(t->getHandle<int64_t>().raw(0));
129 const size_t numElements(t->size());
130 pDataInt32 = new int32_t[numElements];
131 LOG_AND_RETURN_IF_NOT(
132 ERROR, pDataInt32,
133 "Failed to allocate temporary storage for Int64 tensor",
134 NNPI_INVALID_PARAM);
135 for (size_t i = 0; i < numElements; i++) {
136 pDataInt32[i] = static_cast<int32_t>(pDataInt64[i]);
137 }
138 pRawData = static_cast<void *>(pDataInt32);
139 } break;
140 default:
141 LOG_AND_RETURN_IF_NOT(ERROR, 0, "Unhandled tensor data type",
142 NNPI_INVALID_PARAM);
143 break;
144 }
145 auto res = nnpiNetworkAddTensor(network_, name.c_str(), &desc, pRawData);
146
147 if (pDataInt32) {
148 delete[] pDataInt32;
149 }
150 return res;
151 }
152
addTensor(std::string name,const NNPITensorDesc & desc,const void * pData)153 NNPIErrorCode glow::NNPIImporter::addTensor(std::string name,
154 const NNPITensorDesc &desc,
155 const void *pData) {
156 auto res = nnpiNetworkAddTensor(network_, name.c_str(), &desc, pData);
157 return res;
158 }
159
addValueIfTensor(Value * v)160 NNPIErrorCode glow::NNPIImporter::addValueIfTensor(Value *v) {
161 LOG_AND_RETURN_IF_NOT(ERROR, v, "Trying to add NULL value",
162 NNPI_INVALID_PARAM);
163 auto *weight = llvm::dyn_cast<WeightVar>(v);
164 if (weight &&
165 weight->getMutability() == WeightVar::MutabilityKind::Constant &&
166 constants_.count(v->getName())) {
167 // Add a tensor.
168 return addTensor(v->getName().begin());
169 }
170 return NNPI_NO_ERROR;
171 }
172
addValue(std::string name,const glow::Type * vType,bool alternativeLayout,bool input,bool output,const std::string & scaleTensor,const std::string & offsetTensor,bool forceSymlowp)173 NNPIErrorCode glow::NNPIImporter::addValue(
174 std::string name, const glow::Type *vType, bool alternativeLayout,
175 bool input, bool output, const std::string &scaleTensor,
176 const std::string &offsetTensor, bool forceSymlowp) {
177 if (definedTensors_.count(name) && !alternativeLayout && !forceSymlowp &&
178 !input && !output) {
179 // The value was already defined and unless we're forcing a change in
180 // layout/input/output/etc. Don't redefine it.
181 return NNPI_NO_ERROR;
182 } else {
183 definedTensors_.insert(name);
184 }
185
186 NNPITensorDesc desc;
187 desc.attributes.value = 0;
188 desc.attributes.input = input;
189 desc.attributes.output = output;
190 updateDescQuantFromGlow(*vType, desc, scaleTensor, offsetTensor,
191 forceSymlowp || compileOptions_.useSymlowp);
192 updateDescDimsFromGlow(vType->dims(), desc, alternativeLayout);
193
194 const void *pRawData(nullptr);
195 if (constants_.count(name)) {
196 desc.attributes.constant = 1;
197 const Tensor *t = constants_.at(name);
198 switch (t->getType().getElementType()) {
199 case glow::ElemKind::FloatTy:
200 pRawData = &(t->getHandle<float>().raw(0));
201 break;
202 case glow::ElemKind::Float16Ty:
203 pRawData = &(t->getHandle<float16_t>().raw(0));
204 break;
205 case glow::ElemKind::Int64ITy:
206 pRawData = &(t->getHandle<int64_t>().raw(0));
207 break;
208 case glow::ElemKind::Int8QTy:
209 pRawData = &(t->getHandle<int8_t>().raw(0));
210 break;
211 case glow::ElemKind::BoolTy:
212 pRawData = &(t->getHandle<uint8_t>().raw(0));
213 break;
214 case glow::ElemKind::Int32QTy:
215 default:
216 LOG_AND_RETURN_IF_NOT(ERROR, 0, "Unhandled tensor data type",
217 NNPI_INVALID_PARAM);
218 break;
219 }
220 }
221
222 return nnpiNetworkAddTensor(network_, name.c_str(), &desc, pRawData);
223 }
224
updateDescDimsFromGlow(const llvm::ArrayRef<size_t> glowDims,NNPITensorDesc & desc,bool alternativeLayout)225 void glow::NNPIImporter::updateDescDimsFromGlow(
226 const llvm::ArrayRef<size_t> glowDims, NNPITensorDesc &desc,
227 bool alternativeLayout) {
228 desc.numDims = glowDims.size();
229 for (size_t d = 0; d < desc.numDims; d++) {
230 desc.dims[d] = glowDims[d];
231 }
232 switch (desc.numDims) {
233 case 6:
234 desc.layout = NNPI_LAYOUT_ANY;
235 break;
236 case 5:
237 desc.layout = alternativeLayout ? NNPI_LAYOUT_NDHWC : NNPI_LAYOUT_ANY;
238 break;
239 case 4:
240 desc.layout = alternativeLayout ? NNPI_LAYOUT_NHWC : NNPI_LAYOUT_ANY;
241 break;
242 case 3:
243 desc.layout = NNPI_LAYOUT_CHW;
244 break;
245 case 2:
246 desc.layout = alternativeLayout ? NNPI_LAYOUT_CN : NNPI_LAYOUT_NC;
247 break;
248 case 1:
249 desc.layout = NNPI_LAYOUT_C;
250 break;
251 case 0: // Special case for Caffe/Pytorch scalar.
252 desc.layout = NNPI_LAYOUT_C;
253 desc.numDims = 1;
254 desc.dims[0] = 1;
255 break;
256 default:
257 LOG(ERROR) << "Invalid number of dims";
258 break;
259 }
260 }
261
isBufferZero(T * buffer,size_t s)262 template <class T> bool isBufferZero(T *buffer, size_t s) {
263 for (size_t i = 0; i < s; i++) {
264 if (!(buffer[i] == (T)0))
265 return false;
266 }
267 return true;
268 }
269
zeroes(const std::string & name) const270 bool glow::NNPIImporter::zeroes(const std::string &name) const {
271 LOG_AND_RETURN_IF_NOT(ERROR, constants_.count(name), "Can't find tensor",
272 false);
273 const Tensor *t = constants_.at(name);
274 switch (t->getType().getElementType()) {
275 case glow::ElemKind::FloatTy:
276 return t->getHandle<float>().isZero();
277 case glow::ElemKind::Float16Ty: {
278 // Using isZero here leads to ambiguous overload for operator*
279 // for now manually check if the buffer is all zeros.
280 const auto dims = t->dims();
281 size_t s = 1;
282 for (size_t i = 0, e = dims.size(); i < e; i++) {
283 s *= dims[i];
284 }
285 const float16_t *pRawData = reinterpret_cast<const float16_t *>(
286 &(t->getHandle<float16_t>().raw(0)));
287 for (size_t i = 0; i < s; i++) {
288 if (!(pRawData[i] == (float16_t)0)) {
289 return false;
290 }
291 }
292 return true;
293 }
294 case glow::ElemKind::Int64ITy:
295 return t->getHandle<int64_t>().isZero();
296 case glow::ElemKind::Int32ITy:
297 return t->getHandle<int32_t>().isZero();
298 default:
299 LOG_AND_RETURN_IF_NOT(ERROR, 0, "Unhandled tensor data type", false);
300 break;
301 }
302
303 return false;
304 }
305
updateDescQuantFromGlow(const glow::Type & t,NNPITensorDesc & desc,const std::string & scaleTensor,const std::string & offsetTensor,bool forceSymlowp)306 void glow::NNPIImporter::updateDescQuantFromGlow(
307 const glow::Type &t, NNPITensorDesc &desc, const std::string &scaleTensor,
308 const std::string &offsetTensor, bool forceSymlowp) {
309 // Start with blanket defaults.
310 desc.quantParams.params.gemlowp.scale = 1.f;
311 desc.quantParams.params.gemlowp.offset = 0;
312 switch (t.getElementType()) {
313 case glow::ElemKind::FloatTy:
314 LOG_ERROR_IF_NOT((scaleTensor.empty() && offsetTensor.empty()))
315 << "Scales and offsets provided for Float";
316 desc.quantParams.precision = NNPI_PRECISION_FLOAT32;
317 desc.quantParams.type = NNPI_QUANTIZATION_NONE;
318 break;
319 case glow::ElemKind::Float16Ty:
320 LOG_ERROR_IF_NOT((scaleTensor.empty() && offsetTensor.empty()))
321 << "Scales and offsets provided for Float16";
322 desc.quantParams.precision = NNPI_PRECISION_FLOAT16;
323 desc.quantParams.type = NNPI_QUANTIZATION_NONE;
324 break;
325 case glow::ElemKind::Int64ITy:
326 LOG_ERROR_IF_NOT((scaleTensor.empty() && offsetTensor.empty()))
327 << "Scales and offsets provided for Int64";
328 desc.quantParams.precision = NNPI_PRECISION_INT32;
329 desc.quantParams.type = NNPI_QUANTIZATION_NONE;
330 break;
331 case glow::ElemKind::Int8QTy:
332 desc.quantParams.precision = NNPI_PRECISION_INT8;
333 // If we have scales tensor, this is PCQ case.
334 if (!scaleTensor.empty()) {
335 // If there is no offsets, or Symlowp workaround is used and all offsets
336 // are zero, the quantization type is SYMLOWP_PCQ.
337 if (offsetTensor.empty() || (forceSymlowp && zeroes(offsetTensor))) {
338 desc.quantParams.type = NNPI_QUANTIZATION_SYMLOWP_PCQ;
339 std::strncpy(desc.quantParams.params.symlowpPCQ.scalesTensor,
340 scaleTensor.c_str(),
341 sizeof(desc.quantParams.params.symlowpPCQ.scalesTensor));
342 } else { // Both scales and offsets are present.
343 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP_PCQ;
344 std::strncpy(desc.quantParams.params.gemmlowpPCQ.scalesTensor,
345 scaleTensor.c_str(),
346 sizeof(desc.quantParams.params.gemmlowpPCQ.scalesTensor));
347 std::strncpy(desc.quantParams.params.gemmlowpPCQ.offsetsTensor,
348 offsetTensor.c_str(),
349 sizeof(desc.quantParams.params.gemmlowpPCQ.offsetsTensor));
350 }
351 } else {
352 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP;
353 desc.quantParams.params.gemlowp.scale = t.getScale();
354 desc.quantParams.params.gemlowp.offset = t.getOffset();
355 if (forceSymlowp && (t.getOffset() == 0)) {
356 // WA use SYMLOWP for zero offset tensors.
357 DBG("SYMLOWP WA");
358 desc.quantParams.type = NNPI_QUANTIZATION_SYMLOWP;
359 desc.quantParams.params.symlowp.scale = t.getScale();
360 }
361 }
362
363 break;
364 case glow::ElemKind::UInt8QTy:
365 desc.quantParams.precision = NNPI_PRECISION_UINT8;
366 if (!scaleTensor.empty()) {
367 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP_PCQ;
368 std::strncpy(desc.quantParams.params.gemmlowpPCQ.scalesTensor,
369 scaleTensor.c_str(),
370 sizeof(desc.quantParams.params.gemmlowpPCQ.scalesTensor));
371 std::strncpy(desc.quantParams.params.gemmlowpPCQ.offsetsTensor,
372 offsetTensor.c_str(),
373 sizeof(desc.quantParams.params.gemmlowpPCQ.offsetsTensor));
374 } else {
375 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP;
376 desc.quantParams.params.gemlowp.scale = t.getScale();
377 desc.quantParams.params.gemlowp.offset = t.getOffset();
378 }
379 break;
380 case glow::ElemKind::UInt8FusedQTy:
381 desc.quantParams.precision = NNPI_PRECISION_UINT8;
382 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP_PCQ_FUSED;
383 break;
384 case glow::ElemKind::UInt8FusedFP16QTy:
385 desc.quantParams.precision = NNPI_PRECISION_UINT8;
386 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP_PCQ_FUSED_FP16;
387 break;
388 case glow::ElemKind::UInt4FusedFP16QTy:
389 desc.quantParams.precision = NNPI_PRECISION_UINT8;
390 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP_PCQ_4BIT_FUSED_FP16;
391 break;
392 case glow::ElemKind::Int32ITy:
393 desc.quantParams.precision = NNPI_PRECISION_INT32;
394 desc.quantParams.type = NNPI_QUANTIZATION_NONE;
395 break;
396 case glow::ElemKind::Int32QTy:
397 desc.quantParams.precision = NNPI_PRECISION_INT32;
398 if (forceSymlowp && t.getOffset() == 0) {
399 desc.quantParams.type = NNPI_QUANTIZATION_SYMLOWP;
400 } else {
401 desc.quantParams.type = NNPI_QUANTIZATION_GEMMLOWP;
402 }
403 desc.quantParams.params.gemlowp.scale = t.getScale();
404 desc.quantParams.params.gemlowp.offset = t.getOffset();
405 // This will be overwritten in addTensor for Int32QTy->Int8QTy WA.
406 break;
407 case glow::ElemKind::BoolTy:
408 desc.quantParams.precision = NNPI_PRECISION_BOOLEAN;
409 desc.quantParams.type = NNPI_QUANTIZATION_NONE;
410 break;
411 default:
412 LOG(ERROR) << "Unhandled tensor data type";
413 break;
414 }
415 }
416
isVariableUsingAlternativeLayout(Storage * v)417 bool glow::NNPIImporter::isVariableUsingAlternativeLayout(Storage *v) {
418 for (const auto &user : v->getUsers()) {
419 switch (user.getUser()->getKind()) {
420 case Kinded::Kind::ConvolutionNodeKind:
421 case Kinded::Kind::Convolution3DNodeKind:
422 case Kinded::Kind::AvgPoolNodeKind:
423 case Kinded::Kind::MaxPoolNodeKind:
424 return true;
425 case Kinded::Kind::FullyConnectedNodeKind:
426 return (v->getType()->dims().size() == 4);
427 default: // Do nothing.
428 break;
429 }
430 }
431 return false;
432 }
433
434 NNPIErrorCode
addIAExtentionPath(const std::string & extPath)435 glow::NNPIImporter::addIAExtentionPath(const std::string &extPath) {
436 LOG_AND_RETURN_IF(ERROR, extPath.empty(), "Check if empty IA extension path.",
437 NNPI_INVALID_PARAM);
438 std::ifstream extensionFile(extPath.c_str());
439 LOG_AND_RETURN_IF_NOT(ERROR, extensionFile, "IA extension path not found.",
440 NNPI_INVALID_RESOURCE_NAME);
441 iaExtensionPaths_.push_back(extPath);
442 return NNPI_NO_ERROR;
443 }
444
importFunction(Function * F,const BackendOptions & opts)445 NNPINetwork glow::NNPIImporter::importFunction(Function *F,
446 const BackendOptions &opts) {
447 if (compileOptions_.normalizeLayerNames) {
448 std::map<std::string, uint32_t> type2count;
449 std::map<std::string, glow::Node *> nodes;
450 for (auto &N : F->getNodes()) {
451 nodes[N.getName()] = &N;
452 }
453 auto *module = F->getParent();
454 std::string prefix;
455
456 if (module->getFunctions().size() > 1) {
457 uint32_t netID = 0;
458 for (const auto &function : module->getFunctions()) {
459 if (function == F) {
460 break;
461 }
462 netID++;
463 }
464 prefix = std::string("Net") + std::to_string(netID) + "_";
465 }
466 for (auto &pN : nodes) {
467 std::string kindStr = pN.second->getKindName();
468 if (type2count.count(kindStr) == 0) {
469 type2count[kindStr] = 0;
470 }
471 auto counter = type2count[kindStr]++;
472 auto newName = prefix + kindStr + "_" + std::to_string(counter);
473 pN.second->setName(newName);
474 }
475 }
476
477 // Clear internals.
478 constants_.clear();
479 readTensors_.clear();
480 writeTensors_.clear();
481 definedTensors_.clear();
482 DBG_MEM_USAGE("ImportFunction <<");
483 // Add constants.
484 for (const auto &c : F->getParent()->getConstants()) {
485 DBG("Importing Constant: " << c->getName().str() << " ("
486 << nodeValueName(c->getOutput()) << ") ["
487 << c->getKindName() << "]");
488 std::string name = nodeValueName(c->getOutput());
489 constants_.emplace(name, &c->getPayload());
490 DBG_MEM_USAGE("ImportFunction: Add Constant Tensor: " << name);
491 LOG_NNPI_IF_ERROR_RETURN_INVALID_HANDLE(
492 addTensor(nodeValueName(c->getOutput())), "Failed to add constant");
493 }
494
495 // Per node handling.
496 for (auto &N : F->getNodes()) {
497 // Check this type is handled.
498 if (nodeImporters_.count(N.getKindName()) == 0) {
499 DBG("-------------------------------------------------");
500 DBG("Unhandled node type: " << N.getKindName());
501 N.dump();
502 DBG("-------------------------------------------------");
503 return NNPI_INVALID_NNPIHANDLE;
504 }
505
506 DBG("Importing Node: " << N.getName().str() << " (" << N.getKindName()
507 << ")");
508 // Set node inputs and outputs.
509 for (unsigned i = 0, e = N.getNumInputs(); i < e; i++) {
510 auto inVal = N.getNthInput(i);
511 DBG(" Input: " << nodeValueName(inVal));
512 }
513 for (unsigned r = 0, e = N.getNumResults(); r < e; r++) {
514 auto resVal = N.getNthResult(r);
515 LOG_NNPI_IF_ERROR_RETURN_INVALID_HANDLE(
516 addValue(nodeValueName(resVal), resVal.getType()),
517 "Failed to add intermediate");
518 DBG(" Output: " << nodeValueName(resVal));
519 }
520 DBG_MEM_USAGE("ImportFunction import node: " << N.getKindName());
521 // Import node.
522 LOG_NNPI_IF_ERROR_RETURN_INVALID_HANDLE(
523 nodeImporters_.at(N.getKindName())->importNode(&N, *this),
524 "Failed to import node");
525 }
526
527 // Handle placeholders (inputs/outputs).
528 for (auto *v : F->getParent()->getPlaceholders()) {
529 bool inputVar(readTensors_.count(v->getName()) &&
530 !writeTensors_.count(v->getName()));
531 bool outputVar(!readTensors_.count(v->getName()) &&
532 writeTensors_.count(v->getName()));
533 if (inputVar || outputVar) {
534 LOG_NNPI_IF_ERROR_RETURN_INVALID_HANDLE(
535 addValue(v->getName(), v->getType(),
536 isVariableUsingAlternativeLayout(v), inputVar, outputVar),
537 "Failed to add placeholder");
538 DBG("[--IO--] Setting IO variable: " << v->getName().str() << ", R:"
539 << inputVar << ", W:" << outputVar
540 << ", U:" << v->getNumUsers());
541 } else {
542 DBG("[--IO--] Unused Placeholder: " << v->getName().str());
543 }
544 }
545
546 DBG_MEM_USAGE("ImportFunction call nnpiNetworkBuild");
547 // Build network.
548 NNPINetwork net;
549 NNPIErrorCode res = nnpiNetworkBuild(network_);
550 if (res != NNPI_NO_ERROR) {
551 LOG(INFO) << "Failed to build network";
552 LOG_NNPI_IF_ERROR(nnpiNetworkDestroy(network_),
553 "Failed to destroy NNPI network");
554 net = NNPI_INVALID_NNPIHANDLE;
555 } else {
556 net = network_;
557 }
558
559 // Detach network from importer (if failed to build then it's already
560 // destroyed, otherwise relinquish ownership to the backend).
561 network_ = NNPI_INVALID_NNPIHANDLE;
562 DBG_MEM_USAGE("ImportFunction done >>");
563 return net;
564 }
565
566 // Node Importers ////////////////////////////////////////////////////////
567 template <class ConvType = ConvolutionNode, size_t convDims = 2>
568 class ConvolutionNodeImporter : public INNPINodeImporter {
569 public:
importNode(Node * n,NNPIImporter & importer)570 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
571 auto *glowConv = llvm::dyn_cast<ConvType>(n);
572
573 std::string convStr = (convDims == 2) ? "Conv" : "Conv3D";
574 LOG_AND_RETURN_IF_NOT(ERROR, glowConv, "Bad node type", NNPI_INVALID_PARAM);
575
576 LOG_AND_RETURN_IF_NOT(ERROR, glowConv->getKernels().size() == convDims,
577 "[" + convStr + "] Invalid number of kernel sizes",
578 NNPI_INVALID_PARAM);
579 LOG_AND_RETURN_IF_NOT(ERROR, glowConv->getPads().size() == 2 * convDims,
580 "[" + convStr + "] Invalid number of pads",
581 NNPI_INVALID_PARAM);
582 LOG_AND_RETURN_IF_NOT(ERROR, glowConv->getStrides().size() == convDims,
583 "[" + convStr + "] Invalid number of strides",
584 NNPI_INVALID_PARAM);
585
586 uint32_t kernel[convDims];
587 uint32_t paddingStart[convDims];
588 uint32_t paddingEnd[convDims];
589 uint32_t stride[convDims];
590 uint32_t dilation[convDims];
591
592 ConvolutionNode *conv2DNode = llvm::dyn_cast<ConvolutionNode>(glowConv);
593 for (size_t i = 0; i < convDims; i++) {
594 kernel[i] = glowConv->getKernels()[i];
595 stride[i] = glowConv->getStrides()[i];
596 if (conv2DNode) {
597 paddingStart[i] = glowConv->getPads()[i];
598 paddingEnd[i] = glowConv->getPads()[convDims + i];
599 dilation[i] = conv2DNode->getDilation();
600 } else {
601 paddingStart[i] = glowConv->getPads()[i * 2];
602 paddingEnd[i] = glowConv->getPads()[i * 2 + 1];
603 dilation[i] = 1;
604 }
605 }
606
607 LOG_NNPI_IF_ERROR_RETURN_VALUE(
608 importer.addTensor(nodeValueName(glowConv->getFilter()),
609 /* alternativeLayout */ true),
610 "Failed to add tensor to NNPI");
611 LOG_NNPI_IF_ERROR_RETURN_VALUE(
612 importer.addTensor(nodeValueName(glowConv->getBias())),
613 "Failed to add tensor to NNPI");
614
615 // Overwrite input/output values for layout.
616 LOG_NNPI_IF_ERROR_RETURN_VALUE(
617 importer.addValue(nodeValueName(glowConv->getInput()),
618 glowConv->getInput().getType(),
619 /* alternativeLayout */ true),
620 "Failed to add tensor to NNPI");
621 LOG_NNPI_IF_ERROR_RETURN_VALUE(
622 importer.addValue(nodeValueName(glowConv->getResult()),
623 glowConv->getResult().getType(),
624 /* alternativeLayout */ true),
625 "Failed to add tensor to NNPI");
626
627 importer.setUsedTensors({nodeValueName(glowConv->getInput()),
628 nodeValueName(glowConv->getFilter()),
629 nodeValueName(glowConv->getBias())},
630 {nodeValueName(glowConv->getResult())});
631
632 return nnpiNetworkAddConvolutionOp(
633 importer.getNetwork(), glowConv->getName().begin(),
634 nodeValueName(glowConv->getInput()).c_str(),
635 nodeValueName(glowConv->getResult()).c_str(),
636 nodeValueName(glowConv->getFilter()).c_str(),
637 glowConv->getBias() ? nodeValueName(glowConv->getBias()).c_str()
638 : nullptr,
639 kernel, paddingStart, paddingEnd, stride, dilation, convDims,
640 glowConv->getGroup());
641 }
642 };
643
644 class TransposeNodeImporter : public INNPINodeImporter {
645 public:
importNode(Node * n,NNPIImporter & importer)646 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
647 auto *glowTranspose = llvm::dyn_cast<TransposeNode>(n);
648 LOG_AND_RETURN_IF_NOT(ERROR, glowTranspose, "Bad node type",
649 NNPI_INVALID_PARAM);
650
651 const auto &glowOrder = glowTranspose->getShuffle();
652 LOG_AND_RETURN_IF_NOT(ERROR, glowOrder.size() <= NNPI_MAX_DIMS,
653 "Bad dimansion", NNPI_INVALID_DIMS);
654
655 uint32_t nnpiOrder[NNPI_MAX_DIMS];
656 for (size_t i = 0, e = glowOrder.size(); i < e; i++) {
657 nnpiOrder[i] = glowOrder[i];
658 }
659
660 importer.setUsedTensors({nodeValueName(glowTranspose->getInput())},
661 {nodeValueName(glowTranspose->getResult())});
662
663 return nnpiNetworkAddTransposeOp(
664 importer.getNetwork(), glowTranspose->getName().begin(),
665 nodeValueName(glowTranspose->getInput()).c_str(),
666 nodeValueName(glowTranspose->getResult()).c_str(), nnpiOrder,
667 glowOrder.size());
668 }
669 };
670
671 template <class PoolNodeType, NNPI_POOLING_TYPE poolType>
672 class PoolNodeImporter : public INNPINodeImporter {
673 public:
importNode(Node * n,NNPIImporter & importer)674 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
675 auto *glowPool = llvm::dyn_cast<PoolNodeType>(n);
676 int inputDimension = glowPool->getInput().dims().size();
677 int numDims = inputDimension - 2;
678 LOG_AND_RETURN_IF_NOT(ERROR, numDims == 2 || numDims == 3,
679 "Input dimension is incorrect", NNPI_INVALID_PARAM);
680
681 std::string poolStr = (numDims == 2) ? "Pool" : "Pool3D";
682 LOG_AND_RETURN_IF_NOT(ERROR, glowPool, "Bad node type", NNPI_INVALID_PARAM);
683
684 LOG_AND_RETURN_IF_NOT(ERROR, glowPool->getKernels().size() == numDims,
685 "[" + poolStr + "] Invalid number of kernel sizes",
686 NNPI_INVALID_PARAM);
687 LOG_AND_RETURN_IF_NOT(ERROR, glowPool->getPads().size() == 2 * numDims,
688 "[" + poolStr + "] Invalid number of pads",
689 NNPI_INVALID_PARAM);
690 LOG_AND_RETURN_IF_NOT(ERROR, glowPool->getStrides().size() == numDims,
691 "[" + poolStr + "] Invalid number of strides",
692 NNPI_INVALID_PARAM);
693
694 std::vector<uint32_t> kernel(numDims);
695 std::vector<uint32_t> paddingStart(numDims);
696 std::vector<uint32_t> paddingEnd(numDims);
697 std::vector<uint32_t> stride(numDims);
698
699 for (size_t i = 0; i < numDims; i++) {
700 kernel[i] = glowPool->getKernels()[i];
701 stride[i] = glowPool->getStrides()[i];
702 if (numDims == 2) {
703 paddingStart[i] = glowPool->getPads()[i];
704 paddingEnd[i] = glowPool->getPads()[numDims + i];
705 } else {
706 paddingStart[i] = glowPool->getPads()[i * 2];
707 paddingEnd[i] = glowPool->getPads()[i * 2 + 1];
708 }
709 }
710
711 // Overwrite input/output values for layout.
712 LOG_NNPI_IF_ERROR_RETURN_VALUE(
713 importer.addValue(nodeValueName(glowPool->getInput()),
714 glowPool->getInput().getType(),
715 /* alternativeLayout */ true),
716 "Failed to add tensor to NNPI");
717 LOG_NNPI_IF_ERROR_RETURN_VALUE(
718 importer.addValue(nodeValueName(glowPool->getResult()),
719 glowPool->getResult().getType(),
720 /* alternativeLayout */ true),
721 "Failed to add tensor to NNPI");
722
723 importer.setUsedTensors({nodeValueName(glowPool->getInput())},
724 {nodeValueName(glowPool->getResult())});
725
726 return nnpiNetworkAddPoolingOp(
727 importer.getNetwork(), glowPool->getName().begin(),
728 nodeValueName(glowPool->getInput()).c_str(),
729 nodeValueName(glowPool->getResult()).c_str(), NULL, kernel.data(),
730 paddingStart.data(), paddingEnd.data(), stride.data(), numDims,
731 poolType, 0, 0);
732 }
733 };
734
735 template <class AdaptivePoolNodeType, NNPI_POOLING_TYPE poolType>
736 class AdaptivePoolNodeImporter : public INNPINodeImporter {
737 public:
importNode(Node * n,NNPIImporter & importer)738 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
739 auto *glowPool = llvm::dyn_cast<AdaptivePoolNodeType>(n);
740 LOG_AND_RETURN_IF_NOT(ERROR, glowPool, "Bad node type", NNPI_INVALID_PARAM);
741
742 // Overwrite input/output values for layout.
743 LOG_NNPI_IF_ERROR_RETURN_VALUE(
744 importer.addValue(nodeValueName(glowPool->getInput()),
745 glowPool->getInput().getType(),
746 /* alternativeLayout */ true),
747 "Failed to add tensor to NNPI");
748 LOG_NNPI_IF_ERROR_RETURN_VALUE(
749 importer.addValue(nodeValueName(glowPool->getResult()),
750 glowPool->getResult().getType(),
751 /* alternativeLayout */ true),
752 "Failed to add tensor to NNPI");
753
754 importer.setUsedTensors({nodeValueName(glowPool->getInput())},
755 {nodeValueName(glowPool->getResult())});
756
757 return nnpiNetworkAddAdaptivePoolingOp(
758 importer.getNetwork(), glowPool->getName().begin(),
759 nodeValueName(glowPool->getInput()).c_str(),
760 nodeValueName(glowPool->getResult()).c_str(), poolType);
761 }
762 };
763
764 class FullyConnectedNodeImporter : public INNPINodeImporter {
765 public:
importNode(Node * n,NNPIImporter & importer)766 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
767 auto *glowFC = llvm::dyn_cast<FullyConnectedNode>(n);
768 LOG_AND_RETURN_IF_NOT(ERROR, glowFC, "Bad node type", NNPI_INVALID_PARAM);
769
770 LOG_NNPI_IF_ERROR_RETURN_VALUE(
771 importer.addTensor(nodeValueName(glowFC->getWeights()),
772 /* alternativeLayout */ true),
773 "Failed to add tensor to NNPI");
774
775 // Overwrite input/output values for layout.
776 const auto *input = glowFC->getInput().getNode();
777 LOG_NNPI_IF_ERROR_RETURN_VALUE(
778 importer.addValue(input->getName(), input->getType(0),
779 input->getType(0)->dims().size() == 4),
780 "Failed to add tensor to NNPI");
781 const auto *result = glowFC->getResult().getNode();
782 LOG_NNPI_IF_ERROR_RETURN_VALUE(
783 importer.addValue(result->getName(), result->getType(0),
784 result->getType(0)->dims().size() == 4),
785 "Failed to add tensor to NNPI");
786
787 importer.setUsedTensors({nodeValueName(glowFC->getInput()),
788 nodeValueName(glowFC->getWeights()),
789 nodeValueName(glowFC->getBias())},
790 {nodeValueName(glowFC->getResult())});
791
792 return nnpiNetworkAddFullyConnectedOp(
793 importer.getNetwork(), glowFC->getName().begin(),
794 nodeValueName(glowFC->getInput()).c_str(),
795 nodeValueName(glowFC->getResult()).c_str(),
796 nodeValueName(glowFC->getWeights()).c_str(),
797 glowFC->getBias() ? nodeValueName(glowFC->getBias()).c_str() : nullptr);
798 }
799 };
800
801 class SoftMaxNodeImporter : public INNPINodeImporter {
802 public:
importNode(Node * n,NNPIImporter & importer)803 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
804 auto *glowSM = llvm::dyn_cast<SoftMaxNode>(n);
805 LOG_AND_RETURN_IF_NOT(ERROR, glowSM, "Bad node type", NNPI_INVALID_PARAM);
806
807 importer.setUsedTensors({nodeValueName(glowSM->getInput())},
808 {nodeValueName(glowSM->getResult())});
809
810 return nnpiNetworkAddSoftmaxOp(importer.getNetwork(),
811 glowSM->getName().begin(),
812 nodeValueName(glowSM->getInput()).c_str(),
813 nodeValueName(glowSM->getResult()).c_str(),
814 1); // Defaulting to axis 1 (C).
815 }
816 };
817
818 class SaveNodeImporter : public INNPINodeImporter {
819 public:
importNode(Node * n,NNPIImporter & importer)820 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
821 auto *glowSave = llvm::dyn_cast<SaveNode>(n);
822 LOG_AND_RETURN_IF_NOT(ERROR, glowSave, "Bad node type", NNPI_INVALID_PARAM);
823
824 importer.setUsedTensors({nodeValueName(glowSave->getInput())},
825 {nodeValueName(glowSave->getOutput())});
826
827 return nnpiNetworkAddCopyOp(importer.getNetwork(),
828 glowSave->getName().begin(),
829 nodeValueName(glowSave->getInput()).c_str(),
830 nodeValueName(glowSave->getOutput()).c_str());
831 }
832 };
833
834 class ReluNodeImporter : public INNPINodeImporter {
835 public:
importNode(Node * n,NNPIImporter & importer)836 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
837 auto *glowRelu = llvm::dyn_cast<ReluNode>(n);
838 LOG_AND_RETURN_IF_NOT(ERROR, glowRelu, "Bad node type", NNPI_INVALID_PARAM);
839
840 importer.setUsedTensors({nodeValueName(glowRelu->getInput())},
841 {nodeValueName(glowRelu->getResult())});
842
843 return nnpiNetworkAddReluOp(importer.getNetwork(),
844 glowRelu->getName().begin(),
845 nodeValueName(glowRelu->getInput()).c_str(),
846 nodeValueName(glowRelu->getResult()).c_str());
847 }
848 };
849
850 class PReluNodeImporter : public INNPINodeImporter {
851 public:
importNode(Node * n,NNPIImporter & importer)852 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
853 auto *glowPRelu = llvm::dyn_cast<PReluNode>(n);
854 LOG_AND_RETURN_IF_NOT(ERROR, glowPRelu, "Bad node type",
855 NNPI_INVALID_PARAM);
856
857 importer.setUsedTensors({nodeValueName(glowPRelu->getInput()),
858 nodeValueName(glowPRelu->getSlope())},
859 {nodeValueName(glowPRelu->getResult())});
860
861 return nnpiNetworkAddPReluOp(importer.getNetwork(),
862 glowPRelu->getName().begin(),
863 nodeValueName(glowPRelu->getInput()).c_str(),
864 nodeValueName(glowPRelu->getResult()).c_str(),
865 nodeValueName(glowPRelu->getSlope()).c_str());
866 }
867 };
868
869 template <class EltwiseNodeType, NNPI_ELTWISE_TYPE eltwiseType>
870 class BinaryEltwiseNodeImporter : public INNPINodeImporter {
871 public:
importNode(Node * n,NNPIImporter & importer)872 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
873 auto *glowEltwise = llvm::dyn_cast<EltwiseNodeType>(n);
874 LOG_AND_RETURN_IF_NOT(ERROR, glowEltwise, "Bad node type",
875 NNPI_INVALID_PARAM);
876
877 importer.setUsedTensors({nodeValueName(glowEltwise->getRHS()),
878 nodeValueName(glowEltwise->getLHS())},
879 {nodeValueName(glowEltwise->getResult())});
880
881 NNPIObjectName inputNames[2];
882 snprintf(inputNames[0], NNPI_MAX_STRING_LEN, "%s",
883 nodeValueName(glowEltwise->getLHS()).c_str());
884 snprintf(inputNames[1], NNPI_MAX_STRING_LEN, "%s",
885 nodeValueName(glowEltwise->getRHS()).c_str());
886 return nnpiNetworkAddElementwiseOp(
887 importer.getNetwork(), glowEltwise->getName().begin(), inputNames, 2,
888 nodeValueName(glowEltwise->getResult()).c_str(), eltwiseType);
889 }
890 };
891
892 template <class EltwiseNodeType, NNPI_ELTWISE_TYPE eltwiseType>
893 class UnaryEltwiseNodeImporter : public INNPINodeImporter {
894 public:
importNode(Node * n,NNPIImporter & importer)895 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
896 auto *glowEltwise = llvm::dyn_cast<EltwiseNodeType>(n);
897 LOG_AND_RETURN_IF_NOT(ERROR, glowEltwise, "Bad node type",
898 NNPI_INVALID_PARAM);
899
900 importer.setUsedTensors(
901 {
902 nodeValueName(glowEltwise->getInput()),
903 },
904 {nodeValueName(glowEltwise->getResult())});
905
906 NNPIObjectName inputNames[1];
907 snprintf(inputNames[0], NNPI_MAX_STRING_LEN, "%s",
908 nodeValueName(glowEltwise->getInput()).c_str());
909 return nnpiNetworkAddElementwiseOp(
910 importer.getNetwork(), glowEltwise->getName().begin(), inputNames, 1,
911 nodeValueName(glowEltwise->getResult()).c_str(), eltwiseType);
912 }
913 };
914
915 class ReshapeNodeImporter : public INNPINodeImporter {
916 public:
importNode(Node * n,NNPIImporter & importer)917 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
918 auto *glowReshape = llvm::dyn_cast<ReshapeNode>(n);
919 LOG_AND_RETURN_IF_NOT(ERROR, glowReshape, "Bad node type",
920 NNPI_INVALID_PARAM);
921
922 NNPITensorDesc desc;
923 importer.updateDescDimsFromGlow(glowReshape->getResult().getType()->dims(),
924 desc);
925
926 importer.setUsedTensors({nodeValueName(glowReshape->getInput())},
927 {nodeValueName(glowReshape->getResult())});
928
929 return nnpiNetworkAddReshapeOp(
930 importer.getNetwork(), glowReshape->getName().begin(),
931 nodeValueName(glowReshape->getInput()).c_str(),
932 nodeValueName(glowReshape->getResult()).c_str(), &desc);
933 }
934 };
935
936 template <typename TypedNode>
937 class ConvertNodeImporter : public INNPINodeImporter {
938 public:
importNode(Node * n,NNPIImporter & importer)939 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
940 auto *glowTypedNode = llvm::dyn_cast<TypedNode>(n);
941 LOG_AND_RETURN_IF_NOT(ERROR, glowTypedNode, "Bad node type",
942 NNPI_INVALID_PARAM);
943
944 importer.setUsedTensors({nodeValueName(glowTypedNode->getInput())},
945 {nodeValueName(glowTypedNode->getResult())});
946
947 return nnpiNetworkAddConvertOp(
948 importer.getNetwork(), glowTypedNode->getName().begin(),
949 nodeValueName(glowTypedNode->getInput()).c_str(),
950 nodeValueName(glowTypedNode->getResult()).c_str());
951 }
952 };
953
954 template <class MatMulNodeType>
955 class MatMulNodeImporter : public INNPINodeImporter {
956 public:
importNode(Node * n,NNPIImporter & importer)957 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
958 auto *glowMatMul = llvm::dyn_cast<MatMulNodeType>(n);
959 LOG_AND_RETURN_IF_NOT(ERROR, glowMatMul, "Bad node type",
960 NNPI_INVALID_PARAM);
961
962 importer.setUsedTensors({nodeValueName(glowMatMul->getLHS()),
963 nodeValueName(glowMatMul->getRHS())},
964 {nodeValueName(glowMatMul->getResult())});
965
966 return nnpiNetworkAddMatMulOp(
967 importer.getNetwork(), glowMatMul->getName().begin(),
968 nodeValueName(glowMatMul->getLHS()).c_str(),
969 nodeValueName(glowMatMul->getRHS()).c_str(),
970 nodeValueName(glowMatMul->getResult()).c_str());
971 }
972 };
973
974 class SliceNodeImporter : public INNPINodeImporter {
975 public:
importNode(Node * n,NNPIImporter & importer)976 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
977 auto *glowSlice = llvm::dyn_cast<SliceNode>(n);
978 LOG_AND_RETURN_IF_NOT(ERROR, glowSlice, "Bad node type",
979 NNPI_INVALID_PARAM);
980
981 const auto &sliceOffset = glowSlice->getStart();
982 int32_t startOffset[NNPI_MAX_DIMS] = {0};
983 int32_t endOffset[NNPI_MAX_DIMS] = {0};
984 auto *srcType = glowSlice->getInput().getType();
985 auto *dstType = glowSlice->getResult().getType();
986 LOG_AND_RETURN_IF_NOT(ERROR,
987 srcType->dims().size() == dstType->dims().size(),
988 "Bad dimansion", NNPI_INVALID_DIMS);
989 LOG_AND_RETURN_IF_NOT(ERROR, srcType->dims().size() == sliceOffset.size(),
990 "Bad dimansion", NNPI_INVALID_DIMS);
991
992 for (size_t i = 0, e = sliceOffset.size(); i < e; i++) {
993 startOffset[i] = sliceOffset[i];
994 endOffset[i] = startOffset[i] + dstType->dims()[i];
995 }
996
997 importer.setUsedTensors({nodeValueName(glowSlice->getInput())},
998 {nodeValueName(glowSlice->getResult())});
999
1000 return nnpiNetworkAddSliceOp(
1001 importer.getNetwork(), glowSlice->getName().begin(),
1002 nodeValueName(glowSlice->getInput()).c_str(),
1003 nodeValueName(glowSlice->getResult()).c_str(), startOffset, endOffset,
1004 nullptr, uint32_t(sliceOffset.size()));
1005 }
1006 };
1007
1008 class SigmoidNodeImporter : public INNPINodeImporter {
1009 public:
importNode(Node * n,NNPIImporter & importer)1010 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1011 auto *glowSigmoid = llvm::dyn_cast<SigmoidNode>(n);
1012 LOG_AND_RETURN_IF_NOT(ERROR, glowSigmoid, "Bad node type",
1013 NNPI_INVALID_PARAM);
1014
1015 importer.setUsedTensors({nodeValueName(glowSigmoid->getInput())},
1016 {nodeValueName(glowSigmoid->getResult())});
1017
1018 return nnpiNetworkAddSigmoidOp(
1019 importer.getNetwork(), glowSigmoid->getName().begin(),
1020 nodeValueName(glowSigmoid->getInput()).c_str(),
1021 nodeValueName(glowSigmoid->getResult()).c_str());
1022 }
1023 };
1024
1025 class TanhNodeImporter : public INNPINodeImporter {
1026 public:
importNode(Node * n,NNPIImporter & importer)1027 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1028 auto *glowTanh = llvm::dyn_cast<TanhNode>(n);
1029 LOG_AND_RETURN_IF_NOT(ERROR, glowTanh, "Bad node type", NNPI_INVALID_PARAM);
1030
1031 importer.setUsedTensors({nodeValueName(glowTanh->getInput())},
1032 {nodeValueName(glowTanh->getResult())});
1033
1034 return nnpiNetworkAddTanhOp(importer.getNetwork(),
1035 glowTanh->getName().begin(),
1036 nodeValueName(glowTanh->getInput()).c_str(),
1037 nodeValueName(glowTanh->getResult()).c_str());
1038 }
1039 };
1040
1041 class TopkNodeImporter : public INNPINodeImporter {
1042 public:
importNode(Node * n,NNPIImporter & importer)1043 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1044 auto *glowTopk = llvm::dyn_cast<TopKNode>(n);
1045 LOG_AND_RETURN_IF_NOT(ERROR, glowTopk, "Bad node type", NNPI_INVALID_PARAM);
1046
1047 importer.setUsedTensors({nodeValueName(glowTopk->getInput())},
1048 {nodeValueName(glowTopk->getValues()),
1049 nodeValueName(glowTopk->getIndices())});
1050 return nnpiNetworkAddTopkOp(
1051 importer.getNetwork(), glowTopk->getName().begin(),
1052 nodeValueName(glowTopk->getInput()).c_str(),
1053 nodeValueName(glowTopk->getValues()).c_str(),
1054 nodeValueName(glowTopk->getIndices()).c_str(), glowTopk->getK(),
1055 -1); // No Axis parameter in Glow - using -1 by default.
1056 }
1057 };
1058
1059 class ConcatNodeImporter : public INNPINodeImporter {
1060 public:
importNode(Node * n,NNPIImporter & importer)1061 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1062 auto *glowConcat = llvm::dyn_cast<ConcatNode>(n);
1063 LOG_AND_RETURN_IF_NOT(ERROR, glowConcat, "Bad node type",
1064 NNPI_INVALID_PARAM);
1065
1066 auto numInputs = glowConcat->getNumInputs();
1067 NNPIObjectName *inputs = new NNPIObjectName[numInputs];
1068 LOG_AND_RETURN_IF_NOT(ERROR, inputs, "No inputs", NNPI_INVALID_PARAM);
1069 std::unordered_set<std::string> inputTensors;
1070
1071 for (unsigned i = 0; i < numInputs; i++) {
1072 auto nvName = nodeValueName(glowConcat->getNthInput(i));
1073 strncpy(inputs[i], nvName.c_str(), sizeof(NNPIObjectName));
1074 inputTensors.insert(nvName);
1075 }
1076
1077 importer.setUsedTensors(inputTensors,
1078 {nodeValueName(glowConcat->getResult())});
1079
1080 auto res = nnpiNetworkAddConcatOp(
1081 importer.getNetwork(), glowConcat->getName().begin(), inputs, numInputs,
1082 nodeValueName(glowConcat->getResult()).c_str(), glowConcat->getDim());
1083 delete[] inputs;
1084 return res;
1085 }
1086 };
1087
1088 class TileNodeImporter : public INNPINodeImporter {
1089 public:
importNode(Node * n,NNPIImporter & importer)1090 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1091 auto *glowTile = llvm::dyn_cast<TileNode>(n);
1092 LOG_AND_RETURN_IF_NOT(ERROR, glowTile, "Bad node type", NNPI_INVALID_PARAM);
1093
1094 importer.setUsedTensors({nodeValueName(glowTile->getInput())},
1095 {nodeValueName(glowTile->getResult())});
1096
1097 auto numDims = glowTile->getInput().getType()->dims().size();
1098 std::vector<int32_t> repeats(numDims, 1);
1099 auto axis = glowTile->getAxis();
1100 LOG_AND_RETURN_IF_NOT(ERROR, axis >= 0 && axis < numDims,
1101 "tile axis is invalid", NNPI_INVALID_PARAM);
1102 repeats[axis] = glowTile->getCount();
1103 NNPITensorDesc desc;
1104 desc.attributes.value = 0;
1105 desc.attributes.constant = 1;
1106 desc.numDims = 1;
1107 desc.dims[0] = numDims;
1108 desc.quantParams.precision = NNPI_PRECISION_INT32;
1109 desc.quantParams.type = NNPI_QUANTIZATION_NONE;
1110 desc.layout = NNPI_LAYOUT_ANY;
1111
1112 auto repeatsTensorName = glowTile->getName().str() + "_repeats";
1113
1114 importer.addTensor(repeatsTensorName, desc, repeats.data());
1115
1116 return nnpiNetworkAddTileOp(
1117 importer.getNetwork(), glowTile->getName().begin(),
1118 nodeValueName(glowTile->getInput()).c_str(), repeatsTensorName.c_str(),
1119 nodeValueName(glowTile->getResult()).c_str());
1120 }
1121 };
1122
1123 class GatherNodeImporter : public INNPINodeImporter {
1124 public:
importNode(Node * n,NNPIImporter & importer)1125 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1126 auto *glowGather = llvm::dyn_cast<GatherNode>(n);
1127 LOG_AND_RETURN_IF_NOT(ERROR, glowGather, "Bad node type",
1128 NNPI_INVALID_PARAM);
1129
1130 importer.setUsedTensors({nodeValueName(glowGather->getData()),
1131 nodeValueName(glowGather->getIndices())},
1132 {nodeValueName(glowGather->getResult())});
1133
1134 return nnpiNetworkAddGatherOp(
1135 importer.getNetwork(), glowGather->getName().begin(),
1136 nodeValueName(glowGather->getData()).c_str(),
1137 nodeValueName(glowGather->getIndices()).c_str(),
1138 nodeValueName(glowGather->getResult()).c_str(),
1139 glowGather->getBatchDims());
1140 }
1141 };
1142
1143 class ArgMaxNodeImporter : public INNPINodeImporter {
1144 public:
importNode(Node * n,NNPIImporter & importer)1145 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1146 auto *glowArgMax = llvm::dyn_cast<ArgMaxNode>(n);
1147 LOG_AND_RETURN_IF_NOT(ERROR, glowArgMax, "Bad node type",
1148 NNPI_INVALID_PARAM);
1149
1150 importer.setUsedTensors({nodeValueName(glowArgMax->getInput())},
1151 {nodeValueName(glowArgMax->getResult())});
1152
1153 uint32_t axis = glowArgMax->getAxis();
1154 auto keepDims = glowArgMax->getKeepDims() ? 1 : 0;
1155 return nnpiNetworkAddReduceOp(
1156 importer.getNetwork(), glowArgMax->getName().begin(),
1157 nodeValueName(glowArgMax->getInput()).c_str(),
1158 nodeValueName(glowArgMax->getResult()).c_str(), NNPI_REDUCE_ARG_MAX,
1159 &axis, 1, keepDims);
1160 }
1161 };
1162
1163 class ReduceAddNodeImporter : public INNPINodeImporter {
1164 public:
importNode(Node * n,NNPIImporter & importer)1165 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1166 auto *glowReduce = llvm::dyn_cast<BatchedReduceAddNode>(n);
1167 LOG_AND_RETURN_IF_NOT(ERROR, glowReduce, "Bad node type",
1168 NNPI_INVALID_PARAM);
1169
1170 importer.setUsedTensors({nodeValueName(glowReduce->getBatch())},
1171 {nodeValueName(glowReduce->getResult())});
1172
1173 uint32_t axis = glowReduce->getAxis();
1174 return nnpiNetworkAddReduceOp(
1175 importer.getNetwork(), glowReduce->getName().begin(),
1176 nodeValueName(glowReduce->getBatch()).c_str(),
1177 nodeValueName(glowReduce->getResult()).c_str(), NNPI_REDUCE_SUM, &axis,
1178 1, 0);
1179 }
1180 };
1181
1182 template <class ReduceNodeType, NNPI_REDUCE_TYPE reduceType>
1183 class ReduceMultAxesNodeImporter : public INNPINodeImporter {
1184 public:
importNode(Node * n,NNPIImporter & importer)1185 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1186 auto *glowReduce = llvm::dyn_cast<ReduceNodeType>(n);
1187 LOG_AND_RETURN_IF_NOT(ERROR, glowReduce, "Bad node type",
1188 NNPI_INVALID_PARAM);
1189
1190 importer.setUsedTensors({nodeValueName(glowReduce->getBatch())},
1191 {nodeValueName(glowReduce->getResult())});
1192
1193 LOG_AND_RETURN_IF_NOT(ERROR, glowReduce->getAxes().size() == 1,
1194 "Bad axis value", NNPI_INVALID_PARAM);
1195 uint32_t axis = glowReduce->getAxes()[0];
1196 return nnpiNetworkAddReduceOp(
1197 importer.getNetwork(), glowReduce->getName().begin(),
1198 nodeValueName(glowReduce->getBatch()).c_str(),
1199 nodeValueName(glowReduce->getResult()).c_str(), reduceType, &axis, 1,
1200 0);
1201 }
1202 };
1203
1204 class LogNodeImporter : public INNPINodeImporter {
1205 public:
importNode(Node * n,NNPIImporter & importer)1206 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1207 auto *glowLog = llvm::dyn_cast<LogNode>(n);
1208 LOG_AND_RETURN_IF_NOT(ERROR, glowLog, "Bad node type", NNPI_INVALID_PARAM);
1209
1210 importer.setUsedTensors({nodeValueName(glowLog->getInput())},
1211 {nodeValueName(glowLog->getResult())});
1212
1213 return nnpiNetworkAddLogOp(importer.getNetwork(),
1214 glowLog->getName().begin(),
1215 nodeValueName(glowLog->getInput()).c_str(),
1216 nodeValueName(glowLog->getResult()).c_str());
1217 }
1218 };
1219
1220 class SplatNodeImporter : public INNPINodeImporter {
1221 public:
importNode(Node * n,NNPIImporter & importer)1222 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1223 auto *glowSplat = llvm::dyn_cast<SplatNode>(n);
1224 LOG_AND_RETURN_IF_NOT(ERROR, glowSplat, "Bad node type",
1225 NNPI_INVALID_PARAM);
1226
1227 importer.setUsedTensors({}, {nodeValueName(glowSplat->getResult())});
1228 auto *destType = glowSplat->getResult().getType();
1229 int32_t numDims = static_cast<int32_t>(destType->dims().size());
1230 float glowSplatValue = glowSplat->getValue();
1231
1232 std::vector<dim_t> finalShapeFilledWithOnes(numDims, 1);
1233
1234 auto tileInputTensorName = NNPIImporter::internalName_ +
1235 glowSplat->getName().str() + "_Tile_input";
1236
1237 if (destType->getElementType() != ElemKind::FloatTy) {
1238 NNPITensorDesc convertInputDesc;
1239 convertInputDesc.attributes.value = 0;
1240 convertInputDesc.attributes.constant = 1;
1241 convertInputDesc.quantParams.precision = NNPI_PRECISION_FLOAT32;
1242 convertInputDesc.quantParams.type = NNPI_QUANTIZATION_NONE;
1243 importer.updateDescDimsFromGlow(finalShapeFilledWithOnes,
1244 convertInputDesc);
1245
1246 auto convertInputTensorName = NNPIImporter::internalName_ +
1247 glowSplat->getName().str() +
1248 "_Tile_Convert_input";
1249 LOG_NNPI_IF_ERROR_RETURN_VALUE(importer.addTensor(convertInputTensorName,
1250 convertInputDesc,
1251 &glowSplatValue),
1252 "Failed to add tensor");
1253
1254 auto convertName = NNPIImporter::internalName_ +
1255 glowSplat->getName().str() + "_Tile_Convert";
1256 Type convertOutputType =
1257 Type::newShape(*destType, finalShapeFilledWithOnes);
1258 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1259 importer.addValue(tileInputTensorName, &convertOutputType),
1260 "Failed to add value");
1261
1262 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1263 nnpiNetworkAddConvertOp(importer.getNetwork(), convertName.c_str(),
1264 convertInputTensorName.c_str(),
1265 tileInputTensorName.c_str()),
1266 "Failed to add layer");
1267 } else {
1268 NNPITensorDesc tileInputDesc;
1269 tileInputDesc.attributes.value = 0;
1270 tileInputDesc.attributes.constant = 1;
1271 tileInputDesc.quantParams.precision = NNPI_PRECISION_FLOAT32;
1272 tileInputDesc.quantParams.type = NNPI_QUANTIZATION_NONE;
1273 importer.updateDescDimsFromGlow(finalShapeFilledWithOnes, tileInputDesc);
1274
1275 LOG_NNPI_IF_ERROR_RETURN_VALUE(importer.addTensor(tileInputTensorName,
1276 tileInputDesc,
1277 &glowSplatValue),
1278 "Failed to add tensor");
1279 }
1280
1281 NNPITensorDesc repeatsDesc;
1282 repeatsDesc.attributes.value = 0;
1283 repeatsDesc.attributes.constant = 1;
1284 repeatsDesc.quantParams.precision = NNPI_PRECISION_INT32;
1285 repeatsDesc.quantParams.type = NNPI_QUANTIZATION_NONE;
1286 importer.updateDescDimsFromGlow({destType->dims().size()}, repeatsDesc);
1287 auto repeatsTensorName = NNPIImporter::internalName_ +
1288 glowSplat->getName().str() + "_Tile_repeats";
1289 std::vector<int32_t> dims;
1290 for (int i = 0; i < numDims; i++) {
1291 dims.push_back(destType->dims()[i]);
1292 }
1293
1294 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1295 importer.addTensor(repeatsTensorName, repeatsDesc, dims.data()),
1296 "Failed to add tensor");
1297
1298 auto tileNodeName =
1299 NNPIImporter::internalName_ + glowSplat->getName().str() + "_Tile";
1300
1301 return nnpiNetworkAddTileOp(importer.getNetwork(), tileNodeName.c_str(),
1302 tileInputTensorName.c_str(),
1303 repeatsTensorName.c_str(),
1304 nodeValueName(glowSplat->getResult()).c_str());
1305 }
1306 };
1307
1308 class SLSNodeImporter : public INNPINodeImporter {
1309 public:
importNode(Node * n,NNPIImporter & importer)1310 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1311 auto *glowSLS = llvm::dyn_cast<SparseLengthsSumNode>(n);
1312 LOG_AND_RETURN_IF_NOT(ERROR, glowSLS, "Bad node type", NNPI_INVALID_PARAM);
1313
1314 importer.setUsedTensors(
1315 {
1316 nodeValueName(glowSLS->getData()),
1317 nodeValueName(glowSLS->getIndices()),
1318 nodeValueName(glowSLS->getLengths()),
1319 },
1320 {nodeValueName(glowSLS->getResult())});
1321
1322 NNPI_LENGTH_TYPE lengthType;
1323 LOG_AND_RETURN_IF_NOT(ERROR,
1324 NNPIImporter::convertLengthsModeToLengthType(
1325 glowSLS->getLengthsMode(), lengthType) ==
1326 NNPI_NO_ERROR,
1327 "Unhandled SLS length type", NNPI_INVALID_PARAM);
1328
1329 return nnpiNetworkAddSparseLengthsWeightedSumOp(
1330 importer.getNetwork(), glowSLS->getName().begin(),
1331 nodeValueName(glowSLS->getData()).c_str(),
1332 nodeValueName(glowSLS->getResult()).c_str(), NULL,
1333 nodeValueName(glowSLS->getIndices()).c_str(),
1334 nodeValueName(glowSLS->getLengths()).c_str(), false, false,
1335 glowSLS->getAvgLength(), lengthType,
1336 /* force to IA */ false);
1337 }
1338 };
1339
1340 class SLWSNodeImporter : public INNPINodeImporter {
1341 public:
importNode(Node * n,NNPIImporter & importer)1342 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1343 auto *glowSLWS = llvm::dyn_cast<SparseLengthsWeightedSumNode>(n);
1344 LOG_AND_RETURN_IF_NOT(ERROR, glowSLWS, "Bad node type", NNPI_INVALID_PARAM);
1345
1346 importer.setUsedTensors(
1347 {
1348 nodeValueName(glowSLWS->getData()),
1349 nodeValueName(glowSLWS->getWeights()),
1350 nodeValueName(glowSLWS->getIndices()),
1351 nodeValueName(glowSLWS->getLengths()),
1352 },
1353 {nodeValueName(glowSLWS->getResult())});
1354
1355 NNPI_LENGTH_TYPE lengthType;
1356 LOG_AND_RETURN_IF_NOT(ERROR,
1357 NNPIImporter::convertLengthsModeToLengthType(
1358 glowSLWS->getLengthsMode(), lengthType) ==
1359 NNPI_NO_ERROR,
1360 "Unhandled SLS length type", NNPI_INVALID_PARAM);
1361
1362 return nnpiNetworkAddSparseLengthsWeightedSumOp(
1363 importer.getNetwork(), glowSLWS->getName().begin(),
1364 nodeValueName(glowSLWS->getData()).c_str(),
1365 nodeValueName(glowSLWS->getResult()).c_str(),
1366 nodeValueName(glowSLWS->getWeights()).c_str(),
1367 nodeValueName(glowSLWS->getIndices()).c_str(),
1368 nodeValueName(glowSLWS->getLengths()).c_str(), false, false,
1369 glowSLWS->getAvgLength(), lengthType,
1370 /* force to IA */ false);
1371 }
1372 };
1373
1374 class EmbeddingBagNodeImporter : public INNPINodeImporter {
1375 public:
importNode(Node * n,NNPIImporter & importer)1376 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1377 auto *glowEmbeddingBag = llvm::dyn_cast<EmbeddingBagNode>(n);
1378 LOG_AND_RETURN_IF_NOT(ERROR, glowEmbeddingBag, "Bad node type",
1379 NNPI_INVALID_PARAM);
1380
1381 bool hasEndOffset = glowEmbeddingBag->getHasEndOffset();
1382 LOG_AND_RETURN_IF_NOT(ERROR, hasEndOffset,
1383 "[EmbeddingBag] hasEndOffset must be true",
1384 NNPI_INVALID_PARAM);
1385
1386 importer.setUsedTensors(
1387 {
1388 nodeValueName(glowEmbeddingBag->getData()),
1389 nodeValueName(glowEmbeddingBag->getWeights()),
1390 nodeValueName(glowEmbeddingBag->getIndices()),
1391 nodeValueName(glowEmbeddingBag->getOffsets()),
1392 },
1393 {nodeValueName(glowEmbeddingBag->getResult())});
1394
1395 NNPI_LENGTH_TYPE lengthType;
1396 LOG_AND_RETURN_IF_NOT(ERROR,
1397 NNPIImporter::convertLengthsModeToLengthType(
1398 glowEmbeddingBag->getLengthsMode(), lengthType) ==
1399 NNPI_NO_ERROR,
1400 "Unhandled SLS length type", NNPI_INVALID_PARAM);
1401
1402 return nnpiNetworkAddSparseLengthsWeightedSumOp(
1403 importer.getNetwork(), glowEmbeddingBag->getName().begin(),
1404 nodeValueName(glowEmbeddingBag->getData()).c_str(),
1405 nodeValueName(glowEmbeddingBag->getResult()).c_str(),
1406 nodeValueName(glowEmbeddingBag->getWeights()).c_str(),
1407 nodeValueName(glowEmbeddingBag->getIndices()).c_str(),
1408 nodeValueName(glowEmbeddingBag->getOffsets()).c_str(), false, true,
1409 glowEmbeddingBag->getAvgLength(), lengthType,
1410 /* force to IA */ false);
1411 }
1412 };
1413
1414 class EmbeddingBagByteRowwiseOffsetsNodeImporter : public INNPINodeImporter {
1415 public:
importNode(Node * n,NNPIImporter & importer)1416 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1417 auto *glowEBBRO = llvm::dyn_cast<EmbeddingBagByteRowwiseOffsetsNode>(n);
1418 LOG_AND_RETURN_IF_NOT(ERROR, glowEBBRO, "Bad node type",
1419 NNPI_INVALID_PARAM);
1420
1421 bool hasEndOffset = glowEBBRO->getHasEndOffset();
1422 LOG_AND_RETURN_IF_NOT(ERROR, hasEndOffset,
1423 "[EmbeddingBag] hasEndOffset must be true",
1424 NNPI_INVALID_PARAM);
1425
1426 importer.setUsedTensors(
1427 {
1428 nodeValueName(glowEBBRO->getData()),
1429 nodeValueName(glowEBBRO->getWeights()),
1430 nodeValueName(glowEBBRO->getIndices()),
1431 nodeValueName(glowEBBRO->getOffsets()),
1432 },
1433 {nodeValueName(glowEBBRO->getResult())});
1434
1435 bool usFp32Accum = !(glowEBBRO->getUseFP16Accumulation() &&
1436 (glowEBBRO->getResult().getType()->getElementType() ==
1437 glow::ElemKind::Float16Ty));
1438
1439 NNPI_LENGTH_TYPE lengthType;
1440 LOG_AND_RETURN_IF_NOT(ERROR,
1441 NNPIImporter::convertLengthsModeToLengthType(
1442 glowEBBRO->getLengthsMode(), lengthType) ==
1443 NNPI_NO_ERROR,
1444 "Unhandled SLS length type", NNPI_INVALID_PARAM);
1445
1446 return nnpiNetworkAddSparseLengthsWeightedSumOp(
1447 importer.getNetwork(), glowEBBRO->getName().begin(),
1448 nodeValueName(glowEBBRO->getData()).c_str(),
1449 nodeValueName(glowEBBRO->getResult()).c_str(),
1450 nodeValueName(glowEBBRO->getWeights()).c_str(),
1451 nodeValueName(glowEBBRO->getIndices()).c_str(),
1452 nodeValueName(glowEBBRO->getOffsets()).c_str(), usFp32Accum, true,
1453 glowEBBRO->getAvgLength(), lengthType,
1454 /* force to IA */ false);
1455 }
1456 };
1457
1458 class SelectNodeImporter : public INNPINodeImporter {
1459 public:
importNode(Node * n,NNPIImporter & importer)1460 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1461 auto *glowSelect = llvm::dyn_cast<SelectNode>(n);
1462 LOG_AND_RETURN_IF_NOT(ERROR, glowSelect, "Bad node type",
1463 NNPI_INVALID_PARAM);
1464
1465 importer.setUsedTensors(
1466 {
1467 nodeValueName(glowSelect->getLHS()),
1468 nodeValueName(glowSelect->getRHS()),
1469 nodeValueName(glowSelect->getCond()),
1470 },
1471 {nodeValueName(glowSelect->getResult())});
1472 return nnpiNetworkAddSelectOp(
1473 importer.getNetwork(), glowSelect->getName().begin(),
1474 nodeValueName(glowSelect->getLHS()).c_str(), // True values.
1475 nodeValueName(glowSelect->getRHS()).c_str(), // False values.
1476 nodeValueName(glowSelect->getCond()).c_str(),
1477 nodeValueName(glowSelect->getResult()).c_str());
1478 }
1479 };
1480
1481 class LRNNodeImporter : public INNPINodeImporter {
1482 public:
importNode(Node * n,NNPIImporter & importer)1483 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1484 auto *glowLRN = llvm::dyn_cast<LocalResponseNormalizationNode>(n);
1485 LOG_AND_RETURN_IF_NOT(ERROR, glowLRN, "Bad node type", NNPI_INVALID_PARAM);
1486
1487 // Overwrite input/output values for layout.
1488 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1489 importer.addValue(nodeValueName(glowLRN->getInput()),
1490 glowLRN->getInput().getType(),
1491 /* alternativeLayout */ true),
1492 "Failed to add tensor to NNPI");
1493 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1494 importer.addValue(nodeValueName(glowLRN->getResult()),
1495 glowLRN->getResult().getType(),
1496 /* alternativeLayout */ true),
1497 "Failed to add tensor to NNPI");
1498
1499 importer.setUsedTensors({nodeValueName(glowLRN->getInput())},
1500 {nodeValueName(glowLRN->getResult())});
1501
1502 return nnpiNetworkAddLRNOp(
1503 importer.getNetwork(), glowLRN->getName().begin(),
1504 nodeValueName(glowLRN->getInput()).c_str(),
1505 nodeValueName(glowLRN->getResult()).c_str(),
1506 NNPI_LRN_TYPE::NNPI_LRN_ACROSS_CHANNELS, glowLRN->getAlpha(),
1507 glowLRN->getBeta(), glowLRN->getK(),
1508 (2 * glowLRN->getHalfWindowSize() + 1));
1509 }
1510 };
1511
1512 class RQFCNodeImporter : public INNPINodeImporter {
1513 public:
importNode(Node * n,NNPIImporter & importer)1514 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1515 auto *glowRowwiseFC = llvm::dyn_cast<RowwiseQuantizedFullyConnectedNode>(n);
1516 LOG_AND_RETURN_IF_NOT(ERROR, glowRowwiseFC, "Bad node type",
1517 NNPI_INVALID_PARAM);
1518 LOG_AND_RETURN_IF_NOT(
1519 ERROR,
1520 !(glowRowwiseFC->getOffsets()) ||
1521 importer.zeroes(nodeValueName(glowRowwiseFC->getOffsets()).c_str()),
1522 "Bad offset value", NNPI_INVALID_PARAM);
1523
1524 // Create the weights with no offset tensor.
1525 // Assert weights & biases have no offset or all zeroes.
1526
1527 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1528 importer.addTensor(nodeValueName(glowRowwiseFC->getWeights()),
1529 /* alternativeLayout */ false,
1530 nodeValueName(glowRowwiseFC->getScales()),
1531 nodeValueName(glowRowwiseFC->getOffsets()),
1532 /* forceSymlowp */ true),
1533 "Failed to add tensor to NNPI");
1534
1535 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1536 importer.addTensor(nodeValueName(glowRowwiseFC->getBias()),
1537 /* alternativeLayout */ false, {}, {},
1538 /* forceSymlowp */ true),
1539 "Failed to add tensor to NNPI");
1540
1541 // Overwrite input/output values for layout.
1542 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1543 importer.addValue(nodeValueName(glowRowwiseFC->getInput()),
1544 glowRowwiseFC->getInput().getType(),
1545 glowRowwiseFC->getInput().getType()->dims().size() ==
1546 4),
1547 "Failed to add tensor to NNPI");
1548 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1549 importer.addValue(nodeValueName(glowRowwiseFC->getResult()),
1550 glowRowwiseFC->getResult().getType(),
1551 glowRowwiseFC->getResult().getType()->dims().size() ==
1552 4),
1553 "Failed to add tensor to NNPI");
1554
1555 importer.setUsedTensors(
1556 {
1557 nodeValueName(glowRowwiseFC->getInput()),
1558 nodeValueName(glowRowwiseFC->getWeights()),
1559 nodeValueName(glowRowwiseFC->getBias()),
1560 },
1561 {
1562 nodeValueName(glowRowwiseFC->getResult()),
1563 });
1564 return nnpiNetworkAddFullyConnectedOp(
1565 importer.getNetwork(), glowRowwiseFC->getName().begin(),
1566 nodeValueName(glowRowwiseFC->getInput()).c_str(),
1567 nodeValueName(glowRowwiseFC->getResult()).c_str(),
1568 nodeValueName(glowRowwiseFC->getWeights()).c_str(),
1569 glowRowwiseFC->getBias()
1570 ? nodeValueName(glowRowwiseFC->getBias()).c_str()
1571 : nullptr);
1572 }
1573 };
1574
1575 class ChannelwiseQuantizedConvolutionNodeImporter : public INNPINodeImporter {
1576 public:
importNode(Node * n,NNPIImporter & importer)1577 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1578
1579 auto *glowChannelwiseQuantizedConv =
1580 llvm::dyn_cast<ChannelwiseQuantizedConvolutionNode>(n);
1581 LOG_AND_RETURN_IF_NOT(ERROR, glowChannelwiseQuantizedConv, "Bad node type",
1582 NNPI_INVALID_PARAM);
1583 LOG_AND_RETURN_IF_NOT(
1584 ERROR,
1585 !(glowChannelwiseQuantizedConv->getFilterOffsets()) ||
1586 importer.zeroes(
1587 nodeValueName(glowChannelwiseQuantizedConv->getFilterOffsets())
1588 .c_str()),
1589 "Bad offset value", NNPI_INVALID_PARAM);
1590
1591 const uint32_t SPATIAL_DIMS2 = 2;
1592 LOG_AND_RETURN_IF_NOT(
1593 ERROR,
1594 glowChannelwiseQuantizedConv->getKernels().size() == SPATIAL_DIMS2,
1595 "[Conv] Invalid number of kernel sizes", NNPI_INVALID_PARAM);
1596 LOG_AND_RETURN_IF_NOT(ERROR,
1597 glowChannelwiseQuantizedConv->getPads().size() ==
1598 2 * SPATIAL_DIMS2,
1599 "[Conv] Invalid number of pads", NNPI_INVALID_PARAM);
1600 LOG_AND_RETURN_IF_NOT(
1601 ERROR,
1602 glowChannelwiseQuantizedConv->getStrides().size() == SPATIAL_DIMS2,
1603 "[Conv] Invalid number of strides", NNPI_INVALID_PARAM);
1604
1605 uint32_t kernel[SPATIAL_DIMS2] = {
1606 glowChannelwiseQuantizedConv->getKernels()[0],
1607 glowChannelwiseQuantizedConv->getKernels()[1]};
1608 uint32_t paddingStart[SPATIAL_DIMS2] = {
1609 glowChannelwiseQuantizedConv->getPads()[0],
1610 glowChannelwiseQuantizedConv->getPads()[1]};
1611 uint32_t paddingEnd[SPATIAL_DIMS2] = {
1612 glowChannelwiseQuantizedConv->getPads()[2],
1613 glowChannelwiseQuantizedConv->getPads()[3]};
1614 uint32_t stride[SPATIAL_DIMS2] = {
1615 glowChannelwiseQuantizedConv->getStrides()[0],
1616 glowChannelwiseQuantizedConv->getStrides()[1]};
1617 uint32_t dilation[SPATIAL_DIMS2] = {1, 1}; // No dilation, default values
1618
1619 // Create the weights with no offset tensor.
1620 // Assert weights & biases have no offset or all zeroes.
1621
1622 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1623 importer.addTensor(
1624 nodeValueName(glowChannelwiseQuantizedConv->getFilter()),
1625 /* alternativeLayout */ true,
1626 nodeValueName(glowChannelwiseQuantizedConv->getFilterScales()),
1627 nodeValueName(glowChannelwiseQuantizedConv->getFilterOffsets()),
1628 /* forceSymlowp */ true),
1629 "Failed to add tensor to NNPI");
1630
1631 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1632 importer.addTensor(
1633 nodeValueName(glowChannelwiseQuantizedConv->getBias()),
1634 /* alternativeLayout */ false, {}, {},
1635 /* forceSymlowp */ true),
1636 "Failed to add tensor to NNPI");
1637
1638 // Overwrite input/output values for layout.
1639 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1640 importer.addValue(
1641 nodeValueName(glowChannelwiseQuantizedConv->getInput()),
1642 glowChannelwiseQuantizedConv->getInput().getType(),
1643 glowChannelwiseQuantizedConv->getInput().getType()->dims().size() ==
1644 4),
1645 "Failed to add tensor to NNPI");
1646
1647 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1648 importer.addValue(
1649 nodeValueName(glowChannelwiseQuantizedConv->getResult()),
1650 glowChannelwiseQuantizedConv->getResult().getType(),
1651 glowChannelwiseQuantizedConv->getResult()
1652 .getType()
1653 ->dims()
1654 .size() == 4),
1655 "Failed to add tensor to NNPI");
1656
1657 importer.setUsedTensors(
1658 {
1659 nodeValueName(glowChannelwiseQuantizedConv->getInput()),
1660 nodeValueName(glowChannelwiseQuantizedConv->getFilter()),
1661 nodeValueName(glowChannelwiseQuantizedConv->getBias()),
1662 },
1663 {
1664 nodeValueName(glowChannelwiseQuantizedConv->getResult()),
1665 });
1666
1667 return nnpiNetworkAddConvolutionOp(
1668 importer.getNetwork(), glowChannelwiseQuantizedConv->getName().begin(),
1669 nodeValueName(glowChannelwiseQuantizedConv->getInput()).c_str(),
1670 nodeValueName(glowChannelwiseQuantizedConv->getResult()).c_str(),
1671 nodeValueName(glowChannelwiseQuantizedConv->getFilter()).c_str(),
1672 glowChannelwiseQuantizedConv->getBias()
1673 ? nodeValueName(glowChannelwiseQuantizedConv->getBias()).c_str()
1674 : nullptr,
1675 kernel, paddingStart, paddingEnd, stride, dilation, SPATIAL_DIMS2,
1676 glowChannelwiseQuantizedConv->getGroup());
1677 }
1678 };
1679
1680 class ReplaceNaNNodeImporter : public INNPINodeImporter {
1681 public:
importNode(Node * n,NNPIImporter & importer)1682 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1683 auto *glowReplaceNan = llvm::dyn_cast<ReplaceNaNNode>(n);
1684 LOG_AND_RETURN_IF_NOT(ERROR, glowReplaceNan, "Bad node type",
1685 NNPI_INVALID_PARAM);
1686
1687 importer.setUsedTensors({nodeValueName(glowReplaceNan->getInput())},
1688 {nodeValueName(glowReplaceNan->getResult())});
1689 return nnpiNetworkAddReplaceNanOp(
1690 importer.getNetwork(), glowReplaceNan->getName().begin(),
1691 nodeValueName(glowReplaceNan->getInput()).c_str(),
1692 nodeValueName(glowReplaceNan->getResult()).c_str(),
1693 glowReplaceNan->getValue());
1694 }
1695 };
1696
1697 class GatherRangesNodeImporter : public INNPINodeImporter {
1698 public:
importNode(Node * n,NNPIImporter & importer)1699 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1700 auto *glowGatherRanges = llvm::dyn_cast<GatherRangesNode>(n);
1701 LOG_AND_RETURN_IF_NOT(ERROR, glowGatherRanges, "Bad node type",
1702 NNPI_INVALID_PARAM);
1703
1704 importer.setUsedTensors({nodeValueName(glowGatherRanges->getData()),
1705 nodeValueName(glowGatherRanges->getRanges())},
1706 {nodeValueName(glowGatherRanges->getOutput()),
1707 nodeValueName(glowGatherRanges->getLengths())});
1708 return nnpiNetworkAddGatherRangesOp(
1709 importer.getNetwork(), glowGatherRanges->getName().begin(),
1710 nodeValueName(glowGatherRanges->getData()).c_str(),
1711 nodeValueName(glowGatherRanges->getRanges()).c_str(),
1712 nodeValueName(glowGatherRanges->getOutput()).c_str(),
1713 nodeValueName(glowGatherRanges->getLengths()).c_str());
1714 }
1715 };
1716
1717 class BatchAddNodeImporter : public INNPINodeImporter {
1718 public:
importNode(Node * n,NNPIImporter & importer)1719 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1720 auto *glowBatchAdd = llvm::dyn_cast<BatchedAddNode>(n);
1721 LOG_AND_RETURN_IF_NOT(ERROR, glowBatchAdd, "Bad node type",
1722 NNPI_INVALID_PARAM);
1723
1724 NNPIObjectName inputNames[2];
1725 snprintf(inputNames[0], NNPI_MAX_STRING_LEN, "%s",
1726 nodeValueName(glowBatchAdd->getBatch()).c_str());
1727 snprintf(inputNames[1], NNPI_MAX_STRING_LEN, "%s",
1728 nodeValueName(glowBatchAdd->getSlice()).c_str());
1729 importer.setUsedTensors({nodeValueName(glowBatchAdd->getBatch()),
1730 nodeValueName(glowBatchAdd->getSlice())},
1731 {nodeValueName(glowBatchAdd->getResult())});
1732 return nnpiNetworkAddElementwiseOp(
1733 importer.getNetwork(), glowBatchAdd->getName().begin(), inputNames, 2,
1734 nodeValueName(glowBatchAdd->getResult()).c_str(), NNPI_ELTWISE_ADD);
1735 }
1736 };
1737
1738 class RQSLWSNodeImporter : public INNPINodeImporter {
1739 public:
importNode(Node * n,NNPIImporter & importer)1740 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1741 auto *glowSLWS =
1742 llvm::dyn_cast<RowwiseQuantizedSparseLengthsWeightedSumNode>(n);
1743 LOG_AND_RETURN_IF_NOT(ERROR, glowSLWS, "Bad node type", NNPI_INVALID_PARAM);
1744
1745 LOG_NNPI_IF_ERROR_RETURN_VALUE(
1746 importer.addTensor(nodeValueName(glowSLWS->getData()),
1747 /* alternativeLayout */ false,
1748 nodeValueName(glowSLWS->getScales()),
1749 nodeValueName(glowSLWS->getOffsets()),
1750 /* force to IA */ false),
1751 "Failed to add tensor to NNPI");
1752
1753 importer.setUsedTensors(
1754 {
1755 nodeValueName(glowSLWS->getData()),
1756 nodeValueName(glowSLWS->getWeights()),
1757 nodeValueName(glowSLWS->getIndices()),
1758 nodeValueName(glowSLWS->getLengths()),
1759 },
1760 {nodeValueName(glowSLWS->getResult())});
1761
1762 bool usFp32Accum = !(glowSLWS->getUseFP16Accumulation() &&
1763 (glowSLWS->getResult().getType()->getElementType() ==
1764 glow::ElemKind::Float16Ty));
1765
1766 NNPI_LENGTH_TYPE lengthType;
1767 LOG_AND_RETURN_IF_NOT(ERROR,
1768 NNPIImporter::convertLengthsModeToLengthType(
1769 glowSLWS->getLengthsMode(), lengthType) ==
1770 NNPI_NO_ERROR,
1771 "Unhandled SLS length type", NNPI_INVALID_PARAM);
1772
1773 return nnpiNetworkAddSparseLengthsWeightedSumOp(
1774 importer.getNetwork(), glowSLWS->getName().begin(),
1775 nodeValueName(glowSLWS->getData()).c_str(),
1776 nodeValueName(glowSLWS->getResult()).c_str(),
1777 nodeValueName(glowSLWS->getWeights()).c_str(),
1778 nodeValueName(glowSLWS->getIndices()).c_str(),
1779 nodeValueName(glowSLWS->getLengths()).c_str(), usFp32Accum, false,
1780 glowSLWS->getAvgLength(), lengthType,
1781 /* force to IA */ false);
1782 }
1783 };
1784
1785 class FRQSLSNodeImporter : public INNPINodeImporter {
1786 public:
importNode(Node * n,NNPIImporter & importer)1787 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1788 auto *glowSLWS =
1789 llvm::dyn_cast<FusedRowwiseQuantizedSparseLengthsSumNode>(n);
1790 LOG_AND_RETURN_IF_NOT(ERROR, glowSLWS, "Bad node type", NNPI_INVALID_PARAM);
1791
1792 importer.setUsedTensors(
1793 {
1794 nodeValueName(glowSLWS->getData()),
1795 nodeValueName(glowSLWS->getIndices()),
1796 nodeValueName(glowSLWS->getLengths()),
1797 },
1798 {nodeValueName(glowSLWS->getResult())});
1799
1800 bool usFp32Accum = !(glowSLWS->getUseFP16Accumulation() &&
1801 (glowSLWS->getResult().getType()->getElementType() ==
1802 glow::ElemKind::Float16Ty));
1803
1804 NNPI_LENGTH_TYPE lengthType;
1805 LOG_AND_RETURN_IF_NOT(ERROR,
1806 NNPIImporter::convertLengthsModeToLengthType(
1807 glowSLWS->getLengthsMode(), lengthType) ==
1808 NNPI_NO_ERROR,
1809 "Unhandled SLS length type", NNPI_INVALID_PARAM);
1810
1811 return nnpiNetworkAddSparseLengthsWeightedSumOp(
1812 importer.getNetwork(), glowSLWS->getName().begin(),
1813 nodeValueName(glowSLWS->getData()).c_str(),
1814 nodeValueName(glowSLWS->getResult()).c_str(), NULL,
1815 nodeValueName(glowSLWS->getIndices()).c_str(),
1816 nodeValueName(glowSLWS->getLengths()).c_str(), usFp32Accum, false,
1817 glowSLWS->getAvgLength(), lengthType,
1818 /* force to IA */ false);
1819 }
1820 };
1821
1822 class FRQSLWSNodeImporter : public INNPINodeImporter {
1823 public:
importNode(Node * n,NNPIImporter & importer)1824 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1825 auto *glowSLWS =
1826 llvm::dyn_cast<FusedRowwiseQuantizedSparseLengthsWeightedSumNode>(n);
1827 LOG_AND_RETURN_IF_NOT(ERROR, glowSLWS, "Bad node type", NNPI_INVALID_PARAM);
1828
1829 importer.setUsedTensors(
1830 {
1831 nodeValueName(glowSLWS->getData()),
1832 nodeValueName(glowSLWS->getWeights()),
1833 nodeValueName(glowSLWS->getIndices()),
1834 nodeValueName(glowSLWS->getLengths()),
1835 },
1836 {nodeValueName(glowSLWS->getResult())});
1837
1838 bool usFp32Accum = !(glowSLWS->getUseFP16Accumulation() &&
1839 (glowSLWS->getResult().getType()->getElementType() ==
1840 glow::ElemKind::Float16Ty));
1841
1842 NNPI_LENGTH_TYPE lengthType;
1843 LOG_AND_RETURN_IF_NOT(ERROR,
1844 NNPIImporter::convertLengthsModeToLengthType(
1845 glowSLWS->getLengthsMode(), lengthType) ==
1846 NNPI_NO_ERROR,
1847 "Unhandled SLS length type", NNPI_INVALID_PARAM);
1848
1849 return nnpiNetworkAddSparseLengthsWeightedSumOp(
1850 importer.getNetwork(), glowSLWS->getName().begin(),
1851 nodeValueName(glowSLWS->getData()).c_str(),
1852 nodeValueName(glowSLWS->getResult()).c_str(),
1853 nodeValueName(glowSLWS->getWeights()).c_str(),
1854 nodeValueName(glowSLWS->getIndices()).c_str(),
1855 nodeValueName(glowSLWS->getLengths()).c_str(), usFp32Accum, false,
1856 glowSLWS->getAvgLength(), lengthType,
1857 /* force to IA */ false);
1858 }
1859 };
1860
1861 class LengthsRangeFillNodeImporter : public INNPINodeImporter {
1862 public:
importNode(Node * n,NNPIImporter & importer)1863 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1864 auto *glowLengthsRangesfill = llvm::dyn_cast<LengthsRangeFillNode>(n);
1865 LOG_AND_RETURN_IF_NOT(ERROR, glowLengthsRangesfill, "Bad node type",
1866 NNPI_INVALID_PARAM);
1867
1868 importer.setUsedTensors(
1869 {nodeValueName(glowLengthsRangesfill->getLengths())},
1870 {nodeValueName(glowLengthsRangesfill->getResult())});
1871
1872 return nnpiNetworkAddLengthsRangeFillOp(
1873 importer.getNetwork(), glowLengthsRangesfill->getName().begin(),
1874 nodeValueName(glowLengthsRangesfill->getLengths()).c_str(),
1875 nodeValueName(glowLengthsRangesfill->getResult()).c_str());
1876 }
1877 };
1878
1879 class SpaceToDepthNodeImporter : public INNPINodeImporter {
1880 public:
importNode(Node * n,NNPIImporter & importer)1881 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1882 auto *glowSpaceToDepth = llvm::dyn_cast<SpaceToDepthNode>(n);
1883 LOG_AND_RETURN_IF_NOT(ERROR, glowSpaceToDepth, "Bad node type",
1884 NNPI_INVALID_PARAM);
1885
1886 importer.setUsedTensors({nodeValueName(glowSpaceToDepth->getInput())},
1887 {nodeValueName(glowSpaceToDepth->getResult())});
1888
1889 return nnpiNetworkAddSpaceToDepthOp(
1890 importer.getNetwork(), glowSpaceToDepth->getName().begin(),
1891 nodeValueName(glowSpaceToDepth->getInput()).c_str(),
1892 nodeValueName(glowSpaceToDepth->getResult()).c_str(),
1893 glowSpaceToDepth->getBlockSize());
1894 }
1895 };
1896
1897 class BatchOneHotNodeImporter : public INNPINodeImporter {
1898 public:
importNode(Node * n,NNPIImporter & importer)1899 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1900 auto *glowBatchOneHot = llvm::dyn_cast<BatchOneHotNode>(n);
1901 LOG_AND_RETURN_IF_NOT(ERROR, glowBatchOneHot, "Bad node type",
1902 NNPI_INVALID_PARAM);
1903
1904 importer.setUsedTensors(
1905 {
1906 nodeValueName(glowBatchOneHot->getData()),
1907 nodeValueName(glowBatchOneHot->getLengths()),
1908 nodeValueName(glowBatchOneHot->getValues()),
1909 },
1910 {nodeValueName(glowBatchOneHot->getResult())});
1911
1912 return nnpiNetworkAddBatchOneHotOp(
1913 importer.getNetwork(), glowBatchOneHot->getName().begin(),
1914 nodeValueName(glowBatchOneHot->getData()).c_str(),
1915 nodeValueName(glowBatchOneHot->getLengths()).c_str(),
1916 nodeValueName(glowBatchOneHot->getValues()).c_str(),
1917 nodeValueName(glowBatchOneHot->getResult()).c_str());
1918 }
1919 };
1920
1921 class NNPICustomIANodeImporter : public INNPINodeImporter {
1922 public:
importNode(Node * n,NNPIImporter & importer)1923 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1924 auto *glowIA = llvm::dyn_cast<NNPICustomIANode>(n);
1925 LOG_AND_RETURN_IF_NOT(ERROR, glowIA, "Bad node type", NNPI_INVALID_PARAM);
1926
1927 auto numInputs = glowIA->getInputs().size();
1928 NNPIObjectName inputs[numInputs];
1929 LOG_AND_RETURN_IF_NOT(ERROR, inputs, "No inputs", NNPI_INVALID_PARAM);
1930 std::unordered_set<std::string> inputTensors;
1931 uint32_t i = 0;
1932 for (const auto &nv : glowIA->getInputs()) {
1933 auto nvName = nodeValueName(nv);
1934 strncpy(inputs[i++], nvName.c_str(), sizeof(NNPIObjectName));
1935 inputTensors.insert(nvName);
1936 }
1937
1938 uint32_t numOutputs = 1;
1939 NNPIObjectName outputs[numOutputs];
1940 LOG_AND_RETURN_IF_NOT(ERROR, outputs, "No outputs", NNPI_INVALID_PARAM);
1941 std::unordered_set<std::string> outputTensors;
1942 auto nvName = nodeValueName(glowIA->getResult());
1943 strncpy(outputs[0], nvName.c_str(), sizeof(NNPIObjectName));
1944 outputTensors.insert(nvName);
1945
1946 importer.setUsedTensors(inputTensors, outputTensors);
1947 NNPIErrorCode error = importer.addIAExtentionPath(glowIA->getIAPath());
1948 LOG_AND_RETURN_IF_NOT(ERROR, error == NNPI_NO_ERROR,
1949 "Failed to store IA extension", NNPI_INVALID_PARAM);
1950
1951 auto res = nnpiNetworkAddCustomIAOp(
1952 importer.getNetwork(), glowIA->getName().begin(), numInputs, inputs,
1953 numOutputs, outputs, glowIA->getKernelName().c_str());
1954 return res;
1955 }
1956 };
1957 class NNPICustomDSPNodeImporter : public INNPINodeImporter {
1958 public:
importNode(Node * n,NNPIImporter & importer)1959 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
1960 auto *glowDSP = llvm::dyn_cast<NNPICustomDSPNode>(n);
1961 LOG_AND_RETURN_IF_NOT(ERROR, glowDSP, "Bad node type", NNPI_INVALID_PARAM);
1962
1963 auto numInputs = glowDSP->getInputs().size();
1964 NNPIObjectName *inputs = new NNPIObjectName[numInputs];
1965 LOG_AND_RETURN_IF_NOT(ERROR, inputs, "No inputs", NNPI_INVALID_PARAM);
1966 std::unordered_set<std::string> inputTensors;
1967 uint32_t i = 0;
1968 for (const auto &nv : glowDSP->getInputs()) {
1969 auto nvName = nodeValueName(nv);
1970 strncpy(inputs[i++], nvName.c_str(), sizeof(NNPIObjectName));
1971 inputTensors.insert(nvName);
1972 }
1973
1974 uint32_t numOutputs = 1;
1975 NNPIObjectName *outputs = new NNPIObjectName[numOutputs];
1976 LOG_AND_RETURN_IF_NOT(ERROR, outputs, "No outputs", NNPI_INVALID_PARAM);
1977 std::unordered_set<std::string> outputTensors;
1978 auto nvName = nodeValueName(glowDSP->getResult());
1979 strncpy(outputs[0], nvName.c_str(), sizeof(NNPIObjectName));
1980 outputTensors.insert(nvName);
1981
1982 importer.setUsedTensors(inputTensors, outputTensors);
1983
1984 const auto *kpConstant =
1985 glowDSP->getParent()->getParent()->getConstantByName(
1986 glowDSP->getKernelParams().getNode()->getName());
1987 LOG_AND_RETURN_IF_NOT(ERROR, kpConstant, "Kernel Params must be constant",
1988 NNPI_INVALID_PARAM);
1989
1990 const auto *wcConstant =
1991 glowDSP->getParent()->getParent()->getConstantByName(
1992 glowDSP->getWalkConfig().getNode()->getName());
1993 LOG_AND_RETURN_IF_NOT(ERROR, wcConstant, "Walk Config must be constant",
1994 NNPI_INVALID_PARAM);
1995
1996 const Tensor *kpTensor = &kpConstant->getPayload();
1997 const Tensor *wcTensor = &wcConstant->getPayload();
1998 auto res = nnpiNetworkAddCustomDspOp(
1999 importer.getNetwork(), glowDSP->getName().begin(), inputs, numInputs,
2000 outputs, numOutputs, kpTensor->getUnsafePtr(),
2001 kpTensor->getSizeInBytes(),
2002 reinterpret_cast<const NNPIWalkConfig *>(wcTensor->getUnsafePtr()),
2003 glowDSP->getPrivateAreaSize(), glowDSP->getKernelName().c_str(),
2004 reinterpret_cast<const NNPICustomDspIceRefCallback>(
2005 glowDSP->getICERefCallback()));
2006 delete[] inputs;
2007 delete[] outputs;
2008 return res;
2009 }
2010 };
2011
2012 class ClipNodeImporter : public INNPINodeImporter {
2013 public:
importNode(Node * n,NNPIImporter & importer)2014 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
2015 auto *glowClip = llvm::dyn_cast<ClipNode>(n);
2016 LOG_AND_RETURN_IF_NOT(ERROR, glowClip, "Bad node type", NNPI_INVALID_PARAM);
2017
2018 importer.setUsedTensors({nodeValueName(glowClip->getInput())},
2019 {nodeValueName(glowClip->getResult())});
2020
2021 return nnpiNetworkAddClipOp(importer.getNetwork(),
2022 glowClip->getName().begin(),
2023 nodeValueName(glowClip->getInput()).c_str(),
2024 nodeValueName(glowClip->getResult()).c_str(),
2025 glowClip->getMin(), glowClip->getMax());
2026 }
2027 };
2028
2029 class BatchNormalizationNodeImporter : public INNPINodeImporter {
2030 public:
importNode(Node * n,NNPIImporter & importer)2031 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
2032 auto *glowBN = llvm::dyn_cast<BatchNormalizationNode>(n);
2033 LOG_AND_RETURN_IF_NOT(ERROR, glowBN, "Bad node type", NNPI_INVALID_PARAM);
2034
2035 importer.setUsedTensors(
2036 {nodeValueName(glowBN->getInput()), nodeValueName(glowBN->getScale())},
2037 {nodeValueName(glowBN->getBias()), nodeValueName(glowBN->getMean()),
2038 nodeValueName(glowBN->getVar()), nodeValueName(glowBN->getResult())});
2039
2040 return nnpiNetworkAddBatchNormOp(
2041 importer.getNetwork(), glowBN->getName().begin(),
2042 nodeValueName(glowBN->getInput()).c_str(),
2043 nodeValueName(glowBN->getResult()).c_str(),
2044 nodeValueName(glowBN->getMean()).c_str(),
2045 nodeValueName(glowBN->getVar()).c_str(),
2046 nodeValueName(glowBN->getScale()).c_str(),
2047 nodeValueName(glowBN->getBias()).c_str(), glowBN->getEpsilon());
2048 }
2049 };
2050
2051 class LayerNormalizationNodeImporter : public INNPINodeImporter {
2052 public:
importNode(Node * n,NNPIImporter & importer)2053 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
2054 auto *glowLN = llvm::dyn_cast<LayerNormalizationNode>(n);
2055 LOG_AND_RETURN_IF_NOT(ERROR, glowLN, "Bad node type", NNPI_INVALID_PARAM);
2056
2057 importer.setUsedTensors(
2058 {nodeValueName(glowLN->getInput()), nodeValueName(glowLN->getScale())},
2059 {nodeValueName(glowLN->getBias()), nodeValueName(glowLN->getResult())});
2060
2061 llvm::SmallVector<uint32_t, 4> normShape(glowLN->getScale().dims().begin(),
2062 glowLN->getScale().dims().end());
2063
2064 return nnpiNetworkAddLayerNormOp(
2065 importer.getNetwork(), glowLN->getName().begin(),
2066 nodeValueName(glowLN->getInput()).c_str(),
2067 nodeValueName(glowLN->getResult()).c_str(),
2068 nodeValueName(glowLN->getScale()).c_str(),
2069 nodeValueName(glowLN->getBias()).c_str(), normShape.data(),
2070 normShape.size(), glowLN->getEpsilon());
2071 }
2072 };
2073
2074 class LogitNodeImporter : public INNPINodeImporter {
2075 public:
importNode(Node * n,NNPIImporter & importer)2076 NNPIErrorCode importNode(Node *n, NNPIImporter &importer) override {
2077 auto *glowLogit = llvm::dyn_cast<LogitNode>(n);
2078 LOG_AND_RETURN_IF_NOT(ERROR, glowLogit, "Bad node type",
2079 NNPI_INVALID_PARAM);
2080
2081 importer.setUsedTensors({nodeValueName(glowLogit->getInput()),
2082 nodeValueName(glowLogit->getResult())});
2083
2084 return nnpiNetworkAddLogitOp(
2085 importer.getNetwork(), glowLogit->getName().begin(),
2086 nodeValueName(glowLogit->getInput()).c_str(),
2087 nodeValueName(glowLogit->getResult()).c_str(), glowLogit->getEpsilon());
2088 }
2089 };
2090 //////////////////////////////////////////////////////////////////////////
2091 namespace {
2092 std::unordered_map<
2093 std::string,
2094 std::unique_ptr<INNPINodeImporter>>::value_type importerInit[] = {
2095 {"", nullptr},
2096 {"Convolution",
2097 glow::make_unique<ConvolutionNodeImporter<ConvolutionNode, 2>>()},
2098 {"Convolution3D",
2099 glow::make_unique<ConvolutionNodeImporter<Convolution3DNode, 3>>()},
2100 {"Transpose", glow::make_unique<TransposeNodeImporter>()},
2101 {"MaxPool",
2102 glow::make_unique<PoolNodeImporter<glow::MaxPoolNode, NNPI_POOL_MAX>>()},
2103 {"AvgPool",
2104 glow::make_unique<PoolNodeImporter<glow::AvgPoolNode, NNPI_POOL_AVG>>()},
2105 {"AdaptiveAvgPool",
2106 glow::make_unique<
2107 AdaptivePoolNodeImporter<glow::AdaptiveAvgPoolNode, NNPI_POOL_AVG>>()},
2108 {"FullyConnected", glow::make_unique<FullyConnectedNodeImporter>()},
2109 {"SoftMax", glow::make_unique<SoftMaxNodeImporter>()},
2110 {"Save", glow::make_unique<SaveNodeImporter>()},
2111 {"Relu", glow::make_unique<ReluNodeImporter>()},
2112 {"PRelu", glow::make_unique<PReluNodeImporter>()},
2113 {"Exp", glow::make_unique<
2114 UnaryEltwiseNodeImporter<glow::ExpNode, NNPI_ELTWISE_EXP>>()},
2115 {"Max", glow::make_unique<
2116 BinaryEltwiseNodeImporter<glow::MaxNode, NNPI_ELTWISE_MAX>>()},
2117 {"Min", glow::make_unique<
2118 BinaryEltwiseNodeImporter<glow::MinNode, NNPI_ELTWISE_MIN>>()},
2119 {"Add", glow::make_unique<
2120 BinaryEltwiseNodeImporter<glow::AddNode, NNPI_ELTWISE_ADD>>()},
2121 {"Mul", glow::make_unique<
2122 BinaryEltwiseNodeImporter<glow::MulNode, NNPI_ELTWISE_MUL>>()},
2123 {"Div", glow::make_unique<
2124 BinaryEltwiseNodeImporter<glow::DivNode, NNPI_ELTWISE_DIV>>()},
2125 {"Sub", glow::make_unique<
2126 BinaryEltwiseNodeImporter<glow::SubNode, NNPI_ELTWISE_SUB>>()},
2127 {"Pow", glow::make_unique<
2128 BinaryEltwiseNodeImporter<glow::PowNode, NNPI_ELTWISE_POW>>()},
2129 {"CmpEQ",
2130 glow::make_unique<
2131 BinaryEltwiseNodeImporter<glow::CmpEQNode, NNPI_ELTWISE_EQ>>()},
2132 {"CmpLTE",
2133 glow::make_unique<
2134 BinaryEltwiseNodeImporter<glow::CmpLTENode, NNPI_ELTWISE_LTE>>()},
2135 {"CmpLT",
2136 glow::make_unique<
2137 BinaryEltwiseNodeImporter<glow::CmpLTNode, NNPI_ELTWISE_LESS>>()},
2138 {"ArgMax", glow::make_unique<ArgMaxNodeImporter>()},
2139 {"Reshape", glow::make_unique<ReshapeNodeImporter>()},
2140 {"Quantize", glow::make_unique<ConvertNodeImporter<QuantizeNode>>()},
2141 {"Dequantize", glow::make_unique<ConvertNodeImporter<DequantizeNode>>()},
2142 {"RescaleQuantized",
2143 glow::make_unique<ConvertNodeImporter<RescaleQuantizedNode>>()},
2144 {"ConvertTo", glow::make_unique<ConvertNodeImporter<ConvertToNode>>()},
2145 {"MatMul", glow::make_unique<MatMulNodeImporter<MatMulNode>>()},
2146 {"BatchMatMul", glow::make_unique<MatMulNodeImporter<BatchMatMulNode>>()},
2147 {"Slice", glow::make_unique<SliceNodeImporter>()},
2148 {"Sigmoid", glow::make_unique<SigmoidNodeImporter>()},
2149 {"Tanh", glow::make_unique<TanhNodeImporter>()},
2150 {"Concat", glow::make_unique<ConcatNodeImporter>()},
2151 {"Tile", glow::make_unique<TileNodeImporter>()},
2152 {"Gather", glow::make_unique<GatherNodeImporter>()},
2153 {"BatchedReduceAdd", glow::make_unique<ReduceAddNodeImporter>()},
2154 {"Log", glow::make_unique<LogNodeImporter>()},
2155 {"TopK", glow::make_unique<TopkNodeImporter>()},
2156 {"BatchedReduceMean",
2157 glow::make_unique<ReduceMultAxesNodeImporter<glow::BatchedReduceMeanNode,
2158 NNPI_REDUCE_MEAN>>()},
2159 {"BatchedReduceMin",
2160 glow::make_unique<ReduceMultAxesNodeImporter<glow::BatchedReduceMinNode,
2161 NNPI_REDUCE_MIN>>()},
2162 {"Splat", glow::make_unique<SplatNodeImporter>()},
2163 {"SparseLengthsWeightedSum", glow::make_unique<SLWSNodeImporter>()},
2164 {"SparseLengthsSum", glow::make_unique<SLSNodeImporter>()},
2165 {"FusedRowwiseQuantizedSparseLengthsSum",
2166 glow::make_unique<FRQSLSNodeImporter>()},
2167 {"Select", glow::make_unique<SelectNodeImporter>()},
2168 {"LocalResponseNormalization", glow::make_unique<LRNNodeImporter>()},
2169 {"RowwiseQuantizedFullyConnected", glow::make_unique<RQFCNodeImporter>()},
2170 {"ReplaceNaN", glow::make_unique<ReplaceNaNNodeImporter>()},
2171 {"GatherRanges", glow::make_unique<GatherRangesNodeImporter>()},
2172 {"BatchedAdd", glow::make_unique<BatchAddNodeImporter>()},
2173 {"RowwiseQuantizedSparseLengthsWeightedSum",
2174 glow::make_unique<RQSLWSNodeImporter>()},
2175 {"FusedRowwiseQuantizedSparseLengthsWeightedSum",
2176 glow::make_unique<FRQSLWSNodeImporter>()},
2177 {"LengthsRangeFill", glow::make_unique<LengthsRangeFillNodeImporter>()},
2178 {"BatchOneHot", glow::make_unique<BatchOneHotNodeImporter>()},
2179 {"NNPICustomDSP", glow::make_unique<NNPICustomDSPNodeImporter>()},
2180 {"NNPICustomIA", glow::make_unique<NNPICustomIANodeImporter>()},
2181 {"SpaceToDepth", glow::make_unique<SpaceToDepthNodeImporter>()},
2182 {"Clip", glow::make_unique<ClipNodeImporter>()},
2183 {"BatchNormalization", glow::make_unique<BatchNormalizationNodeImporter>()},
2184 {"LayerNormalization", glow::make_unique<LayerNormalizationNodeImporter>()},
2185 {"ChannelwiseQuantizedConvolution",
2186 glow::make_unique<ChannelwiseQuantizedConvolutionNodeImporter>()},
2187 {"EmbeddingBag", glow::make_unique<EmbeddingBagNodeImporter>()},
2188 {"EmbeddingBagByteRowwiseOffsets",
2189 glow::make_unique<EmbeddingBagByteRowwiseOffsetsNodeImporter>()},
2190 {"Logit", glow::make_unique<LogitNodeImporter>()},
2191 };
2192 }
2193
2194 const std::unordered_map<std::string, std::unique_ptr<INNPINodeImporter>>
2195 NNPIImporter::nodeImporters_ = {
2196 std::make_move_iterator(std::begin(importerInit)),
2197 std::make_move_iterator(std::end(importerInit))};
2198