1 // Tencent is pleased to support the open source community by making ncnn available.
2 //
3 // Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.
4 //
5 // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6 // in compliance with the License. You may obtain a copy of the License at
7 //
8 // https://opensource.org/licenses/BSD-3-Clause
9 //
10 // Unless required by applicable law or agreed to in writing, software distributed
11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
13 // specific language governing permissions and limitations under the License.
14 
15 #include "pass_level1.h"
16 
17 #include "../utils.h"
18 
19 namespace pnnx {
20 
21 class LSTM : public FuseModulePass
22 {
23 public:
match_type_str() const24     const char* match_type_str() const
25     {
26         return "__torch__.torch.nn.modules.rnn.LSTM";
27     }
28 
type_str() const29     const char* type_str() const
30     {
31         return "nn.LSTM";
32     }
33 
write(Operator * op,const std::shared_ptr<torch::jit::Graph> & graph,const torch::jit::Module & mod) const34     void write(Operator* op, const std::shared_ptr<torch::jit::Graph>& graph, const torch::jit::Module& mod) const
35     {
36         //         mod.dump(true, true, true);
37 
38         //         graph->dump();
39 
40         const torch::jit::Node* lstm = find_node_by_kind(graph, "aten::lstm");
41 
42         const torch::jit::Node* return_tuple = find_node_by_kind(graph, "prim::TupleConstruct");
43         if (return_tuple && return_tuple->inputs().size() == 3 && lstm->outputs().size() == 3
44                 && return_tuple->inputs()[0] == lstm->outputs()[1] && return_tuple->inputs()[1] == lstm->outputs()[2] && return_tuple->inputs()[2] == lstm->outputs()[0])
45         {
46             // mark the swapped output tuple
47             // we would restore the fine order in pass_level3/fuse_rnn_unpack
48             fprintf(stderr, "swapped detected !\n");
49             op->params["pnnx_rnn_output_swapped"] = 1;
50         }
51 
52         //         for (auto aa : lstm->schema().arguments())
53         //         {
54         //             fprintf(stderr, "arg %s\n", aa.name().c_str());
55         //         }
56 
57         const auto& weight_ih_l0 = mod.attr("weight_ih_l0").toTensor();
58 
59         op->params["input_size"] = weight_ih_l0.size(1);
60         op->params["hidden_size"] = weight_ih_l0.size(0) / 4;
61         op->params["num_layers"] = lstm->namedInput("num_layers");
62         op->params["bias"] = lstm->namedInput("has_biases");
63         op->params["batch_first"] = lstm->namedInput("batch_first");
64         op->params["bidirectional"] = lstm->namedInput("bidirectional");
65 
66         const int num_layers = op->params["num_layers"].i;
67         const bool bias = op->params["bias"].b;
68         const bool bidirectional = op->params["bidirectional"].b;
69 
70         for (int k = 0; k < num_layers; k++)
71         {
72             std::string weight_ih_lk_key = std::string("weight_ih_l") + std::to_string(k);
73             std::string weight_hh_lk_key = std::string("weight_hh_l") + std::to_string(k);
74 
75             op->attrs[weight_ih_lk_key] = mod.attr(weight_ih_lk_key).toTensor();
76             op->attrs[weight_hh_lk_key] = mod.attr(weight_hh_lk_key).toTensor();
77 
78             if (bias)
79             {
80                 std::string bias_ih_lk_key = std::string("bias_ih_l") + std::to_string(k);
81                 std::string bias_hh_lk_key = std::string("bias_hh_l") + std::to_string(k);
82 
83                 op->attrs[bias_ih_lk_key] = mod.attr(bias_ih_lk_key).toTensor();
84                 op->attrs[bias_hh_lk_key] = mod.attr(bias_hh_lk_key).toTensor();
85             }
86 
87             if (bidirectional)
88             {
89                 std::string weight_ih_lk_reverse_key = std::string("weight_ih_l") + std::to_string(k) + "_reverse";
90                 std::string weight_hh_lk_reverse_key = std::string("weight_hh_l") + std::to_string(k) + "_reverse";
91 
92                 op->attrs[weight_ih_lk_reverse_key] = mod.attr(weight_ih_lk_reverse_key).toTensor();
93                 op->attrs[weight_hh_lk_reverse_key] = mod.attr(weight_hh_lk_reverse_key).toTensor();
94 
95                 if (bias)
96                 {
97                     std::string bias_ih_lk_reverse_key = std::string("bias_ih_l") + std::to_string(k) + "_reverse";
98                     std::string bias_hh_lk_reverse_key = std::string("bias_hh_l") + std::to_string(k) + "_reverse";
99 
100                     op->attrs[bias_ih_lk_reverse_key] = mod.attr(bias_ih_lk_reverse_key).toTensor();
101                     op->attrs[bias_hh_lk_reverse_key] = mod.attr(bias_hh_lk_reverse_key).toTensor();
102                 }
103             }
104         }
105     }
106 };
107 
108 REGISTER_GLOBAL_PNNX_FUSE_MODULE_PASS(LSTM)
109 
110 } // namespace pnnx
111