1 // This file is part of OpenCV project.
2 // It is subject to the license terms in the LICENSE file found in the top-level directory
3 // of this distribution and at http://opencv.org/license.html.
4 //
5 // Copyright (C) 2021 Intel Corporation
6
7 #include <ade/util/zip_range.hpp> // zip_range, indexed
8
9 #include <opencv2/gapi/util/throw.hpp> // throw_error
10 #include <opencv2/gapi/python/python.hpp>
11
12 #include "api/gbackend_priv.hpp"
13 #include "backends/common/gbackend.hpp"
14
GPythonKernel(cv::gapi::python::Impl run)15 cv::gapi::python::GPythonKernel::GPythonKernel(cv::gapi::python::Impl run)
16 : m_run(run)
17 {
18 }
19
operator ()(const cv::gapi::python::GPythonContext & ctx)20 cv::GRunArgs cv::gapi::python::GPythonKernel::operator()(const cv::gapi::python::GPythonContext& ctx)
21 {
22 return m_run(ctx);
23 }
24
GPythonFunctor(const char * id,const cv::gapi::python::GPythonFunctor::Meta & meta,const cv::gapi::python::Impl & impl)25 cv::gapi::python::GPythonFunctor::GPythonFunctor(const char* id,
26 const cv::gapi::python::GPythonFunctor::Meta &meta,
27 const cv::gapi::python::Impl& impl)
28 : gapi::GFunctor(id), impl_{GPythonKernel{impl}, meta}
29 {
30 }
31
impl() const32 cv::GKernelImpl cv::gapi::python::GPythonFunctor::impl() const
33 {
34 return impl_;
35 }
36
backend() const37 cv::gapi::GBackend cv::gapi::python::GPythonFunctor::backend() const
38 {
39 return cv::gapi::python::backend();
40 }
41
42 namespace {
43
44 struct PythonUnit
45 {
name__anonf4b9b12f0111::PythonUnit46 static const char *name() { return "PythonUnit"; }
47 cv::gapi::python::GPythonKernel kernel;
48 };
49
50 using PythonModel = ade::TypedGraph
51 < cv::gimpl::Op
52 , PythonUnit
53 >;
54
55 using ConstPythonModel = ade::ConstTypedGraph
56 < cv::gimpl::Op
57 , PythonUnit
58 >;
59
60 class GPythonExecutable final: public cv::gimpl::GIslandExecutable
61 {
62 virtual void run(std::vector<InObj> &&,
63 std::vector<OutObj> &&) override;
64
allocatesOutputs() const65 virtual bool allocatesOutputs() const override { return true; }
66 // Return an empty RMat since we will reuse the input.
67 // There is no need to allocate and copy 4k image here.
allocate(const cv::GMatDesc &) const68 virtual cv::RMat allocate(const cv::GMatDesc&) const override { return {}; }
69
canReshape() const70 virtual bool canReshape() const override { return true; }
reshape(ade::Graph &,const cv::GCompileArgs &)71 virtual void reshape(ade::Graph&, const cv::GCompileArgs&) override {
72 // Do nothing here
73 }
74
75 public:
76 GPythonExecutable(const ade::Graph &,
77 const std::vector<ade::NodeHandle> &);
78
79 const ade::Graph& m_g;
80 cv::gimpl::GModel::ConstGraph m_gm;
81 cv::gapi::python::GPythonKernel m_kernel;
82 ade::NodeHandle m_op;
83
84 cv::GTypesInfo m_out_info;
85 cv::GMetaArgs m_in_metas;
86 cv::gimpl::Mag m_res;
87 };
88
packArg(cv::gimpl::Mag & m_res,const cv::GArg & arg)89 static cv::GArg packArg(cv::gimpl::Mag& m_res, const cv::GArg &arg)
90 {
91 // No API placeholders allowed at this point
92 // FIXME: this check has to be done somewhere in compilation stage.
93 GAPI_Assert( arg.kind != cv::detail::ArgKind::GMAT
94 && arg.kind != cv::detail::ArgKind::GSCALAR
95 && arg.kind != cv::detail::ArgKind::GARRAY
96 && arg.kind != cv::detail::ArgKind::GOPAQUE
97 && arg.kind != cv::detail::ArgKind::GFRAME);
98
99 if (arg.kind != cv::detail::ArgKind::GOBJREF)
100 {
101 // All other cases - pass as-is, with no transformations to GArg contents.
102 return arg;
103 }
104 GAPI_Assert(arg.kind == cv::detail::ArgKind::GOBJREF);
105
106 // Wrap associated CPU object (either host or an internal one)
107 // FIXME: object can be moved out!!! GExecutor faced that.
108 const cv::gimpl::RcDesc &ref = arg.get<cv::gimpl::RcDesc>();
109 switch (ref.shape)
110 {
111 case cv::GShape::GMAT: return cv::GArg(m_res.slot<cv::Mat>() [ref.id]);
112 case cv::GShape::GSCALAR: return cv::GArg(m_res.slot<cv::Scalar>()[ref.id]);
113 // Note: .at() is intentional for GArray and GOpaque as objects MUST be already there
114 // (and constructed by either bindIn/Out or resetInternal)
115 case cv::GShape::GARRAY: return cv::GArg(m_res.slot<cv::detail::VectorRef>().at(ref.id));
116 case cv::GShape::GOPAQUE: return cv::GArg(m_res.slot<cv::detail::OpaqueRef>().at(ref.id));
117 case cv::GShape::GFRAME: return cv::GArg(m_res.slot<cv::MediaFrame>().at(ref.id));
118 default:
119 cv::util::throw_error(std::logic_error("Unsupported GShape type"));
120 break;
121 }
122 }
123
writeBack(cv::GRunArg & arg,cv::GRunArgP & out)124 static void writeBack(cv::GRunArg& arg, cv::GRunArgP& out)
125 {
126 switch (arg.index())
127 {
128 case cv::GRunArg::index_of<cv::Mat>():
129 {
130 auto& rmat = *cv::util::get<cv::RMat*>(out);
131 rmat = cv::make_rmat<cv::gimpl::RMatAdapter>(cv::util::get<cv::Mat>(arg));
132 break;
133 }
134 case cv::GRunArg::index_of<cv::Scalar>():
135 {
136 *cv::util::get<cv::Scalar*>(out) = cv::util::get<cv::Scalar>(arg);
137 break;
138 }
139 case cv::GRunArg::index_of<cv::detail::OpaqueRef>():
140 {
141 auto& oref = cv::util::get<cv::detail::OpaqueRef>(arg);
142 cv::util::get<cv::detail::OpaqueRef>(out).mov(oref);
143 break;
144 }
145 case cv::GRunArg::index_of<cv::detail::VectorRef>():
146 {
147 auto& vref = cv::util::get<cv::detail::VectorRef>(arg);
148 cv::util::get<cv::detail::VectorRef>(out).mov(vref);
149 break;
150 }
151 default:
152 GAPI_Assert(false && "Unsupported output type");
153 }
154 }
155
run(std::vector<InObj> && input_objs,std::vector<OutObj> && output_objs)156 void GPythonExecutable::run(std::vector<InObj> &&input_objs,
157 std::vector<OutObj> &&output_objs)
158 {
159 const auto &op = m_gm.metadata(m_op).get<cv::gimpl::Op>();
160 for (auto& it : input_objs) cv::gimpl::magazine::bindInArg(m_res, it.first, it.second);
161
162 using namespace std::placeholders;
163 cv::GArgs inputs;
164 ade::util::transform(op.args,
165 std::back_inserter(inputs),
166 std::bind(&packArg, std::ref(m_res), _1));
167
168
169 cv::gapi::python::GPythonContext ctx{inputs, m_in_metas, m_out_info};
170 auto outs = m_kernel(ctx);
171
172 for (auto&& it : ade::util::zip(outs, output_objs))
173 {
174 writeBack(std::get<0>(it), std::get<1>(it).second);
175 }
176 }
177
178 class GPythonBackendImpl final: public cv::gapi::GBackend::Priv
179 {
unpackKernel(ade::Graph & graph,const ade::NodeHandle & op_node,const cv::GKernelImpl & impl)180 virtual void unpackKernel(ade::Graph &graph,
181 const ade::NodeHandle &op_node,
182 const cv::GKernelImpl &impl) override
183 {
184 PythonModel gm(graph);
185 const auto &kernel = cv::util::any_cast<cv::gapi::python::GPythonKernel>(impl.opaque);
186 gm.metadata(op_node).set(PythonUnit{kernel});
187 }
188
compile(const ade::Graph & graph,const cv::GCompileArgs &,const std::vector<ade::NodeHandle> & nodes) const189 virtual EPtr compile(const ade::Graph &graph,
190 const cv::GCompileArgs &,
191 const std::vector<ade::NodeHandle> &nodes) const override
192 {
193 return EPtr{new GPythonExecutable(graph, nodes)};
194 }
195
controlsMerge() const196 virtual bool controlsMerge() const override
197 {
198 return true;
199 }
200
allowsMerge(const cv::gimpl::GIslandModel::Graph &,const ade::NodeHandle &,const ade::NodeHandle &,const ade::NodeHandle &) const201 virtual bool allowsMerge(const cv::gimpl::GIslandModel::Graph &,
202 const ade::NodeHandle &,
203 const ade::NodeHandle &,
204 const ade::NodeHandle &) const override
205 {
206 return false;
207 }
208 };
209
GPythonExecutable(const ade::Graph & g,const std::vector<ade::NodeHandle> & nodes)210 GPythonExecutable::GPythonExecutable(const ade::Graph& g,
211 const std::vector<ade::NodeHandle>& nodes)
212 : m_g(g), m_gm(m_g)
213 {
214 using namespace cv::gimpl;
215 const auto is_op = [this](const ade::NodeHandle &nh)
216 {
217 return m_gm.metadata(nh).get<NodeType>().t == NodeType::OP;
218 };
219
220 auto it = std::find_if(nodes.begin(), nodes.end(), is_op);
221 GAPI_Assert(it != nodes.end() && "No operators found for this island?!");
222
223 ConstPythonModel cag(m_g);
224
225 m_op = *it;
226 m_kernel = cag.metadata(m_op).get<PythonUnit>().kernel;
227
228 // Ensure this the only op in the graph
229 if (std::any_of(it+1, nodes.end(), is_op))
230 {
231 cv::util::throw_error
232 (std::logic_error
233 ("Internal error: Python subgraph has multiple operations"));
234 }
235
236 m_out_info.reserve(m_op->outEdges().size());
237 for (const auto &e : m_op->outEdges())
238 {
239 const auto& out_data = m_gm.metadata(e->dstNode()).get<cv::gimpl::Data>();
240 m_out_info.push_back(cv::GTypeInfo{out_data.shape, out_data.kind, out_data.ctor});
241 }
242
243 const auto& op = m_gm.metadata(m_op).get<cv::gimpl::Op>();
244 m_in_metas.resize(op.args.size());
245 GAPI_Assert(m_op->inEdges().size() > 0);
246 for (const auto &in_eh : m_op->inEdges())
247 {
248 const auto& input_port = m_gm.metadata(in_eh).get<Input>().port;
249 const auto& input_nh = in_eh->srcNode();
250 const auto& input_meta = m_gm.metadata(input_nh).get<Data>().meta;
251 m_in_metas.at(input_port) = input_meta;
252 }
253 }
254
255 } // anonymous namespace
256
backend()257 cv::gapi::GBackend cv::gapi::python::backend()
258 {
259 static cv::gapi::GBackend this_backend(std::make_shared<GPythonBackendImpl>());
260 return this_backend;
261 }
262