1 /* ***** BEGIN LICENSE BLOCK *****
2  * This file is part of openfx-misc <https://github.com/devernay/openfx-misc>,
3  * Copyright (C) 2013-2018 INRIA
4  *
5  * openfx-misc is free software: you can redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; either version 2 of the License, or
8  * (at your option) any later version.
9  *
10  * openfx-misc is distributed in the hope that it will be useful,
11  * but WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13  * GNU General Public License for more details.
14  *
15  * You should have received a copy of the GNU General Public License
16  * along with openfx-misc.  If not, see <http://www.gnu.org/licenses/gpl-2.0.html>
17  * ***** END LICENSE BLOCK ***** */
18 
19 /*
20  * OFX CImgInpaint plugin.
21  */
22 
23 
24 #include <memory>
25 #include <cmath>
26 #include <cstring>
27 #if defined(_WIN32) || defined(__WIN32__) || defined(WIN32)
28 #include <windows.h>
29 #endif
30 
31 #include "ofxsProcessing.H"
32 #include "ofxsMaskMix.h"
33 #include "ofxsMacros.h"
34 #include "ofxsCoords.h"
35 #include "ofxsCopier.h"
36 
37 #include "CImgFilter.h"
38 
39 #ifdef PLUGIN_PACK_GPL2
40 
41 using namespace OFX;
42 
43 OFXS_NAMESPACE_ANONYMOUS_ENTER
44 
45 #define kPluginName          "InpaintCImg"
46 #define kPluginGrouping      "Filter"
47 #define kPluginDescription \
48     "Inpaint (a.k.a. content-aware fill) the areas indicated by the Mask input using patch-based inpainting.\n" \
49     "Be aware that this filter may produce different results on each frame of a video, even if there is little change in the video content. To inpaint areas with lots of details, it may be better to inpaint on a single frame and paste the inpainted area on other frames (if a transform is also required to match the other frames, it may be computed by tracking).\n" \
50     "\n" \
51     "A tutorial on using this filter can be found at http://blog.patdavid.net/2014/02/getting-around-in-gimp-gmic-inpainting.html\n" \
52     "The algorithm is described in the two following publications:\n" \
53     "\"A Smarter Examplar-based Inpainting Algorithm using Local and Global Heuristics " \
54     "for more Geometric Coherence.\" " \
55     "(M. Daisy, P. Buyssens, D. Tschumperlé, O. Lezoray). " \
56     "IEEE International Conference on Image Processing (ICIP'14), Paris/France, Oct. 2014\n" \
57     "and\n" \
58     "\"A Fast Spatial Patch Blending Algorithm for Artefact Reduction in Pattern-based " \
59     "Image Inpainting.\" " \
60     "(M. Daisy, D. Tschumperlé, O. Lezoray). " \
61     "SIGGRAPH Asia 2013 Technical Briefs, Hong-Kong, November 2013.\n" \
62     "\n" \
63     "Uses the 'inpaint' plugin from the CImg library.\n" \
64     "CImg is a free, open-source library distributed under the CeCILL-C " \
65     "(close to the GNU LGPL) or CeCILL (compatible with the GNU GPL) licenses. " \
66     "It can be used in commercial applications (see http://cimg.eu). " \
67     "The 'inpaint' CImg plugin is distributed under the CeCILL (compatible with the GNU GPL) license."
68 
69 #define kPluginIdentifier    "eu.cimg.Inpaint"
70 // History:
71 // version 1.0: initial version
72 #define kPluginVersionMajor 1 // Incrementing this number means that you have broken backwards compatibility of the plug-in.
73 #define kPluginVersionMinor 0 // Increment this when you have fixed a bug or made it faster.
74 
75 #define kSupportsComponentRemapping 1
76 #define kSupportsTiles 0 // requires the whole image to search for patches, which may be far away (how far exactly?)
77 #define kSupportsMultiResolution 1
78 #define kSupportsRenderScale 1
79 #define kSupportsMultipleClipPARs false
80 #define kSupportsMultipleClipDepths false
81 #define kRenderThreadSafety eRenderFullySafe
82 #ifdef cimg_use_openmp
83 #define kHostFrameThreading false
84 #else
85 #define kHostFrameThreading true
86 #endif
87 #define kSupportsRGBA true
88 #define kSupportsRGB true
89 #define kSupportsXY true
90 #define kSupportsAlpha true
91 
92 #define kParamPatchSize "patchSize"
93 #define kParamPatchSizeLabel "Patch Size" //, "Patch size in pixels."
94 #define kParamPatchSizeDefault 7 // 1-64
95 
96 #define kParamLookupSize "lookupSize"
97 #define kParamLookupSizeLabel "Lookup Size" //, "Lookup region size in pixels."
98 #define kParamLookupSizeDefault 16 // 1-32
99 
100 #define kParamLookupFactor "lookupFactor"
101 #define kParamLookupFactorLabel "Lookup Factor"
102 #define kParamLookupFactorDefault 0.1 // 0-1
103 
104 #define kParamBlendSize "blendSize"
105 #define kParamBlendSizeLabel "Blend Size"
106 #define kParamBlendSizeDefault 1.2 // 0-5
107 
108 #define kParamBlendThreshold "blendThreshold"
109 #define kParamBlendThresholdLabel "Blend Threshold"
110 #define kParamBlendThresholdDefault 0. // 0-1
111 
112 #define kParamBlendDecay "blendDecay"
113 #define kParamBlendDecayLabel "Blend Decay"
114 #define kParamBlendDecayDefault 0.05 // 0-0.5
115 
116 #define kParamBlendScales "blendScales"
117 #define kParamBlendScalesLabel "Blend Scales"
118 #define kParamBlendScalesDefault 10 // 1-20 (int)
119 
120 #define kParamIsBlendOuter "isBlendOuter"
121 #define kParamIsBlendOuterLabel "Allow Outer Blending"
122 #define kParamIsBlendOuterDefault true
123 
124 
125 /// Inpaint plugin
126 struct CImgInpaintParams
127 {
128     int patch_size;
129     double lookup_size;
130     double lookup_factor;
131     //int lookup_increment=1;
132     double blend_size;
133     double blend_threshold;
134     double blend_decay;
135     int blend_scales;
136     bool is_blend_outer;
137 };
138 
139 class CImgInpaintPlugin
140     : public CImgFilterPluginHelper<CImgInpaintParams, false>
141 {
142 public:
143 
CImgInpaintPlugin(OfxImageEffectHandle handle)144     CImgInpaintPlugin(OfxImageEffectHandle handle)
145         : CImgFilterPluginHelper<CImgInpaintParams, false>(handle, /*usesMask=*/true, kSupportsComponentRemapping, kSupportsTiles, kSupportsMultiResolution, kSupportsRenderScale, /*defaultUnpremult=*/ true)
146     {
147         _patch_size      = fetchIntParam(kParamPatchSize);
148         _lookup_size     = fetchDoubleParam(kParamLookupSize);
149         _lookup_factor   = fetchDoubleParam(kParamLookupFactor);
150         _blend_size      = fetchDoubleParam(kParamBlendSize);
151         _blend_threshold = fetchDoubleParam(kParamBlendThreshold);
152         _blend_decay     = fetchDoubleParam(kParamBlendDecay);
153         _blend_scales    = fetchIntParam(kParamBlendScales);
154         _is_blend_outer  = fetchBooleanParam(kParamIsBlendOuter);
155         assert(_patch_size && _lookup_size && _lookup_factor && _blend_size && _blend_threshold && _blend_decay && _blend_scales && _is_blend_outer);
156     }
157 
getValuesAtTime(double time,CImgInpaintParams & params)158     virtual void getValuesAtTime(double time,
159                                  CImgInpaintParams& params) OVERRIDE FINAL
160     {
161         _patch_size->getValueAtTime(time, params.patch_size);
162         _lookup_size->getValueAtTime(time, params.lookup_size);
163         _lookup_factor->getValueAtTime(time, params.lookup_factor);
164         _blend_size->getValueAtTime(time, params.blend_size);
165         _blend_threshold->getValueAtTime(time, params.blend_threshold);
166         _blend_decay->getValueAtTime(time, params.blend_decay);
167         _blend_scales->getValueAtTime(time, params.blend_scales);
168         _is_blend_outer->getValueAtTime(time, params.is_blend_outer);
169     }
170 
171     // compute the roi required to compute rect, given params. This roi is then intersected with the image rod.
172     // only called if mix != 0.
getRoI(const OfxRectI & rect,const OfxPointD &,const CImgInpaintParams &,OfxRectI * roi)173     virtual void getRoI(const OfxRectI& rect,
174                         const OfxPointD& /*renderScale*/,
175                         const CImgInpaintParams& /*params*/,
176                         OfxRectI* roi) OVERRIDE FINAL
177     {
178         int delta_pix = 0; // does not support tiles anyway
179         assert(!kSupportsTiles);
180 
181         roi->x1 = rect.x1 - delta_pix;
182         roi->x2 = rect.x2 + delta_pix;
183         roi->y1 = rect.y1 - delta_pix;
184         roi->y2 = rect.y2 + delta_pix;
185     }
186 
render(const RenderArguments & args,const CImgInpaintParams & params,int,int,cimg_library::CImg<cimgpix_t> & mask,cimg_library::CImg<cimgpix_t> & cimg,int)187     virtual void render(const RenderArguments &args,
188                         const CImgInpaintParams& params,
189                         int /*x1*/,
190                         int /*y1*/,
191                         cimg_library::CImg<cimgpix_t>& mask,
192                         cimg_library::CImg<cimgpix_t>& cimg,
193                         int /*alphaChannel*/) OVERRIDE FINAL
194     {
195         printf("%dx%d\n",cimg.width(), cimg.height());
196         // PROCESSING.
197         // This is the only place where the actual processing takes place
198         if ( (params.patch_size <= 0) || (params.lookup_size <= 0.) || cimg.is_empty() ) {
199             return;
200         }
201         // binarize the mask (because inpaint casts it to an int)
202         cimg_for(mask, ptrd, cimgpix_t) {
203             *ptrd = (*ptrd > 0);
204         }
205         cimg.inpaint_patch(mask,
206                            (int)std::ceil(params.patch_size * args.renderScale.x),
207                            (int)std::ceil(params.patch_size * params.lookup_size * args.renderScale.x),
208                            params.lookup_factor,
209                            /*params.lookup_increment=*/1,
210                            (int)(params.blend_size * params.patch_size * args.renderScale.x),
211                            (float)params.blend_threshold,
212                            (float)params.blend_decay,
213                            params.blend_scales,
214                            params.is_blend_outer);
215     }
216 
isIdentity(const IsIdentityArguments &,const CImgInpaintParams & params)217     virtual bool isIdentity(const IsIdentityArguments & /*args*/,
218                             const CImgInpaintParams& params) OVERRIDE FINAL
219     {
220         return (params.patch_size <= 0) || (params.lookup_size <= 0.);
221     };
222 
223     /*
224     virtual void changedParam(const InstanceChangedArgs &args,
225                               const std::string &paramName) OVERRIDE FINAL
226     {
227         CImgFilterPluginHelper<CImgInpaintParams, false>::changedParam(args, paramName);
228     }
229      */
230 
231 private:
232 
233     // params
234     IntParam *_patch_size;
235     DoubleParam *_lookup_size;
236     DoubleParam *_lookup_factor;
237     DoubleParam *_blend_size;
238     DoubleParam *_blend_threshold;
239     DoubleParam *_blend_decay;
240     IntParam *_blend_scales;
241     BooleanParam *_is_blend_outer;
242 };
243 
244 
245 mDeclarePluginFactory(CImgInpaintPluginFactory, {ofxsThreadSuiteCheck();}, {});
246 
247 void
describe(ImageEffectDescriptor & desc)248 CImgInpaintPluginFactory::describe(ImageEffectDescriptor& desc)
249 {
250     // basic labels
251     desc.setLabel(kPluginName);
252     desc.setPluginGrouping(kPluginGrouping);
253     desc.setPluginDescription(kPluginDescription);
254 
255     // add supported context
256     desc.addSupportedContext(eContextFilter);
257     desc.addSupportedContext(eContextGeneral);
258 
259     // add supported pixel depths
260     //desc.addSupportedBitDepth(eBitDepthUByte);
261     //desc.addSupportedBitDepth(eBitDepthUShort);
262     desc.addSupportedBitDepth(eBitDepthFloat);
263 
264     // set a few flags
265     desc.setSingleInstance(false);
266     desc.setHostFrameThreading(kHostFrameThreading);
267     desc.setSupportsMultiResolution(kSupportsMultiResolution);
268     desc.setSupportsTiles(kSupportsTiles);
269     desc.setTemporalClipAccess(false);
270     desc.setRenderTwiceAlways(true);
271     desc.setSupportsMultipleClipPARs(kSupportsMultipleClipPARs);
272     desc.setSupportsMultipleClipDepths(kSupportsMultipleClipDepths);
273     desc.setRenderThreadSafety(kRenderThreadSafety);
274 }
275 
276 void
describeInContext(ImageEffectDescriptor & desc,ContextEnum context)277 CImgInpaintPluginFactory::describeInContext(ImageEffectDescriptor& desc,
278                                            ContextEnum context)
279 {
280     // create the clips and params
281     PageParamDescriptor *page = CImgInpaintPlugin::describeInContextBegin(desc, context,
282                                                                          kSupportsRGBA,
283                                                                          kSupportsRGB,
284                                                                          kSupportsXY,
285                                                                          kSupportsAlpha,
286                                                                          kSupportsTiles,
287                                                                          /*processRGB=*/ true,
288                                                                          /*processAlpha*/ false,
289                                                                          /*processIsSecret=*/ false);
290 
291     {
292         IntParamDescriptor *param = desc.defineIntParam(kParamPatchSize);
293         param->setLabel(kParamPatchSizeLabel);
294         param->setRange(1, 64);
295         param->setDisplayRange(1, 64);
296         param->setDefault(kParamPatchSizeDefault);
297         if (page) {
298             page->addChild(*param);
299         }
300     }
301     {
302         DoubleParamDescriptor *param = desc.defineDoubleParam(kParamLookupSize);
303         param->setLabel(kParamLookupSizeLabel);
304         param->setRange(1., 32.);
305         param->setDisplayRange(1., 32.);
306         param->setDefault(kParamLookupSizeDefault);
307         if (page) {
308             page->addChild(*param);
309         }
310     }
311     {
312         DoubleParamDescriptor *param = desc.defineDoubleParam(kParamLookupFactor);
313         param->setLabel(kParamLookupFactorLabel);
314         param->setRange(0., 1.);
315         param->setDisplayRange(0., 1.);
316         param->setDefault(kParamLookupFactorDefault);
317         param->setIncrement(0.01);
318         if (page) {
319             page->addChild(*param);
320         }
321     }
322     {
323         DoubleParamDescriptor *param = desc.defineDoubleParam(kParamBlendSize);
324         param->setLabel(kParamBlendSizeLabel);
325         param->setRange(0., 5.);
326         param->setDisplayRange(0., 5.);
327         param->setDefault(kParamBlendSizeDefault);
328         param->setIncrement(0.05);
329         if (page) {
330             page->addChild(*param);
331         }
332     }
333     {
334         DoubleParamDescriptor *param = desc.defineDoubleParam(kParamBlendThreshold);
335         param->setLabel(kParamBlendThresholdLabel);
336         param->setRange(0., 1.);
337         param->setDisplayRange(0., 1.);
338         param->setDefault(kParamBlendThresholdDefault);
339         param->setIncrement(0.05);
340         if (page) {
341             page->addChild(*param);
342         }
343     }
344     {
345         DoubleParamDescriptor *param = desc.defineDoubleParam(kParamBlendDecay);
346         param->setLabel(kParamBlendDecayLabel);
347         param->setRange(0., 0.5);
348         param->setDisplayRange(0., 0.5);
349         param->setDefault(kParamBlendDecayDefault);
350         param->setIncrement(0.01);
351         if (page) {
352             page->addChild(*param);
353         }
354     }
355     {
356         IntParamDescriptor *param = desc.defineIntParam(kParamBlendScales);
357         param->setLabel(kParamBlendScalesLabel);
358         param->setRange(1, 20);
359         param->setDisplayRange(1, 20);
360         param->setDefault(kParamBlendScalesDefault);
361         if (page) {
362             page->addChild(*param);
363         }
364     }
365     {
366         BooleanParamDescriptor *param = desc.defineBooleanParam(kParamIsBlendOuter);
367         param->setLabel(kParamIsBlendOuterLabel);
368         param->setDefault(kParamIsBlendOuterDefault);
369         if (page) {
370             page->addChild(*param);
371         }
372     }
373 
374     CImgInpaintPlugin::describeInContextEnd(desc, context, page);
375 } // CImgInpaintPluginFactory::describeInContext
376 
377 ImageEffect*
createInstance(OfxImageEffectHandle handle,ContextEnum)378 CImgInpaintPluginFactory::createInstance(OfxImageEffectHandle handle,
379                                         ContextEnum /*context*/)
380 {
381     return new CImgInpaintPlugin(handle);
382 }
383 
384 static CImgInpaintPluginFactory p(kPluginIdentifier, kPluginVersionMajor, kPluginVersionMinor);
385 mRegisterPluginFactoryInstance(p)
386 
387 OFXS_NAMESPACE_ANONYMOUS_EXIT
388 
389 #endif // PLUGIN_PACK_GPL2
390