1 // Copyright (C) 2020 by Yuri Victorovich. All rights reserved.
2 
3 #include "main-window.h"
4 #include "plugin-interface.h"
5 #include "plugin-manager.h"
6 #include "model-functions.h"
7 
8 #include "util.h"
9 #include "misc.h"
10 #include "nn-types.h"
11 #include "tensor.h"
12 #include "nn-operators.h"
13 #include "image.h"
14 #include "compute.h"
15 #include "svg-graphics-generator.h"
16 #include "svg-push-button.h"
17 #include "model-views/merge-dequantize-operators.h"
18 
19 #include <QByteArray>
20 #include <QEvent>
21 #include <QWheelEvent>
22 #include <QDebug>
23 #include <QPushButton>
24 #include <QFontMetrics>
25 #include <QDesktopWidget>
26 #include <QApplication>
27 #include <QFileDialog>
28 #include <QPixmap>
29 #include <QElapsedTimer>
30 #include <QClipboard>
31 #include <QMimeData>
32 #include <QScrollBar>
33 #include <QSettings>
34 #include <QVariant>
35 
36 #include <assert.h>
37 #include <half.hpp> // to instantiate DataTable2D with the float16 type
38 #include <stdlib.h> // only for ::getenv
39 
40 #include <map>
41 #include <memory>
42 #include <algorithm>
43 
44 #if defined(USE_PERFTOOLS)
45 #include <gperftools/malloc_extension.h>
46 #endif
47 
48 /// local enums and values
49 
50 enum ConvolutionEffect {
51 	ConvolutionEffect_None,
52 	ConvolutionEffect_Blur_3x3,
53 	ConvolutionEffect_Blur_5x5,
54 	ConvolutionEffect_Gaussian_3x3,
55 	ConvolutionEffect_Motion_3x3,
56 	ConvolutionEffect_Sharpen_3x3
57 };
58 #define THR 1./13.
59 #define S01 1./16.
60 #define S02 2./16.
61 #define S04 4./16.
62 #define TTT 1./3.
63 static const std::map<ConvolutionEffect, std::tuple<TensorShape,std::vector<float>>> convolutionEffects = {
64 	{ConvolutionEffect_None, {{},{}}},
65 	{ConvolutionEffect_Blur_3x3, {{3,3,3,3}, {
66 		0.0,0.0,0.0, 0.2,0.0,0.0, 0.0,0.0,0.0,
67 		0.2,0.0,0.0, 0.2,0.0,0.0, 0.2,0.0,0.0,
68 		0.0,0.0,0.0, 0.2,0.0,0.0, 0.0,0.0,0.0,
69 
70 		0.0,0.0,0.0, 0.0,0.2,0.0, 0.0,0.0,0.0,
71 		0.0,0.2,0.0, 0.0,0.2,0.0, 0.0,0.2,0.0,
72 		0.0,0.0,0.0, 0.0,0.2,0.0, 0.0,0.0,0.0,
73 
74 		0.0,0.0,0.0, 0.0,0.0,0.2, 0.0,0.0,0.0,
75 		0.0,0.0,0.2, 0.0,0.0,0.2, 0.0,0.0,0.2,
76 		0.0,0.0,0.0, 0.0,0.0,0.2, 0.0,0.0,0.0
77 	}}},
78 	{ConvolutionEffect_Blur_5x5, {{3,5,5,3}, {
79 		0.0,0.0,0.0, 0.0,0.0,0.0, THR,0.0,0.0, 0.0,0.0,0.0, 0.0,0.0,0.0,
80 		0.0,0.0,0.0, THR,0.0,0.0, THR,0.0,0.0, THR,0.0,0.0, 0.0,0.0,0.0,
81 		THR,0.0,0.0, THR,0.0,0.0, THR,0.0,0.0, THR,0.0,0.0, THR,0.0,0.0,
82 		0.0,0.0,0.0, THR,0.0,0.0, THR,0.0,0.0, THR,0.0,0.0, 0.0,0.0,0.0,
83 		0.0,0.0,0.0, 0.0,0.0,0.0, THR,0.0,0.0, 0.0,0.0,0.0, 0.0,0.0,0.0,
84 
85 		0.0,0.0,0.0, 0.0,0.0,0.0, 0.0,THR,0.0, 0.0,0.0,0.0, 0.0,0.0,0.0,
86 		0.0,0.0,0.0, 0.0,THR,0.0, 0.0,THR,0.0, 0.0,THR,0.0, 0.0,0.0,0.0,
87 		0.0,THR,0.0, 0.0,THR,0.0, 0.0,THR,0.0, 0.0,THR,0.0, 0.0,THR,0.0,
88 		0.0,0.0,0.0, 0.0,THR,0.0, 0.0,THR,0.0, 0.0,THR,0.0, 0.0,0.0,0.0,
89 		0.0,0.0,0.0, 0.0,0.0,0.0, 0.0,THR,0.0, 0.0,0.0,0.0, 0.0,0.0,0.0,
90 
91 		0.0,0.0,0.0, 0.0,0.0,0.0, 0.0,0.0,THR, 0.0,0.0,0.0, 0.0,0.0,0.0,
92 		0.0,0.0,0.0, 0.0,0.0,THR, 0.0,0.0,THR, 0.0,0.0,THR, 0.0,0.0,0.0,
93 		0.0,0.0,THR, 0.0,0.0,THR, 0.0,0.0,THR, 0.0,0.0,THR, 0.0,0.0,THR,
94 		0.0,0.0,0.0, 0.0,0.0,THR, 0.0,0.0,THR, 0.0,0.0,THR, 0.0,0.0,0.0,
95 		0.0,0.0,0.0, 0.0,0.0,0.0, 0.0,0.0,THR, 0.0,0.0,0.0, 0.0,0.0,0.0
96 	}}},
97 	{ConvolutionEffect_Gaussian_3x3, {{3,3,3,3}, {
98 		S01,0.0,0.0, S02,0.0,0.0, S01,0.0,0.0,
99 		S02,0.0,0.0, S04,0.0,0.0, S02,0.0,0.0,
100 		S01,0.0,0.0, S02,0.0,0.0, S01,0.0,0.0,
101 
102 		0.0,S01,0.0, 0.0,S02,0.0, 0.0,S01,0.0,
103 		0.0,S02,0.0, 0.0,S04,0.0, 0.0,S02,0.0,
104 		0.0,S01,0.0, 0.0,S02,0.0, 0.0,S01,0.0,
105 
106 		0.0,0.0,S01, 0.0,0.0,S02, 0.0,0.0,S01,
107 		0.0,0.0,S02, 0.0,0.0,S04, 0.0,0.0,S02,
108 		0.0,0.0,S01, 0.0,0.0,S02, 0.0,0.0,S01
109 	}}},
110 	{ConvolutionEffect_Motion_3x3, {{3,3,3,3}, {
111 		TTT,0.0,0.0, 0.0,0.0,0.0, 0.0,0.0,0.0,
112 		0.0,0.0,0.0, TTT,0.0,0.0, 0.0,0.0,0.0,
113 		0.0,0.0,0.0, 0.0,0.0,0.0, TTT,0.0,0.0,
114 
115 		0.0,TTT,0.0, 0.0,0.0,0.0, 0.0,0.0,0.0,
116 		0.0,0.0,0.0, 0.0,TTT,0.0, 0.0,0.0,0.0,
117 		0.0,0.0,0.0, 0.0,0.0,0.0, 0.0,TTT,0.0,
118 
119 		0.0,0.0,TTT, 0.0,0.0,0.0, 0.0,0.0,0.0,
120 		0.0,0.0,0.0, 0.0,0.0,TTT, 0.0,0.0,0.0,
121 		0.0,0.0,0.0, 0.0,0.0,0.0, 0.0,0.0,TTT
122 	}}},
123 	{ConvolutionEffect_Sharpen_3x3, {{3,3,3,3}, {
124 		-1.,0.0,0.0, -1.,0.0,0.0, -1.,0.0,0.0,
125 		-1.,0.0,0.0, 9.0,0.0,0.0, -1.,0.0,0.0,
126 		-1.,0.0,0.0, -1.,0.0,0.0, -1.,0.0,0.0,
127 
128 		0.0,-1.,0.0, 0.0,-1.,0.0, 0.0,-1.,0.0,
129 		0.0,-1.,0.0, 0.0,9.0,0.0, 0.0,-1.,0.0,
130 		0.0,-1.,0.0, 0.0,-1.,0.0, 0.0,-1.,0.0,
131 
132 		0.0,0.0,-1., 0.0,0.0,-1., 0.0,0.0,-1.,
133 		0.0,0.0,-1., 0.0,0.0,9.0, 0.0,0.0,-1.,
134 		0.0,0.0,-1., 0.0,0.0,-1., 0.0,0.0,-1.
135 	}}}
136 };
137 #undef THR
138 #undef S01
139 #undef S02
140 #undef S04
141 #undef TTT
142 
MainWindow()143 MainWindow::MainWindow()
144 : mainSplitter(this)
145 ,   svgScrollArea(&mainSplitter)
146 ,     nnWidget(&mainSplitter)
147 ,   rhsWidget(&mainSplitter)
148 ,      rhsLayout(&rhsWidget)
149 ,      sourceWidget(tr("Source Data"), &rhsWidget)
150 ,        sourceLayout(&sourceWidget)
151 ,        sourceDetails(&sourceWidget)
152 ,          sourceDetailsLayout(&sourceDetails)
153 ,          sourceImageFileNameLabel(tr("File name:"), &sourceDetails)
154 ,          sourceImageFileNameText(&sourceDetails)
155 ,          sourceImageFileSizeLabel(tr("File size:"), &sourceDetails)
156 ,          sourceImageFileSizeText(&sourceDetails)
157 ,          sourceImageSizeLabel(tr("Image size:"), &sourceDetails)
158 ,          sourceImageSizeText(&sourceDetails)
159 ,          sourceImageCurrentRegionLabel(tr("Current region:"), &sourceDetails)
160 ,          sourceImageCurrentRegionText(&sourceDetails)
161 ,          outputInterpretationSummaryLineEdit(&sourceDetails)
162 ,          scaleImageWidget(&sourceDetails)
163 ,            scaleImageLayout(&scaleImageWidget)
164 ,            spacerScaleWidget(&scaleImageWidget)
165 ,            scaleImageLabel(tr("Scale image:"), &scaleImageWidget)
166 ,            scaleImageSpinBoxes(&scaleImageWidget)
167 ,          sourceApplyEffectsWidget(tr("Apply Effects"), &sourceDetails)
168 ,            sourceApplyEffectsLayout(&sourceApplyEffectsWidget)
169 ,            sourceEffectFlipHorizontallyLabel(tr("Flip horizontally"), &sourceApplyEffectsWidget)
170 ,            sourceEffectFlipHorizontallyCheckBox(&sourceApplyEffectsWidget)
171 ,            sourceEffectFlipVerticallyLabel(tr("Flip vertically"), &sourceApplyEffectsWidget)
172 ,            sourceEffectFlipVerticallyCheckBox(&sourceApplyEffectsWidget)
173 ,            sourceEffectMakeGrayscaleLabel(tr("Make grayscale"), &sourceApplyEffectsWidget)
174 ,            sourceEffectMakeGrayscaleCheckBox(&sourceApplyEffectsWidget)
175 ,            sourceEffectConvolutionLabel(tr("Convolution"), &sourceApplyEffectsWidget)
176 ,            sourceEffectConvolutionParamsWidget(&sourceApplyEffectsWidget)
177 ,              sourceEffectConvolutionParamsLayout(&sourceEffectConvolutionParamsWidget)
178 ,              sourceEffectConvolutionTypeComboBox(&sourceEffectConvolutionParamsWidget)
179 ,              sourceEffectConvolutionCountComboBox(&sourceEffectConvolutionParamsWidget)
180 ,          computeWidget(&sourceDetails)
181 ,            computeLayout(&computeWidget)
182 ,            computeButton(tr("Compute"), &computeWidget)
183 ,            computeRegionComboBox(&computeWidget)
184 ,          computeByWidget(&sourceDetails)
185 ,            computeByLayout(&computeByWidget)
186 ,            inputNormalizationLabel(tr("Normalization"), &computeByWidget)
187 ,            inputNormalizationRangeComboBox(&computeByWidget)
188 ,            spacer1Widget(&computeByWidget)
189 ,            computationTimeLabel(QString(tr("Computed in %1")).arg(tr(": n/a")), &computeByWidget)
190 ,            spacer2Widget(&computeByWidget)
191 ,            outputInterpretationLabel(tr("Interpret as"), &computeByWidget)
192 ,            outputInterpretationKindComboBox(&computeByWidget)
193 ,            spacer3Widget(&computeByWidget)
194 ,            clearComputationResults(tr("Clear"), &computeByWidget)
195 ,        sourceImageStack(&sourceWidget)
196 ,          sourceImageScrollArea(&sourceImageStack)
197 ,            sourceImage(&sourceImageScrollArea)
198 ,      nnDetailsStack(&rhsWidget)
199 ,        nnNetworkDetails(tr("Neural Network Details"), &nnDetailsStack)
200 ,          nnNetworkDetailsLayout(&nnNetworkDetails)
201 ,          nnNetworkDescriptionLabel(tr("Description"), &nnNetworkDetails)
202 ,          nnNetworkDescriptionText(&nnNetworkDetails)
203 ,          nnNetworkComplexityLabel(tr("Complexity"), &nnNetworkDetails)
204 ,          nnNetworkComplexityText(&nnNetworkDetails)
205 ,          nnNetworkFileSizeLabel(tr("File size"), &nnNetworkDetails)
206 ,          nnNetworkFileSizeText(&nnNetworkDetails)
207 ,          nnNetworkNumberInsOutsLabel(tr("Number of inputs/outputs"), &nnNetworkDetails)
208 ,          nnNetworkNumberInsOutsText(&nnNetworkDetails)
209 ,          nnNetworkNumberOperatorsLabel(tr("Number of operators"), &nnNetworkDetails)
210 ,          nnNetworkNumberOperatorsText(&nnNetworkDetails)
211 ,          nnNetworkStaticDataLabel(tr("Amount of static data"), &nnNetworkDetails)
212 ,          nnNetworkStaticDataText(&nnNetworkDetails)
213 ,          nnNetworkOperatorsListLabel(tr("Operators"), &nnNetworkDetails)
214 ,          nnNetworkOperatorsListWidget(&nnNetworkDetails)
215 ,        nnOperatorDetails(&nnDetailsStack)
216 ,          nnOperatorDetailsLayout(&nnOperatorDetails)
217 ,          nnOperatorTypeLabel(tr("Operator Type"), &nnOperatorDetails)
218 ,          nnOperatorTypeValue(&nnOperatorDetails)
219 ,          nnOperatorOptionsLabel(tr("Options"), &nnOperatorDetails)
220 ,          nnOperatorInputsLabel(tr("Inputs"), &nnOperatorDetails)
221 ,          nnOperatorOutputsLabel(tr("Outputs"), &nnOperatorDetails)
222 ,          nnOperatorComplexityLabel(tr("Complexity"), &nnOperatorDetails)
223 ,          nnOperatorComplexityValue(&nnOperatorDetails)
224 ,          nnOperatorStaticDataLabel(tr("Static data"), &nnOperatorDetails)
225 ,          nnOperatorStaticDataValue(&nnOperatorDetails)
226 ,          nnOperatorDataRatioLabel(tr("Data ratio"), &nnOperatorDetails)
227 ,          nnOperatorDataRatioValue(&nnOperatorDetails)
228 ,          nnOperatorDetailsSpacer(&nnOperatorDetails)
229 ,        nnTensorDetails(&nnDetailsStack)
230 ,          nnCurrentTensorId(-1)
231 ,          nnTensorDetailsLayout(&nnTensorDetails)
232 ,          nnTensorKindLabel(tr("Kind"), &nnTensorDetails)
233 ,          nnTensorKindValue(&nnTensorDetails)
234 ,          nnTensorShapeLabel(tr("Shape"), &nnTensorDetails)
235 ,          nnTensorShapeValue(&nnTensorDetails)
236 ,          nnTensorTypeLabel(tr("Type"), &nnTensorDetails)
237 ,          nnTensorTypeValue(&nnTensorDetails)
238 ,          nnTensorSaveDataButton(&nnTensorDetails)
239 ,          nnTensorDataPlaceholder(tr("No Tensor Data Available"), &nnTensorDetails)
240 ,          nnTensorDataPlaceholder1DnotImplemented(tr("1D Data View Not Yet Implemented"), &nnTensorDetails)
241 ,     noNnIsOpenGroupBox(tr("No Neural Network File is Open"), &rhsWidget)
242 ,       noNnIsOpenLayout(&noNnIsOpenGroupBox)
243 ,       noNnIsOpenWidget(&noNnIsOpenGroupBox)
244 , menuBar(this)
245 , statusBar(this)
246 #if defined(USE_PERFTOOLS)
247 ,   memoryUseLabel(&statusBar)
248 ,   memoryUseTimer(&statusBar)
249 #endif
250 , plugin(nullptr)
251 , scaleImageWidthPct(0)
252 , scaleImageHeightPct(0)
253 , self(0)
254 {
255 	// window size and position
256 	if (true) { // set it to center on the screen until we will have persistent app options
257 		QDesktopWidget *desktop = QApplication::desktop();
258 		resize(desktop->width()*3/4, desktop->height()*3/4); // initialize our window to be 3/4 of the size of the desktop
259 		move((desktop->width() - width())/2, (desktop->height() - height())/2);
260 	}
261 
262 	// set up widgets
263 	setCentralWidget(&mainSplitter);
264 	mainSplitter.addWidget(&svgScrollArea);
265 	  svgScrollArea.setWidget(&nnWidget);
266 	mainSplitter.addWidget(&rhsWidget);
267 
268 	rhsLayout.addWidget(&sourceWidget);
269 	  sourceLayout.addWidget(&sourceDetails);
270 	    sourceDetailsLayout.addWidget(&sourceImageFileNameLabel, 0/*row*/, 0/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
271 	    sourceDetailsLayout.addWidget(&sourceImageFileNameText,  0/*row*/, 1/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
272 	    sourceDetailsLayout.addWidget(&sourceImageFileSizeLabel, 1/*row*/, 0/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
273 	    sourceDetailsLayout.addWidget(&sourceImageFileSizeText,  1/*row*/, 1/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
274 	    sourceDetailsLayout.addWidget(&sourceImageSizeLabel,     2/*row*/, 0/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
275 	    sourceDetailsLayout.addWidget(&sourceImageSizeText,      2/*row*/, 1/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
276 	    sourceDetailsLayout.addWidget(&sourceImageCurrentRegionLabel, 3/*row*/, 0/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
277 	    sourceDetailsLayout.addWidget(&sourceImageCurrentRegionText,  3/*row*/, 1/*col*/, 1/*rowSpan*/, 1/*columnSpan*/);
278 	    sourceDetailsLayout.addWidget(&outputInterpretationSummaryLineEdit, 0/*row*/, 2/*col*/, 2/*rowSpan*/, 2/*columnSpan*/);
279 	    sourceDetailsLayout.addWidget(&scaleImageWidget,         4/*row*/, 2/*col*/, 1/*rowSpan*/, 2/*columnSpan*/);
280 	      scaleImageLayout.addWidget(&spacerScaleWidget);
281 	      scaleImageLayout.addWidget(&scaleImageLabel);
282 	      scaleImageLayout.addWidget(&scaleImageSpinBoxes);
283 	    sourceDetailsLayout.addWidget(&sourceApplyEffectsWidget, 5/*row*/, 0/*col*/, 1/*rowSpan*/, 4/*columnSpan*/);
284 	      sourceApplyEffectsLayout.addWidget(&sourceEffectFlipHorizontallyLabel,    0/*row*/, 0/*column*/);
285 	      sourceApplyEffectsLayout.addWidget(&sourceEffectFlipHorizontallyCheckBox, 0/*row*/, 1/*column*/);
286 	      sourceApplyEffectsLayout.addWidget(&sourceEffectFlipVerticallyLabel,      1/*row*/, 0/*column*/);
287 	      sourceApplyEffectsLayout.addWidget(&sourceEffectFlipVerticallyCheckBox,   1/*row*/, 1/*column*/);
288 	      sourceApplyEffectsLayout.addWidget(&sourceEffectMakeGrayscaleLabel,       2/*row*/, 0/*column*/);
289 	      sourceApplyEffectsLayout.addWidget(&sourceEffectMakeGrayscaleCheckBox,    2/*row*/, 1/*column*/);
290 	      sourceApplyEffectsLayout.addWidget(&sourceEffectConvolutionLabel,         3/*row*/, 0/*column*/);
291 	      sourceApplyEffectsLayout.addWidget(&sourceEffectConvolutionParamsWidget,  3/*row*/, 1/*column*/);
292 	        sourceEffectConvolutionParamsLayout.addWidget(&sourceEffectConvolutionTypeComboBox);
293 	        sourceEffectConvolutionParamsLayout.addWidget(&sourceEffectConvolutionCountComboBox);
294 	    sourceDetailsLayout.addWidget(&computeWidget,            6/*row*/, 0/*col*/, 1/*rowSpan*/, 4/*columnSpan*/);
295 	      computeLayout.addWidget(&computeButton);
296 	      computeLayout.addWidget(&computeRegionComboBox);
297 	    sourceDetailsLayout.addWidget(&computeByWidget,          7/*row*/, 0/*col*/, 1/*rowSpan*/, 4/*columnSpan*/);
298 	      computeByLayout.addWidget(&inputNormalizationLabel);
299 	      computeByLayout.addWidget(&inputNormalizationRangeComboBox);
300 	      computeByLayout.addWidget(&inputNormalizationColorOrderComboBox);
301 	      computeByLayout.addWidget(&spacer1Widget);
302 	      computeByLayout.addWidget(&computationTimeLabel);
303 	      computeByLayout.addWidget(&spacer2Widget);
304 	      computeByLayout.addWidget(&outputInterpretationLabel);
305 	      computeByLayout.addWidget(&outputInterpretationKindComboBox);
306 	      computeByLayout.addWidget(&spacer3Widget);
307 	      computeByLayout.addWidget(&clearComputationResults);
308 	  sourceLayout.addWidget(&sourceImageStack);
309 	    sourceImageStack.addWidget(&sourceImageScrollArea);
310 	      sourceImageScrollArea.setWidget(&sourceImage);
311 	rhsLayout.addWidget(&nnDetailsStack);
312 	rhsLayout.addWidget(&noNnIsOpenGroupBox);
313 	  noNnIsOpenLayout.addWidget(&noNnIsOpenWidget);
314 	nnDetailsStack.addWidget(&nnNetworkDetails);
315 		nnNetworkDetailsLayout.addWidget(&nnNetworkDescriptionLabel,      0/*row*/, 0/*col*/);
316 		nnNetworkDetailsLayout.addWidget(&nnNetworkDescriptionText,       0/*row*/, 1/*col*/);
317 		nnNetworkDetailsLayout.addWidget(&nnNetworkComplexityLabel,       1/*row*/, 0/*col*/);
318 		nnNetworkDetailsLayout.addWidget(&nnNetworkComplexityText,        1/*row*/, 1/*col*/);
319 		nnNetworkDetailsLayout.addWidget(&nnNetworkFileSizeLabel,         2/*row*/, 0/*col*/);
320 		nnNetworkDetailsLayout.addWidget(&nnNetworkFileSizeText,          2/*row*/, 1/*col*/);
321 		nnNetworkDetailsLayout.addWidget(&nnNetworkNumberInsOutsLabel,    3/*row*/, 0/*col*/);
322 		nnNetworkDetailsLayout.addWidget(&nnNetworkNumberInsOutsText,     3/*row*/, 1/*col*/);
323 		nnNetworkDetailsLayout.addWidget(&nnNetworkNumberOperatorsLabel,  4/*row*/, 0/*col*/);
324 		nnNetworkDetailsLayout.addWidget(&nnNetworkNumberOperatorsText,   4/*row*/, 1/*col*/);
325 		nnNetworkDetailsLayout.addWidget(&nnNetworkStaticDataLabel,       5/*row*/, 0/*col*/);
326 		nnNetworkDetailsLayout.addWidget(&nnNetworkStaticDataText,        5/*row*/, 1/*col*/);
327 		nnNetworkDetailsLayout.addWidget(&nnNetworkOperatorsListLabel,    6/*row*/, 0/*col*/);
328 		nnNetworkDetailsLayout.addWidget(&nnNetworkOperatorsListWidget,   7/*row*/, 0/*col*/,  1/*rowSpan*/, 2/*columnSpan*/);
329 		nnTensorDetailsLayout .addWidget(&nnTensorKindLabel,              0/*row*/, 0/*col*/);
330 		nnTensorDetailsLayout .addWidget(&nnTensorKindValue,              0/*row*/, 1/*col*/);
331 		nnTensorDetailsLayout .addWidget(&nnTensorShapeLabel,             1/*row*/, 0/*col*/);
332 		nnTensorDetailsLayout .addWidget(&nnTensorShapeValue,             1/*row*/, 1/*col*/);
333 		nnTensorDetailsLayout .addWidget(&nnTensorTypeLabel,              2/*row*/, 0/*col*/);
334 		nnTensorDetailsLayout .addWidget(&nnTensorTypeValue,              2/*row*/, 1/*col*/);
335 		nnTensorDetailsLayout .addWidget(&nnTensorSaveDataButton,         0/*row*/, 2/*col*/,  2/*rowSpan*/, 1/*columnSpan*/);
336 		for (auto l : {&nnTensorDataPlaceholder, &nnTensorDataPlaceholder1DnotImplemented})
337 			nnTensorDetailsLayout.addWidget(l,                        3/*row*/, 0/*col*/,  1/*rowSpan*/, 3/*columnSpan*/);
338 	nnDetailsStack.addWidget(&nnOperatorDetails);
339 	nnDetailsStack.addWidget(&nnTensorDetails);
340 
341 	svgScrollArea.setWidgetResizable(true);
342 	svgScrollArea.setHorizontalScrollBarPolicy(Qt::ScrollBarAlwaysOn);
343 	svgScrollArea.setVerticalScrollBarPolicy(Qt::ScrollBarAlwaysOn);
344 
345 	sourceImageScrollArea.setWidgetResizable(true);
346 	sourceImageScrollArea.setHorizontalScrollBarPolicy(Qt::ScrollBarAsNeeded);
347 	sourceImageScrollArea.setVerticalScrollBarPolicy(Qt::ScrollBarAsNeeded);
348 
349 	setMenuBar(&menuBar);
350 	setStatusBar(&statusBar);
351 #if defined(USE_PERFTOOLS)
352 	statusBar.addWidget(&memoryUseLabel);
353 #endif
354 
355 	// alignment
356 	svgScrollArea.setAlignment(Qt::AlignHCenter|Qt::AlignVCenter);
357 	for (auto w : {&sourceImageFileNameLabel, &sourceImageFileSizeLabel, &sourceImageSizeLabel, &sourceImageCurrentRegionLabel, &scaleImageLabel})
358 		w->setAlignment(Qt::AlignRight|Qt::AlignVCenter);
359 	for (auto w : {&sourceImageFileNameText, &sourceImageFileSizeText, &sourceImageSizeText})
360 		w->setAlignment(Qt::AlignLeft|Qt::AlignVCenter);
361 	outputInterpretationSummaryLineEdit.setAlignment(Qt::AlignRight);
362 	for (auto w : {&sourceEffectFlipHorizontallyLabel, &sourceEffectFlipVerticallyLabel, &sourceEffectMakeGrayscaleLabel, &sourceEffectConvolutionLabel})
363 		w->setAlignment(Qt::AlignRight|Qt::AlignVCenter);
364 	for (auto w : {&inputNormalizationLabel, &computationTimeLabel})
365 		w->setAlignment(Qt::AlignRight|Qt::AlignVCenter);
366 	sourceImage.setAlignment(Qt::AlignHCenter|Qt::AlignVCenter);
367 	for (auto l : {&nnTensorDataPlaceholder, &nnTensorDataPlaceholder1DnotImplemented})
368 		l->setAlignment(Qt::AlignCenter|Qt::AlignVCenter);
369 
370 	{ // double the font size in the summary
371 		QFont font = outputInterpretationSummaryLineEdit.font();
372 		font.setPointSize(2*font.pointSize());
373 		font.setBold(true);
374 		outputInterpretationSummaryLineEdit.setFont(font);
375 	}
376 
377 	// tooltips
378 	sourceImageFileNameLabel            .setToolTip(tr("File name of the input image"));
379 	sourceImageFileNameText             .setToolTip(tr("File name of the input image"));
380 	sourceImageFileSizeLabel            .setToolTip(tr("File size of the input image"));
381 	sourceImageFileSizeText             .setToolTip(tr("File size of the input image"));
382 	sourceImageSizeLabel                .setToolTip(tr("Input image size"));
383 	sourceImageSizeText                 .setToolTip(tr("Input image size"));
384 	sourceImageCurrentRegionLabel       .setToolTip(tr("Currently selected region of the image"));
385 	sourceImageCurrentRegionText        .setToolTip(tr("Currently selected region of the image"));
386 	for (QWidget *w : {(QWidget*)&scaleImageLabel,(QWidget*)&scaleImageSpinBoxes})
387 		w->                          setToolTip(tr("Scale the image to fit the screen, or to select its area for NN computation"));
388 	sourceApplyEffectsWidget            .setToolTip(tr("Apply effects to the image"));
389 	sourceEffectFlipHorizontallyLabel   .setToolTip(tr("Flip the image horizontally"));
390 	sourceEffectFlipHorizontallyCheckBox.setToolTip(tr("Flip the image horizontally"));
391 	sourceEffectFlipVerticallyLabel     .setToolTip(tr("Flip the image vertically"));
392 	sourceEffectFlipVerticallyCheckBox  .setToolTip(tr("Flip the image vertically"));
393 	sourceEffectMakeGrayscaleLabel      .setToolTip(tr("Make the image grayscale"));
394 	sourceEffectMakeGrayscaleCheckBox   .setToolTip(tr("Make the image grayscale"));
395 	sourceEffectConvolutionLabel        .setToolTip(tr("Apply convolution to the image"));
396 	sourceEffectConvolutionTypeComboBox .setToolTip(tr("Convolution type to apply to the image"));
397 	sourceEffectConvolutionCountComboBox.setToolTip(tr("How many times to apply the convolution"));
398 	computeButton                       .setToolTip(tr("Perform neural network computation for the currently selected image as input"));
399 	computeRegionComboBox               .setToolTip(tr("Choose what region of the image to compute on: the visible area or the whole image"));
400 	inputNormalizationLabel             .setToolTip(tr("Specify how does this NN expect its input data be normalized"));
401 	inputNormalizationRangeComboBox     .setToolTip(tr("Specify what value range does this NN expect its input data be normalized to"));
402 	inputNormalizationColorOrderComboBox.setToolTip(tr("Specify what color order does this NN expect its input data be supplied in"));
403 	computationTimeLabel                .setToolTip(tr("Show how long did the the NN computation take"));
404 	for (QWidget *w : {(QWidget*)&outputInterpretationLabel,(QWidget*)&outputInterpretationKindComboBox})
405 		w->                          setToolTip(tr("How to interpret the computation result?"));
406 	clearComputationResults             .setToolTip(tr("Clear computation results"));
407 	sourceImage                         .setToolTip(tr("Image currently used as NN input"));
408 	// network page
409 	for (auto l : {&nnNetworkDescriptionLabel,&nnNetworkDescriptionText})
410 		l->                          setToolTip(tr("Network description as specified in the NN file, if any"));
411 	for (auto l : {&nnNetworkComplexityLabel,&nnNetworkComplexityText})
412 		l->                          setToolTip(tr("Network complexity, i.e. how many operations between simple numbers are required to compute this network"));
413 	for (auto l : {&nnNetworkFileSizeLabel,&nnNetworkFileSizeText})
414 		l->                          setToolTip(tr("Network complexity, i.e. how many operations between simple numbers are required to compute this network"));
415 	for (auto l : {&nnNetworkNumberInsOutsLabel,&nnNetworkNumberInsOutsText})
416 		l->                          setToolTip(tr("Number of inputs and outputs in this network"));
417 	for (auto l : {&nnNetworkNumberOperatorsLabel,&nnNetworkNumberOperatorsText})
418 		l->                          setToolTip(tr("Number of operators in this network"));
419 	for (auto l : {&nnNetworkStaticDataLabel,&nnNetworkStaticDataText})
420 		l->                          setToolTip(tr("Amount of static data supplied for operators in the network"));
421 	for (auto w : {(QWidget*)&nnNetworkOperatorsListLabel,(QWidget*)&nnNetworkOperatorsListWidget})
422 		w->                          setToolTip(tr("List of operators in the model with details about them"));
423 	// operator page
424 	for (auto l : {&nnOperatorComplexityLabel,&nnOperatorComplexityValue})
425 		l->                          setToolTip(tr("Complexity of the currently selected NN in FLOPS"));
426 	// tensor page
427 	for (auto l : {&nnTensorKindLabel,&nnTensorKindValue})
428 		l->                          setToolTip(tr("What kind of tensor this is"));
429 	for (auto l : {&nnTensorShapeLabel,&nnTensorShapeValue})
430 		l->                          setToolTip(tr("Shape of the tensor describes how meny dimensions does it have and sizes in each dimension"));
431 	for (auto l : {&nnTensorTypeLabel,&nnTensorTypeValue})
432 		l->                          setToolTip(tr("Type of data in this tensor"));
433 	nnTensorSaveDataButton              .setToolTip(tr("Press to save the data for this tensor in the json format"));
434 
435 	// size policies
436 	svgScrollArea                        .setSizeAdjustPolicy(QAbstractScrollArea::AdjustToContents);
437 	sourceWidget                         .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Fixed);
438 	sourceImageFileNameLabel             .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
439 	sourceImageFileNameText              .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
440 	sourceImageFileSizeLabel             .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
441 	sourceImageFileSizeText              .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
442 	sourceImageSizeLabel                 .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
443 	sourceImageSizeText                  .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
444 	sourceImageCurrentRegionLabel        .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
445 	sourceImageCurrentRegionText         .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
446 	scaleImageWidget                     .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
447 	spacerScaleWidget                    .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Minimum);
448 	scaleImageLabel                      .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Fixed);
449 	scaleImageSpinBoxes                  .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Fixed);
450 	sourceApplyEffectsWidget             .setSizePolicy(QSizePolicy::MinimumExpanding, QSizePolicy::Maximum);
451 	for (QWidget *w : {&sourceEffectConvolutionParamsWidget, (QWidget*)&sourceEffectConvolutionTypeComboBox, (QWidget*)&sourceEffectConvolutionCountComboBox})
452 		w->setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum);
453 	computeWidget                        .setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Maximum);
454 	computeButton                        .setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Maximum);
455 	computeRegionComboBox                .setSizePolicy(QSizePolicy::Fixed,   QSizePolicy::Maximum);
456 	inputNormalizationLabel              .setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum); //The sizeHint() is a maximum
457 	inputNormalizationRangeComboBox      .setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum);
458 	inputNormalizationColorOrderComboBox .setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum);
459 	spacer1Widget                        .setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Maximum);
460 	computationTimeLabel                 .setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum);
461 	spacer2Widget                        .setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Maximum);
462 	outputInterpretationLabel            .setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum);
463 	outputInterpretationKindComboBox     .setSizePolicy(QSizePolicy::Maximum, QSizePolicy::Maximum);
464 	spacer3Widget                        .setSizePolicy(QSizePolicy::Minimum, QSizePolicy::Maximum);
465 	sourceImageStack                     .setSizePolicy(QSizePolicy::Fixed,   QSizePolicy::Fixed);
466 	for (auto *w : {&nnNetworkDescriptionLabel, &nnNetworkDescriptionText, &nnNetworkComplexityLabel, &nnNetworkComplexityText,
467 	                &nnNetworkFileSizeLabel, &nnNetworkFileSizeText, &nnNetworkNumberInsOutsLabel, &nnNetworkNumberInsOutsText,
468 	                &nnNetworkNumberOperatorsLabel, &nnNetworkNumberOperatorsText, &nnNetworkStaticDataLabel, &nnNetworkStaticDataText, &nnNetworkOperatorsListLabel,
469 	                &nnOperatorTypeLabel, &nnOperatorTypeValue, &nnOperatorOptionsLabel, &nnOperatorInputsLabel, &nnOperatorOutputsLabel,
470 	                &nnOperatorComplexityLabel, &nnOperatorComplexityValue, &nnOperatorStaticDataLabel, &nnOperatorStaticDataValue, &nnOperatorDataRatioLabel, &nnOperatorDataRatioValue,
471 	                &nnTensorKindLabel, &nnTensorKindValue, &nnTensorShapeLabel, &nnTensorShapeValue, &nnTensorTypeLabel})
472 		w->                           setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Maximum);
473 	nnNetworkOperatorsListWidget         .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Minimum);
474 	nnOperatorDetailsSpacer              .setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Minimum);
475 	for (auto l : {&nnTensorDataPlaceholder, &nnTensorDataPlaceholder1DnotImplemented})
476 		l->                           setSizePolicy(QSizePolicy::Minimum,   QSizePolicy::Minimum);
477 
478 	// margins and spacing
479 	rhsLayout.setSpacing(0);
480 	for (QLayout *l : {&scaleImageLayout, &sourceEffectConvolutionParamsLayout, &computeByLayout})
481 		l->setContentsMargins(0,0,0,0);
482 	for (QLayout *l : {&sourceEffectConvolutionParamsLayout, &computeByLayout})
483 		l->setSpacing(0);
484 	for (auto w : {&spacer1Widget, &spacer2Widget, &spacer3Widget})
485 		w->setMinimumWidth(10);
486 	for (auto l : {&nnNetworkDetailsLayout, &nnOperatorDetailsLayout, &nnTensorDetailsLayout})
487 		l->setVerticalSpacing(0);
488 
489 	// widget options and flags
490 	updateSectionWidgetsVisibility();
491 	sourceEffectConvolutionCountComboBox.setEnabled(false); // is only enabled when some convoulution is chosen
492 	nnNetworkStaticDataText.setWordWrap(true); // allow word wrap because text is long in this label
493 	outputInterpretationSummaryLineEdit.setWordWrap(true);
494 	nnTensorDataPlaceholder1DnotImplemented.hide();
495 
496 	// widget states
497 	updateResultInterpretationSummaryText(false/*enable*/, tr("n/a"), tr("n/a"));
498 
499 	// fill lists
500 	sourceEffectConvolutionTypeComboBox.addItem(tr("None"),          ConvolutionEffect_None);
501 	sourceEffectConvolutionTypeComboBox.addItem(tr("Blur (3x3)"),    ConvolutionEffect_Blur_3x3);
502 	sourceEffectConvolutionTypeComboBox.addItem(tr("Blur (5x5)"),    ConvolutionEffect_Blur_5x5);
503 	sourceEffectConvolutionTypeComboBox.addItem(tr("Gauss (3x3)"),   ConvolutionEffect_Gaussian_3x3);
504 	sourceEffectConvolutionTypeComboBox.addItem(tr("Motion (3x3)"),  ConvolutionEffect_Motion_3x3);
505 	sourceEffectConvolutionTypeComboBox.addItem(tr("Sharpen (3x3)"), ConvolutionEffect_Sharpen_3x3);
506 	for (unsigned c = 1; c <= 20; c++)
507 		sourceEffectConvolutionCountComboBox.addItem(QString("x%1").arg(c), c);
508 
509 	computeRegionComboBox.addItem("on the visible region");
510 	computeRegionComboBox.addItem("on the whole image");
511 
512 	inputNormalizationRangeComboBox.addItem("0..1",         InputNormalizationRange_0_1); // default
513 	inputNormalizationRangeComboBox.addItem("0..255",       InputNormalizationRange_0_255);
514 	inputNormalizationRangeComboBox.addItem("0..128",       InputNormalizationRange_0_128);
515 	inputNormalizationRangeComboBox.addItem("0..64",        InputNormalizationRange_0_64);
516 	inputNormalizationRangeComboBox.addItem("0..32",        InputNormalizationRange_0_32);
517 	inputNormalizationRangeComboBox.addItem("0..16",        InputNormalizationRange_0_16);
518 	inputNormalizationRangeComboBox.addItem("0..8",         InputNormalizationRange_0_8);
519 	inputNormalizationRangeComboBox.addItem("-1..1",        InputNormalizationRange_M1_P1);
520 	inputNormalizationRangeComboBox.addItem("-½..½",        InputNormalizationRange_M05_P05);
521 	inputNormalizationRangeComboBox.addItem("¼..¾",         InputNormalizationRange_14_34);
522 	inputNormalizationRangeComboBox.addItem("ImageNet",     InputNormalizationRange_ImageNet);
523 	//
524 	inputNormalizationColorOrderComboBox.addItem("RGB",     InputNormalizationColorOrder_RGB); // default
525 	inputNormalizationColorOrderComboBox.addItem("BGR",     InputNormalizationColorOrder_BGR);
526 	//
527 	outputInterpretationKindComboBox.addItem("Undefined",        OutputInterpretationKind_Undefined);
528 	outputInterpretationKindComboBox.addItem("ImageNet-1001",    OutputInterpretationKind_ImageNet1001);
529 	outputInterpretationKindComboBox.addItem("ImageNet-1000",    OutputInterpretationKind_ImageNet1000);
530 	outputInterpretationKindComboBox.addItem("No/Yes",           OutputInterpretationKind_NoYes);
531 	outputInterpretationKindComboBox.addItem("Yes/No",           OutputInterpretationKind_YesNo);
532 	outputInterpretationKindComboBox.addItem("Per-Pixel",        OutputInterpretationKind_PixelClassification);
533 	outputInterpretationKindComboBox.addItem("Image conversion", OutputInterpretationKind_ImageConversion);
534 
535 	// fonts
536 	for (auto widget : {&nnNetworkDescriptionLabel, &nnNetworkComplexityLabel, &nnNetworkFileSizeLabel, &nnNetworkNumberInsOutsLabel, &nnNetworkNumberOperatorsLabel,
537 	                    &nnNetworkStaticDataLabel, &nnNetworkOperatorsListLabel,
538 	                    &nnOperatorTypeLabel, &nnOperatorOptionsLabel, &nnOperatorInputsLabel, &nnOperatorOutputsLabel, &nnOperatorComplexityLabel,
539 	                    &nnOperatorStaticDataLabel, &nnOperatorDataRatioLabel,
540 	                    &nnTensorKindLabel, &nnTensorShapeLabel, &nnTensorTypeLabel})
541 		widget->setStyleSheet("font-weight: bold;");
542 	for (auto l : {&nnTensorDataPlaceholder, &nnTensorDataPlaceholder1DnotImplemented})
543 		l->setStyleSheet("color: gray; font: 35pt;");
544 
545 	// connect signals
546 	connect(&nnWidget, &NnWidget::clickedOnOperator, [this](PluginInterface::OperatorId operatorId) {
547 		showOperatorDetails(operatorId);
548 		nnNetworkOperatorsListWidget.selectOperator(operatorId);
549 	});
550 	connect(&nnWidget, &NnWidget::clickedOnTensorEdge, [this](PluginInterface::TensorId tensorId) {
551 		showTensorDetails(tensorId);
552 	});
553 	connect(&nnWidget, &NnWidget::clickedOnInput, [this](PluginInterface::TensorId tensorId) {
554 		showInputDetails(tensorId);
555 	});
556 	connect(&nnWidget, &NnWidget::clickedOnOutput, [this](PluginInterface::TensorId tensorId) {
557 		showOutputDetails(tensorId);
558 	});
559 	connect(&nnWidget, &NnWidget::clickedOnBlankSpace, [this]() {
560 		showNetworkDetails();
561 	});
562 	connect(&nnNetworkOperatorsListWidget, &OperatorsListWidget::operatorSelected, [](PluginInterface::OperatorId operatorId) {
563 		PRINT("TODO OperatorsListWidget::operatorSelected oid=" << operatorId)
564 	});
565 	connect(&scaleImageSpinBoxes, &ScaleImageWidget::scalingFactorChanged, [this](unsigned widthFactor, unsigned heightFactor) {
566 		assert(scaleImageWidthPct!=0 && scaleImageHeightPct!=0); // scaling percentages are initially set when the image is open/pasted/etc
567 		self++;
568 
569 		inputParamsChanged(); // scaling change invalidates computation results because this changes the image ares on which computation is performed
570 
571 		// accept percentages set by the user
572 		scaleImageWidthPct = widthFactor;
573 		scaleImageHeightPct = heightFactor;
574 
575 		// update the image on screen accordingly
576 		updateSourceImageOnScreen();
577 		updateCurrentRegionText();
578 
579 		self--;
580 	});
581 	connect(&sourceEffectFlipHorizontallyCheckBox, &QCheckBox::stateChanged, [this](int) {
582 		effectsChanged();
583 	});
584 	connect(&sourceEffectFlipVerticallyCheckBox, &QCheckBox::stateChanged, [this](int) {
585 		effectsChanged();
586 	});
587 	connect(&sourceEffectMakeGrayscaleCheckBox, &QCheckBox::stateChanged, [this](int) {
588 		effectsChanged();
589 	});
590 	connect(&sourceEffectConvolutionTypeComboBox, QOverload<int>::of(&QComboBox::activated), [this](int index) {
591 		effectsChanged();
592 		sourceEffectConvolutionCountComboBox.setEnabled(index>0);
593 	});
594 	connect(&sourceEffectConvolutionCountComboBox, QOverload<int>::of(&QComboBox::activated), [this](int) {
595 		if (sourceEffectConvolutionTypeComboBox.currentIndex() != 0)
596 			effectsChanged();
597 	});
598 	connect(&computeButton, &QAbstractButton::pressed, [this]() {
599 		QElapsedTimer timer;
600 		timer.start();
601 
602 		// allocate tensors array
603 		if (!tensorData) {
604 			tensorData.reset(new std::vector<std::shared_ptr<const float>>);
605 			tensorData->resize(model->numTensors());
606 		}
607 
608 		// computation arguments
609 		bool doVisibleRegion = computeRegionComboBox.currentIndex()==0;
610 		std::array<unsigned,4> imageRegion = doVisibleRegion ? getVisibleImageRegion() : std::array<unsigned,4>{0,0, sourceTensorShape[1]-1,sourceTensorShape[0]-1};
611 		InputNormalization inputNormalization = {
612 			(InputNormalizationRange)inputNormalizationRangeComboBox.currentData().toUInt(),
613 			(InputNormalizationColorOrder)inputNormalizationColorOrderComboBox.currentData().toUInt()
614 		};
615 		auto cbTensorComputed = [](PluginInterface::TensorId tensorId) {
616 			//PRINT("Tensor DONE: tid=" << tensorId)
617 		};
618 		auto cbWarningMessage = [this](const std::string &msg) {
619 			Util::warningOk(this, S2Q(msg));
620 		};
621 
622 		// find input data and convert it to the required format
623 		std::map<PluginInterface::TensorId, std::shared_ptr<const float>> modelInputs;
624 		bool succ = Compute::buildComputeInputs(model.get(),
625 			imageRegion, inputNormalization,
626 			sourceTensorDataAsUsed, sourceTensorShape,
627 			modelInputs,
628 			cbTensorComputed,cbWarningMessage);
629 		if (!succ) {
630 			PRINT("WARNING couldn't prepare arguments for the computation")
631 			return;
632 		}
633 
634 		// fill the input data into tensors
635 		Compute::fillInputs(modelInputs, tensorData);
636 
637 		// compute
638 		succ = Compute::compute(model.get(), tensorData, cbTensorComputed,cbWarningMessage);
639 		if (!succ) {
640 			PRINT("WARNING computation didn't succeed")
641 			return;
642 		}
643 
644 		// computation succeeded
645 		if (nnCurrentTensorId!=-1 && model->isTensorComputed(nnCurrentTensorId)) {
646 			if (!nnTensorData2D) {
647 				showNnTensorData2D();
648 			} else {
649 				nnTensorData2D->dataChanged((*tensorData.get())[nnCurrentTensorId].get());
650 				nnTensorData2D->setEnabled(true);
651 			}
652 		}
653 		updateResultInterpretation();
654 		computationTimeLabel.setText(QString("Computed in %1").arg(QString("%1 ms").arg(S2Q(Util::formatUIntHumanReadable(timer.elapsed())))));
655 	});
656 	connect(&computeRegionComboBox, QOverload<int>::of(&QComboBox::activated), [this](int) {
657 		clearComputedTensorData(Temporary);
658 		updateResultInterpretation();
659 	});
660 	connect(&inputNormalizationRangeComboBox, QOverload<int>::of(&QComboBox::activated), [this](int) {
661 		inputNormalizationChanged();
662 	});
663 	connect(&inputNormalizationColorOrderComboBox, QOverload<int>::of(&QComboBox::activated), [this](int) {
664 		inputNormalizationChanged();
665 	});
666 	connect(&outputInterpretationKindComboBox, QOverload<int>::of(&QComboBox::activated), [this](int) {
667 		updateResultInterpretation();
668 	});
669 	connect(&clearComputationResults, &QAbstractButton::pressed, [this]() {
670 		clearComputedTensorData(Temporary);
671 		removeTableIfAny();
672 		updateResultInterpretation();
673 	});
674 	connect(sourceImageScrollArea.horizontalScrollBar(), &QAbstractSlider::valueChanged, [this]() {
675 		if (!self && haveImageOpen()) // scrollbars still send signals after the image was closed
676 			updateCurrentRegionText();
677 	});
678 	connect(sourceImageScrollArea.verticalScrollBar(), &QAbstractSlider::valueChanged, [this]() {
679 		if (!self && haveImageOpen())
680 			updateCurrentRegionText();
681 	});
682 	connect(&noNnIsOpenWidget, &NoNnIsOpenWidget::openNeuralNetworkFilePressed, [this]() {
683 		onOpenNeuralNetworkFileUserIntent();
684 	});
685 	connect(&nnTensorSaveDataButton, &QAbstractButton::pressed, [this]() {
686 		PluginInterface::TensorId tensorId = nnTensorSaveDataButton.property("tensorId").toUInt();
687 		PRINT("Saving data for tensor#" << tensorId)
688 		if (model->getTensorHasData(tensorId) || (tensorData && (*tensorData.get())[tensorId])) {
689 			Tensor::saveTensorDataAsJson(
690 				model->getTensorShape(tensorId),
691 				model->getTensorHasData(tensorId) ? model->getTensorDataF32(tensorId) : (*tensorData.get())[tensorId].get(),
692 				CSTR("tensor#" << tensorId << ".json") // match the name with one in compute.cpp
693 			);
694 		} else {
695 			Util::warningOk(this, QString(tr("Data for tensor#%1 isn't available")).arg(tensorId));
696 		}
697 	});
698 
699 	// monitor memory use
700 #if defined(USE_PERFTOOLS)
701 	connect(&memoryUseTimer, &QTimer::timeout, [this]() {
702 		size_t inuseBytes = 0;
703 		(void)MallocExtension::instance()->GetNumericProperty("generic.current_allocated_bytes", &inuseBytes);
704 		memoryUseLabel.setText(QString(tr("Memory use: %1 bytes")).arg(S2Q(Util::formatUIntHumanReadable(inuseBytes))));
705 	});
706 	memoryUseTimer.start(1000);
707 #endif
708 
709 	// add menus
710 	auto fileMenu = menuBar.addMenu(tr("&File"));
711 	fileMenu->addAction(tr("Open Image"), [this]() {
712 		QString fileName = QFileDialog::getOpenFileName(this,
713 			tr("Open image file"), "",
714 			tr("Image (*.png);;All Files (*)")
715 		);
716 		if (!fileName.isEmpty())
717 			openImageFile(fileName);
718 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_O));
719 	fileMenu->addAction(tr("Open Neural Network File"), [this]() {
720 		onOpenNeuralNetworkFileUserIntent();
721 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_N)); // non-standard because this is our custom operation
722 	fileMenu->addSeparator();
723 	fileMenu->addAction(tr("Take Screenshot"), [this]() {
724 		openImagePixmap(Util::getScreenshot(true/*hideOurWindows*/), tr("screenshot"));
725 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_R)); // non-standard
726 	fileMenu->addAction(tr("Paste Image"), [this]() {
727 		const QClipboard *clipboard = QApplication::clipboard();
728 		const QMimeData *mimeData = clipboard->mimeData();
729 
730 		if (mimeData->hasImage()) {
731 			auto pixmap = qvariant_cast<QPixmap>(mimeData->imageData());
732 			if (pixmap.height() != 0)
733 				openImagePixmap(pixmap, tr("paste from clipboard"));
734 			else // see https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=242932
735 				Util::warningOk(this, QString(tr("No image to paste, clipboard contains an empty image")));
736 		} else {
737 			auto formats = mimeData->formats();
738 			if (!formats.empty())
739 				Util::warningOk(this, QString(tr("%1:\n• %2"))
740 					.arg(tr("No image to paste, clipboard can be interpreted as"))
741 					.arg(mimeData->formats().join("\n• ")));
742 			else
743 				Util::warningOk(this, QString(tr("No image to paste, clipboard s empty")));
744 		}
745 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_V));
746 	fileMenu->addSeparator();
747 	fileMenu->addAction(tr("Copy Image"), [this]() {
748 		if (sourceTensorDataAsUsed)
749 			QApplication::clipboard()->setPixmap(Image::toQPixmap(sourceTensorDataAsUsed.get(), sourceTensorShape), QClipboard::Clipboard);
750 		else
751 			Util::warningOk(this, QString(tr("Can't copy the image: no image in currently open")));
752 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_C));
753 	fileMenu->addAction(tr("Save Image As"), [this]() {
754 		QString fileName = QFileDialog::getSaveFileName(this,
755 			tr("Save image as file"), ""
756 		);
757 		if (!fileName.isEmpty()) { // save the visible region, same as NN computation normally sees
758 			std::array<unsigned,4> imageRegion = getVisibleImageRegion();
759 			Image::writePngImageFile( // for simplicity - extract the region whether this is needed or not
760 				std::unique_ptr<float>(Image::regionOfImage(sourceTensorDataAsUsed.get(), sourceTensorShape, imageRegion)).get(),
761 				{imageRegion[3]-imageRegion[1]+1, imageRegion[2]-imageRegion[0]+1, sourceTensorShape[2]},
762 				Q2S(fileName.endsWith(".png") ? fileName : fileName+".png")
763 			);
764 		}
765 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_S));
766 	fileMenu->addSeparator();
767 	fileMenu->addAction(tr("Close Image"), [this]() {
768 		clearInputImageDisplay();
769 		clearEffects();
770 		clearComputedTensorData(Permanent); // closing image invalidates computation results
771 		updateResultInterpretation();
772 		updateSectionWidgetsVisibility();
773 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_W)); // like "close tab" in chrome
774 	fileMenu->addAction(tr("Close Neural Network"), [this]() {
775 		if (model)
776 			closeNeuralNetwork();
777 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_Z));
778 	fileMenu->addSeparator();
779 	fileMenu->addAction(tr("Quit"), []() {
780 		QApplication::quit();
781 	})->setShortcut(QKeySequence(Qt::CTRL + Qt::Key_Q));
782 
783 	auto viewMenu = menuBar.addMenu(tr("&View"));
784 	viewMenu->addAction(tr("Full Screen"), [this]() {
785 		if (isFullScreen())
786 			showNormal();
787 		else
788 			showFullScreen();
789 	})->setShortcut(QKeySequence(Qt::Key_F11));
790 
791 	// icon
792 	setWindowIcon(QPixmap::fromImage(Util::svgToImage(SvgGraphics::generateNnAppIcon(), QSize(128,128), QPainter::CompositionMode_SourceOver)));
793 
794 	// restore geometry
795 	restoreGeometry(appSettings.value("MainWindow.geometry", QByteArray()).toByteArray());
796 }
797 
~MainWindow()798 MainWindow::~MainWindow() {
799 	if (model) {
800 		model.reset(nullptr);
801 		pluginInterface.reset(nullptr);
802 		PluginManager::unloadPlugin(plugin);
803 	}
804 
805 	// save geometry
806 	appSettings.setValue("MainWindow.geometry", saveGeometry());
807 }
808 
loadModelFile(const QString & filePath)809 bool MainWindow::loadModelFile(const QString &filePath) {
810 	// helpers
811 	auto endsWith = [](const std::string &fullString, const std::string &ending) {
812 		return
813 			(fullString.length() >= ending.length()+1)
814 			&&
815 			(0 == fullString.compare(fullString.length()-ending.length(), ending.length(), ending));
816 	};
817 	auto fileNameToPluginName = [&](const QString &filePath) {
818 		if (endsWith(Q2S(filePath), ".tflite"))
819 			return "tf-lite";
820 		else
821 			return (const char*)nullptr;
822 	};
823 
824 	// file name -> plugin name
825 	auto pluginName = fileNameToPluginName(filePath);
826 	if (pluginName == nullptr)
827 		return Util::warningOk(this, QString("%1 '%2'").arg(tr("Couldn't find a plugin to open the file")).arg(filePath));
828 
829 	// load the plugin
830 	plugin = PluginManager::loadPlugin(pluginName);
831 	if (!plugin)
832 		FAIL(Q2S(QString("%1 '%2'").arg(tr("failed to load the plugin")).arg(pluginName)))
833 	pluginInterface.reset(PluginManager::getInterface(plugin)());
834 
835 	// load the model
836 	if (pluginInterface->open(Q2S(filePath)))
837 		PRINT("loaded the model '" << Q2S(filePath) << "' successfully")
838 	else
839 		FAIL("failed to load the model '" << Q2S(filePath) << "'")
840 	if (pluginInterface->numModels() != 1)
841 		FAIL("multi-model files aren't supported yet")
842 	model.reset(pluginInterface->getModel(0));
843 
844 	// add ModelViews::MergeDequantizeOperators
845 	if (!::getenv("NN_INSIGHT_NO_MERGE_DEQUANTIZE_OPERATORS")) // XXX TODO need to have a UI-based options screen for such choices
846 		model.reset(new ModelViews::MergeDequantizeOperators(model.release()));
847 
848 	// render the model as SVG image
849 	nnWidget.open(model.get());
850 	nnNetworkOperatorsListWidget.setNnModel(model.get());
851 	updateSectionWidgetsVisibility();
852 
853 	// guess the output interpretation type
854 	Util::selectComboBoxItemWithItemData(outputInterpretationKindComboBox, (int)ModelFunctions::guessOutputInterpretationKind(model.get()));
855 
856 	// switch NN details to show the whole network info page
857 	updateNetworkDetailsPage();
858 	nnDetailsStack.setCurrentIndex(/*page#*/0);
859 
860 	// set window title
861 	setWindowTitle(QString("NN Insight: %1 (%2)").arg(filePath).arg(S2Q(Util::formatFlops(ModelFunctions::computeModelFlops(model.get())))));
862 
863 	return true; // success
864 }
865 
866 /// private methods
867 
haveImageOpen() const868 bool MainWindow::haveImageOpen() const {
869 	return (bool)sourceTensorDataAsLoaded;
870 }
871 
showNetworkDetails()872 void MainWindow::showNetworkDetails() {
873 	nnDetailsStack.setCurrentIndex(/*page#*/0);
874 }
875 
showOperatorDetails(PluginInterface::OperatorId operatorId)876 void MainWindow::showOperatorDetails(PluginInterface::OperatorId operatorId) {
877 	// switch to the details page, set title
878 	nnDetailsStack.setCurrentIndex(/*page#1*/1);
879 	nnOperatorDetails.setTitle(QString(tr("NN Operator#%1")).arg(operatorId+1));
880 
881 	// clear items
882 	while (nnOperatorDetailsLayout.count() > 0)
883 		nnOperatorDetailsLayout.removeItem(nnOperatorDetailsLayout.itemAt(0));
884 	tempDetailWidgets.clear();
885 
886 	// helper
887 	auto addTensorLines = [this](auto &tensors, unsigned &row) {
888 		for (auto tensorId : tensors) {
889 			row++;
890 			// tensor number
891 			auto label = makeTextSelectable(new QLabel(QString(tr("tensor#%1:")).arg(tensorId), &nnOperatorDetails));
892 			label->setToolTip(tr("Tensor number"));
893 			label->setAlignment(Qt::AlignRight|Qt::AlignVCenter);
894 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
895 			nnOperatorDetailsLayout.addWidget(label,         row,   0/*column*/);
896 			// tensor name
897 			label = makeTextSelectable(new QLabel(S2Q(model->getTensorName(tensorId)), &nnOperatorDetails));
898 			label->setToolTip(tr("Tensor name"));
899 			label->setAlignment(Qt::AlignLeft|Qt::AlignVCenter);
900 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
901 			nnOperatorDetailsLayout.addWidget(label,         row,   1/*column*/);
902 			// tensor shape
903 			auto describeShape = [](const TensorShape &shape, PluginInterface::DataType dataType) {
904 				auto flatSize = Tensor::flatSize(shape);
905 				return STR(shape <<
906 				         " (" <<
907 				             Util::formatUIntHumanReadable(flatSize) << " " << dataType << " " << Q2S(tr("values")) << ", " <<
908 				             Util::formatUIntHumanReadable(flatSize*sizeof(float)) << " " << Q2S(tr("bytes")) <<
909 				          ")"
910 				);
911 			};
912 			label = makeTextSelectable(new QLabel(S2Q(describeShape(model->getTensorShape(tensorId), model->getTensorType(tensorId))), &nnOperatorDetails));
913 			label->setToolTip(tr("Tensor shape and data size"));
914 			label->setAlignment(Qt::AlignLeft|Qt::AlignVCenter);
915 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
916 			nnOperatorDetailsLayout.addWidget(label,         row,   2/*column*/);
917 			// has buffer? is variable?
918 			label = makeTextSelectable(
919 				new QLabel(QString("<%1>").arg(S2Q(ModelFunctions::tensorKind(model.get(), tensorId))),
920 				&nnOperatorDetails));
921 			label->setToolTip(tr("Tensor type"));
922 			label->setAlignment(Qt::AlignCenter|Qt::AlignVCenter);
923 			label->setStyleSheet("font: italic");
924 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
925 			nnOperatorDetailsLayout.addWidget(label,         row,   3/*column*/);
926 			// button
927 			auto hasStaticData = model->getTensorHasData(tensorId);
928 			if (hasStaticData || (tensorData && (*tensorData.get())[tensorId])) {
929 				auto button = new SvgPushButton(SvgGraphics::generateTableIcon(), &nnOperatorDetails);
930 				button->setContentsMargins(0,0,0,0);
931 				button->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::Preferred);
932 				button->setMaximumSize(QFontMetrics(button->font()).size(Qt::TextSingleLine, "XX")+QSize(4,4));
933 				button->setToolTip(tr("Show the tensor data as a table"));
934 				tempDetailWidgets.push_back(std::unique_ptr<QWidget>(button));
935 				nnOperatorDetailsLayout.addWidget(button,         row,   4/*column*/);
936 				connect(button, &QAbstractButton::pressed, [this,tensorId]() {
937 					showTensorDetails(tensorId);
938 				});
939 			}
940 		}
941 	};
942 
943 	// read operator inputs/outputs from the model
944 	std::vector<PluginInterface::TensorId> inputs, outputs;
945 	model->getOperatorIo(operatorId, inputs, outputs);
946 
947 	// add items
948 	unsigned row = 0;
949 	nnOperatorDetailsLayout.addWidget(&nnOperatorTypeLabel,          row,   0/*column*/);
950 	nnOperatorDetailsLayout.addWidget(&nnOperatorTypeValue,          row,   1/*column*/);
951 	row++;
952 	nnOperatorDetailsLayout.addWidget(&nnOperatorOptionsLabel,       row,   0/*column*/);
953 	{
954 		std::unique_ptr<PluginInterface::OperatorOptionsList> opts(model->getOperatorOptions(operatorId));
955 		for (auto &opt : *opts) {
956 			row++;
957 			// option name
958 			auto label = makeTextSelectable(new QLabel(S2Q(STR(opt.name)), &nnOperatorDetails));
959 			label->setToolTip("Option name");
960 			label->setAlignment(Qt::AlignRight|Qt::AlignVCenter);
961 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
962 			nnOperatorDetailsLayout.addWidget(label,               row,   0/*column*/);
963 			// option type
964 			label = makeTextSelectable(new QLabel(S2Q(STR("<" << opt.value.type << ">")), &nnOperatorDetails));
965 			label->setToolTip(tr("Option type"));
966 			label->setAlignment(Qt::AlignLeft|Qt::AlignVCenter);
967 			label->setStyleSheet("font: italic");
968 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
969 			nnOperatorDetailsLayout.addWidget(label,               row,   1/*column*/);
970 			// option value
971 			label = makeTextSelectable(new QLabel(S2Q(STR(opt.value)), &nnOperatorDetails));
972 			label->setToolTip(tr("Option value"));
973 			label->setAlignment(Qt::AlignLeft|Qt::AlignVCenter);
974 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
975 			nnOperatorDetailsLayout.addWidget(label,               row,   2/*column*/);
976 		}
977 		if (opts->empty()) {
978 			row++;
979 			auto label = makeTextSelectable(new QLabel("-none-", &nnOperatorDetails));
980 			label->setAlignment(Qt::AlignRight|Qt::AlignVCenter);
981 			tempDetailWidgets.push_back(std::unique_ptr<QWidget>(label));
982 			nnOperatorDetailsLayout.addWidget(label,               row,   0/*column*/);
983 		}
984 	}
985 	row++;
986 	nnOperatorDetailsLayout.addWidget(&nnOperatorInputsLabel,        row,   0/*column*/);
987 	addTensorLines(inputs, row);
988 	row++;
989 	nnOperatorDetailsLayout.addWidget(&nnOperatorOutputsLabel,       row,   0/*column*/);
990 	addTensorLines(outputs, row);
991 	row++;
992 	nnOperatorDetailsLayout.addWidget(&nnOperatorComplexityLabel,    row,   0/*column*/);
993 	nnOperatorDetailsLayout.addWidget(&nnOperatorComplexityValue,    row,   1/*column*/);
994 	row++;
995 	nnOperatorDetailsLayout.addWidget(&nnOperatorStaticDataLabel,    row,   0/*column*/);
996 	nnOperatorDetailsLayout.addWidget(&nnOperatorStaticDataValue,    row,   1/*column*/);
997 	row++;
998 	nnOperatorDetailsLayout.addWidget(&nnOperatorDataRatioLabel,     row,   0/*column*/);
999 	nnOperatorDetailsLayout.addWidget(&nnOperatorDataRatioValue,     row,   1/*column*/);
1000 	row++;
1001 	nnOperatorDetailsLayout.addWidget(&nnOperatorDetailsSpacer,      row,   0/*column*/,  1/*rowSpan*/, 4/*columnSpan*/);
1002 
1003 	// set texts
1004 	nnOperatorTypeValue.setText(S2Q(STR(model->getOperatorKind(operatorId))));
1005 	nnOperatorComplexityValue.setText(S2Q(Util::formatFlops(ModelFunctions::computeOperatorFlops(model.get(), operatorId))));
1006 	unsigned unused;
1007 	nnOperatorStaticDataValue.setText(QString("%1 bytes")
1008 		.arg(S2Q(Util::formatUIntHumanReadable(ModelFunctions::sizeOfOperatorStaticData(model.get(), operatorId, unused)))));
1009 	float dataRateIncreaseOboveInput, modelInputToOut;
1010 	nnOperatorDataRatioValue.setText(S2Q(ModelFunctions::dataRatioOfOperatorStr(model.get(), operatorId, dataRateIncreaseOboveInput, modelInputToOut)));
1011 	nnOperatorDataRatioValue.setStyleSheet(dataRateIncreaseOboveInput>1 ? "QLabel{color: red;}" : "QLabel{color: black;}");
1012 }
1013 
showTensorDetails(PluginInterface::TensorId tensorId)1014 void MainWindow::showTensorDetails(PluginInterface::TensorId tensorId) {
1015 	showTensorDetails(tensorId, ""/*no label*/);
1016 }
1017 
showTensorDetails(PluginInterface::TensorId tensorId,const char * label)1018 void MainWindow::showTensorDetails(PluginInterface::TensorId tensorId, const char *label) {
1019 	nnDetailsStack.setCurrentIndex(/*page#*/2);
1020 	nnTensorDetails.setTitle(QString("%1Tensor#%2: %3").arg(label).arg(tensorId).arg(S2Q(model->getTensorName(tensorId))));
1021 	nnTensorKindValue .setText(S2Q(ModelFunctions::tensorKind(model.get(), tensorId)));
1022 	nnTensorShapeValue.setText(S2Q(STR(model->getTensorShape(tensorId))));
1023 	nnTensorTypeValue .setText(S2Q(STR(model->getTensorType(tensorId))));
1024 	nnTensorSaveDataButton.setText(QString(tr("Save Tensor #%1 Data")).arg(tensorId));
1025 	nnTensorSaveDataButton.setProperty("tensorId", QVariant(tensorId));
1026 	// tensor data table
1027 	if (tensorId != nnCurrentTensorId) {
1028 		nnCurrentTensorId = tensorId;
1029 		if (nnTensorData2D)
1030 			clearNnTensorData2D();
1031 		if (model->getTensorHasData(nnCurrentTensorId) || (tensorData && (*tensorData)[nnCurrentTensorId]))
1032 			showNnTensorData2D();
1033 	}
1034 }
1035 
showInputDetails(PluginInterface::TensorId tensorId)1036 void MainWindow::showInputDetails(PluginInterface::TensorId tensorId) {
1037 	showTensorDetails(tensorId, "Input "/*label*/);
1038 }
1039 
showOutputDetails(PluginInterface::TensorId tensorId)1040 void MainWindow::showOutputDetails(PluginInterface::TensorId tensorId) {
1041 	showTensorDetails(tensorId, "Output "/*label*/);
1042 }
1043 
removeTableIfAny()1044 void MainWindow::removeTableIfAny() {
1045 	if (nnTensorData2D)
1046 		clearNnTensorData2D();
1047 }
1048 
openImageFile(const QString & imageFileName)1049 void MainWindow::openImageFile(const QString &imageFileName) {
1050 	// clear the previous image data if any
1051 	clearInputImageDisplay();
1052 	clearEffects();
1053 	clearComputedTensorData(Permanent); // opening image invalidates computation results
1054 	updateResultInterpretation();
1055 	// read the image as tensor
1056 	sourceTensorDataAsLoaded.reset(Image::readPngImageFile(Q2S(imageFileName), sourceTensorShape));
1057 	sourceTensorDataAsUsed = sourceTensorDataAsLoaded;
1058 	// enable widgets, show image
1059 	updateSectionWidgetsVisibility();
1060 	updateSourceImageOnScreen();
1061 	// set info on the screen
1062 	sourceImageFileNameText.setText(imageFileName);
1063 	sourceImageFileSizeText.setText(QString("%1 bytes").arg(S2Q(Util::formatUIntHumanReadable(Util::getFileSize(imageFileName)))));
1064 	sourceImageSizeText.setText(S2Q(STR(sourceTensorShape)));
1065 	updateCurrentRegionText();
1066 	// focus
1067 	computeButton.setFocus();
1068 }
1069 
openImagePixmap(const QPixmap & imagePixmap,const QString & sourceName)1070 void MainWindow::openImagePixmap(const QPixmap &imagePixmap, const QString &sourceName) {
1071 	// clear the previous image data if any
1072 	clearInputImageDisplay();
1073 	clearEffects();
1074 	clearComputedTensorData(Permanent); // opening image invalidates computation results
1075 	updateResultInterpretation();
1076 	// read the image as tensor
1077 	sourceTensorDataAsLoaded.reset(Image::readPixmap(imagePixmap, sourceTensorShape, [this,&sourceName](const std::string &msg) {
1078 		PRINT("WARNING: failed in " << Q2S(sourceName) << ": " << msg)
1079 		Util::warningOk(this, S2Q(msg));
1080 	}));
1081 	if (!sourceTensorDataAsLoaded) // message should have been called above
1082 		return;
1083 	if (0) { // TMP: scale down a huge screenshot 1/6
1084 		TensorShape sourceTensorShapeNew = {sourceTensorShape[0]/6, sourceTensorShape[1]/6, sourceTensorShape[2]};
1085 		sourceTensorDataAsLoaded.reset(Image::resizeImage(sourceTensorDataAsLoaded.get(), sourceTensorShape, sourceTensorShapeNew));
1086 		sourceTensorShape = sourceTensorShapeNew;
1087 	}
1088 	sourceTensorDataAsUsed = sourceTensorDataAsLoaded;
1089 	// enable widgets, show image
1090 	updateSectionWidgetsVisibility();
1091 	updateSourceImageOnScreen();
1092 	// set info on the screen
1093 	sourceImageFileNameText.setText(QString("{%1}").arg(sourceName));
1094 	sourceImageFileSizeText.setText(QString("{%1}").arg(sourceName));
1095 	sourceImageSizeText.setText(S2Q(STR(sourceTensorShape)));
1096 	updateCurrentRegionText();
1097 	// focus
1098 	computeButton.setFocus();
1099 }
1100 
clearInputImageDisplay()1101 void MainWindow::clearInputImageDisplay() {
1102 	updateSectionWidgetsVisibility();
1103 	sourceImage.setPixmap(QPixmap());
1104 	sourceTensorDataAsLoaded = nullptr;
1105 	sourceTensorDataAsUsed = nullptr;
1106 	sourceTensorShape = TensorShape();
1107 	tensorData.reset(nullptr);
1108 	scaleImageWidthPct = 0;
1109 	scaleImageHeightPct = 0;
1110 }
1111 
clearComputedTensorData(HowLong howLong)1112 void MainWindow::clearComputedTensorData(HowLong howLong) {
1113 	// clear table-like display of data about to be invalidated
1114 	if (howLong == Temporary) {
1115 		if (nnTensorData2D && model->isTensorComputed(nnCurrentTensorId))
1116 			nnTensorData2D->setEnabled(false);
1117 	} else
1118 		removeTableIfAny();
1119 	// clear tensor data
1120 	tensorData.reset(nullptr);
1121 }
1122 
effectsChanged()1123 void MainWindow::effectsChanged() {
1124 	inputParamsChanged(); // effects change invalidates computation results
1125 
1126 	// all available effects that can be applied
1127 	bool flipHorizontally = sourceEffectFlipHorizontallyCheckBox.isChecked();
1128 	bool flipVertically   = sourceEffectFlipVerticallyCheckBox.isChecked();
1129 	bool makeGrayscale    = sourceEffectMakeGrayscaleCheckBox.isChecked();
1130 	auto convolution      = convolutionEffects.find((ConvolutionEffect)sourceEffectConvolutionTypeComboBox.currentData().toUInt())->second;
1131 
1132 	// any effects to apply?
1133 	if (flipHorizontally || flipVertically || makeGrayscale || !std::get<1>(convolution).empty()) {
1134 		sourceTensorDataAsUsed.reset(applyEffects(sourceTensorDataAsLoaded.get(), sourceTensorShape,
1135 			flipHorizontally, flipVertically, makeGrayscale, convolution,sourceEffectConvolutionCountComboBox.currentData().toUInt()));
1136 	} else {
1137 		sourceTensorDataAsUsed = sourceTensorDataAsLoaded;
1138 	}
1139 
1140 	updateSourceImageOnScreen();
1141 }
1142 
inputNormalizationChanged()1143 void MainWindow::inputNormalizationChanged() {
1144 	inputParamsChanged(); // input normalization change invalidates computation results
1145 }
1146 
inputParamsChanged()1147 void MainWindow::inputParamsChanged() {
1148 	clearComputedTensorData(Temporary); // effects change invalidates computation results
1149 	if (nnTensorData2D && model->isTensorComputed(nnCurrentTensorId))
1150 		nnTensorData2D->setEnabled(false); // gray out the table because its tensor data is cleared
1151 	updateResultInterpretation();
1152 }
1153 
applyEffects(const float * image,const TensorShape & shape,bool flipHorizontally,bool flipVertically,bool makeGrayscale,const std::tuple<TensorShape,std::vector<float>> & convolution,unsigned convolutionCount) const1154 float* MainWindow::applyEffects(const float *image, const TensorShape &shape,
1155 	bool flipHorizontally, bool flipVertically, bool makeGrayscale,
1156 	const std::tuple<TensorShape,std::vector<float>> &convolution, unsigned convolutionCount) const
1157 {
1158 	assert(shape.size()==3);
1159 	assert(flipHorizontally || flipVertically || makeGrayscale || !std::get<1>(convolution).empty());
1160 
1161 	unsigned idx = 0; // idx=0 is "image", idx can be 0,1,2
1162 	std::unique_ptr<float> withEffects[2]; // idx=1 and idx=2 are allocatable "images"
1163 
1164 	auto idxNext = [](unsigned idx) {
1165 		return (idx+1)<3 ? idx+1 : 1;
1166 	};
1167 	auto src = [&](unsigned idx) {
1168 		if (idx==0)
1169 			return image;
1170 		else
1171 			return (const float*)withEffects[idx-1].get();
1172 	};
1173 	auto dst = [&](unsigned idx) {
1174 		auto &we = withEffects[idxNext(idx)-1];
1175 		if (!we)
1176 			we.reset(new float[Tensor::flatSize(shape)]);
1177 		return we.get();
1178 	};
1179 
1180 	if (flipHorizontally) {
1181 		Image::flipHorizontally(shape, src(idx), dst(idx));
1182 		idx = idxNext(idx);
1183 	}
1184 	if (flipVertically) {
1185 		Image::flipVertically(shape, src(idx), dst(idx));
1186 		idx = idxNext(idx);
1187 	}
1188 	if (makeGrayscale) {
1189 		Image::makeGrayscale(shape, src(idx), dst(idx));
1190 		idx = idxNext(idx);
1191 	}
1192 	if (!std::get<1>(convolution).empty()) {
1193 		TensorShape shapeWithBatch = shape;
1194 		shapeWithBatch.insert(shapeWithBatch.begin(), 1/*batch*/);
1195 		auto clip = [](float *a, size_t sz) {
1196 			for (auto ae = a+sz; a<ae; a++)
1197 				if (*a < 0.)
1198 					*a = 0.;
1199 				else if (*a > 255.)
1200 					*a = 255.;
1201 		};
1202 		const static float bias[3] = {0,0,0};
1203 		for (unsigned i = 1; i <= convolutionCount; i++) {
1204 			float *d = dst(idx);
1205 			NnOperators::Conv2D(
1206 				shapeWithBatch, src(idx),
1207 				std::get<0>(convolution), std::get<1>(convolution).data(),
1208 				{3}, bias, // no bias
1209 				shapeWithBatch, d,
1210 				std::get<0>(convolution)[2]/2, std::get<0>(convolution)[1]/2, // padding, paddings not matching kernel size work but cause image shifts
1211 				1,1, // strides
1212 				1,1  // dilation factors
1213 			);
1214 			clip(d, Tensor::flatSize(shapeWithBatch)); // we have to clip the result because otherwise some values are out of range 0..255.
1215 			idx = idxNext(idx);
1216 		}
1217 	}
1218 
1219 	return withEffects[idx-1].release();
1220 }
1221 
clearEffects()1222 void MainWindow::clearEffects() {
1223 	sourceEffectFlipHorizontallyCheckBox.setChecked(false);
1224 	sourceEffectFlipVerticallyCheckBox  .setChecked(false);
1225 	sourceEffectMakeGrayscaleCheckBox   .setChecked(false);
1226 	sourceEffectConvolutionTypeComboBox .setCurrentIndex(0);
1227 	sourceEffectConvolutionCountComboBox.setCurrentIndex(0);
1228 	sourceEffectConvolutionCountComboBox.setEnabled(false);
1229 }
1230 
updateNetworkDetailsPage()1231 void MainWindow::updateNetworkDetailsPage() {
1232 	auto numInPlural = [](unsigned num, const QString &strSingle, const QString &strPlural) {
1233 		return (num%10==1) ? strSingle : strPlural;
1234 	};
1235 
1236 	nnNetworkDescriptionText   .setText(S2Q(pluginInterface->modelDescription()));
1237 	nnNetworkComplexityText    .setText(S2Q(Util::formatFlops(ModelFunctions::computeModelFlops(model.get()))));
1238 	nnNetworkFileSizeText      .setText(QString(tr("%1 bytes")).arg(S2Q(Util::formatUIntHumanReadable(Util::getFileSize(S2Q(pluginInterface->filePath()))))));
1239 	nnNetworkNumberInsOutsText .setText(QString("%1 %2, %3 %4")
1240 		.arg(model->numInputs())
1241 		.arg(numInPlural(model->numInputs(), tr("input"), tr("inputs")))
1242 		.arg(model->numOutputs())
1243 		.arg(numInPlural(model->numOutputs(), tr("output"), tr("outputs")))
1244 	);
1245 	nnNetworkNumberOperatorsText .setText(QString("%1 %2")
1246 		.arg(model->numOperators())
1247 		.arg(numInPlural(model->numOperators(), tr("operator"), tr("operators")))
1248 	);
1249 	unsigned staticDataTensors = 0;
1250 	size_t   maxStaticDataPerOperator = 0;
1251 	auto sizeOfStaticData = ModelFunctions::sizeOfModelStaticData(model.get(), staticDataTensors, maxStaticDataPerOperator);
1252 	nnNetworkStaticDataText.setText(QString(tr("%1 bytes in %2 %3, average %4 bytes per operator, max %5 bytes per operator"))
1253 		.arg(S2Q(Util::formatUIntHumanReadable(sizeOfStaticData)))
1254 		.arg(staticDataTensors)
1255 		.arg(numInPlural(staticDataTensors, tr("tensor"), tr("tensors")))
1256 		.arg(model->numOperators()!=0 ? S2Q(Util::formatUIntHumanReadable(sizeOfStaticData/model->numOperators())) : tr("n/a"))
1257 		.arg(S2Q(Util::formatUIntHumanReadable(maxStaticDataPerOperator)))
1258 	);
1259 }
1260 
updateSourceImageOnScreen()1261 void MainWindow::updateSourceImageOnScreen() {
1262 	{ // fix image size to the size of details to its left, so that it would nicely align to them
1263 		auto height = sourceDetails.height();
1264 		sourceImageScrollArea.setMinimumSize(QSize(height,height));
1265 		sourceImageScrollArea.setMaximumSize(QSize(height,height));
1266 	}
1267 
1268 	// decide on the scaling percentage
1269 	if (scaleImageWidthPct == 0)  {
1270 		// compute the percentage
1271 		assert(sourceTensorShape.size() == 3);
1272 		auto screenSize = sourceImageScrollArea.minimumSize();
1273 		float scaleFactorWidth  = (float)screenSize.width()/(float)sourceTensorShape[1];
1274 		float scaleFactorHeight = (float)screenSize.height()/(float)sourceTensorShape[0];
1275 		float scaleFactor = std::min(scaleFactorWidth, scaleFactorHeight);
1276 		if (scaleFactor < 0.01) // we only scale down to 1%
1277 			scaleFactor = 0.01;
1278 		if (scaleFactor > ScaleImageWidget::maxValue) // do not exceed a max value set by ScaleImageWidget
1279 			scaleFactor = ScaleImageWidget::maxValue;
1280 		scaleImageWidthPct  = scaleFactor*100;
1281 		scaleImageHeightPct = scaleFactor*100;
1282 
1283 		// set the same percentage in the scaling widget
1284 		scaleImageSpinBoxes.setFactor(scaleImageWidthPct);
1285 	}
1286 
1287 	// generate and set the pixmap
1288 	QPixmap pixmap;
1289 	if (scaleImageWidthPct != 100 || scaleImageHeightPct != 100) {
1290 		assert(sourceTensorShape.size() == 3);
1291 		TensorShape resizedShape = {
1292 			sourceTensorShape[0]*scaleImageHeightPct/100,
1293 			sourceTensorShape[1]*scaleImageWidthPct/100,
1294 			sourceTensorShape[2]
1295 		};
1296 		std::unique_ptr<float> resizedImage(Image::resizeImage(sourceTensorDataAsUsed.get(), sourceTensorShape, resizedShape));
1297 		pixmap = Image::toQPixmap(resizedImage.get(), resizedShape);
1298 	} else
1299 		pixmap = Image::toQPixmap(sourceTensorDataAsUsed.get(), sourceTensorShape);
1300 
1301 	// memorize the center of sourceImage
1302 	bool hadPixmap = sourceImage.pixmap()!=nullptr && sourceImage.pixmap()->width()>0;
1303 	QPoint ptCenterPrev = hadPixmap ? sourceImage.mapToGlobal(QPoint(sourceImage.width()/2,sourceImage.height()/2)) : QPoint(0,0);
1304 	// set new pixmap
1305 	sourceImage.setPixmap(pixmap);
1306 	sourceImage.resize(pixmap.width(), pixmap.height());
1307 	// resize the stack according to the source image
1308 	//sourceImageStack.resize(sourceImage.size());
1309 	// if it had a pixmap previously, keep the center still
1310 	if (hadPixmap) {
1311 		QPoint ptCenterCurr = hadPixmap ? sourceImage.mapToGlobal(QPoint(sourceImage.width()/2,sourceImage.height()/2)) : QPoint(0,0);
1312 		sourceImageScrollArea.horizontalScrollBar()->setValue(sourceImageScrollArea.horizontalScrollBar()->value() + (ptCenterCurr.x()-ptCenterPrev.x()));
1313 		sourceImageScrollArea.verticalScrollBar()  ->setValue(sourceImageScrollArea.verticalScrollBar()->value() + (ptCenterCurr.y()-ptCenterPrev.y()));
1314 	}
1315 }
1316 
updateCurrentRegionText()1317 void MainWindow::updateCurrentRegionText() {
1318 	auto region = getVisibleImageRegion();
1319 	if (region[0]!=0 || region[1]!=0 || region[2]+1!=sourceTensorShape[1] || region[3]+1!=sourceTensorShape[0])
1320 		sourceImageCurrentRegionText.setText(QString("[%1x%2,%3x%4]")
1321 			.arg(region[0])
1322 			.arg(region[1])
1323 			.arg(region[2]-region[0]+1)
1324 			.arg(region[3]-region[1]+1)
1325 		);
1326 	else
1327 		sourceImageCurrentRegionText.setText("<whole image>");
1328 }
1329 
updateResultInterpretation()1330 void MainWindow::updateResultInterpretation() {
1331 	bool computedResultExists = model && (bool)tensorData && (*tensorData)[model->getOutputs()[0]].get();
1332 
1333 	// helpers
1334 	auto makeRed = [&]() {
1335 		Util::setWidgetColor(&outputInterpretationKindComboBox, "red");
1336 	};
1337 	auto makeBlack = [&]() {
1338 		Util::setWidgetColor(&outputInterpretationKindComboBox, "black");
1339 	};
1340 	auto interpretBasedOnLabelsList = [&](const char *listFile, unsigned idx0, unsigned idx1) {
1341 		// interpret results based on the labels list
1342 		auto outputTensorId = model->getOutputs()[0];
1343 		auto result = (*tensorData)[outputTensorId].get();
1344 		auto resultShape = model->getTensorShape(outputTensorId);
1345 		assert(resultShape.size()==2 && resultShape[0]==1); // [B,C] with B=1
1346 		if (resultShape[1] != idx1-idx0)
1347 			return false; // failed to interpret it
1348 
1349 		// compute the likelihood array
1350 		typedef std::tuple<unsigned/*order num*/,float/*likelihood*/> Likelihood;
1351 		std::vector<Likelihood> likelihoods;
1352 		for (unsigned i = 0, ie = resultShape[1]; i<ie; i++)
1353 			likelihoods.push_back({i,result[i]});
1354 		std::sort(likelihoods.begin(), likelihoods.end(), [](const Likelihood &a, const Likelihood &b) {return std::get<1>(a) > std::get<1>(b);});
1355 
1356 		// load labels
1357 		auto labels = Util::readListFromFile(listFile);
1358 		assert(labels.size() >= idx1-idx0);
1359 
1360 		// report top few labels to the user
1361 		std::ostringstream ss;
1362 		for (unsigned i = 0, ie = std::min(unsigned(10), idx1-idx0); i<ie; i++)
1363 			ss << (i>0 ? "\n" : "") << "• " << Q2S(labels[idx0+std::get<0>(likelihoods[i])]) << " = " << std::get<1>(likelihoods[i]);
1364 		updateResultInterpretationSummaryText(
1365 			true/*enable*/,
1366 			QString("%1 (%2)").arg(labels[idx0+std::get<0>(likelihoods[0])]).arg(std::get<1>(likelihoods[0])),
1367 			S2Q(ss.str())
1368 		);
1369 
1370 		return true; // success
1371 	};
1372 	auto interpretAsImageNet = [&](unsigned count) {
1373 		return interpretBasedOnLabelsList(":/nn-labels/imagenet-labels.txt", count==1000 ? 1:0, 1001); // skip the first label of 1001 labels when count=1000
1374 	};
1375 	auto interpretAsNoYes = [&](bool reversed) {
1376 		return interpretBasedOnLabelsList(!reversed ? ":/nn-labels/no-yes.txt" : ":/nn-labels/yes-no.txt", 0, 2);
1377 	};
1378 	auto interpretAsPixelClassification = [&]() {
1379 		// TODO
1380 		return false;
1381 	};
1382 	auto interpretAsImageConversion = [&]() {
1383 		// get tensors and shapes
1384 		auto inputTensorId = model->getInputs()[0]; // look at the first model input
1385 		auto outputTensorId = model->getOutputs()[0]; // look at the first model output
1386 		auto output = (*tensorData)[outputTensorId].get();
1387 		auto inputShape = model->getTensorShape(inputTensorId);
1388 		auto outputShape = model->getTensorShape(outputTensorId);
1389 
1390 		// can this be an image?
1391 		if (!Tensor::canBeAnImage(outputShape))
1392 			return false;
1393 		// does the image size match the input?
1394 		if (outputShape[0]!=inputShape[0] || outputShape[1]!=inputShape[1])
1395 			return false; // we don't support mismatching image sizes for now
1396 
1397 		// create the widget
1398 		interpretationImage.reset(new QLabel(&sourceImageStack));
1399 		interpretationImage->setAlignment(Qt::AlignHCenter|Qt::AlignVCenter);
1400 		interpretationImage->setToolTip(tr("Image resulting from the conversion of the NN input image"));
1401 		interpretationImage->resize(sourceImageStack.size());
1402 		sourceImageStack.addWidget(interpretationImage.get());
1403 
1404 		// resize the image
1405 		TensorShape resizedShape({{(unsigned)interpretationImage->height(), (unsigned)interpretationImage->width(), outputShape[2]}});
1406 		std::unique_ptr<float> outputResized(Image::resizeImage(output, outputShape, resizedShape));
1407 
1408 		// convert the output to a pixmap and set it in the widget
1409 		QPixmap pixmap = Image::toQPixmap(outputResized.get(), resizedShape);
1410 		interpretationImage->setPixmap(pixmap);
1411 
1412 		// switch to interpretationImage
1413 		sourceImageStack.setCurrentIndex(1);
1414 
1415 		return true;
1416 	};
1417 	auto interpretAs = [&](OutputInterpretationKind kind) {
1418 		// clear the previous one
1419 		interpretationImage.reset(nullptr);
1420 
1421 		// try to set
1422 		switch (kind) {
1423 		case OutputInterpretationKind_Undefined:
1424 			// no nothing
1425 			return true;
1426 		case OutputInterpretationKind_ImageNet1001:
1427 			return interpretAsImageNet(1001);
1428 		case OutputInterpretationKind_ImageNet1000:
1429 			return interpretAsImageNet(1000);
1430 		case OutputInterpretationKind_NoYes:
1431 			return interpretAsNoYes(false);
1432 		case OutputInterpretationKind_YesNo:
1433 			return interpretAsNoYes(true);
1434 		case OutputInterpretationKind_PixelClassification:
1435 			return interpretAsPixelClassification();
1436 		case OutputInterpretationKind_ImageConversion:
1437 			return interpretAsImageConversion();
1438 		}
1439 	};
1440 
1441 	updateResultInterpretationSummaryText(false/*enable*/, tr("n/a"), tr("n/a"));
1442 
1443 	if (!computedResultExists) {
1444 		makeBlack();
1445 		interpretationImage.reset(nullptr);
1446 	} else if (interpretAs((OutputInterpretationKind)outputInterpretationKindComboBox.itemData(outputInterpretationKindComboBox.currentIndex()).toInt()))
1447 		makeBlack();
1448 	else
1449 		makeRed();
1450 }
1451 
updateResultInterpretationSummaryText(bool enable,const QString & oneLine,const QString & details)1452 void MainWindow::updateResultInterpretationSummaryText(bool enable, const QString &oneLine, const QString &details) {
1453 	outputInterpretationSummaryLineEdit.setVisible(enable);
1454 	outputInterpretationSummaryLineEdit.setText(oneLine);
1455 	outputInterpretationSummaryLineEdit.setToolTip(QString(tr("Result interpretation:\n%1")).arg(details));
1456 }
1457 
getVisibleImageRegion() const1458 std::array<unsigned,4> MainWindow::getVisibleImageRegion() const {
1459 	assert(haveImageOpen());
1460 	auto visibleRegion = sourceImage.visibleRegion().boundingRect();
1461 	auto size = sourceImage.size();
1462 	assert(visibleRegion.left() >= 0);
1463 	assert(visibleRegion.right() <= size.width());
1464 	assert(visibleRegion.top() >= 0);
1465 	assert(visibleRegion.bottom() <= size.height());
1466 
1467 	// region relative to the resized image normalized to 0..1
1468 	float region[4] = {
1469 		(float)visibleRegion.left()/(float)size.width(),
1470 		(float)visibleRegion.top()/(float)size.height(),
1471 		(float)(visibleRegion.right()+1)/(float)size.width(),
1472 		(float)(visibleRegion.bottom()+1)/(float)size.height()
1473 	};
1474 
1475 	// scale it to the original image
1476 	auto sourceWidth = sourceTensorShape[1];
1477 	auto sourceHeight = sourceTensorShape[0];
1478 	return std::array<unsigned,4>{
1479 		(unsigned)(region[0]*sourceWidth),
1480 		(unsigned)(region[1]*sourceHeight),
1481 		(unsigned)(region[2]*sourceWidth-1),
1482 		(unsigned)(region[3]*sourceHeight-1)
1483 	};
1484 }
1485 
updateSectionWidgetsVisibility()1486 void MainWindow::updateSectionWidgetsVisibility() {
1487 	bool haveImage = (bool)sourceTensorDataAsLoaded;
1488 	bool haveNn    = (model.get() != nullptr);
1489 
1490 	sourceWidget      .setVisible(haveImage);
1491 	nnDetailsStack    .setVisible(haveNn);
1492 	noNnIsOpenGroupBox.setVisible(!haveNn);
1493 
1494 	//nnDetailsStack.setSizePolicy(QSizePolicy::Preferred, haveImage ? QSizePolicy::Fixed : QSizePolicy::Minimum);
1495 }
1496 
onOpenNeuralNetworkFileUserIntent()1497 void MainWindow::onOpenNeuralNetworkFileUserIntent() {
1498 	QString fileName = QFileDialog::getOpenFileName(this,
1499 		tr("Open neural network file"), "",
1500 		tr("Neural Network (*.tflite);;All Files (*)")
1501 	);
1502 	if (!fileName.isEmpty()) {
1503 		if (model)
1504 			closeNeuralNetwork();
1505 		loadModelFile(fileName);
1506 		updateSectionWidgetsVisibility();
1507 	}
1508 }
1509 
closeNeuralNetwork()1510 void MainWindow::closeNeuralNetwork() {
1511 	clearComputedTensorData(Permanent);
1512 	updateResultInterpretation();
1513 	nnWidget.close();
1514 	nnNetworkOperatorsListWidget.clearNnModel();
1515 	pluginInterface.reset(nullptr);
1516 	PluginManager::unloadPlugin(plugin);
1517 	model = nullptr;
1518 	plugin = nullptr;
1519 	// update screen
1520 	updateSectionWidgetsVisibility();
1521 }
1522 
makeTextSelectable(QLabel * label)1523 QLabel* MainWindow::makeTextSelectable(QLabel *label) {
1524 	label->setTextInteractionFlags(Qt::TextSelectableByMouse);
1525 	return label;
1526 }
1527 
showNnTensorData2D()1528 void MainWindow::showNnTensorData2D() {
1529 	assert(nnCurrentTensorId >= 0);
1530 	if (Tensor::numMultiDims(model->getTensorShape(nnCurrentTensorId)) >= 2) {
1531 		switch (model->getTensorType(nnCurrentTensorId)) {
1532 		case PluginInterface::DataType_Float16:
1533 			assert(!model->isTensorComputed(nnCurrentTensorId)); // we don't yet support computed tensors of type float16 because tensorData always has float32
1534 			nnTensorData2D.reset(new DataTable2D<half_float::half>(
1535 				model->getTensorShape(nnCurrentTensorId),
1536 				(const half_float::half*)model->getTensorData(nnCurrentTensorId),
1537 				&nnTensorDetails
1538 			));
1539 			break;
1540 		case PluginInterface::DataType_Float32:
1541 			nnTensorData2D.reset(new DataTable2D<float>(
1542 				model->getTensorShape(nnCurrentTensorId),
1543 				model->isTensorComputed(nnCurrentTensorId) ? (*tensorData.get())[nnCurrentTensorId].get() : model->getTensorDataF32(nnCurrentTensorId),
1544 				&nnTensorDetails
1545 			));
1546 			break;
1547 		case PluginInterface::DataType_Float64:
1548 			assert(!model->isTensorComputed(nnCurrentTensorId)); // we don't yet support computed tensors of the type float64 because tensorData always has float32
1549 			nnTensorData2D.reset(new DataTable2D<double>(
1550 				model->getTensorShape(nnCurrentTensorId),
1551 				static_cast<const double*>(model->getTensorData(nnCurrentTensorId)),
1552 				&nnTensorDetails
1553 			));
1554 			break;
1555 		case PluginInterface::DataType_Int8:
1556 			assert(!model->isTensorComputed(nnCurrentTensorId)); // we don't yet support computed tensors of the type int8 because tensorData always has float32
1557 			nnTensorData2D.reset(new DataTable2D<int8_t>(
1558 				model->getTensorShape(nnCurrentTensorId),
1559 				static_cast<const int8_t*>(model->getTensorData(nnCurrentTensorId)),
1560 				&nnTensorDetails
1561 			));
1562 			break;
1563 		case PluginInterface::DataType_UInt8:
1564 			assert(!model->isTensorComputed(nnCurrentTensorId)); // we don't yet support computed tensors of the type uint8 because tensorData always has float32
1565 			nnTensorData2D.reset(new DataTable2D<uint8_t>(
1566 				model->getTensorShape(nnCurrentTensorId),
1567 				static_cast<const uint8_t*>(model->getTensorData(nnCurrentTensorId)),
1568 				&nnTensorDetails
1569 			));
1570 			break;
1571 		case PluginInterface::DataType_Int16:
1572 			assert(!model->isTensorComputed(nnCurrentTensorId)); // we don't yet support computed tensors of the type int16 because tensorData always has float32
1573 			nnTensorData2D.reset(new DataTable2D<int16_t>(
1574 				model->getTensorShape(nnCurrentTensorId),
1575 				static_cast<const int16_t*>(model->getTensorData(nnCurrentTensorId)),
1576 				&nnTensorDetails
1577 			));
1578 			break;
1579 		case PluginInterface::DataType_Int32:
1580 			assert(!model->isTensorComputed(nnCurrentTensorId)); // we don't yet support computed tensors of the type int32 because tensorData always has float32
1581 			nnTensorData2D.reset(new DataTable2D<int32_t>(
1582 				model->getTensorShape(nnCurrentTensorId),
1583 				static_cast<const int32_t*>(model->getTensorData(nnCurrentTensorId)),
1584 				&nnTensorDetails
1585 			));
1586 			break;
1587 		case PluginInterface::DataType_Int64:
1588 			assert(!model->isTensorComputed(nnCurrentTensorId)); // we don't yet support computed tensors of the type int64 because tensorData always has float32
1589 			nnTensorData2D.reset(new DataTable2D<int64_t>(
1590 				model->getTensorShape(nnCurrentTensorId),
1591 				static_cast<const int64_t*>(model->getTensorData(nnCurrentTensorId)),
1592 				&nnTensorDetails
1593 			));
1594 			break;
1595 		default:
1596 			FAIL("unsupported tensor data type " << model->getTensorType(nnCurrentTensorId))
1597 		}
1598 		nnTensorDetailsLayout.addWidget(nnTensorData2D.get(), 3/*row*/, 0/*col*/,  1/*rowSpan*/, 2/*columnSpan*/);
1599 		nnTensorData2D.get()->setSizePolicy(QSizePolicy::Minimum,   QSizePolicy::Minimum);
1600 	} else {
1601 		nnTensorDataPlaceholder1DnotImplemented.show();
1602 	}
1603 	nnTensorDataPlaceholder.hide();
1604 }
1605 
clearNnTensorData2D()1606 void MainWindow::clearNnTensorData2D() {
1607 	nnTensorData2D.reset(nullptr);
1608 	nnTensorDataPlaceholder.show();
1609 	nnTensorDataPlaceholder1DnotImplemented.hide();
1610 }
1611