1 /*
2 	Actiona
3     Copyright (C) 2005 Jonathan Mercier-Ganady
4 
5 	Actiona is free software: you can redistribute it and/or modify
6 	it under the terms of the GNU General Public License as published by
7 	the Free Software Foundation, either version 3 of the License, or
8 	(at your option) any later version.
9 
10 	Actiona is distributed in the hope that it will be useful,
11 	but WITHOUT ANY WARRANTY; without even the implied warranty of
12 	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 	GNU General Public License for more details.
14 
15 	You should have received a copy of the GNU General Public License
16 	along with this program. If not, see <http://www.gnu.org/licenses/>.
17 
18 	Contact : jmgr@jmgr.info
19 */
20 
21 #include "findimageinstance.h"
22 #include "opencvalgorithms.h"
23 #include "code/point.h"
24 #include "screenshooter.h"
25 
26 #include <QPixmap>
27 #include <QApplication>
28 #include <QDesktopWidget>
29 
30 namespace Actions
31 {
32     Tools::StringListPair FindImageInstance::sources =
33     {
34         {
35             QStringLiteral("screenshot"),
36             QStringLiteral("window"),
37             QStringLiteral("image")
38         },
39         {
40             QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Screenshot")),
41             QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Window")),
42             QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Image"))
43         }
44     };
45     Tools::StringListPair FindImageInstance::methods =
46     {
47         {
48             QStringLiteral("correlationcoefficient"),
49             QStringLiteral("crosscorrelation"),
50             QStringLiteral("squareddifference")
51         },
52         {
53             QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Correlation Coefficient")),
54             QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Cross Correlation")),
55             QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Squared Difference"))
56         }
57     };
58 
FindImageInstance(const ActionTools::ActionDefinition * definition,QObject * parent)59 	FindImageInstance::FindImageInstance(const ActionTools::ActionDefinition *definition, QObject *parent)
60 		: ActionTools::ActionInstance(definition, parent),
61 		  mOpenCVAlgorithms(new ActionTools::OpenCVAlgorithms(this)),
62           mMethod(CorrelationCoefficientMethod),
63 		  mWindowRelativePosition(false),
64           mConfidenceMinimum(0),
65           mSource(ScreenshotSource),
66           mMaximumMatches(1),
67           mDownPyramidCount(0),
68           mSearchExpansion(0)
69 	{
70         connect(mOpenCVAlgorithms, static_cast<void (ActionTools::OpenCVAlgorithms::*)(const ActionTools::MatchingPointList &)>(&ActionTools::OpenCVAlgorithms::finished),
71                 this, &FindImageInstance::searchFinished);
72         connect(&mWaitTimer, &QTimer::timeout, this, &FindImageInstance::startSearching);
73 
74         mWaitTimer.setSingleShot(true);
75 	}
76 
77     FindImageInstance::~FindImageInstance() = default;
78 
startExecution()79 	void FindImageInstance::startExecution()
80 	{
81 		bool ok = true;
82 
83 		mSource = evaluateListElement<Source>(ok, sources, QStringLiteral("source"));
84 		mImageToFind = evaluateImage(ok, QStringLiteral("imageToFind"));
85 		mIfFound = evaluateIfAction(ok, QStringLiteral("ifFound"));
86 		mIfNotFound = evaluateIfAction(ok, QStringLiteral("ifNotFound"));
87 		mPositionVariableName = evaluateVariable(ok, QStringLiteral("position"));
88 		mMethod = evaluateListElement<Method>(ok, methods, QStringLiteral("method"));
89 		mWindowRelativePosition = evaluateBoolean(ok, QStringLiteral("windowRelativePosition"));
90 		mConfidenceMinimum = evaluateInteger(ok, QStringLiteral("confidenceMinimum"));
91 		mMaximumMatches = evaluateInteger(ok, QStringLiteral("maximumMatches"));
92 		mDownPyramidCount = evaluateInteger(ok, QStringLiteral("downPyramidCount"));
93 		mSearchExpansion = evaluateInteger(ok, QStringLiteral("searchExpansion"));
94 		mConfidenceVariableName = evaluateVariable(ok, QStringLiteral("confidence"));
95 		mSearchDelay = evaluateInteger(ok, QStringLiteral("searchDelay"));
96 
97 		if(!ok)
98 			return;
99 
100 		validateParameterRange(ok, mConfidenceMinimum, QStringLiteral("confidenceMinimum"), tr("minimum confidence"), 0, 100);
101 		validateParameterRange(ok, mMaximumMatches, QStringLiteral("maximumMatches"), tr("maximum matches"), 1);
102 		validateParameterRange(ok, mDownPyramidCount, QStringLiteral("downPyramidCount"), tr("downsampling"), 1);
103 		validateParameterRange(ok, mSearchExpansion, QStringLiteral("searchExpansion"), tr("search expansion"), 1);
104 
105 		if(!ok)
106 			return;
107 
108         if(mImageToFind.isNull())
109 		{
110             emit executionException(ActionTools::ActionException::InvalidParameterException, tr("Invalid image to find"));
111 
112 			return;
113 		}
114 
115         startSearching();
116     }
117 
stopExecution()118     void FindImageInstance::stopExecution()
119     {
120         mOpenCVAlgorithms->cancelSearch();
121 
122         mWaitTimer.stop();
123     }
124 
startSearching()125     void FindImageInstance::startSearching()
126     {
127         mOpenCVAlgorithms->cancelSearch();
128 
129         mImagesToSearchIn.clear();
130 
131         switch(mSource)
132         {
133         case ScreenshotSource:
134             mImagesToSearchIn = ActionTools::ScreenShooter::captureScreens();
135             break;
136         case WindowSource:
137             {
138                 bool ok = true;
139 
140 				QString windowName = evaluateString(ok, QStringLiteral("windowName"));
141 
142                 if(!ok)
143                     return;
144 
145                 mWindows = ActionTools::WindowHandle::findWindows(QRegExp(windowName, Qt::CaseSensitive, QRegExp::WildcardUnix));
146 
147                 if(mWindows.isEmpty())
148                 {
149                     emit executionException(ActionTools::ActionException::InvalidParameterException, tr("Unable to find any window named %1").arg(windowName));
150 
151                     return;
152                 }
153 
154                 mImagesToSearchIn = ActionTools::ScreenShooter::captureWindows(mWindows);
155             }
156             break;
157         case ImageSource:
158             {
159                 bool ok = true;
160 
161 				QImage imageToSearchIn = evaluateImage(ok, QStringLiteral("imageToSearchIn"));
162 
163                 if(!ok)
164                     return;
165 
166                 if(imageToSearchIn.isNull())
167                 {
168                     emit executionException(ActionTools::ActionException::InvalidParameterException, tr("Invalid image to search in"));
169 
170                     return;
171                 }
172 
173                 mImagesToSearchIn.append(std::make_pair(QPixmap::fromImage(imageToSearchIn), imageToSearchIn.rect()));
174             }
175             break;
176         }
177 
178         QList<QImage> sourceImages;
179         sourceImages.reserve(mImagesToSearchIn.size());
180 
181         for(const auto &imageToSearchIn: mImagesToSearchIn)
182             sourceImages.append(imageToSearchIn.first.toImage());
183 
184         if(!mOpenCVAlgorithms->findSubImageAsync(sourceImages,
185                                                  mImageToFind,
186                                                  mConfidenceMinimum,
187                                                  mMaximumMatches,
188                                                  mDownPyramidCount,
189                                                  mSearchExpansion,
190                                                  static_cast<ActionTools::OpenCVAlgorithms::AlgorithmMethod>(mMethod)))
191         {
192             emit executionException(ErrorWhileSearchingException, tr("Error while searching: %1").arg(mOpenCVAlgorithms->errorString()));
193 
194             return;
195         }
196     }
197 
searchFinished(const ActionTools::MatchingPointList & matchingPointList)198 	void FindImageInstance::searchFinished(const ActionTools::MatchingPointList &matchingPointList)
199 	{
200         bool ok = true;
201 
202         if(matchingPointList.empty())
203         {
204 			setCurrentParameter(QStringLiteral("ifNotFound"), QStringLiteral("line"));
205 
206             QString line = evaluateSubParameter(ok, mIfNotFound.actionParameter());
207             if(!ok)
208                 return;
209 
210             if(mIfNotFound.action() == ActionTools::IfActionValue::GOTO)
211             {
212                 setNextLine(line);
213 
214                 executionEnded();
215             }
216             else if(mIfNotFound.action() == ActionTools::IfActionValue::CALLPROCEDURE)
217             {
218                 if(!callProcedure(line))
219                     return;
220 
221                 executionEnded();
222             }
223             else if(mIfNotFound.action() == ActionTools::IfActionValue::WAIT)
224             {
225                 mWaitTimer.start(mSearchDelay);
226             }
227             else
228                 executionEnded();
229 
230             return;
231         }
232 
233 		if(mMaximumMatches == 1)
234 		{
235             const ActionTools::MatchingPoint &bestMatchingPoint = matchingPointList.first();
236             QPoint position = bestMatchingPoint.position;
237 
238             if(mSource != WindowSource || !mWindowRelativePosition)
239                 position += mImagesToSearchIn.at(bestMatchingPoint.imageIndex).second.topLeft();
240 
241 			setVariable(mPositionVariableName, Code::Point::constructor(position, scriptEngine()));
242             setVariable(mConfidenceVariableName, bestMatchingPoint.confidence);
243 		}
244 		else
245 		{
246 			QScriptValue arrayResult = scriptEngine()->newArray(matchingPointList.size());
247             QScriptValue arrayConfidenceResult = scriptEngine()->newArray(matchingPointList.size());
248 
249 			for(int i = 0; i < matchingPointList.size(); ++i)
250             {
251                 const ActionTools::MatchingPoint &matchingPoint = matchingPointList.at(i);
252                 QPoint position = matchingPoint.position;
253 
254                 if(mSource != WindowSource || !mWindowRelativePosition)
255                     position += mImagesToSearchIn.at(matchingPoint.imageIndex).second.topLeft();
256 
257                 arrayResult.setProperty(i, Code::Point::constructor(position, scriptEngine()));
258                 arrayConfidenceResult.setProperty(i, matchingPoint.confidence);
259             }
260 
261 			setVariable(mPositionVariableName, arrayResult);
262             setVariable(mConfidenceVariableName, arrayConfidenceResult);
263 		}
264 
265 		setCurrentParameter(QStringLiteral("ifFound"), QStringLiteral("line"));
266 
267         QString line = evaluateSubParameter(ok, mIfFound.actionParameter());
268         if(!ok)
269             return;
270 
271         if(mIfFound.action() == ActionTools::IfActionValue::GOTO)
272         {
273             setNextLine(line);
274 
275             executionEnded();
276         }
277         else if(mIfFound.action() == ActionTools::IfActionValue::CALLPROCEDURE)
278         {
279             if(!callProcedure(line))
280                 return;
281 
282             executionEnded();
283         }
284         else if(mIfFound.action() == ActionTools::IfActionValue::WAIT)
285         {
286             mWaitTimer.start(mSearchDelay);
287         }
288         else
289             executionEnded();
290     }
291 }
292 
293