1 /* 2 Actiona 3 Copyright (C) 2005 Jonathan Mercier-Ganady 4 5 Actiona is free software: you can redistribute it and/or modify 6 it under the terms of the GNU General Public License as published by 7 the Free Software Foundation, either version 3 of the License, or 8 (at your option) any later version. 9 10 Actiona is distributed in the hope that it will be useful, 11 but WITHOUT ANY WARRANTY; without even the implied warranty of 12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 GNU General Public License for more details. 14 15 You should have received a copy of the GNU General Public License 16 along with this program. If not, see <http://www.gnu.org/licenses/>. 17 18 Contact : jmgr@jmgr.info 19 */ 20 21 #include "findimageinstance.h" 22 #include "opencvalgorithms.h" 23 #include "code/point.h" 24 #include "screenshooter.h" 25 26 #include <QPixmap> 27 #include <QApplication> 28 #include <QDesktopWidget> 29 30 namespace Actions 31 { 32 Tools::StringListPair FindImageInstance::sources = 33 { 34 { 35 QStringLiteral("screenshot"), 36 QStringLiteral("window"), 37 QStringLiteral("image") 38 }, 39 { 40 QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Screenshot")), 41 QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Window")), 42 QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Image")) 43 } 44 }; 45 Tools::StringListPair FindImageInstance::methods = 46 { 47 { 48 QStringLiteral("correlationcoefficient"), 49 QStringLiteral("crosscorrelation"), 50 QStringLiteral("squareddifference") 51 }, 52 { 53 QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Correlation Coefficient")), 54 QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Cross Correlation")), 55 QStringLiteral(QT_TRANSLATE_NOOP("FindImageInstance::sources", "Squared Difference")) 56 } 57 }; 58 FindImageInstance(const ActionTools::ActionDefinition * definition,QObject * parent)59 FindImageInstance::FindImageInstance(const ActionTools::ActionDefinition *definition, QObject *parent) 60 : ActionTools::ActionInstance(definition, parent), 61 mOpenCVAlgorithms(new ActionTools::OpenCVAlgorithms(this)), 62 mMethod(CorrelationCoefficientMethod), 63 mWindowRelativePosition(false), 64 mConfidenceMinimum(0), 65 mSource(ScreenshotSource), 66 mMaximumMatches(1), 67 mDownPyramidCount(0), 68 mSearchExpansion(0) 69 { 70 connect(mOpenCVAlgorithms, static_cast<void (ActionTools::OpenCVAlgorithms::*)(const ActionTools::MatchingPointList &)>(&ActionTools::OpenCVAlgorithms::finished), 71 this, &FindImageInstance::searchFinished); 72 connect(&mWaitTimer, &QTimer::timeout, this, &FindImageInstance::startSearching); 73 74 mWaitTimer.setSingleShot(true); 75 } 76 77 FindImageInstance::~FindImageInstance() = default; 78 startExecution()79 void FindImageInstance::startExecution() 80 { 81 bool ok = true; 82 83 mSource = evaluateListElement<Source>(ok, sources, QStringLiteral("source")); 84 mImageToFind = evaluateImage(ok, QStringLiteral("imageToFind")); 85 mIfFound = evaluateIfAction(ok, QStringLiteral("ifFound")); 86 mIfNotFound = evaluateIfAction(ok, QStringLiteral("ifNotFound")); 87 mPositionVariableName = evaluateVariable(ok, QStringLiteral("position")); 88 mMethod = evaluateListElement<Method>(ok, methods, QStringLiteral("method")); 89 mWindowRelativePosition = evaluateBoolean(ok, QStringLiteral("windowRelativePosition")); 90 mConfidenceMinimum = evaluateInteger(ok, QStringLiteral("confidenceMinimum")); 91 mMaximumMatches = evaluateInteger(ok, QStringLiteral("maximumMatches")); 92 mDownPyramidCount = evaluateInteger(ok, QStringLiteral("downPyramidCount")); 93 mSearchExpansion = evaluateInteger(ok, QStringLiteral("searchExpansion")); 94 mConfidenceVariableName = evaluateVariable(ok, QStringLiteral("confidence")); 95 mSearchDelay = evaluateInteger(ok, QStringLiteral("searchDelay")); 96 97 if(!ok) 98 return; 99 100 validateParameterRange(ok, mConfidenceMinimum, QStringLiteral("confidenceMinimum"), tr("minimum confidence"), 0, 100); 101 validateParameterRange(ok, mMaximumMatches, QStringLiteral("maximumMatches"), tr("maximum matches"), 1); 102 validateParameterRange(ok, mDownPyramidCount, QStringLiteral("downPyramidCount"), tr("downsampling"), 1); 103 validateParameterRange(ok, mSearchExpansion, QStringLiteral("searchExpansion"), tr("search expansion"), 1); 104 105 if(!ok) 106 return; 107 108 if(mImageToFind.isNull()) 109 { 110 emit executionException(ActionTools::ActionException::InvalidParameterException, tr("Invalid image to find")); 111 112 return; 113 } 114 115 startSearching(); 116 } 117 stopExecution()118 void FindImageInstance::stopExecution() 119 { 120 mOpenCVAlgorithms->cancelSearch(); 121 122 mWaitTimer.stop(); 123 } 124 startSearching()125 void FindImageInstance::startSearching() 126 { 127 mOpenCVAlgorithms->cancelSearch(); 128 129 mImagesToSearchIn.clear(); 130 131 switch(mSource) 132 { 133 case ScreenshotSource: 134 mImagesToSearchIn = ActionTools::ScreenShooter::captureScreens(); 135 break; 136 case WindowSource: 137 { 138 bool ok = true; 139 140 QString windowName = evaluateString(ok, QStringLiteral("windowName")); 141 142 if(!ok) 143 return; 144 145 mWindows = ActionTools::WindowHandle::findWindows(QRegExp(windowName, Qt::CaseSensitive, QRegExp::WildcardUnix)); 146 147 if(mWindows.isEmpty()) 148 { 149 emit executionException(ActionTools::ActionException::InvalidParameterException, tr("Unable to find any window named %1").arg(windowName)); 150 151 return; 152 } 153 154 mImagesToSearchIn = ActionTools::ScreenShooter::captureWindows(mWindows); 155 } 156 break; 157 case ImageSource: 158 { 159 bool ok = true; 160 161 QImage imageToSearchIn = evaluateImage(ok, QStringLiteral("imageToSearchIn")); 162 163 if(!ok) 164 return; 165 166 if(imageToSearchIn.isNull()) 167 { 168 emit executionException(ActionTools::ActionException::InvalidParameterException, tr("Invalid image to search in")); 169 170 return; 171 } 172 173 mImagesToSearchIn.append(std::make_pair(QPixmap::fromImage(imageToSearchIn), imageToSearchIn.rect())); 174 } 175 break; 176 } 177 178 QList<QImage> sourceImages; 179 sourceImages.reserve(mImagesToSearchIn.size()); 180 181 for(const auto &imageToSearchIn: mImagesToSearchIn) 182 sourceImages.append(imageToSearchIn.first.toImage()); 183 184 if(!mOpenCVAlgorithms->findSubImageAsync(sourceImages, 185 mImageToFind, 186 mConfidenceMinimum, 187 mMaximumMatches, 188 mDownPyramidCount, 189 mSearchExpansion, 190 static_cast<ActionTools::OpenCVAlgorithms::AlgorithmMethod>(mMethod))) 191 { 192 emit executionException(ErrorWhileSearchingException, tr("Error while searching: %1").arg(mOpenCVAlgorithms->errorString())); 193 194 return; 195 } 196 } 197 searchFinished(const ActionTools::MatchingPointList & matchingPointList)198 void FindImageInstance::searchFinished(const ActionTools::MatchingPointList &matchingPointList) 199 { 200 bool ok = true; 201 202 if(matchingPointList.empty()) 203 { 204 setCurrentParameter(QStringLiteral("ifNotFound"), QStringLiteral("line")); 205 206 QString line = evaluateSubParameter(ok, mIfNotFound.actionParameter()); 207 if(!ok) 208 return; 209 210 if(mIfNotFound.action() == ActionTools::IfActionValue::GOTO) 211 { 212 setNextLine(line); 213 214 executionEnded(); 215 } 216 else if(mIfNotFound.action() == ActionTools::IfActionValue::CALLPROCEDURE) 217 { 218 if(!callProcedure(line)) 219 return; 220 221 executionEnded(); 222 } 223 else if(mIfNotFound.action() == ActionTools::IfActionValue::WAIT) 224 { 225 mWaitTimer.start(mSearchDelay); 226 } 227 else 228 executionEnded(); 229 230 return; 231 } 232 233 if(mMaximumMatches == 1) 234 { 235 const ActionTools::MatchingPoint &bestMatchingPoint = matchingPointList.first(); 236 QPoint position = bestMatchingPoint.position; 237 238 if(mSource != WindowSource || !mWindowRelativePosition) 239 position += mImagesToSearchIn.at(bestMatchingPoint.imageIndex).second.topLeft(); 240 241 setVariable(mPositionVariableName, Code::Point::constructor(position, scriptEngine())); 242 setVariable(mConfidenceVariableName, bestMatchingPoint.confidence); 243 } 244 else 245 { 246 QScriptValue arrayResult = scriptEngine()->newArray(matchingPointList.size()); 247 QScriptValue arrayConfidenceResult = scriptEngine()->newArray(matchingPointList.size()); 248 249 for(int i = 0; i < matchingPointList.size(); ++i) 250 { 251 const ActionTools::MatchingPoint &matchingPoint = matchingPointList.at(i); 252 QPoint position = matchingPoint.position; 253 254 if(mSource != WindowSource || !mWindowRelativePosition) 255 position += mImagesToSearchIn.at(matchingPoint.imageIndex).second.topLeft(); 256 257 arrayResult.setProperty(i, Code::Point::constructor(position, scriptEngine())); 258 arrayConfidenceResult.setProperty(i, matchingPoint.confidence); 259 } 260 261 setVariable(mPositionVariableName, arrayResult); 262 setVariable(mConfidenceVariableName, arrayConfidenceResult); 263 } 264 265 setCurrentParameter(QStringLiteral("ifFound"), QStringLiteral("line")); 266 267 QString line = evaluateSubParameter(ok, mIfFound.actionParameter()); 268 if(!ok) 269 return; 270 271 if(mIfFound.action() == ActionTools::IfActionValue::GOTO) 272 { 273 setNextLine(line); 274 275 executionEnded(); 276 } 277 else if(mIfFound.action() == ActionTools::IfActionValue::CALLPROCEDURE) 278 { 279 if(!callProcedure(line)) 280 return; 281 282 executionEnded(); 283 } 284 else if(mIfFound.action() == ActionTools::IfActionValue::WAIT) 285 { 286 mWaitTimer.start(mSearchDelay); 287 } 288 else 289 executionEnded(); 290 } 291 } 292 293