1 2 /** 3 * Header file for Khan's deghosting algorithm 4 * Copyright (C) 2009 Lukáš Jirkovský <l.jirkovsky@gmail.com> 5 * 6 * This program is free software; you can redistribute it and/or 7 * modify it under the terms of the GNU General Public License 8 * as published by the Free Software Foundation; either version 2 9 * of the License, or (at your option) any later version. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program; if not, write to the Free Software 18 *Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 19 */ 20 21 #ifndef KHAN_H_ 22 #define KHAN_H_ 23 24 #include "deghosting.h" 25 // for AlgTinyVector, NormalizeFunctor and LogarithmFunctor 26 #include "support.h" 27 #include "algtinyvector.h" 28 29 // needed for RGB2Lab 30 #include <vigra/imageinfo.hxx> 31 #include <vigra/transformimage.hxx> 32 #include <vigra/colorconversions.hxx> 33 34 // for resampleImage 35 #include <vigra/resizeimage.hxx> 36 37 // for RGBvalue used in hat function 38 #include <vigra/rgbvalue.hxx> 39 // for importImage und importImageAlpha 40 #include <vigra_ext/impexalpha.hxx> 41 42 // needed for Kh() 43 #define PI 3.14159265358979323846 44 45 // number of pixels to look at in all directions 46 // ie. 1 for neighbourhood of size 3x3, 2 for 5x5 etc. 47 #define NEIGHB_DIST 1 48 49 #if defined _WIN32 50 #define snprintf _snprintf 51 #endif 52 // define for use atan based kernel function 53 // leave undefined for gaussian normal distribution function 54 //#define ATAN_KH 55 56 namespace deghosting 57 { 58 template <class PixelType> 59 class ImageTypes { 60 public: 61 typedef vigra::FImage ImageType; 62 typedef std::shared_ptr<ImageType> ImagePtr; 63 typedef vigra::BasicImage<float> ProcessImageType; 64 typedef std::shared_ptr<ProcessImageType> ProcessImageTypePtr; 65 }; 66 67 template <class PixelType> 68 class ImageTypes<vigra::RGBValue<PixelType> > { 69 public: 70 typedef vigra::FRGBImage ImageType; 71 typedef std::shared_ptr<ImageType> ImagePtr; 72 typedef vigra::BasicImage<vigra::AlgTinyVector<float, 3> > ProcessImageType; 73 typedef std::shared_ptr<ProcessImageType> ProcessImageTypePtr; 74 }; 75 76 template <class PixelType> 77 class Khan : public Deghosting, private ImageTypes<PixelType> 78 { 79 public: 80 Khan(std::vector<std::string>& inputFiles, const uint16_t flags, const uint16_t debugFlags, int iterations, double sigma, int verbosity); 81 virtual std::vector<FImagePtr> createWeightMasks() override; ~Khan()82 ~Khan() {} 83 protected: 84 typedef typename ImageTypes<PixelType>::ImageType ImageType; 85 typedef typename ImageTypes<PixelType>::ProcessImageType ProcessImageType; 86 typedef typename ImageTypes<PixelType>::ProcessImageType::traverser ProcessImageTraverser; 87 typedef typename ImageTypes<PixelType>::ProcessImageType::PixelType ProcessImagePixelType; 88 typedef typename ImageTypes<PixelType>::ProcessImageTypePtr ProcessImageTypePtr; 89 typedef typename vigra::NumericTraits<PixelType>::isScalar srcIsScalar; 90 91 // Kh() things 92 // (2*pi)^(1/2) 93 double PIPOW; 94 // 1/Kh denominator 95 double denom; 96 // sigma in gauusian density function 97 double sigma; 98 99 // other necessary stuff 100 std::vector<ProcessImageTypePtr> processImages; 101 std::vector<FImagePtr> weights; 102 103 /** set sigma 104 * sets sigma for gaussian weigting function 105 */ 106 void setSigma(double sigma); 107 108 /** transform image using EMoR response 109 * @param inputFile filename of image to be transformed 110 * @param *pInputImg FRGBImage to be transformed 111 */ 112 //void linearizeRGB(std::string, FRGBImage* pInputImg); 113 114 /** kernel function 115 * Standard probability density function 116 */ 117 inline float Kh(ProcessImagePixelType x); 118 119 /** convert image for internal use 120 * if input image is RGB then convert it to L*a*b 121 * if input image is grayscale then only copy image 122 */ 123 void convertImage(ImageType * in, ProcessImageTypePtr & out, vigra::VigraFalseType); 124 void convertImage(ImageType * in, ProcessImageTypePtr & out, vigra::VigraTrueType); 125 126 /** import RGB image 127 */ 128 void importRGBImage(vigra::ImageImportInfo & info, ImageType * img, vigra::VigraFalseType); 129 void importRGBImage(vigra::ImageImportInfo & info, ImageType * img, vigra::VigraTrueType); 130 131 /** function to preprocess input image 132 * This function loads image, linearize it using EMoR (FIXME), 133 * tranform it using logarithm or gamma if input images are HDR 134 */ 135 void preprocessImage(unsigned int i, FImagePtr &weight, ProcessImageTypePtr &output); 136 }; 137 138 template <class PixelType> Khan(std::vector<std::string> & newInputFiles,const uint16_t newFlags,const uint16_t newDebugFlags,int newIterations,double newSigma,int newVerbosity)139 Khan<PixelType>::Khan(std::vector<std::string>& newInputFiles, const uint16_t newFlags, const uint16_t newDebugFlags, 140 int newIterations, double newSigma, int newVerbosity) { 141 try { 142 Deghosting::loadImages(newInputFiles); 143 Deghosting::setFlags(newFlags); 144 Deghosting::setDebugFlags(newDebugFlags); 145 Deghosting::setIterationNum(newIterations); 146 Deghosting::setVerbosity(newVerbosity); 147 148 // I don't know why, but sigma for HDR input have to approximately 10 times smaller 149 // FIXME: Maybe it would be better to use different sigma for different images in case both HDR and LDR are mixed 150 const char * fileType= vigra::ImageImportInfo(newInputFiles[0].c_str()).getFileType(); 151 if ( (!strcmp(fileType,"TIFF") && strcmp(fileType,"UINT8")) || !strcmp(fileType,"EXR") || !strcmp(fileType,"FLOAT")) { 152 setSigma(newSigma/10); 153 } else { 154 setSigma(newSigma); 155 } 156 157 for (unsigned int i=0; i<5; i++) 158 Deghosting::response.push_back(0); 159 PIPOW = sigma*std::sqrt(2*PI); 160 denom = 1/PIPOW; 161 } catch (...) { 162 throw; 163 } 164 } 165 166 template <class PixelType> setSigma(double newSigma)167 void Khan<PixelType>::setSigma(double newSigma) { 168 sigma = newSigma; 169 } 170 171 template <class PixelType> Kh(ProcessImagePixelType x)172 float Khan<PixelType>::Kh(ProcessImagePixelType x) { 173 #ifdef ATAN_KH 174 // good choice for sigma for this function is around 600 175 return std::atan(-(x*x)+sigma)/PI + 0.5; 176 #else 177 // good choice for sigma for this function is around 30 178 return (std::exp(-(x*x)/(2*sigma*sigma)) * denom); 179 #endif 180 } 181 182 /*void Khan::linearizeRGB(std::string inputFile,FRGBImage *pInputImg) { 183 HuginBase::SrcPanoImage panoImg(inputFile); 184 panoImg.setResponseType(HuginBase::SrcPanoImage::RESPONSE_EMOR); 185 panoImg.setEMoRParams(response); 186 // response transform functor 187 HuginBase::Photometric::InvResponseTransform<RGBValue<float>, 188 RGBValue<float> > invResponse(panoImg); 189 invResponse.enforceMonotonicity(); 190 191 // iterator to the upper left corner 192 FRGBImage::traverser imgIterSourceY = pInputImg->upperLeft(); 193 // iterator to he lower right corner 194 FRGBImage::traverser imgIterEnd = pInputImg->lowerRight(); 195 // iterator to the output 196 FRGBImage::traverser imgIterOut = pInputImg->upperLeft(); 197 // loop through the image 198 for (int y=1; imgIterSourceY.y != imgIterEnd.y; ++imgIterSourceY.y, ++imgIterOut.y, ++y) { 199 // iterator to the input 200 FRGBImage::traverser sx = imgIterSourceY; 201 // iterator to the output 202 FRGBImage::traverser dx = imgIterOut; 203 for (int x=1; sx.x != imgIterEnd.x; ++sx.x, ++dx.x, ++x) { 204 // transform image using response 205 *dx = vigra_ext::zeroNegative(invResponse(*sx, hugin_utils::FDiff2D(x, y))); 206 } 207 } 208 }*/ 209 210 // RGB 211 template <class PixelType> convertImage(ImageType * in,ProcessImageTypePtr & out,vigra::VigraFalseType)212 void Khan<PixelType>::convertImage(ImageType * in, ProcessImageTypePtr & out, vigra::VigraFalseType) { 213 vigra::RGB2LabFunctor<float> RGB2Lab; 214 vigra::transformImage(vigra::srcImageRange(*in), vigra::destImage(*out), RGB2Lab); 215 } 216 217 // grayscale 218 template <class PixelType> convertImage(ImageType * in,ProcessImageTypePtr & out,vigra::VigraTrueType)219 void Khan<PixelType>::convertImage(ImageType* in, ProcessImageTypePtr& out, vigra::VigraTrueType) { 220 vigra::copyImage(srcImageRange(*in), destImage(*out)); 221 } 222 223 // load image and convert it to grayscale 224 template <class PixelType> importRGBImage(vigra::ImageImportInfo & info,ImageType * img,vigra::VigraTrueType)225 void Khan<PixelType>::importRGBImage(vigra::ImageImportInfo & info, ImageType * img, vigra::VigraTrueType) { 226 // NOTE: I guess this is not optimal, but it works 227 vigra::RGBToGrayAccessor<vigra::FRGBImage::PixelType> color2gray; 228 vigra::FRGBImage tmpImg(getOutputROI().size()); 229 vigra::Point2D offset = vigra::Point2D(info.getPosition()); 230 offset -= getOutputROI().upperLeft(); 231 if (info.numBands() == 4) { 232 vigra::BImage imgAlpha(tmpImg.size()); 233 vigra::importImageAlpha(info, destImage(tmpImg, offset), destImage(imgAlpha, offset)); 234 } else { 235 vigra::importImage(info, destImage(tmpImg, offset)); 236 } 237 vigra::transformImage(vigra::srcImageRange(tmpImg, color2gray), vigra::destImage(*img), log(vigra::functor::Arg1() + vigra::functor::Param(1.0f))); 238 } 239 240 // only load image 241 template <class PixelType> importRGBImage(vigra::ImageImportInfo & info,ImageType * img,vigra::VigraFalseType)242 void Khan<PixelType>::importRGBImage(vigra::ImageImportInfo & info, ImageType * img, vigra::VigraFalseType) { 243 vigra::Point2D offset = vigra::Point2D(info.getPosition()); 244 offset -= getOutputROI().upperLeft(); 245 if (info.numBands() == 4) { 246 vigra::BImage imgAlpha(img->size()); 247 vigra::importImageAlpha(info, destImage(*img, offset), destImage(imgAlpha, offset)); 248 } else { 249 vigra::importImage(info, destImage(*img, offset)); 250 } 251 } 252 253 template <class PixelType> preprocessImage(unsigned int i,FImagePtr & weight,ProcessImageTypePtr & output)254 void Khan<PixelType>::preprocessImage(unsigned int i, FImagePtr &weight, ProcessImageTypePtr &output) { 255 vigra::ImageImportInfo imgInfo(inputFiles[i]); 256 ImageType * pInputImg = new ImageType(getOutputROI().size()); 257 weight = FImagePtr(new vigra::FImage(getOutputROI().size())); 258 output = ProcessImageTypePtr(new ProcessImageType(getOutputROI().size())); 259 260 // import image 261 // NOTE: Maybe alpha can be of some use but I don't 262 // know about any now 263 if (imgInfo.isColor()) { 264 importRGBImage(imgInfo, pInputImg, srcIsScalar()); 265 } else { 266 importImage(imgInfo, destImage(*pInputImg)); 267 } 268 269 // linearize RGB and convert it to L*a*b image 270 //linearizeRGB(inputFiles[i], pInputImg); 271 272 // take logarithm or gamma correction if the input images are HDR 273 // I'm not sure if it's the right way how to 274 // find out if they are HDR 275 const char * fileType= imgInfo.getFileType(); 276 if ( (!strcmp(fileType,"TIFF") && strcmp(fileType,"UINT8")) || !strcmp(fileType,"EXR") || !strcmp(fileType,"FLOAT")) { 277 // use gamma 2.2 278 if (flags & ADV_GAMMA) { 279 // GammaFunctor is only in vigra 1.6 GRRR 280 // I have to use BrightnessContrastFunctor 281 // TODO: change to the GammaFunctor in the future 282 vigra::FindMinMax<float> minmax; 283 vigra::inspectImage(vigra::srcImageRange(*pInputImg), minmax); 284 vigra::transformImage(vigra::srcImageRange(*pInputImg), vigra::destImage(*pInputImg), vigra::BrightnessContrastFunctor<PixelType>(0.45f, 1.0, minmax.min, minmax.max)); 285 } else { 286 // take logarithm 287 vigra::transformImage(vigra::srcImageRange(*pInputImg), vigra::destImage(*pInputImg), LogarithmFunctor<PixelType>(1.0)); 288 } 289 } 290 291 // generate initial weights 292 vigra::transformImage(vigra::srcImageRange(*pInputImg), vigra::destImage(*weight), HatFunctor<PixelType>()); 293 294 convertImage(pInputImg, output, srcIsScalar()); 295 296 delete pInputImg; 297 pInputImg = 0; 298 } 299 300 template <class PixelType> createWeightMasks()301 std::vector<FImagePtr> Khan<PixelType>::createWeightMasks() { 302 for (unsigned int i = 0; i < inputFiles.size(); i++) { 303 FImagePtr weight; 304 ProcessImageTypePtr processImage; 305 preprocessImage(i, weight, processImage); 306 processImages.push_back(processImage); 307 weights.push_back(weight); 308 309 // save init weights 310 if (debugFlags & SAVE_INITWEIGHTS) { 311 char tmpfn[100]; 312 snprintf(tmpfn, 99, "init_weights_%u.tiff", i); 313 vigra::ImageExportInfo exWeights(tmpfn); 314 vigra::exportImage(vigra::srcImageRange(*weight), exWeights.setPixelType("UINT8")); 315 } 316 } 317 318 float maxWeight = 0; 319 // image size 320 const int origWidth = weights[0]->width(); 321 const int origHeight = weights[0]->height(); 322 323 // if we doing scaling, we have to backup L*a*b images of original size 324 std::vector<ProcessImageTypePtr> backupLab; 325 if (flags & ADV_MULTIRES) { 326 for (unsigned int i = 0; i < processImages.size(); i++) { 327 // backup original size L*a*b 328 backupLab.push_back(processImages[i]); 329 } 330 } 331 332 if (verbosity > 0) 333 std::cout << "Running khan algorithm" << std::endl; 334 // and we can run khan algorithm 335 // khan iteration 336 for (int it = 0; it < iterations; it++) { 337 if (verbosity > 0) 338 std::cout << "iteration " << it+1 << std::endl; 339 // copy weights from previous iteration 340 if (verbosity > 1) 341 std::cout << "copying weights from previous iteration" << std::endl; 342 343 std::vector<FImagePtr> prevWeights; 344 for (unsigned int i = 0; i < weights.size(); i++) { 345 // scale weights to the requied size 346 if (flags & ADV_MULTIRES) { 347 // it would be better to use resampleImage, but it seems to be present only in VIGRA 1.6 348 // so let's use some of the resizeImageINTERPOLATION() functions 349 350 // compute width 351 int resized_width = origWidth / ( iterations/(it+1) ); 352 //compute height 353 int resized_height = origHeight / ( iterations/(it+1) ); 354 // destination images 355 vigra::FImage resizedWeight; 356 ProcessImageType resizedLab; 357 // it's not worthy to scale to less than 100px per side 358 if (resized_width > 100 && resized_height > 100) { 359 // create destination image of desired size 360 resizedWeight = vigra::FImage(vigra::Size2D(resized_width,resized_height)); 361 resizedLab = ProcessImageType(vigra::Size2D(resized_width, resized_height)); 362 } else if (origWidth >= 100 && origHeight >= 100) { 363 // resize it to the smallest value (ie 100px for the shorter side) 364 if (origWidth >= origHeight) { 365 resizedWeight = vigra::FImage(vigra::Size2D(100 * origWidth / origHeight, 100)); 366 resizedLab = ProcessImageType(vigra::Size2D(100 * origWidth / origHeight, 100)); 367 } else { 368 resizedWeight = vigra::FImage(vigra::Size2D(100, 100 * origHeight / origWidth)); 369 resizedLab = ProcessImageType(vigra::Size2D(100, 100 * origHeight / origWidth)); 370 } 371 } else { 372 // don't scale at all 373 // just copy weights as if no scaling seting was applied 374 goto DONTSCALE; 375 } 376 377 // No interpolation – only for testing 378 resizeImageNoInterpolation(srcImageRange(*weights[i]), destImageRange(resizedWeight)); 379 resizeImageNoInterpolation(srcImageRange(*backupLab[i]), destImageRange(resizedLab)); 380 381 FImagePtr tmp(new vigra::FImage(resizedWeight)); 382 prevWeights.push_back(tmp); 383 processImages[i] = ProcessImageTypePtr(new ProcessImageType(resizedLab)); 384 weights[i] = FImagePtr(new vigra::FImage(resizedWeight)); 385 } else { 386 DONTSCALE: 387 FImagePtr tmp(new vigra::FImage(*weights[i])); 388 prevWeights.push_back(tmp); 389 } 390 } 391 392 // loop through all images 393 for (unsigned int i = 0; i < processImages.size(); i++) { 394 if (verbosity > 1) 395 std::cout << "processing image " << i+1 << std::endl; 396 397 // vector storing pixel data 398 ProcessImagePixelType X; 399 // sums for eq. 6 400 double wpqssum = 0; 401 double wpqsKhsum = 0; 402 // image size 403 const int width = processImages[i]->width(); 404 const int height = processImages[i]->height(); 405 406 // iterator to the upper left corner 407 ProcessImageTraverser sy = processImages[i]->upperLeft(); 408 // iterator to the lower right corner 409 ProcessImageTraverser send = processImages[i]->lowerRight(); 410 // iterator to the weight image left corner 411 vigra::FImage::traverser wy = weights[i]->upperLeft(); 412 // loop through the row 413 for (int y=0; sy.y != send.y; ++sy.y, ++wy.y, ++y) { 414 // iterator to the source (L*a*b image) 415 ProcessImageTraverser sx = sy; 416 // iterator to the weight 417 vigra::FImage::traverser wx = wy; 418 // loop over the pixels 419 for (int x=0; sx.x != send.x; ++sx.x, ++wx.x, ++x) { 420 if (verbosity > 2) 421 std::cout << "processing pixel (" << x+1 << "," << y+1 << ")" << std::endl; 422 // set pixel vector 423 X = *sx; 424 425 // loop through all layers 426 for (unsigned int j = 0; j < processImages.size(); j++) { 427 if (verbosity > 2) 428 std::cout << "processing layer " << j << std::endl; 429 430 // iterator to the neighbour 431 ProcessImageTraverser neighby = processImages[j]->upperLeft(); 432 // iterator to the weight 433 vigra::FImage::traverser weighty = prevWeights[j]->upperLeft(); 434 // pixel offset 435 int ndy = -NEIGHB_DIST; 436 // move them to the upper bound 437 // find the best upper bound 438 if (y-NEIGHB_DIST < 0) { 439 ndy = -y; 440 } 441 else { 442 neighby.y += y-NEIGHB_DIST; 443 weighty.y += y-NEIGHB_DIST; 444 } 445 446 // iterate through neighbourhoods y axis 447 int maxDisty = (height - y) > NEIGHB_DIST ? NEIGHB_DIST : (height - y-1); 448 for (; ndy <= maxDisty; ++neighby.y, ++weighty.y, ++ndy) { 449 ProcessImageTraverser neighbx = neighby; 450 vigra::FImage::traverser weightx = weighty; 451 // pixel offset 452 int ndx = -NEIGHB_DIST; 453 // move them to the upper bound 454 // find the best upper bound 455 if (x-NEIGHB_DIST < 0) { 456 ndx = -x; 457 } 458 else { 459 neighbx.x += x-NEIGHB_DIST; 460 weightx.x += x-NEIGHB_DIST; 461 } 462 // iterate through neighbourhoods x axis 463 int maxDistx = (width - x) > NEIGHB_DIST ? NEIGHB_DIST : (width - x-1); 464 for (; ndx <= maxDistx; ++neighbx.x, ++weightx.x, ++ndx) { 465 if (verbosity > 3) 466 std::cout << "(" << ndx << "," << ndy << ")"; 467 // now we can construct pixel vector 468 // should omit the middle pixel, ie use only neighbours 469 if (ndx != 0 || ndy != 0) { 470 wpqsKhsum += (*weightx * Kh(X-(*neighbx))); 471 wpqssum += *weightx; 472 } 473 474 maxDistx = (width - x) > NEIGHB_DIST ? NEIGHB_DIST : (width - x-1); 475 } 476 if (verbosity > 3) 477 std::cout << std::endl; 478 479 maxDisty = (height - y) > NEIGHB_DIST ? NEIGHB_DIST : (height - y-1); 480 } 481 } 482 483 if (verbosity > 2) 484 std::cout << "computing new weight" << std::endl; 485 // compute probability and set weight 486 //std::cout << "P=" << (float) wpqsKhsum/wpqssum << std::endl; 487 if (wpqssum > 0) 488 { 489 if (flags & ADV_ONLYP) 490 *wx = (float)wpqsKhsum / wpqssum; 491 else 492 *wx *= (float)wpqsKhsum / wpqssum; 493 if (maxWeight < *wx) 494 maxWeight = *wx; 495 }; 496 wpqsKhsum = wpqssum = 0; 497 498 } 499 } 500 } 501 } 502 503 if (verbosity > 1) 504 std::cout << "normalizing weights" << std::endl; 505 double factor = 255.0f/maxWeight; 506 for (unsigned int i=0; i<weights.size(); ++i) { 507 transformImage(srcImageRange(*(weights[i])), destImage(*(weights[i])), NormalizeFunctor<float>(factor)); 508 } 509 return weights; 510 } 511 } 512 513 #endif /* KHAN_H_ */ 514