1 /* 2 * Copyright (C) 2010-2011 Simon Andreas Eugster (simon.eu@gmail.com) 3 * This file is a Frei0r plugin. 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License as published by 7 * the Free Software Foundation; either version 2 of the License, or 8 * (at your option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software 17 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 18 */ 19 20 /** 21 LIGHT GRAFFITI / LIGHT PAINTING / LUMASOL 22 23 24 This effect is intended to simulate what happens when you use a shutter speed of 25 e.g. 10 seconds for your camera and paint with light, like lamps, in the air 26 -- just with video. It tries to remember bright spots and keeps them in a mask. 27 Areas that are not very bright (i.e. background) will not sum up. 28 29 Originally I saw this effect in some Ford Kuga commercials on YouTube when a friend 30 shew me those. One of them was [1]. No information about how this effect works is given 31 -- only that a guy from the Dutch PIPS:LAB[2] was involved. The technique seems to be 32 slightly different though; whileas this frei0r effect works in post, the original Lumasol 33 effect is said to work directly in-camera. 34 35 Since the technique is fascinating I started writing this Open-Source effect.[3] The general 36 concept is: 37 1. Extract the light by using thresholding (absolute brightness and brightness change relative to 38 the background image fetched from the first frame) 39 2. Store the color in a light mask, and the estimated density in an alpha map (increased every time 40 that a light source hits a pixel to simulate overexposure) 41 3. Paint the light mask over the video image 42 4. Dim the alpha map, and update the background image (moving average), if desired. 43 5. Repeat for the next frame. 44 45 The second approach (LG_ADV) is based on the observation that colour mixing does not work well 46 with the above one that stores colour values and changes the brightness via an alpha map. Therefore 47 the new approach directly sums up colour values detected in the light source and does not use an 48 alpha map. 49 * Transitions are not very smooth out-of-the-box. This is solved by multiplying the light source's 50 RGB value by (r+g+b)/3 (after normalizing them to [0,1]); Darker lights will then be even darker 51 and the transition to the background looks smoother. 52 * Lights may look a little faint regarding color. Therefore the saturation can be increased by a custom 53 factor. Saturation depends on the brightness of the light map; the darker the light is, the more 54 the saturation is increased. This will, within a sensible range, make the lights look more vital. 55 Dimming works by scaling each color values individually. 56 57 If you write your own Light Graffiti effect (e.g. for After Effects) I'd very much appreciate 58 to hear about it! 59 60 -- Simon (Granjow) 61 62 [1] http://www.youtube.com/watch?v=WVaxuIKPKvU 63 [2] http://www.pipslab.org/bio/keez-duyves/ 64 [3] http://kdenlive.org/users/granjow/writing-light-graffiti-effect 65 66 */ 67 #include "frei0r.hpp" 68 69 #include <cmath> 70 #include <cstdio> 71 #include <climits> 72 #include <algorithm> 73 74 #define LG_ADV 75 //#define LG_NO_OVERLAY // Not really working yet 76 //#define LG_DEBUG 77 78 // Macros to extract color components 79 #define GETA(abgr) (((abgr) >> (3*CHAR_BIT)) & 0xFF) 80 #define GETB(abgr) (((abgr) >> (2*CHAR_BIT)) & 0xFF) 81 #define GETG(abgr) (((abgr) >> (1*CHAR_BIT)) & 0xFF) 82 #define GETR(abgr) (((abgr) >> (0*CHAR_BIT)) & 0xFF) 83 84 // Macro to assemble a color in RGBA8888 format 85 #define RGBA(r,g,b,a) ( ((r) << (0*CHAR_BIT)) | ((g) << (1*CHAR_BIT)) | ((b) << (2*CHAR_BIT)) | ((a) << (3*CHAR_BIT)) ) 86 // Component-wise maximum 87 #define MAX(a,b) ( (((((a) >> (0*CHAR_BIT)) & 0xFF) > (((b) >> (0*CHAR_BIT)) & 0xFF)) ? ((a) & (0xFF << (0*CHAR_BIT))) : ((b) & (0xFF << (0*CHAR_BIT)))) \ 88 | (((((a) >> (1*CHAR_BIT)) & 0xFF) > (((b) >> (1*CHAR_BIT)) & 0xFF)) ? ((a) & (0xFF << (1*CHAR_BIT))) : ((b) & (0xFF << (1*CHAR_BIT)))) \ 89 | (((((a) >> (2*CHAR_BIT)) & 0xFF) > (((b) >> (2*CHAR_BIT)) & 0xFF)) ? ((a) & (0xFF << (2*CHAR_BIT))) : ((b) & (0xFF << (2*CHAR_BIT)))) \ 90 | (((((a) >> (3*CHAR_BIT)) & 0xFF) > (((b) >> (3*CHAR_BIT)) & 0xFF)) ? ((a) & (0xFF << (3*CHAR_BIT))) : ((b) & (0xFF << (3*CHAR_BIT)))) ) 91 92 #define CLAMP(a) (((a) < 0) ? 0 : (((a) > 255) ? 255 : (a))) 93 94 #define ALPHA(mask,img) \ 95 ( ( ((uint32_t) ( ((((mask) >> (0*CHAR_BIT)) & 0xFF)/255.0) * ( ((mask) >> (0*CHAR_BIT)) & 0xFF) \ 96 + (1 - (( ((mask) >> (0*CHAR_BIT)) & 0xFF)/255.0)) * ( ((img) >> (0*CHAR_BIT)) & 0xFF) )) << (0*CHAR_BIT)) \ 97 | ( ((uint32_t) ( ((((mask) >> (1*CHAR_BIT)) & 0xFF)/255.0) * ( ((mask) >> (1*CHAR_BIT)) & 0xFF) \ 98 + (1 - (( ((mask) >> (1*CHAR_BIT)) & 0xFF)/255.0)) * ( ((img) >> (1*CHAR_BIT)) & 0xFF) )) << (1*CHAR_BIT)) \ 99 | ( ((uint32_t) ( ((((mask) >> (2*CHAR_BIT)) & 0xFF)/255.0) * ( ((mask) >> (2*CHAR_BIT)) & 0xFF) \ 100 + (1 - (( ((mask) >> (2*CHAR_BIT)) & 0xFF)/255.0)) * ( ((img) >> (2*CHAR_BIT)) & 0xFF) )) << (2*CHAR_BIT)) \ 101 | ( ((uint32_t) ( ((((mask) >> (3*CHAR_BIT)) & 0xFF)/255.0) * ( ((mask) >> (3*CHAR_BIT)) & 0xFF) \ 102 + (1 - (( ((mask) >> (3*CHAR_BIT)) & 0xFF)/255.0)) * ( ((img) >> (3*CHAR_BIT)) & 0xFF) )) << (3*CHAR_BIT)) ) 103 104 // Screen layer mode 105 #define SCREEN1(mask,img) ((uint8_t) (255-(255.0-(mask))*(255.0-(img))/255.0)) 106 107 // Luma calculation. Refer to the SOP/Sat filter. 108 #define REC709Y(r,g,b) (.2126*(r) + .7152*(g) + .0722*(b)) 109 110 struct RGBFloat { 111 float r; 112 float g; 113 float b; 114 }; 115 116 class LightGraffiti : public frei0r::filter 117 { 118 119 public: 120 LightGraffiti(unsigned int width,unsigned int height)121 LightGraffiti(unsigned int width, unsigned int height) : 122 m_lightMask(width*height, 0), 123 m_alphaMap(4*width*height, 0), 124 m_meanInitialized(false) 125 126 { 127 m_mode = Graffiti_LongAvgAlphaCumC; 128 m_dimMode = Dim_Mult; 129 130 #ifdef LG_ADV 131 RGBFloat rgb0; 132 rgb0.r = 0; 133 rgb0.g = 0; 134 rgb0.b = 0; 135 m_rgbLightMask = std::vector<RGBFloat>(width*height, rgb0); 136 137 #ifdef LG_DEBUG 138 for (int i = 0; i < width*height; i++) { 139 if (m_rgbLightMask[i].r != 0 || m_rgbLightMask[i].g != 0 || m_rgbLightMask[i].b != 0) { 140 std::cout << "ERROR: " << m_rgbLightMask[i].r; 141 } 142 } 143 #endif 144 #endif 145 146 #ifdef LG_NO_OVERLAY 147 m_prevMask = std::vector<RGBFloat>(width*height, rgb0); 148 #endif 149 150 register_param(m_pSensitivity, "sensitivity", "Sensitivity of the effect for light (higher sensitivity will lead to brighter lights)"); 151 register_param(m_pBackgroundWeight, "backgroundWeight", "Describes how strong the (accumulated) background should shine through"); 152 register_param(m_pThresholdBrightness, "thresholdBrightness", "Brightness threshold to distinguish between foreground and background"); 153 register_param(m_pThresholdDifference, "thresholdDifference", "Threshold: Difference to background to distinguish between fore- and background"); 154 register_param(m_pThresholdDiffSum, "thresholdDiffSum", "Threshold for sum of differences. Can in most cases be ignored (set to 0)."); 155 register_param(m_pDim, "dim", "Dimming of the light mask"); 156 register_param(m_pSaturation, "saturation", "Saturation of lights"); 157 register_param(m_pLowerOverexposure, "lowerOverexposure", "Prevents some overexposure if the light source stays steady too long (varying speed)"); 158 register_param(m_pStatsBrightness, "statsBrightness", "Display the brightness and threshold, for adjusting the brightness threshold parameter"); 159 register_param(m_pStatsDiff, "statsDifference", "Display the background difference and threshold"); 160 register_param(m_pStatsDiffSum, "statsDiffSum", "Display the sum of the background difference and the threshold"); 161 register_param(m_pReset, "reset", "Reset filter masks"); 162 register_param(m_pTransparentBackground, "transparentBackground", "Make the background transparent"); 163 register_param(m_pBlackReference, "blackReference", "Uses black as background image instead of the first frame."); 164 register_param(m_pLongAlpha, "longAlpha", "Alpha value for moving average"); 165 register_param(m_pNonlinearDim, "nonlinearDim", "Nonlinear dimming (may look more natural)"); 166 m_pLongAlpha = 1/128.0; 167 m_pSensitivity = 1 / 5.; 168 m_pBackgroundWeight = 0; 169 m_pThresholdBrightness = 450 / 765.; 170 m_pThresholdDifference = 0; 171 m_pThresholdDiffSum = 0; 172 m_pDim = 0; 173 m_pSaturation = 1 / 4.; 174 m_pLowerOverexposure = 0; 175 m_pStatsBrightness = false; 176 m_pStatsDiff = false; 177 m_pStatsDiffSum = false; 178 m_pReset = false; 179 m_pTransparentBackground = false; 180 m_pBlackReference = false; 181 m_pLongAlpha = 0; 182 m_pNonlinearDim = 0; 183 184 } 185 ~LightGraffiti()186 ~LightGraffiti() 187 { 188 // I was told that a std::vector does not need to be deleted -- 189 // therefore nothing to do here! 190 } 191 192 enum GraffitiMode { Graffiti_max, Graffiti_max_sum, Graffiti_Y, Graffiti_Avg, Graffiti_Avg2, 193 Graffiti_Avg_Stat, Graffiti_AvgTresh_Stat, Graffiti_Max_Stat, Graffiti_Y_Stat, Graffiti_S_Stat, 194 Graffiti_STresh_Stat, Graffiti_SDiff_Stat, Graffiti_SDiffTresh_Stat, 195 Graffiti_SSqrt_Stat, 196 Graffiti_LongAvg, Graffiti_LongAvg_Stat, Graffiti_LongAvgAlpha, Graffiti_LongAvgAlpha_Stat, 197 Graffiti_LongAvgAlphaCumC }; 198 enum DimMode { Dim_Mult, Dim_Sin }; 199 200 201 202 203 update(double time,uint32_t * out,const uint32_t * in)204 virtual void update(double time, 205 uint32_t* out, 206 const uint32_t* in) 207 { 208 double sensitivity = m_pSensitivity * 5; 209 double thresholdBrightness = m_pThresholdBrightness * 765; 210 double thresholdDifference = m_pThresholdDifference * 255; 211 double thresholdDiffSum = m_pThresholdDiffSum * 765; 212 double saturation = m_pSaturation * 4; 213 double lowerOverexposure = m_pLowerOverexposure * 10; 214 215 // Copy everything to the output image. 216 // Most of the image will very likely not change at all. 217 std::copy(in, in + width*height, out); 218 219 #ifdef LG_ADV 220 RGBFloat rgb0; 221 rgb0.r = 0; 222 rgb0.g = 0; 223 rgb0.b = 0; 224 #endif 225 226 if (m_pNonlinearDim) { 227 m_dimMode = Dim_Sin; 228 } else { 229 m_dimMode = Dim_Mult; 230 } 231 232 233 /* 234 Refresh the background image 235 */ 236 if (!m_meanInitialized || m_pReset) { 237 if (m_pBlackReference) { 238 // Do not use the first frame from the movie as background image but plain black 239 // to calculate the added light. Useful e.g. when dealing with still images. 240 m_longMeanImage = std::vector<float>(width*height*3, 0); 241 } else { 242 m_longMeanImage = std::vector<float>(width*height*3); 243 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 244 m_longMeanImage[3*pixel+0] = GETR(in[pixel]); 245 m_longMeanImage[3*pixel+1] = GETG(in[pixel]); 246 m_longMeanImage[3*pixel+2] = GETB(in[pixel]); 247 } 248 } 249 m_meanInitialized = true; 250 } else { 251 // Calculate the mean image to estimate the background. If alpha is set > 0, bright light sources 252 // moving into the image and standing still will eventually be treated as background. 253 if (m_pLongAlpha > 0) { 254 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 255 m_longMeanImage[3*pixel+0] = (1-m_pLongAlpha) * m_longMeanImage[3*pixel+0] + m_pLongAlpha * GETR(in[pixel]); 256 m_longMeanImage[3*pixel+1] = (1-m_pLongAlpha) * m_longMeanImage[3*pixel+1] + m_pLongAlpha * GETG(in[pixel]); 257 m_longMeanImage[3*pixel+2] = (1-m_pLongAlpha) * m_longMeanImage[3*pixel+2] + m_pLongAlpha * GETB(in[pixel]); 258 } 259 } 260 } 261 262 263 /* 264 Light mask dimming 265 */ 266 if (m_pDim > 0) { 267 // Dims the light mask. Lights will leave fainting trails. 268 269 float factor = 1-m_pDim; 270 271 /* Gnu Octave: 272 range=linspace(0,1,100); 273 % Sin 274 plot(range,sin(range*pi/2).^.5) 275 plot(range,sin(range*pi/2).^.25) 276 */ 277 278 switch (m_dimMode) { 279 280 case Dim_Mult: 281 #ifdef LG_ADV 282 for (size_t i = 0; i < m_rgbLightMask.size(); i++) { 283 m_rgbLightMask[i].r *= factor; 284 m_rgbLightMask[i].g *= factor; 285 m_rgbLightMask[i].b *= factor; 286 } 287 #else 288 for (unsigned int i = 0; i < width*height; i++) { 289 m_alphaMap[4*i + 0] *= factor; 290 m_alphaMap[4*i + 1] *= factor; 291 m_alphaMap[4*i + 2] *= factor; 292 m_alphaMap[4*i + 3] *= factor; 293 } 294 #endif 295 break; 296 297 298 case Dim_Sin: 299 #ifdef LG_ADV 300 for (size_t i = 0; i < m_rgbLightMask.size(); i++) { 301 // Red 302 if (m_rgbLightMask[i].r < 1) { 303 m_rgbLightMask[i].r *= pow(sin(m_rgbLightMask[i].r * M_PI/2), m_pDim) - .01; 304 } else { 305 m_rgbLightMask[i].r *= factor; 306 } 307 if (m_rgbLightMask[i].r < 0) { m_rgbLightMask[i].r = 0; } 308 309 // Green 310 if (m_rgbLightMask[i].g < 1) { 311 m_rgbLightMask[i].g *= pow(sin(m_rgbLightMask[i].g * M_PI/2), m_pDim) - .01; 312 } else { 313 m_rgbLightMask[i].g *= factor; 314 } 315 if (m_rgbLightMask[i].g < 0) { m_rgbLightMask[i].g = 0; } 316 317 // Blue 318 if (m_rgbLightMask[i].b < 1) { 319 m_rgbLightMask[i].b *= pow(sin(m_rgbLightMask[i].b * M_PI/2), m_pDim) - .01; 320 } else { 321 m_rgbLightMask[i].b *= factor; 322 } 323 if (m_rgbLightMask[i].b < 0) { m_rgbLightMask[i].b = 0; } 324 } 325 #else 326 // Attention: Since Graffiti_LongAvgAlphaCumC only makes use of the first alpha channel 327 // the other channels are not calculated here due to efficiency reasons. 328 // May have to be adjusted if required. 329 for (int i = 0; i < width*height; i++) { 330 if (m_alphaMap[4*i + 0] < 1) { 331 m_alphaMap[4*i + 0] *= pow(sin(m_alphaMap[4*i + 0] * M_PI/2), m_pDim) - .01; 332 } else { 333 m_alphaMap[4*i + 0] *= factor; 334 } 335 if (m_alphaMap[4*i + 0] < 0) { m_alphaMap[4*i + 0] = 0; } 336 } 337 #endif 338 break; 339 } 340 341 } 342 343 344 345 /* 346 Reset all masks if desired 347 (mainly for parameter adjustments when working in the NLE) 348 */ 349 if (m_pReset) { 350 #ifdef LG_ADV 351 m_rgbLightMask = std::vector<RGBFloat>(width*height, rgb0); 352 #else 353 std::fill(&m_lightMask[0], &m_lightMask[width*height - 1], 0); 354 std::fill(&m_alphaMap[0], &m_alphaMap[width*height*4 - 1], 0); 355 #endif 356 // m_longMeanImage has been handled above already (set to the current image). 357 } 358 359 360 int r, g, b; 361 int maxDiff, temp, sum; 362 unsigned int min; 363 unsigned int max; 364 float f; 365 float fr, fg, fb, sr, sg, sb, fy, fsat; 366 367 #ifdef LG_DEBUG 368 int deCount = 0; 369 #endif 370 371 372 switch (m_mode) { 373 /* 374 Lots of testing modes here! 375 */ 376 case Graffiti_max: 377 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 378 379 if ( 380 (GETR(out[pixel]) == 0xFF 381 || GETG(out[pixel]) == 0xFF 382 || GETB(out[pixel]) == 0xFF) 383 ){ 384 m_lightMask[pixel] |= out[pixel]; 385 } 386 if (m_lightMask[pixel] != 0) { 387 out[pixel] = m_lightMask[pixel]; 388 } 389 } 390 break; 391 case Graffiti_max_sum: 392 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 393 394 if ( 395 (GETR(out[pixel]) == 0xFF 396 || GETG(out[pixel]) == 0xFF 397 || GETB(out[pixel]) == 0xFF) 398 && 399 (GETR(out[pixel]) 400 + GETG(out[pixel]) 401 + GETB(out[pixel]) 402 > 0xFF + 0xCC + 0xCC) 403 ){ 404 m_lightMask[pixel] |= out[pixel]; 405 } 406 if (m_lightMask[pixel] != 0) { 407 out[pixel] = m_lightMask[pixel]; 408 } 409 } 410 break; 411 412 case Graffiti_Y: 413 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 414 if ( 415 .299*GETR(out[pixel])/255.0 416 + .587 * GETG(out[pixel])/255.0 417 + .114 * GETB(out[pixel])/255.0 418 >= .85 419 ){ 420 m_lightMask[pixel] |= out[pixel]; 421 } 422 if (m_lightMask[pixel] != 0) { 423 out[pixel] = m_lightMask[pixel]; 424 } 425 } 426 break; 427 428 case Graffiti_Max_Stat: 429 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 430 if (GETR(out[pixel]) == 0xFF || GETG(out[pixel]) == 0xFF || GETB(out[pixel]) == 0xFF) { 431 out[pixel] = 0xFFFFFFFF; 432 } else { 433 out[pixel] = 0; 434 } 435 } 436 break; 437 case Graffiti_Y_Stat: 438 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 439 temp = .299*GETR(out[pixel]) + .587 * GETG(out[pixel]) + .114 * GETB(out[pixel]); 440 temp = CLAMP(temp); 441 out[pixel] = RGBA(temp, temp, temp, 0xFF); 442 } 443 break; 444 case Graffiti_S_Stat: 445 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 446 min = GETR(out[pixel]); 447 max = GETR(out[pixel]); 448 if (GETG(out[pixel]) < min) min = GETG(out[pixel]); 449 if (GETG(out[pixel]) > max) max = GETG(out[pixel]); 450 if (GETB(out[pixel]) < min) min = GETB(out[pixel]); 451 if (GETB(out[pixel]) > max) max = GETB(out[pixel]); 452 if (min == 0) { out[pixel] = 0; } 453 else { 454 temp = 255.0*(max-min)/(float)max; 455 temp = CLAMP(temp); 456 out[pixel] = RGBA(temp, temp, temp, 0xFF); 457 } 458 } 459 break; 460 case Graffiti_STresh_Stat: 461 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 462 min = GETR(out[pixel]); 463 max = GETR(out[pixel]); 464 if (GETG(out[pixel]) < min) min = GETG(out[pixel]); 465 if (GETG(out[pixel]) > max) max = GETG(out[pixel]); 466 if (GETB(out[pixel]) < min) min = GETB(out[pixel]); 467 if (GETB(out[pixel]) > max) max = GETB(out[pixel]); 468 if (min == 0 || max < 0x80) { out[pixel] = 0; } 469 else { 470 temp = 255.0*((float)max-min)/max; 471 temp = CLAMP(temp); 472 out[pixel] = RGBA(temp, temp, temp, 0xFF); 473 } 474 } 475 break; 476 case Graffiti_SDiff_Stat: 477 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 478 min = GETR(out[pixel]); 479 max = GETR(out[pixel]); 480 if (GETG(out[pixel]) < min) min = GETG(out[pixel]); 481 if (GETG(out[pixel]) > max) max = GETG(out[pixel]); 482 if (GETB(out[pixel]) < min) min = GETB(out[pixel]); 483 if (GETB(out[pixel]) > max) max = GETB(out[pixel]); 484 int sat; 485 if (min == 0) { sat = 0; } 486 else { 487 temp = 255.0*(max-min)/(float)max; 488 temp = CLAMP(temp); 489 sat = RGBA(temp, temp, temp, 0xFF); 490 } 491 min = m_longMeanImage[3*pixel+0]; 492 max = m_longMeanImage[3*pixel+0]; 493 if (m_longMeanImage[3*pixel+1] < min) min = m_longMeanImage[3*pixel+1]; 494 if (m_longMeanImage[3*pixel+1] > max) max = m_longMeanImage[3*pixel+1]; 495 if (m_longMeanImage[3*pixel+2] < min) min = m_longMeanImage[3*pixel+2]; 496 if (m_longMeanImage[3*pixel+2] > max) max = m_longMeanImage[3*pixel+2]; 497 if (min == 0) { out[pixel] = 0; } 498 else { 499 temp = 255.0*(max-min)/(float)max; 500 temp = CLAMP(temp); 501 out[pixel] = RGBA(temp, temp, temp, 0xFF); 502 } 503 temp = 0x7f + GETR(out[pixel]) - GETR(sat); 504 temp = CLAMP(temp); 505 out[pixel] = RGBA(temp, temp, temp, 0xFF); 506 } 507 break; 508 case Graffiti_SDiffTresh_Stat: 509 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 510 min = GETR(out[pixel]); 511 max = GETR(out[pixel]); 512 if (GETG(out[pixel]) < min) min = GETG(out[pixel]); 513 if (GETG(out[pixel]) > max) max = GETG(out[pixel]); 514 if (GETB(out[pixel]) < min) min = GETB(out[pixel]); 515 if (GETB(out[pixel]) > max) max = GETB(out[pixel]); 516 int sat; 517 if (min == 0) { sat = 0; } 518 else { 519 temp = 255.0*(max-min)/(float)max; 520 temp = CLAMP(temp); 521 sat = RGBA(temp, temp, temp, 0xFF); 522 } 523 if (max < 0x80) { 524 out[pixel] = RGBA(0,0,0,0xFF); 525 } else { 526 min = m_longMeanImage[3*pixel+0]; 527 max = m_longMeanImage[3*pixel+0]; 528 if (m_longMeanImage[3*pixel+1] < min) min = m_longMeanImage[3*pixel+1]; 529 if (m_longMeanImage[3*pixel+1] > max) max = m_longMeanImage[3*pixel+1]; 530 if (m_longMeanImage[3*pixel+2] < min) min = m_longMeanImage[3*pixel+2]; 531 if (m_longMeanImage[3*pixel+2] > max) max = m_longMeanImage[3*pixel+2]; 532 if (min == 0) { out[pixel] = 0; } 533 else { 534 temp = 255.0*(max-min)/(float)max; 535 temp = CLAMP(temp); 536 out[pixel] = RGBA(temp, temp, temp, 0xFF); 537 } 538 temp = 0x7f + GETR(out[pixel]) - GETR(sat); 539 temp = CLAMP(temp); 540 out[pixel] = RGBA(temp, temp, temp, 0xFF); 541 } 542 } 543 break; 544 case Graffiti_SSqrt_Stat: 545 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 546 min = GETR(out[pixel]); 547 max = GETR(out[pixel]); 548 if (GETG(out[pixel]) < min) min = GETG(out[pixel]); 549 if (GETG(out[pixel]) > max) max = GETG(out[pixel]); 550 if (GETB(out[pixel]) < min) min = GETB(out[pixel]); 551 if (GETB(out[pixel]) > max) max = GETB(out[pixel]); 552 if (min == 0) { out[pixel] = 0; } 553 else { 554 temp = 255.0*(max-min)/(float)max/(256.0-max); 555 temp = CLAMP(temp); 556 out[pixel] = RGBA(temp, temp, temp, 0xFF); 557 } 558 } 559 break; 560 case Graffiti_LongAvg_Stat: 561 maxDiff = 0; 562 temp = 0; 563 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 564 r = 0x7f + (GETR(out[pixel]) - m_longMeanImage[3*pixel+0])/2; 565 r = CLAMP(r); 566 g = 0x7f + (GETG(out[pixel]) - m_longMeanImage[3*pixel+1])/2; 567 g = CLAMP(g); 568 b = 0x7f + (GETB(out[pixel]) - m_longMeanImage[3*pixel+2])/2; 569 b = CLAMP(b); 570 571 out[pixel] = RGBA(r,g,b,0xFF); 572 } 573 break; 574 case Graffiti_LongAvg: 575 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 576 577 r = 0x7f + (GETR(out[pixel]) - m_longMeanImage[3*pixel+0]); 578 r = CLAMP(r); 579 max = GETR(out[pixel]); 580 maxDiff = r; 581 temp = r; 582 583 g = 0x7f + (GETG(out[pixel]) - m_longMeanImage[3*pixel+1]); 584 g = CLAMP(g); 585 if (maxDiff < g) maxDiff = g; 586 if (max < GETG(out[pixel])) max = GETG(out[pixel]); 587 temp += g; 588 589 b = 0x7f + (GETB(out[pixel]) - m_longMeanImage[3*pixel+2]); 590 b = CLAMP(b); 591 if (maxDiff < b) maxDiff = b; 592 if (max < GETB(out[pixel])) max = GETB(out[pixel]); 593 temp += b; 594 595 if (maxDiff > 0xe0 && temp > 0xe0 + 0xd0 + 0x80) { 596 m_lightMask[pixel] = MAX(m_lightMask[pixel], out[pixel]); 597 598 m_alphaMap[4*pixel+0] = 2*(GETR(out[pixel])-m_longMeanImage[3*pixel+0]); 599 m_alphaMap[4*pixel+0] = CLAMP(m_alphaMap[4*pixel+0])/255.0; 600 601 m_alphaMap[4*pixel+1] = 2*(GETG(out[pixel])-m_longMeanImage[3*pixel+1]); 602 m_alphaMap[4*pixel+1] = CLAMP(m_alphaMap[4*pixel+1])/255.0; 603 604 m_alphaMap[4*pixel+2] = 2*(GETB(out[pixel])-m_longMeanImage[3*pixel+2]); 605 m_alphaMap[4*pixel+2] = CLAMP(m_alphaMap[4*pixel+2])/255.0; 606 607 m_alphaMap[4*pixel+3] = 1; 608 } 609 610 if (m_lightMask[pixel] != 0) { 611 r = SCREEN1(GETR(out[pixel]), GETR(m_lightMask[pixel])); 612 g = SCREEN1(GETG(out[pixel]), GETG(m_lightMask[pixel])); 613 b = SCREEN1(GETB(out[pixel]), GETB(m_lightMask[pixel])); 614 r = CLAMP(r); 615 g = CLAMP(g); 616 b = CLAMP(b); 617 out[pixel] = RGBA(r,g,b,0xFF); 618 } 619 } 620 break; 621 case Graffiti_LongAvgAlpha_Stat: 622 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 623 624 r = 0x7f + (GETR(out[pixel]) - m_longMeanImage[3*pixel+0]); 625 r = CLAMP(r); 626 max = GETR(out[pixel]); 627 maxDiff = r; 628 temp = r; 629 630 g = 0x7f + (GETG(out[pixel]) - m_longMeanImage[3*pixel+1]); 631 g = CLAMP(g); 632 if (maxDiff < g) maxDiff = g; 633 if (max < GETG(out[pixel])) max = GETG(out[pixel]); 634 temp += g; 635 636 b = 0x7f + (GETB(out[pixel]) - m_longMeanImage[3*pixel+2]); 637 b = CLAMP(b); 638 if (maxDiff < b) maxDiff = b; 639 if (max < GETB(out[pixel])) max = GETB(out[pixel]); 640 temp += b; 641 642 if (maxDiff > 0xe0 && temp > 0xe0 + 0xd0 + 0x80) { 643 m_lightMask[pixel] = MAX(m_lightMask[pixel], out[pixel]); 644 645 f = 2*(GETR(out[pixel])-m_longMeanImage[3*pixel+0]); 646 f = CLAMP(f)/255.0; 647 if (f > m_alphaMap[4*pixel+0]) m_alphaMap[4*pixel+0] = f; 648 649 f = 2*(GETG(out[pixel])-m_longMeanImage[3*pixel+1]); 650 f = CLAMP(f)/255.0; 651 if (f > m_alphaMap[4*pixel+1]) m_alphaMap[4*pixel+1] = f; 652 653 f = 2*(GETB(out[pixel])-m_longMeanImage[3*pixel+2]); 654 f = CLAMP(f)/255.0; 655 if (f > m_alphaMap[4*pixel+2]) m_alphaMap[4*pixel+2] = f; 656 657 m_alphaMap[4*pixel+3] = 1; 658 } 659 r = 255.0*m_alphaMap[4*pixel+0]; 660 g = 255*m_alphaMap[4*pixel+1]; 661 b = 255*m_alphaMap[4*pixel+2]; 662 out[pixel] = RGBA(r,g,b,0xFF); 663 } 664 break; 665 case Graffiti_LongAvgAlpha: 666 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 667 668 r = 0x7f + (GETR(out[pixel]) - m_longMeanImage[3*pixel+0]); 669 r = CLAMP(r); 670 max = GETR(out[pixel]); 671 maxDiff = r; 672 temp = r; 673 674 g = 0x7f + (GETG(out[pixel]) - m_longMeanImage[3*pixel+1]); 675 g = CLAMP(g); 676 if (maxDiff < g) maxDiff = g; 677 if (max < GETG(out[pixel])) max = GETG(out[pixel]); 678 temp += g; 679 680 b = 0x7f + (GETB(out[pixel]) - m_longMeanImage[3*pixel+2]); 681 b = CLAMP(b); 682 if (maxDiff < b) maxDiff = b; 683 if (max < GETB(out[pixel])) max = GETB(out[pixel]); 684 temp += b; 685 686 if (maxDiff > 0xe0 && temp > 0xe0 + 0xd0 + 0x80) { 687 m_lightMask[pixel] = MAX(m_lightMask[pixel], out[pixel]); 688 689 f = 2*(GETR(out[pixel])-m_longMeanImage[3*pixel+0]); 690 f = CLAMP(f)/255.0; 691 f *= f; 692 if (f > m_alphaMap[4*pixel+0]) m_alphaMap[4*pixel+0] = f; 693 694 f = 2*(GETG(out[pixel])-m_longMeanImage[3*pixel+1]); 695 f = CLAMP(f)/255.0; 696 f *= f; 697 if (f > m_alphaMap[4*pixel+1]) m_alphaMap[4*pixel+1] = f; 698 699 f = 2*(GETB(out[pixel])-m_longMeanImage[3*pixel+2]); 700 f = CLAMP(f)/255.0; 701 f *= f; 702 if (f > m_alphaMap[4*pixel+2]) m_alphaMap[4*pixel+2] = f; 703 } 704 if (m_lightMask[pixel] != 0) { 705 r = SCREEN1(GETR(out[pixel]), m_alphaMap[4*pixel+0]*GETR(m_lightMask[pixel])); 706 g = SCREEN1(GETG(out[pixel]), m_alphaMap[4*pixel+1]*GETG(m_lightMask[pixel])); 707 b = SCREEN1(GETB(out[pixel]), m_alphaMap[4*pixel+2]*GETB(m_lightMask[pixel])); 708 r = CLAMP(r); 709 g = CLAMP(g); 710 b = CLAMP(b); 711 out[pixel] = RGBA(r,g,b,0xFF); 712 } 713 } 714 break; 715 case Graffiti_LongAvgAlphaCumC: 716 717 /** 718 Ideas (partially considered) to get a realistic look: 719 * Remember Hue if Saturation > 0.1 (below: Close to white, so Hue might be wrong → remember Saturation as well) 720 * Maximize Saturation for low alpha (opacity) 721 * Make alpha depend on the light source's brightness 722 * If alpha > 1: Simulate overexposure by going towards white 723 * If pixel is bright in another frame: Sum up alpha values (longer exposure) 724 Maybe: Logarithmic scale? → Overexposure becomes harder 725 log(alpha/factor + 1) or sqrt(alpha/factor) 726 */ 727 for (unsigned int pixel = 0; pixel < width*height; pixel++) { 728 729 /* 730 Light detection 731 */ 732 733 // maxDiff: Maximum difference to the mean image 734 // {-255,...,255} 735 // max: Maximum pixel value 736 // {0,...,255} 737 // temp: Sum of all differences 738 // {-3*255,...,3*255} 739 // sum: Sum of all pixel values 740 // {0,...,3*255} 741 742 r = GETR(out[pixel]) - m_longMeanImage[3*pixel+0]; 743 maxDiff = r; 744 max = GETR(out[pixel]); 745 temp = r; 746 747 g = GETG(out[pixel]) - m_longMeanImage[3*pixel+1]; 748 if (max < GETG(out[pixel])) { 749 max = GETG(out[pixel]); 750 } 751 if (maxDiff < g) { 752 maxDiff = g; 753 } 754 temp += g; 755 756 b = GETB(out[pixel]) - m_longMeanImage[3*pixel+2]; 757 if (max < GETB(out[pixel])) { 758 max = GETB(out[pixel]); 759 } 760 if (maxDiff < b) { 761 maxDiff = b; 762 } 763 temp += b; 764 765 sum = GETR(out[pixel]) + GETG(out[pixel]) + GETB(out[pixel]); 766 767 if ( 768 maxDiff > thresholdDifference 769 && temp > thresholdDiffSum 770 && sum > thresholdBrightness 771 // If all requirements are met, then this should be a light source. 772 ) 773 { 774 #ifdef LG_ADV 775 // Just add values as float. Overflows are highly unlikely (3.4E38+ frames ...). 776 fr = CLAMP(r)/255.0; 777 fg = CLAMP(g)/255.0; 778 fb = CLAMP(b)/255.0; 779 780 f = (fr + fg + fb) / 3 * sensitivity; 781 fr *= f; 782 fg *= f; 783 fb *= f; 784 785 #ifdef LG_NO_OVERLAY 786 // std::cout << "fr: " << fr << "; fg: " << fg << "; fb: " << fb << "\n"; 787 fr -= m_prevMask[pixel].r; 788 fg -= m_prevMask[pixel].g; 789 fb -= m_prevMask[pixel].b; 790 m_prevMask[pixel].r += fr; 791 m_prevMask[pixel].g += fg; 792 m_prevMask[pixel].b += fb; 793 // std::cout << "fr2: " << fr << "; fg2: " << fg << "; fb2: " << fb << "\n"; 794 if (fr < 0) { fr = 0; } 795 if (fg < 0) { fg = 0; } 796 if (fb < 0) { fb = 0; } 797 #endif 798 799 m_rgbLightMask[pixel].r += fr; 800 m_rgbLightMask[pixel].g += fg; 801 m_rgbLightMask[pixel].b += fb; 802 803 #else 804 // Store the «additional» light delivered by the light source in the light mask. 805 color = RGBA(CLAMP(r), CLAMP(g), CLAMP(b),0xFF); 806 m_lightMask[pixel] = MAX(m_lightMask[pixel], color); 807 808 // Add the brightness of the light source to the brightness map (alpha map) 809 y = REC709Y(CLAMP(r), CLAMP(g), CLAMP(b)) / 255.0; 810 y = y * sensitivity; 811 m_alphaMap[4*pixel] += y; 812 #endif 813 } else { 814 #ifdef LG_NO_OVERLAY 815 m_prevMask[pixel] = rgb0; 816 #endif 817 } 818 819 820 821 /* 822 Background weight 823 */ 824 if (m_pBackgroundWeight > 0) { 825 // Use part of the background mean. This allows one to have only lights appearing in the video 826 // if people or other objects walk into the video after the first frame (darker, therefore not in the light mask). 827 out[pixel] = RGBA((int) (m_pBackgroundWeight*m_longMeanImage[3*pixel+0] + (1-m_pBackgroundWeight)*GETR(out[pixel])), 828 (int) (m_pBackgroundWeight*m_longMeanImage[3*pixel+1] + (1-m_pBackgroundWeight)*GETG(out[pixel])), 829 (int) (m_pBackgroundWeight*m_longMeanImage[3*pixel+2] + (1-m_pBackgroundWeight)*GETB(out[pixel])), 830 0xFF); 831 } 832 833 834 /* 835 Adding light mask 836 */ 837 #ifdef LG_ADV 838 if ( 839 (m_rgbLightMask[pixel].r != 0 || m_rgbLightMask[pixel].g != 0 || m_rgbLightMask[pixel].b != 0) 840 && !m_pStatsBrightness && !m_pStatsDiff && !m_pStatsDiffSum 841 ) 842 { 843 844 fr = m_rgbLightMask[pixel].r; 845 fg = m_rgbLightMask[pixel].g; 846 fb = m_rgbLightMask[pixel].b; 847 848 if (lowerOverexposure > 0) { 849 // Comparisation of plots with octave: 850 // clf;hold on;plot([0 1],[0 1],'k');plot(range,ones(length(range),1),'k');plot(range,sqrt(range));plot(range,log(1+range),'k');plot(range,log(1+range),'g');plot(range,(log(1+range)/3).^.5,'r');axis equal 851 fr = pow( log(1+fr)/lowerOverexposure, .5 ); 852 fg = pow( log(1+fg)/lowerOverexposure, .5 ); 853 fb = pow( log(1+fb)/lowerOverexposure, .5 ); 854 } 855 856 857 // Calculate overflow between different colours: 858 // A very bright red light source will eventually overflow into other channels. 859 sr = 0; 860 sg = 0; 861 sb = 0; 862 if (fr > 1) { 863 sr += fr - 1; 864 } 865 if (fg > 1) { 866 sg += fg - 1; 867 } 868 if (fb > 1) { 869 sb += fb - 1; 870 } 871 fr += (sg + sb)/2; 872 fg += (sr + sb)/2; 873 fb += (sg + sb)/2; 874 if (fr > 1) { 875 fr = 1; 876 } 877 if (fg > 1) { 878 fg = 1; 879 } 880 if (fb > 1) { 881 fb = 1; 882 } 883 884 // Increase the saturation if the average brightness is below a certain level 885 // Do not use Rec709 Luma since we want to consider all colours to equal parts. 886 fy = (fr + fg + fb) / 3; 887 if (fy < 1 && saturation > 0) { 888 fsat = 1 + saturation*(1-fy); 889 890 fr = fy + fsat * (fr-fy); 891 fg = fy + fsat * (fg-fy); 892 fb = fy + fsat * (fb-fy); 893 } 894 895 // Paint the light on top of the image using addition 896 // Since brightness is equidistant in sRGB, this works fine. 897 r = 255*fr + GETR(out[pixel]); 898 g = 255*fg + GETG(out[pixel]); 899 b = 255*fb + GETB(out[pixel]); 900 r = CLAMP(r); 901 g = CLAMP(g); 902 b = CLAMP(b); 903 out[pixel] = RGBA(r,g,b,0xFF); 904 905 #ifdef LG_DEBUG 906 deCount++; 907 if (deCount < 10) { 908 std::cout << "r: " << m_rgbLightMask[pixel].r << ", fy: " << fy << ", fr: " << fr << ", sr: " << sr << ", R: " << r << ", inR: " << GETR(in[pixel]) << "\n"; 909 } 910 #endif 911 912 } else if (m_pTransparentBackground) { 913 // Transparent background 914 out[pixel] &= RGBA(0xFF, 0xFF, 0xFF, 0); 915 } 916 #else 917 if ( 918 m_lightMask[pixel] != 0 && m_alphaMap[4*pixel + 0] != 0 919 && !m_pStatsBrightness && !m_pStatsDiff && !m_pStatsDiffSum 920 ) 921 { 922 923 f = sqrt(m_alphaMap[4*pixel]); 924 925 r = f * GETR(m_lightMask[pixel]); 926 g = f * GETG(m_lightMask[pixel]); 927 b = f * GETB(m_lightMask[pixel]); 928 929 if (f > 1) { 930 // Simulate overexposure 931 sum = 0; 932 if (r > 255) { 933 sum += r-255; 934 } 935 if (g > 255) { 936 sum += g-255; 937 } 938 if (b > 255) { 939 sum += g-255; 940 } 941 942 if (sum > 0) { 943 sum = sum/10.0; 944 r += sum; 945 g += sum; 946 b += sum; 947 } 948 } else if (f < 1) { 949 // Lower exposure: Stronger colors 950 y = REC709Y(r,g,b); 951 float sat = 2.0; 952 953 r = y + sat * (r-y); 954 g = y + sat * (g-y); 955 b = y + sat * (b-y); 956 } 957 958 959 // Add the light map as additional light to the image 960 r += GETR(out[pixel]); 961 g += GETG(out[pixel]); 962 b += GETB(out[pixel]); 963 r = CLAMP(r); 964 g = CLAMP(g); 965 b = CLAMP(b); 966 out[pixel] = RGBA(r,g,b,0xFF); 967 } else if (m_pTransparentBackground) { 968 // Transparent background 969 out[pixel] &= RGBA(0xFF, 0xFF, 0xFF, 0); 970 } 971 #endif 972 973 974 /* 975 In-video statistics for easier parameter adjustment (thresholds) 976 */ 977 if (m_pStatsBrightness) { 978 // Show the image's brightness and highlight the threshold set by the user 979 980 // Limit maximum brightness to 80% for still being able to distinguish 981 // between «bright spot» (light grey) and «over the threshold» (blue) 982 r = .8*sum/3; 983 g = .8*sum/3; 984 b = .8*sum/3; 985 if (sum > thresholdBrightness) { 986 b = 255; 987 } 988 out[pixel] = RGBA(r,g,b,0xFF); 989 } 990 991 if (m_pStatsDiff) { 992 // As above, but for the brightness difference relative to the background. 993 r = .8*CLAMP(maxDiff); 994 g = r; 995 if (!m_pStatsBrightness) { 996 b = r; 997 } 998 999 if (maxDiff > thresholdDifference) { 1000 g = 255; 1001 } 1002 out[pixel] = RGBA(r,g,b,0xFF); 1003 } 1004 1005 if (m_pStatsDiffSum) { 1006 // As above, for the sum of the differences in each color channel. 1007 r = .8*CLAMP(temp/3.0); 1008 if (!m_pStatsDiff) { 1009 g = r; 1010 } 1011 if (!m_pStatsBrightness) { 1012 b = r; 1013 } 1014 if (temp > thresholdDiffSum) { 1015 r = 255; 1016 } 1017 out[pixel] = RGBA(r,g,b,0xFF); 1018 } 1019 } 1020 break; 1021 default: 1022 break; 1023 } 1024 } 1025 1026 private: 1027 std::vector<uint32_t> m_lightMask; 1028 std::vector<float> m_longMeanImage; 1029 std::vector<float> m_alphaMap; 1030 bool m_meanInitialized; 1031 GraffitiMode m_mode; 1032 DimMode m_dimMode; 1033 1034 #ifdef LG_ADV 1035 std::vector<RGBFloat> m_rgbLightMask; 1036 #endif 1037 #ifdef LG_NO_OVERLAY 1038 std::vector<RGBFloat> m_prevMask; 1039 #endif 1040 1041 double m_pLongAlpha; 1042 double m_pSensitivity; 1043 double m_pBackgroundWeight; 1044 double m_pThresholdBrightness; 1045 double m_pThresholdDifference; 1046 double m_pThresholdDiffSum; 1047 double m_pDim; 1048 double m_pSaturation; 1049 double m_pLowerOverexposure; 1050 bool m_pStatsBrightness; 1051 bool m_pStatsDiff; 1052 bool m_pStatsDiffSum; 1053 bool m_pTransparentBackground; 1054 bool m_pBlackReference; 1055 bool m_pNonlinearDim; 1056 bool m_pReset; 1057 1058 }; 1059 1060 1061 1062 frei0r::construct<LightGraffiti> plugin("Light Graffiti", 1063 "Creates light graffitis from a video by keeping the brightest spots.", 1064 "Simon A. Eugster (Granjow)", 1065 0,3, 1066 F0R_COLOR_MODEL_RGBA8888); 1067