1245f7b51SCorentin Chary /* 2245f7b51SCorentin Chary * QEMU VNC display driver: tight encoding 3245f7b51SCorentin Chary * 4245f7b51SCorentin Chary * From libvncserver/libvncserver/tight.c 5245f7b51SCorentin Chary * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved. 6245f7b51SCorentin Chary * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved. 7245f7b51SCorentin Chary * 8245f7b51SCorentin Chary * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com> 9245f7b51SCorentin Chary * 10245f7b51SCorentin Chary * Permission is hereby granted, free of charge, to any person obtaining a copy 11245f7b51SCorentin Chary * of this software and associated documentation files (the "Software"), to deal 12245f7b51SCorentin Chary * in the Software without restriction, including without limitation the rights 13245f7b51SCorentin Chary * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14245f7b51SCorentin Chary * copies of the Software, and to permit persons to whom the Software is 15245f7b51SCorentin Chary * furnished to do so, subject to the following conditions: 16245f7b51SCorentin Chary * 17245f7b51SCorentin Chary * The above copyright notice and this permission notice shall be included in 18245f7b51SCorentin Chary * all copies or substantial portions of the Software. 19245f7b51SCorentin Chary * 20245f7b51SCorentin Chary * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21245f7b51SCorentin Chary * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22245f7b51SCorentin Chary * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 23245f7b51SCorentin Chary * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24245f7b51SCorentin Chary * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25245f7b51SCorentin Chary * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 26245f7b51SCorentin Chary * THE SOFTWARE. 27245f7b51SCorentin Chary */ 28245f7b51SCorentin Chary 29e16f4c87SPeter Maydell #include "qemu/osdep.h" 30245f7b51SCorentin Chary 31f26e428dSRoy Tam /* This needs to be before jpeglib.h line because of conflict with 32f26e428dSRoy Tam INT32 definitions between jmorecfg.h (included by jpeglib.h) and 33f26e428dSRoy Tam Win32 basetsd.h (included by windows.h). */ 34f26e428dSRoy Tam 35*95f8510eSKshitij Suri #ifdef CONFIG_PNG 362fb0c09fSStefan Weil /* The following define is needed by pngconf.h. Otherwise it won't compile, 370f9668e0SMarc-André Lureau because setjmp.h was already included by osdep.h. */ 382fb0c09fSStefan Weil #define PNG_SKIP_SETJMP_CHECK 39efe556adSCorentin Chary #include <png.h> 40efe556adSCorentin Chary #endif 41245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG 42245f7b51SCorentin Chary #include <jpeglib.h> 43245f7b51SCorentin Chary #endif 44245f7b51SCorentin Chary 451de7afc9SPaolo Bonzini #include "qemu/bswap.h" 46245f7b51SCorentin Chary #include "vnc.h" 47245f7b51SCorentin Chary #include "vnc-enc-tight.h" 485136a052SCorentin Chary #include "vnc-palette.h" 49245f7b51SCorentin Chary 50245f7b51SCorentin Chary /* Compression level stuff. The following array contains various 51245f7b51SCorentin Chary encoder parameters for each of 10 compression levels (0..9). 52245f7b51SCorentin Chary Last three parameters correspond to JPEG quality levels (0..9). */ 53245f7b51SCorentin Chary 54245f7b51SCorentin Chary static const struct { 55245f7b51SCorentin Chary int max_rect_size, max_rect_width; 56245f7b51SCorentin Chary int mono_min_rect_size, gradient_min_rect_size; 57245f7b51SCorentin Chary int idx_zlib_level, mono_zlib_level, raw_zlib_level, gradient_zlib_level; 58245f7b51SCorentin Chary int gradient_threshold, gradient_threshold24; 59245f7b51SCorentin Chary int idx_max_colors_divisor; 60245f7b51SCorentin Chary int jpeg_quality, jpeg_threshold, jpeg_threshold24; 61245f7b51SCorentin Chary } tight_conf[] = { 62245f7b51SCorentin Chary { 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 5, 10000, 23000 }, 63245f7b51SCorentin Chary { 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 10, 8000, 18000 }, 64245f7b51SCorentin Chary { 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 15, 6500, 15000 }, 65245f7b51SCorentin Chary { 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 25, 5000, 12000 }, 66245f7b51SCorentin Chary { 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 37, 4000, 10000 }, 67245f7b51SCorentin Chary { 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 50, 3000, 8000 }, 68245f7b51SCorentin Chary { 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 60, 2000, 5000 }, 69245f7b51SCorentin Chary { 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 }, 70245f7b51SCorentin Chary { 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 }, 71245f7b51SCorentin Chary { 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 } 72245f7b51SCorentin Chary }; 73245f7b51SCorentin Chary 74efe556adSCorentin Chary 75efe556adSCorentin Chary static int tight_send_framebuffer_update(VncState *vs, int x, int y, 76efe556adSCorentin Chary int w, int h); 77efe556adSCorentin Chary 78ce702e93SCorentin Chary #ifdef CONFIG_VNC_JPEG 79ce702e93SCorentin Chary static const struct { 80ce702e93SCorentin Chary double jpeg_freq_min; /* Don't send JPEG if the freq is bellow */ 81ce702e93SCorentin Chary double jpeg_freq_threshold; /* Always send JPEG if the freq is above */ 82ce702e93SCorentin Chary int jpeg_idx; /* Allow indexed JPEG */ 83ce702e93SCorentin Chary int jpeg_full; /* Allow full color JPEG */ 84ce702e93SCorentin Chary } tight_jpeg_conf[] = { 858cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 868cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 878cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 888cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 898cb4a6b7SCorentin Chary { 0, 10, 1, 1 }, 908cb4a6b7SCorentin Chary { 0.1, 10, 1, 1 }, 918cb4a6b7SCorentin Chary { 0.2, 10, 1, 1 }, 928cb4a6b7SCorentin Chary { 0.3, 12, 0, 0 }, 938cb4a6b7SCorentin Chary { 0.4, 14, 0, 0 }, 948cb4a6b7SCorentin Chary { 0.5, 16, 0, 0 }, 95ce702e93SCorentin Chary }; 96ce702e93SCorentin Chary #endif 97ce702e93SCorentin Chary 98*95f8510eSKshitij Suri #ifdef CONFIG_PNG 993941bf6fSCorentin Chary static const struct { 1003941bf6fSCorentin Chary int png_zlib_level, png_filters; 1013941bf6fSCorentin Chary } tight_png_conf[] = { 1023941bf6fSCorentin Chary { 0, PNG_NO_FILTERS }, 1033941bf6fSCorentin Chary { 1, PNG_NO_FILTERS }, 1043941bf6fSCorentin Chary { 2, PNG_NO_FILTERS }, 1053941bf6fSCorentin Chary { 3, PNG_NO_FILTERS }, 1063941bf6fSCorentin Chary { 4, PNG_NO_FILTERS }, 1073941bf6fSCorentin Chary { 5, PNG_ALL_FILTERS }, 1083941bf6fSCorentin Chary { 6, PNG_ALL_FILTERS }, 1093941bf6fSCorentin Chary { 7, PNG_ALL_FILTERS }, 1103941bf6fSCorentin Chary { 8, PNG_ALL_FILTERS }, 1113941bf6fSCorentin Chary { 9, PNG_ALL_FILTERS }, 1123941bf6fSCorentin Chary }; 1133941bf6fSCorentin Chary 114efe556adSCorentin Chary static int send_png_rect(VncState *vs, int x, int y, int w, int h, 1155136a052SCorentin Chary VncPalette *palette); 116efe556adSCorentin Chary 117efe556adSCorentin Chary static bool tight_can_send_png_rect(VncState *vs, int w, int h) 118efe556adSCorentin Chary { 1196bf21f3dSLi Qiang if (vs->tight->type != VNC_ENCODING_TIGHT_PNG) { 120efe556adSCorentin Chary return false; 121efe556adSCorentin Chary } 122efe556adSCorentin Chary 123d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1 || 1249f64916dSGerd Hoffmann vs->client_pf.bytes_per_pixel == 1) { 125efe556adSCorentin Chary return false; 126efe556adSCorentin Chary } 127efe556adSCorentin Chary 128efe556adSCorentin Chary return true; 129efe556adSCorentin Chary } 130efe556adSCorentin Chary #endif 131efe556adSCorentin Chary 132245f7b51SCorentin Chary /* 133245f7b51SCorentin Chary * Code to guess if given rectangle is suitable for smooth image 134245f7b51SCorentin Chary * compression (by applying "gradient" filter or JPEG coder). 135245f7b51SCorentin Chary */ 136245f7b51SCorentin Chary 137249cdb42SBlue Swirl static unsigned int 138245f7b51SCorentin Chary tight_detect_smooth_image24(VncState *vs, int w, int h) 139245f7b51SCorentin Chary { 140245f7b51SCorentin Chary int off; 141245f7b51SCorentin Chary int x, y, d, dx; 142249cdb42SBlue Swirl unsigned int c; 143249cdb42SBlue Swirl unsigned int stats[256]; 144245f7b51SCorentin Chary int pixels = 0; 145245f7b51SCorentin Chary int pix, left[3]; 146249cdb42SBlue Swirl unsigned int errors; 1476bf21f3dSLi Qiang unsigned char *buf = vs->tight->tight.buffer; 148245f7b51SCorentin Chary 149245f7b51SCorentin Chary /* 150245f7b51SCorentin Chary * If client is big-endian, color samples begin from the second 151245f7b51SCorentin Chary * byte (offset 1) of a 32-bit pixel value. 152245f7b51SCorentin Chary */ 1539f64916dSGerd Hoffmann off = vs->client_be; 154245f7b51SCorentin Chary 155245f7b51SCorentin Chary memset(stats, 0, sizeof (stats)); 156245f7b51SCorentin Chary 157245f7b51SCorentin Chary for (y = 0, x = 0; y < h && x < w;) { 158245f7b51SCorentin Chary for (d = 0; d < h - y && d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; 159245f7b51SCorentin Chary d++) { 160245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 161245f7b51SCorentin Chary left[c] = buf[((y+d)*w+x+d)*4+off+c] & 0xFF; 162245f7b51SCorentin Chary } 163245f7b51SCorentin Chary for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; dx++) { 164245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 165245f7b51SCorentin Chary pix = buf[((y+d)*w+x+d+dx)*4+off+c] & 0xFF; 166245f7b51SCorentin Chary stats[abs(pix - left[c])]++; 167245f7b51SCorentin Chary left[c] = pix; 168245f7b51SCorentin Chary } 169245f7b51SCorentin Chary pixels++; 170245f7b51SCorentin Chary } 171245f7b51SCorentin Chary } 172245f7b51SCorentin Chary if (w > h) { 173245f7b51SCorentin Chary x += h; 174245f7b51SCorentin Chary y = 0; 175245f7b51SCorentin Chary } else { 176245f7b51SCorentin Chary x = 0; 177245f7b51SCorentin Chary y += w; 178245f7b51SCorentin Chary } 179245f7b51SCorentin Chary } 180245f7b51SCorentin Chary 181b5299153SGonglei if (pixels == 0) { 182b5299153SGonglei return 0; 183b5299153SGonglei } 184b5299153SGonglei 185245f7b51SCorentin Chary /* 95% smooth or more ... */ 186245f7b51SCorentin Chary if (stats[0] * 33 / pixels >= 95) { 187245f7b51SCorentin Chary return 0; 188245f7b51SCorentin Chary } 189245f7b51SCorentin Chary 190245f7b51SCorentin Chary errors = 0; 191245f7b51SCorentin Chary for (c = 1; c < 8; c++) { 192245f7b51SCorentin Chary errors += stats[c] * (c * c); 193245f7b51SCorentin Chary if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { 194245f7b51SCorentin Chary return 0; 195245f7b51SCorentin Chary } 196245f7b51SCorentin Chary } 197245f7b51SCorentin Chary for (; c < 256; c++) { 198245f7b51SCorentin Chary errors += stats[c] * (c * c); 199245f7b51SCorentin Chary } 200245f7b51SCorentin Chary errors /= (pixels * 3 - stats[0]); 201245f7b51SCorentin Chary 202245f7b51SCorentin Chary return errors; 203245f7b51SCorentin Chary } 204245f7b51SCorentin Chary 205245f7b51SCorentin Chary #define DEFINE_DETECT_FUNCTION(bpp) \ 206245f7b51SCorentin Chary \ 207249cdb42SBlue Swirl static unsigned int \ 208245f7b51SCorentin Chary tight_detect_smooth_image##bpp(VncState *vs, int w, int h) { \ 209245f7b51SCorentin Chary bool endian; \ 210245f7b51SCorentin Chary uint##bpp##_t pix; \ 211245f7b51SCorentin Chary int max[3], shift[3]; \ 212245f7b51SCorentin Chary int x, y, d, dx; \ 213249cdb42SBlue Swirl unsigned int c; \ 214249cdb42SBlue Swirl unsigned int stats[256]; \ 215245f7b51SCorentin Chary int pixels = 0; \ 216245f7b51SCorentin Chary int sample, sum, left[3]; \ 217249cdb42SBlue Swirl unsigned int errors; \ 2186bf21f3dSLi Qiang unsigned char *buf = vs->tight->tight.buffer; \ 219245f7b51SCorentin Chary \ 22077bfcf28SBenjamin Herrenschmidt endian = 0; /* FIXME */ \ 221245f7b51SCorentin Chary \ 222245f7b51SCorentin Chary \ 2239f64916dSGerd Hoffmann max[0] = vs->client_pf.rmax; \ 2249f64916dSGerd Hoffmann max[1] = vs->client_pf.gmax; \ 2259f64916dSGerd Hoffmann max[2] = vs->client_pf.bmax; \ 2269f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; \ 2279f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; \ 2289f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; \ 229245f7b51SCorentin Chary \ 230245f7b51SCorentin Chary memset(stats, 0, sizeof(stats)); \ 231245f7b51SCorentin Chary \ 232245f7b51SCorentin Chary y = 0, x = 0; \ 233245f7b51SCorentin Chary while (y < h && x < w) { \ 234245f7b51SCorentin Chary for (d = 0; d < h - y && \ 235245f7b51SCorentin Chary d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; d++) { \ 236245f7b51SCorentin Chary pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d]; \ 237245f7b51SCorentin Chary if (endian) { \ 238ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \ 239245f7b51SCorentin Chary } \ 240245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 241245f7b51SCorentin Chary left[c] = (int)(pix >> shift[c] & max[c]); \ 242245f7b51SCorentin Chary } \ 243245f7b51SCorentin Chary for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; \ 244245f7b51SCorentin Chary dx++) { \ 245245f7b51SCorentin Chary pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d+dx]; \ 246245f7b51SCorentin Chary if (endian) { \ 247ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \ 248245f7b51SCorentin Chary } \ 249245f7b51SCorentin Chary sum = 0; \ 250245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 251245f7b51SCorentin Chary sample = (int)(pix >> shift[c] & max[c]); \ 252245f7b51SCorentin Chary sum += abs(sample - left[c]); \ 253245f7b51SCorentin Chary left[c] = sample; \ 254245f7b51SCorentin Chary } \ 255245f7b51SCorentin Chary if (sum > 255) { \ 256245f7b51SCorentin Chary sum = 255; \ 257245f7b51SCorentin Chary } \ 258245f7b51SCorentin Chary stats[sum]++; \ 259245f7b51SCorentin Chary pixels++; \ 260245f7b51SCorentin Chary } \ 261245f7b51SCorentin Chary } \ 262245f7b51SCorentin Chary if (w > h) { \ 263245f7b51SCorentin Chary x += h; \ 264245f7b51SCorentin Chary y = 0; \ 265245f7b51SCorentin Chary } else { \ 266245f7b51SCorentin Chary x = 0; \ 267245f7b51SCorentin Chary y += w; \ 268245f7b51SCorentin Chary } \ 269245f7b51SCorentin Chary } \ 270b5299153SGonglei if (pixels == 0) { \ 271b5299153SGonglei return 0; \ 272b5299153SGonglei } \ 273245f7b51SCorentin Chary if ((stats[0] + stats[1]) * 100 / pixels >= 90) { \ 274245f7b51SCorentin Chary return 0; \ 275245f7b51SCorentin Chary } \ 276245f7b51SCorentin Chary \ 277245f7b51SCorentin Chary errors = 0; \ 278245f7b51SCorentin Chary for (c = 1; c < 8; c++) { \ 279245f7b51SCorentin Chary errors += stats[c] * (c * c); \ 280245f7b51SCorentin Chary if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { \ 281245f7b51SCorentin Chary return 0; \ 282245f7b51SCorentin Chary } \ 283245f7b51SCorentin Chary } \ 284245f7b51SCorentin Chary for (; c < 256; c++) { \ 285245f7b51SCorentin Chary errors += stats[c] * (c * c); \ 286245f7b51SCorentin Chary } \ 287245f7b51SCorentin Chary errors /= (pixels - stats[0]); \ 288245f7b51SCorentin Chary \ 289245f7b51SCorentin Chary return errors; \ 290245f7b51SCorentin Chary } 291245f7b51SCorentin Chary 292245f7b51SCorentin Chary DEFINE_DETECT_FUNCTION(16) 293245f7b51SCorentin Chary DEFINE_DETECT_FUNCTION(32) 294245f7b51SCorentin Chary 295245f7b51SCorentin Chary static int 296245f7b51SCorentin Chary tight_detect_smooth_image(VncState *vs, int w, int h) 297245f7b51SCorentin Chary { 298249cdb42SBlue Swirl unsigned int errors; 2996bf21f3dSLi Qiang int compression = vs->tight->compression; 3006bf21f3dSLi Qiang int quality = vs->tight->quality; 301245f7b51SCorentin Chary 302245f7b51SCorentin Chary if (!vs->vd->lossy) { 303245f7b51SCorentin Chary return 0; 304245f7b51SCorentin Chary } 305245f7b51SCorentin Chary 306d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1 || 3079f64916dSGerd Hoffmann vs->client_pf.bytes_per_pixel == 1 || 308245f7b51SCorentin Chary w < VNC_TIGHT_DETECT_MIN_WIDTH || h < VNC_TIGHT_DETECT_MIN_HEIGHT) { 309245f7b51SCorentin Chary return 0; 310245f7b51SCorentin Chary } 311245f7b51SCorentin Chary 3126bf21f3dSLi Qiang if (vs->tight->quality != (uint8_t)-1) { 313245f7b51SCorentin Chary if (w * h < VNC_TIGHT_JPEG_MIN_RECT_SIZE) { 314245f7b51SCorentin Chary return 0; 315245f7b51SCorentin Chary } 316245f7b51SCorentin Chary } else { 317245f7b51SCorentin Chary if (w * h < tight_conf[compression].gradient_min_rect_size) { 318245f7b51SCorentin Chary return 0; 319245f7b51SCorentin Chary } 320245f7b51SCorentin Chary } 321245f7b51SCorentin Chary 3229f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4) { 3236bf21f3dSLi Qiang if (vs->tight->pixel24) { 324245f7b51SCorentin Chary errors = tight_detect_smooth_image24(vs, w, h); 3256bf21f3dSLi Qiang if (vs->tight->quality != (uint8_t)-1) { 326245f7b51SCorentin Chary return (errors < tight_conf[quality].jpeg_threshold24); 327245f7b51SCorentin Chary } 328245f7b51SCorentin Chary return (errors < tight_conf[compression].gradient_threshold24); 329245f7b51SCorentin Chary } else { 330245f7b51SCorentin Chary errors = tight_detect_smooth_image32(vs, w, h); 331245f7b51SCorentin Chary } 332245f7b51SCorentin Chary } else { 333245f7b51SCorentin Chary errors = tight_detect_smooth_image16(vs, w, h); 334245f7b51SCorentin Chary } 3352e7bcdb9SMarkus Armbruster if (quality != (uint8_t)-1) { 336245f7b51SCorentin Chary return (errors < tight_conf[quality].jpeg_threshold); 337245f7b51SCorentin Chary } 338245f7b51SCorentin Chary return (errors < tight_conf[compression].gradient_threshold); 339245f7b51SCorentin Chary } 340245f7b51SCorentin Chary 341245f7b51SCorentin Chary /* 342245f7b51SCorentin Chary * Code to determine how many different colors used in rectangle. 343245f7b51SCorentin Chary */ 344245f7b51SCorentin Chary #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \ 345245f7b51SCorentin Chary \ 346245f7b51SCorentin Chary static int \ 347245f7b51SCorentin Chary tight_fill_palette##bpp(VncState *vs, int x, int y, \ 348245f7b51SCorentin Chary int max, size_t count, \ 349245f7b51SCorentin Chary uint32_t *bg, uint32_t *fg, \ 350095497ffSPeter Lieven VncPalette *palette) { \ 351245f7b51SCorentin Chary uint##bpp##_t *data; \ 352245f7b51SCorentin Chary uint##bpp##_t c0, c1, ci; \ 353245f7b51SCorentin Chary int i, n0, n1; \ 354245f7b51SCorentin Chary \ 3556bf21f3dSLi Qiang data = (uint##bpp##_t *)vs->tight->tight.buffer; \ 356245f7b51SCorentin Chary \ 357245f7b51SCorentin Chary c0 = data[0]; \ 358245f7b51SCorentin Chary i = 1; \ 359245f7b51SCorentin Chary while (i < count && data[i] == c0) \ 360245f7b51SCorentin Chary i++; \ 361245f7b51SCorentin Chary if (i >= count) { \ 362245f7b51SCorentin Chary *bg = *fg = c0; \ 363245f7b51SCorentin Chary return 1; \ 364245f7b51SCorentin Chary } \ 365245f7b51SCorentin Chary \ 366245f7b51SCorentin Chary if (max < 2) { \ 367245f7b51SCorentin Chary return 0; \ 368245f7b51SCorentin Chary } \ 369245f7b51SCorentin Chary \ 370245f7b51SCorentin Chary n0 = i; \ 371245f7b51SCorentin Chary c1 = data[i]; \ 372245f7b51SCorentin Chary n1 = 0; \ 373245f7b51SCorentin Chary for (i++; i < count; i++) { \ 374245f7b51SCorentin Chary ci = data[i]; \ 375245f7b51SCorentin Chary if (ci == c0) { \ 376245f7b51SCorentin Chary n0++; \ 377245f7b51SCorentin Chary } else if (ci == c1) { \ 378245f7b51SCorentin Chary n1++; \ 379245f7b51SCorentin Chary } else \ 380245f7b51SCorentin Chary break; \ 381245f7b51SCorentin Chary } \ 382245f7b51SCorentin Chary if (i >= count) { \ 383245f7b51SCorentin Chary if (n0 > n1) { \ 384245f7b51SCorentin Chary *bg = (uint32_t)c0; \ 385245f7b51SCorentin Chary *fg = (uint32_t)c1; \ 386245f7b51SCorentin Chary } else { \ 387245f7b51SCorentin Chary *bg = (uint32_t)c1; \ 388245f7b51SCorentin Chary *fg = (uint32_t)c0; \ 389245f7b51SCorentin Chary } \ 390245f7b51SCorentin Chary return 2; \ 391245f7b51SCorentin Chary } \ 392245f7b51SCorentin Chary \ 393245f7b51SCorentin Chary if (max == 2) { \ 394245f7b51SCorentin Chary return 0; \ 395245f7b51SCorentin Chary } \ 396245f7b51SCorentin Chary \ 397095497ffSPeter Lieven palette_init(palette, max, bpp); \ 398095497ffSPeter Lieven palette_put(palette, c0); \ 399095497ffSPeter Lieven palette_put(palette, c1); \ 400095497ffSPeter Lieven palette_put(palette, ci); \ 401245f7b51SCorentin Chary \ 402245f7b51SCorentin Chary for (i++; i < count; i++) { \ 403245f7b51SCorentin Chary if (data[i] == ci) { \ 404245f7b51SCorentin Chary continue; \ 405245f7b51SCorentin Chary } else { \ 4065d8efe39SCorentin Chary ci = data[i]; \ 407095497ffSPeter Lieven if (!palette_put(palette, (uint32_t)ci)) { \ 408245f7b51SCorentin Chary return 0; \ 409245f7b51SCorentin Chary } \ 410245f7b51SCorentin Chary } \ 411245f7b51SCorentin Chary } \ 412245f7b51SCorentin Chary \ 413095497ffSPeter Lieven return palette_size(palette); \ 414245f7b51SCorentin Chary } 415245f7b51SCorentin Chary 416245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(8) 417245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(16) 418245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(32) 419245f7b51SCorentin Chary 420245f7b51SCorentin Chary static int tight_fill_palette(VncState *vs, int x, int y, 421245f7b51SCorentin Chary size_t count, uint32_t *bg, uint32_t *fg, 422095497ffSPeter Lieven VncPalette *palette) 423245f7b51SCorentin Chary { 424245f7b51SCorentin Chary int max; 425245f7b51SCorentin Chary 4266bf21f3dSLi Qiang max = count / tight_conf[vs->tight->compression].idx_max_colors_divisor; 427245f7b51SCorentin Chary if (max < 2 && 4286bf21f3dSLi Qiang count >= tight_conf[vs->tight->compression].mono_min_rect_size) { 429245f7b51SCorentin Chary max = 2; 430245f7b51SCorentin Chary } 431245f7b51SCorentin Chary if (max >= 256) { 432245f7b51SCorentin Chary max = 256; 433245f7b51SCorentin Chary } 434245f7b51SCorentin Chary 4359f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) { 436245f7b51SCorentin Chary case 4: 437245f7b51SCorentin Chary return tight_fill_palette32(vs, x, y, max, count, bg, fg, palette); 438245f7b51SCorentin Chary case 2: 439245f7b51SCorentin Chary return tight_fill_palette16(vs, x, y, max, count, bg, fg, palette); 440245f7b51SCorentin Chary default: 441245f7b51SCorentin Chary max = 2; 442245f7b51SCorentin Chary return tight_fill_palette8(vs, x, y, max, count, bg, fg, palette); 443245f7b51SCorentin Chary } 444245f7b51SCorentin Chary return 0; 445245f7b51SCorentin Chary } 446245f7b51SCorentin Chary 447245f7b51SCorentin Chary /* 448245f7b51SCorentin Chary * Converting truecolor samples into palette indices. 449245f7b51SCorentin Chary */ 450245f7b51SCorentin Chary #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \ 451245f7b51SCorentin Chary \ 452245f7b51SCorentin Chary static void \ 453245f7b51SCorentin Chary tight_encode_indexed_rect##bpp(uint8_t *buf, int count, \ 4545136a052SCorentin Chary VncPalette *palette) { \ 455245f7b51SCorentin Chary uint##bpp##_t *src; \ 456245f7b51SCorentin Chary uint##bpp##_t rgb; \ 457245f7b51SCorentin Chary int i, rep; \ 458245f7b51SCorentin Chary uint8_t idx; \ 459245f7b51SCorentin Chary \ 460245f7b51SCorentin Chary src = (uint##bpp##_t *) buf; \ 461245f7b51SCorentin Chary \ 4623f7e51bcSHerongguang (Stephen) for (i = 0; i < count; ) { \ 463efe556adSCorentin Chary \ 464245f7b51SCorentin Chary rgb = *src++; \ 4653f7e51bcSHerongguang (Stephen) i++; \ 466245f7b51SCorentin Chary rep = 0; \ 467245f7b51SCorentin Chary while (i < count && *src == rgb) { \ 468245f7b51SCorentin Chary rep++, src++, i++; \ 469245f7b51SCorentin Chary } \ 4705136a052SCorentin Chary idx = palette_idx(palette, rgb); \ 471245f7b51SCorentin Chary /* \ 472245f7b51SCorentin Chary * Should never happen, but don't break everything \ 473245f7b51SCorentin Chary * if it does, use the first color instead \ 474245f7b51SCorentin Chary */ \ 4757bccf573SBlue Swirl if (idx == (uint8_t)-1) { \ 476245f7b51SCorentin Chary idx = 0; \ 477245f7b51SCorentin Chary } \ 478245f7b51SCorentin Chary while (rep >= 0) { \ 479245f7b51SCorentin Chary *buf++ = idx; \ 480245f7b51SCorentin Chary rep--; \ 481245f7b51SCorentin Chary } \ 482245f7b51SCorentin Chary } \ 483245f7b51SCorentin Chary } 484245f7b51SCorentin Chary 485245f7b51SCorentin Chary DEFINE_IDX_ENCODE_FUNCTION(16) 486245f7b51SCorentin Chary DEFINE_IDX_ENCODE_FUNCTION(32) 487245f7b51SCorentin Chary 488245f7b51SCorentin Chary #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \ 489245f7b51SCorentin Chary \ 490245f7b51SCorentin Chary static void \ 491245f7b51SCorentin Chary tight_encode_mono_rect##bpp(uint8_t *buf, int w, int h, \ 492245f7b51SCorentin Chary uint##bpp##_t bg, uint##bpp##_t fg) { \ 493245f7b51SCorentin Chary uint##bpp##_t *ptr; \ 494245f7b51SCorentin Chary unsigned int value, mask; \ 495245f7b51SCorentin Chary int aligned_width; \ 496245f7b51SCorentin Chary int x, y, bg_bits; \ 497245f7b51SCorentin Chary \ 498245f7b51SCorentin Chary ptr = (uint##bpp##_t *) buf; \ 499245f7b51SCorentin Chary aligned_width = w - w % 8; \ 500245f7b51SCorentin Chary \ 501245f7b51SCorentin Chary for (y = 0; y < h; y++) { \ 502245f7b51SCorentin Chary for (x = 0; x < aligned_width; x += 8) { \ 503245f7b51SCorentin Chary for (bg_bits = 0; bg_bits < 8; bg_bits++) { \ 504245f7b51SCorentin Chary if (*ptr++ != bg) { \ 505245f7b51SCorentin Chary break; \ 506245f7b51SCorentin Chary } \ 507245f7b51SCorentin Chary } \ 508245f7b51SCorentin Chary if (bg_bits == 8) { \ 509245f7b51SCorentin Chary *buf++ = 0; \ 510245f7b51SCorentin Chary continue; \ 511245f7b51SCorentin Chary } \ 512245f7b51SCorentin Chary mask = 0x80 >> bg_bits; \ 513245f7b51SCorentin Chary value = mask; \ 514245f7b51SCorentin Chary for (bg_bits++; bg_bits < 8; bg_bits++) { \ 515245f7b51SCorentin Chary mask >>= 1; \ 516245f7b51SCorentin Chary if (*ptr++ != bg) { \ 517245f7b51SCorentin Chary value |= mask; \ 518245f7b51SCorentin Chary } \ 519245f7b51SCorentin Chary } \ 520245f7b51SCorentin Chary *buf++ = (uint8_t)value; \ 521245f7b51SCorentin Chary } \ 522245f7b51SCorentin Chary \ 523245f7b51SCorentin Chary mask = 0x80; \ 524245f7b51SCorentin Chary value = 0; \ 525245f7b51SCorentin Chary if (x >= w) { \ 526245f7b51SCorentin Chary continue; \ 527245f7b51SCorentin Chary } \ 528245f7b51SCorentin Chary \ 529245f7b51SCorentin Chary for (; x < w; x++) { \ 530245f7b51SCorentin Chary if (*ptr++ != bg) { \ 531245f7b51SCorentin Chary value |= mask; \ 532245f7b51SCorentin Chary } \ 533245f7b51SCorentin Chary mask >>= 1; \ 534245f7b51SCorentin Chary } \ 535245f7b51SCorentin Chary *buf++ = (uint8_t)value; \ 536245f7b51SCorentin Chary } \ 537245f7b51SCorentin Chary } 538245f7b51SCorentin Chary 539245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(8) 540245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(16) 541245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(32) 542245f7b51SCorentin Chary 543245f7b51SCorentin Chary /* 544245f7b51SCorentin Chary * ``Gradient'' filter for 24-bit color samples. 545245f7b51SCorentin Chary * Should be called only when redMax, greenMax and blueMax are 255. 546245f7b51SCorentin Chary * Color components assumed to be byte-aligned. 547245f7b51SCorentin Chary */ 548245f7b51SCorentin Chary 549245f7b51SCorentin Chary static void 550245f7b51SCorentin Chary tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h) 551245f7b51SCorentin Chary { 552245f7b51SCorentin Chary uint32_t *buf32; 553245f7b51SCorentin Chary uint32_t pix32; 554245f7b51SCorentin Chary int shift[3]; 555245f7b51SCorentin Chary int *prev; 556245f7b51SCorentin Chary int here[3], upper[3], left[3], upperleft[3]; 557245f7b51SCorentin Chary int prediction; 558245f7b51SCorentin Chary int x, y, c; 559245f7b51SCorentin Chary 560245f7b51SCorentin Chary buf32 = (uint32_t *)buf; 5616bf21f3dSLi Qiang memset(vs->tight->gradient.buffer, 0, w * 3 * sizeof(int)); 562245f7b51SCorentin Chary 56377bfcf28SBenjamin Herrenschmidt if (1 /* FIXME */) { 5649f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; 5659f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; 5669f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; 567245f7b51SCorentin Chary } else { 5689f64916dSGerd Hoffmann shift[0] = 24 - vs->client_pf.rshift; 5699f64916dSGerd Hoffmann shift[1] = 24 - vs->client_pf.gshift; 5709f64916dSGerd Hoffmann shift[2] = 24 - vs->client_pf.bshift; 571245f7b51SCorentin Chary } 572245f7b51SCorentin Chary 573245f7b51SCorentin Chary for (y = 0; y < h; y++) { 574245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 575245f7b51SCorentin Chary upper[c] = 0; 576245f7b51SCorentin Chary here[c] = 0; 577245f7b51SCorentin Chary } 5786bf21f3dSLi Qiang prev = (int *)vs->tight->gradient.buffer; 579245f7b51SCorentin Chary for (x = 0; x < w; x++) { 580245f7b51SCorentin Chary pix32 = *buf32++; 581245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 582245f7b51SCorentin Chary upperleft[c] = upper[c]; 583245f7b51SCorentin Chary left[c] = here[c]; 584245f7b51SCorentin Chary upper[c] = *prev; 585245f7b51SCorentin Chary here[c] = (int)(pix32 >> shift[c] & 0xFF); 586245f7b51SCorentin Chary *prev++ = here[c]; 587245f7b51SCorentin Chary 588245f7b51SCorentin Chary prediction = left[c] + upper[c] - upperleft[c]; 589245f7b51SCorentin Chary if (prediction < 0) { 590245f7b51SCorentin Chary prediction = 0; 591245f7b51SCorentin Chary } else if (prediction > 0xFF) { 592245f7b51SCorentin Chary prediction = 0xFF; 593245f7b51SCorentin Chary } 594245f7b51SCorentin Chary *buf++ = (char)(here[c] - prediction); 595245f7b51SCorentin Chary } 596245f7b51SCorentin Chary } 597245f7b51SCorentin Chary } 598245f7b51SCorentin Chary } 599245f7b51SCorentin Chary 600245f7b51SCorentin Chary 601245f7b51SCorentin Chary /* 602245f7b51SCorentin Chary * ``Gradient'' filter for other color depths. 603245f7b51SCorentin Chary */ 604245f7b51SCorentin Chary 605245f7b51SCorentin Chary #define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \ 606245f7b51SCorentin Chary \ 607245f7b51SCorentin Chary static void \ 608245f7b51SCorentin Chary tight_filter_gradient##bpp(VncState *vs, uint##bpp##_t *buf, \ 609245f7b51SCorentin Chary int w, int h) { \ 610245f7b51SCorentin Chary uint##bpp##_t pix, diff; \ 611245f7b51SCorentin Chary bool endian; \ 612245f7b51SCorentin Chary int *prev; \ 613245f7b51SCorentin Chary int max[3], shift[3]; \ 614245f7b51SCorentin Chary int here[3], upper[3], left[3], upperleft[3]; \ 615245f7b51SCorentin Chary int prediction; \ 616245f7b51SCorentin Chary int x, y, c; \ 617245f7b51SCorentin Chary \ 6186bf21f3dSLi Qiang memset(vs->tight->gradient.buffer, 0, w * 3 * sizeof(int)); \ 619245f7b51SCorentin Chary \ 62077bfcf28SBenjamin Herrenschmidt endian = 0; /* FIXME */ \ 621245f7b51SCorentin Chary \ 6229f64916dSGerd Hoffmann max[0] = vs->client_pf.rmax; \ 6239f64916dSGerd Hoffmann max[1] = vs->client_pf.gmax; \ 6249f64916dSGerd Hoffmann max[2] = vs->client_pf.bmax; \ 6259f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; \ 6269f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; \ 6279f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; \ 628245f7b51SCorentin Chary \ 629245f7b51SCorentin Chary for (y = 0; y < h; y++) { \ 630245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 631245f7b51SCorentin Chary upper[c] = 0; \ 632245f7b51SCorentin Chary here[c] = 0; \ 633245f7b51SCorentin Chary } \ 6346bf21f3dSLi Qiang prev = (int *)vs->tight->gradient.buffer; \ 635245f7b51SCorentin Chary for (x = 0; x < w; x++) { \ 636245f7b51SCorentin Chary pix = *buf; \ 637245f7b51SCorentin Chary if (endian) { \ 638ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \ 639245f7b51SCorentin Chary } \ 640245f7b51SCorentin Chary diff = 0; \ 641245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 642245f7b51SCorentin Chary upperleft[c] = upper[c]; \ 643245f7b51SCorentin Chary left[c] = here[c]; \ 644245f7b51SCorentin Chary upper[c] = *prev; \ 645245f7b51SCorentin Chary here[c] = (int)(pix >> shift[c] & max[c]); \ 646245f7b51SCorentin Chary *prev++ = here[c]; \ 647245f7b51SCorentin Chary \ 648245f7b51SCorentin Chary prediction = left[c] + upper[c] - upperleft[c]; \ 649245f7b51SCorentin Chary if (prediction < 0) { \ 650245f7b51SCorentin Chary prediction = 0; \ 651245f7b51SCorentin Chary } else if (prediction > max[c]) { \ 652245f7b51SCorentin Chary prediction = max[c]; \ 653245f7b51SCorentin Chary } \ 654245f7b51SCorentin Chary diff |= ((here[c] - prediction) & max[c]) \ 655245f7b51SCorentin Chary << shift[c]; \ 656245f7b51SCorentin Chary } \ 657245f7b51SCorentin Chary if (endian) { \ 658ba5e7f82SIzumi Tsutsui diff = bswap##bpp(diff); \ 659245f7b51SCorentin Chary } \ 660245f7b51SCorentin Chary *buf++ = diff; \ 661245f7b51SCorentin Chary } \ 662245f7b51SCorentin Chary } \ 663245f7b51SCorentin Chary } 664245f7b51SCorentin Chary 665245f7b51SCorentin Chary DEFINE_GRADIENT_FILTER_FUNCTION(16) 666245f7b51SCorentin Chary DEFINE_GRADIENT_FILTER_FUNCTION(32) 667245f7b51SCorentin Chary 668245f7b51SCorentin Chary /* 669245f7b51SCorentin Chary * Check if a rectangle is all of the same color. If needSameColor is 670245f7b51SCorentin Chary * set to non-zero, then also check that its color equals to the 671b0cd712cSStefan Weil * *colorPtr value. The result is 1 if the test is successful, and in 672245f7b51SCorentin Chary * that case new color will be stored in *colorPtr. 673245f7b51SCorentin Chary */ 674245f7b51SCorentin Chary 67594362682SGerd Hoffmann static bool 67694362682SGerd Hoffmann check_solid_tile32(VncState *vs, int x, int y, int w, int h, 67794362682SGerd Hoffmann uint32_t *color, bool samecolor) 67894362682SGerd Hoffmann { 67994362682SGerd Hoffmann VncDisplay *vd = vs->vd; 68094362682SGerd Hoffmann uint32_t *fbptr; 68194362682SGerd Hoffmann uint32_t c; 68294362682SGerd Hoffmann int dx, dy; 68394362682SGerd Hoffmann 68494362682SGerd Hoffmann fbptr = vnc_server_fb_ptr(vd, x, y); 68594362682SGerd Hoffmann 68694362682SGerd Hoffmann c = *fbptr; 68794362682SGerd Hoffmann if (samecolor && (uint32_t)c != *color) { 68894362682SGerd Hoffmann return false; 689245f7b51SCorentin Chary } 690245f7b51SCorentin Chary 69194362682SGerd Hoffmann for (dy = 0; dy < h; dy++) { 69294362682SGerd Hoffmann for (dx = 0; dx < w; dx++) { 69394362682SGerd Hoffmann if (c != fbptr[dx]) { 69494362682SGerd Hoffmann return false; 69594362682SGerd Hoffmann } 69694362682SGerd Hoffmann } 69794362682SGerd Hoffmann fbptr = (uint32_t *) 69894362682SGerd Hoffmann ((uint8_t *)fbptr + vnc_server_fb_stride(vd)); 69994362682SGerd Hoffmann } 70094362682SGerd Hoffmann 70194362682SGerd Hoffmann *color = (uint32_t)c; 70294362682SGerd Hoffmann return true; 70394362682SGerd Hoffmann } 704245f7b51SCorentin Chary 705245f7b51SCorentin Chary static bool check_solid_tile(VncState *vs, int x, int y, int w, int h, 706245f7b51SCorentin Chary uint32_t* color, bool samecolor) 707245f7b51SCorentin Chary { 708d9d2663cSAlex Bennée QEMU_BUILD_BUG_ON(VNC_SERVER_FB_BYTES != 4); 709245f7b51SCorentin Chary return check_solid_tile32(vs, x, y, w, h, color, samecolor); 710245f7b51SCorentin Chary } 711245f7b51SCorentin Chary 712245f7b51SCorentin Chary static void find_best_solid_area(VncState *vs, int x, int y, int w, int h, 713245f7b51SCorentin Chary uint32_t color, int *w_ptr, int *h_ptr) 714245f7b51SCorentin Chary { 715245f7b51SCorentin Chary int dx, dy, dw, dh; 716245f7b51SCorentin Chary int w_prev; 717245f7b51SCorentin Chary int w_best = 0, h_best = 0; 718245f7b51SCorentin Chary 719245f7b51SCorentin Chary w_prev = w; 720245f7b51SCorentin Chary 721245f7b51SCorentin Chary for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) { 722245f7b51SCorentin Chary 723245f7b51SCorentin Chary dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, y + h - dy); 724245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, w_prev); 725245f7b51SCorentin Chary 726245f7b51SCorentin Chary if (!check_solid_tile(vs, x, dy, dw, dh, &color, true)) { 727245f7b51SCorentin Chary break; 728245f7b51SCorentin Chary } 729245f7b51SCorentin Chary 730245f7b51SCorentin Chary for (dx = x + dw; dx < x + w_prev;) { 731245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, x + w_prev - dx); 732245f7b51SCorentin Chary 733245f7b51SCorentin Chary if (!check_solid_tile(vs, dx, dy, dw, dh, &color, true)) { 734245f7b51SCorentin Chary break; 735245f7b51SCorentin Chary } 736245f7b51SCorentin Chary dx += dw; 737245f7b51SCorentin Chary } 738245f7b51SCorentin Chary 739245f7b51SCorentin Chary w_prev = dx - x; 740245f7b51SCorentin Chary if (w_prev * (dy + dh - y) > w_best * h_best) { 741245f7b51SCorentin Chary w_best = w_prev; 742245f7b51SCorentin Chary h_best = dy + dh - y; 743245f7b51SCorentin Chary } 744245f7b51SCorentin Chary } 745245f7b51SCorentin Chary 746245f7b51SCorentin Chary *w_ptr = w_best; 747245f7b51SCorentin Chary *h_ptr = h_best; 748245f7b51SCorentin Chary } 749245f7b51SCorentin Chary 750245f7b51SCorentin Chary static void extend_solid_area(VncState *vs, int x, int y, int w, int h, 751245f7b51SCorentin Chary uint32_t color, int *x_ptr, int *y_ptr, 752245f7b51SCorentin Chary int *w_ptr, int *h_ptr) 753245f7b51SCorentin Chary { 754245f7b51SCorentin Chary int cx, cy; 755245f7b51SCorentin Chary 756245f7b51SCorentin Chary /* Try to extend the area upwards. */ 757245f7b51SCorentin Chary for ( cy = *y_ptr - 1; 758245f7b51SCorentin Chary cy >= y && check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true); 759245f7b51SCorentin Chary cy-- ); 760245f7b51SCorentin Chary *h_ptr += *y_ptr - (cy + 1); 761245f7b51SCorentin Chary *y_ptr = cy + 1; 762245f7b51SCorentin Chary 763245f7b51SCorentin Chary /* ... downwards. */ 764245f7b51SCorentin Chary for ( cy = *y_ptr + *h_ptr; 765245f7b51SCorentin Chary cy < y + h && 766245f7b51SCorentin Chary check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true); 767245f7b51SCorentin Chary cy++ ); 768245f7b51SCorentin Chary *h_ptr += cy - (*y_ptr + *h_ptr); 769245f7b51SCorentin Chary 770245f7b51SCorentin Chary /* ... to the left. */ 771245f7b51SCorentin Chary for ( cx = *x_ptr - 1; 772245f7b51SCorentin Chary cx >= x && check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true); 773245f7b51SCorentin Chary cx-- ); 774245f7b51SCorentin Chary *w_ptr += *x_ptr - (cx + 1); 775245f7b51SCorentin Chary *x_ptr = cx + 1; 776245f7b51SCorentin Chary 777245f7b51SCorentin Chary /* ... to the right. */ 778245f7b51SCorentin Chary for ( cx = *x_ptr + *w_ptr; 779245f7b51SCorentin Chary cx < x + w && 780245f7b51SCorentin Chary check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true); 781245f7b51SCorentin Chary cx++ ); 782245f7b51SCorentin Chary *w_ptr += cx - (*x_ptr + *w_ptr); 783245f7b51SCorentin Chary } 784245f7b51SCorentin Chary 785245f7b51SCorentin Chary static int tight_init_stream(VncState *vs, int stream_id, 786245f7b51SCorentin Chary int level, int strategy) 787245f7b51SCorentin Chary { 7886bf21f3dSLi Qiang z_streamp zstream = &vs->tight->stream[stream_id]; 789245f7b51SCorentin Chary 790245f7b51SCorentin Chary if (zstream->opaque == NULL) { 791245f7b51SCorentin Chary int err; 792245f7b51SCorentin Chary 793245f7b51SCorentin Chary VNC_DEBUG("VNC: TIGHT: initializing zlib stream %d\n", stream_id); 794245f7b51SCorentin Chary VNC_DEBUG("VNC: TIGHT: opaque = %p | vs = %p\n", zstream->opaque, vs); 795245f7b51SCorentin Chary zstream->zalloc = vnc_zlib_zalloc; 796245f7b51SCorentin Chary zstream->zfree = vnc_zlib_zfree; 797245f7b51SCorentin Chary 798245f7b51SCorentin Chary err = deflateInit2(zstream, level, Z_DEFLATED, MAX_WBITS, 799245f7b51SCorentin Chary MAX_MEM_LEVEL, strategy); 800245f7b51SCorentin Chary 801245f7b51SCorentin Chary if (err != Z_OK) { 802245f7b51SCorentin Chary fprintf(stderr, "VNC: error initializing zlib\n"); 803245f7b51SCorentin Chary return -1; 804245f7b51SCorentin Chary } 805245f7b51SCorentin Chary 8066bf21f3dSLi Qiang vs->tight->levels[stream_id] = level; 807245f7b51SCorentin Chary zstream->opaque = vs; 808245f7b51SCorentin Chary } 809245f7b51SCorentin Chary 8106bf21f3dSLi Qiang if (vs->tight->levels[stream_id] != level) { 811245f7b51SCorentin Chary if (deflateParams(zstream, level, strategy) != Z_OK) { 812245f7b51SCorentin Chary return -1; 813245f7b51SCorentin Chary } 8146bf21f3dSLi Qiang vs->tight->levels[stream_id] = level; 815245f7b51SCorentin Chary } 816245f7b51SCorentin Chary return 0; 817245f7b51SCorentin Chary } 818245f7b51SCorentin Chary 819245f7b51SCorentin Chary static void tight_send_compact_size(VncState *vs, size_t len) 820245f7b51SCorentin Chary { 821245f7b51SCorentin Chary int lpc = 0; 822245f7b51SCorentin Chary int bytes = 0; 823245f7b51SCorentin Chary char buf[3] = {0, 0, 0}; 824245f7b51SCorentin Chary 825245f7b51SCorentin Chary buf[bytes++] = len & 0x7F; 826245f7b51SCorentin Chary if (len > 0x7F) { 827245f7b51SCorentin Chary buf[bytes-1] |= 0x80; 828245f7b51SCorentin Chary buf[bytes++] = (len >> 7) & 0x7F; 829245f7b51SCorentin Chary if (len > 0x3FFF) { 830245f7b51SCorentin Chary buf[bytes-1] |= 0x80; 831245f7b51SCorentin Chary buf[bytes++] = (len >> 14) & 0xFF; 832245f7b51SCorentin Chary } 833245f7b51SCorentin Chary } 834245f7b51SCorentin Chary for (lpc = 0; lpc < bytes; lpc++) { 835245f7b51SCorentin Chary vnc_write_u8(vs, buf[lpc]); 836245f7b51SCorentin Chary } 837245f7b51SCorentin Chary } 838245f7b51SCorentin Chary 839245f7b51SCorentin Chary static int tight_compress_data(VncState *vs, int stream_id, size_t bytes, 840245f7b51SCorentin Chary int level, int strategy) 841245f7b51SCorentin Chary { 8426bf21f3dSLi Qiang z_streamp zstream = &vs->tight->stream[stream_id]; 843245f7b51SCorentin Chary int previous_out; 844245f7b51SCorentin Chary 845245f7b51SCorentin Chary if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) { 8466bf21f3dSLi Qiang vnc_write(vs, vs->tight->tight.buffer, vs->tight->tight.offset); 847245f7b51SCorentin Chary return bytes; 848245f7b51SCorentin Chary } 849245f7b51SCorentin Chary 850245f7b51SCorentin Chary if (tight_init_stream(vs, stream_id, level, strategy)) { 851245f7b51SCorentin Chary return -1; 852245f7b51SCorentin Chary } 853245f7b51SCorentin Chary 854245f7b51SCorentin Chary /* reserve memory in output buffer */ 8556bf21f3dSLi Qiang buffer_reserve(&vs->tight->zlib, bytes + 64); 856245f7b51SCorentin Chary 857245f7b51SCorentin Chary /* set pointers */ 8586bf21f3dSLi Qiang zstream->next_in = vs->tight->tight.buffer; 8596bf21f3dSLi Qiang zstream->avail_in = vs->tight->tight.offset; 8606bf21f3dSLi Qiang zstream->next_out = vs->tight->zlib.buffer + vs->tight->zlib.offset; 8616bf21f3dSLi Qiang zstream->avail_out = vs->tight->zlib.capacity - vs->tight->zlib.offset; 8622caa9e9dSMichael Tokarev previous_out = zstream->avail_out; 863245f7b51SCorentin Chary zstream->data_type = Z_BINARY; 864245f7b51SCorentin Chary 865245f7b51SCorentin Chary /* start encoding */ 866245f7b51SCorentin Chary if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) { 867245f7b51SCorentin Chary fprintf(stderr, "VNC: error during tight compression\n"); 868245f7b51SCorentin Chary return -1; 869245f7b51SCorentin Chary } 870245f7b51SCorentin Chary 8716bf21f3dSLi Qiang vs->tight->zlib.offset = vs->tight->zlib.capacity - zstream->avail_out; 8722caa9e9dSMichael Tokarev /* ...how much data has actually been produced by deflate() */ 8732caa9e9dSMichael Tokarev bytes = previous_out - zstream->avail_out; 874245f7b51SCorentin Chary 875245f7b51SCorentin Chary tight_send_compact_size(vs, bytes); 8766bf21f3dSLi Qiang vnc_write(vs, vs->tight->zlib.buffer, bytes); 877245f7b51SCorentin Chary 8786bf21f3dSLi Qiang buffer_reset(&vs->tight->zlib); 879245f7b51SCorentin Chary 880245f7b51SCorentin Chary return bytes; 881245f7b51SCorentin Chary } 882245f7b51SCorentin Chary 883245f7b51SCorentin Chary /* 884245f7b51SCorentin Chary * Subencoding implementations. 885245f7b51SCorentin Chary */ 886245f7b51SCorentin Chary static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret) 887245f7b51SCorentin Chary { 88818e845f7SLi Qiang uint8_t *buf8; 889245f7b51SCorentin Chary uint32_t pix; 890245f7b51SCorentin Chary int rshift, gshift, bshift; 891245f7b51SCorentin Chary 89218e845f7SLi Qiang buf8 = buf; 893245f7b51SCorentin Chary 89477bfcf28SBenjamin Herrenschmidt if (1 /* FIXME */) { 8959f64916dSGerd Hoffmann rshift = vs->client_pf.rshift; 8969f64916dSGerd Hoffmann gshift = vs->client_pf.gshift; 8979f64916dSGerd Hoffmann bshift = vs->client_pf.bshift; 898245f7b51SCorentin Chary } else { 8999f64916dSGerd Hoffmann rshift = 24 - vs->client_pf.rshift; 9009f64916dSGerd Hoffmann gshift = 24 - vs->client_pf.gshift; 9019f64916dSGerd Hoffmann bshift = 24 - vs->client_pf.bshift; 902245f7b51SCorentin Chary } 903245f7b51SCorentin Chary 904245f7b51SCorentin Chary if (ret) { 905245f7b51SCorentin Chary *ret = count * 3; 906245f7b51SCorentin Chary } 907245f7b51SCorentin Chary 908245f7b51SCorentin Chary while (count--) { 90918e845f7SLi Qiang pix = ldl_he_p(buf8); 910245f7b51SCorentin Chary *buf++ = (char)(pix >> rshift); 911245f7b51SCorentin Chary *buf++ = (char)(pix >> gshift); 912245f7b51SCorentin Chary *buf++ = (char)(pix >> bshift); 91318e845f7SLi Qiang buf8 += 4; 914245f7b51SCorentin Chary } 915245f7b51SCorentin Chary } 916245f7b51SCorentin Chary 917efe556adSCorentin Chary static int send_full_color_rect(VncState *vs, int x, int y, int w, int h) 918245f7b51SCorentin Chary { 919245f7b51SCorentin Chary int stream = 0; 9202116eff9SJes Sorensen ssize_t bytes; 921245f7b51SCorentin Chary 922*95f8510eSKshitij Suri #ifdef CONFIG_PNG 923efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) { 924efe556adSCorentin Chary return send_png_rect(vs, x, y, w, h, NULL); 925efe556adSCorentin Chary } 926efe556adSCorentin Chary #endif 927efe556adSCorentin Chary 928245f7b51SCorentin Chary vnc_write_u8(vs, stream << 4); /* no flushing, no filter */ 929245f7b51SCorentin Chary 9306bf21f3dSLi Qiang if (vs->tight->pixel24) { 9316bf21f3dSLi Qiang tight_pack24(vs, vs->tight->tight.buffer, w * h, 9326bf21f3dSLi Qiang &vs->tight->tight.offset); 933245f7b51SCorentin Chary bytes = 3; 934245f7b51SCorentin Chary } else { 9359f64916dSGerd Hoffmann bytes = vs->client_pf.bytes_per_pixel; 936245f7b51SCorentin Chary } 937245f7b51SCorentin Chary 938245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, w * h * bytes, 9396bf21f3dSLi Qiang tight_conf[vs->tight->compression].raw_zlib_level, 940245f7b51SCorentin Chary Z_DEFAULT_STRATEGY); 941245f7b51SCorentin Chary 942245f7b51SCorentin Chary return (bytes >= 0); 943245f7b51SCorentin Chary } 944245f7b51SCorentin Chary 945245f7b51SCorentin Chary static int send_solid_rect(VncState *vs) 946245f7b51SCorentin Chary { 947245f7b51SCorentin Chary size_t bytes; 948245f7b51SCorentin Chary 949245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILL << 4); /* no flushing, no filter */ 950245f7b51SCorentin Chary 9516bf21f3dSLi Qiang if (vs->tight->pixel24) { 9526bf21f3dSLi Qiang tight_pack24(vs, vs->tight->tight.buffer, 1, &vs->tight->tight.offset); 953245f7b51SCorentin Chary bytes = 3; 954245f7b51SCorentin Chary } else { 9559f64916dSGerd Hoffmann bytes = vs->client_pf.bytes_per_pixel; 956245f7b51SCorentin Chary } 957245f7b51SCorentin Chary 9586bf21f3dSLi Qiang vnc_write(vs, vs->tight->tight.buffer, bytes); 959245f7b51SCorentin Chary return 1; 960245f7b51SCorentin Chary } 961245f7b51SCorentin Chary 962efe556adSCorentin Chary static int send_mono_rect(VncState *vs, int x, int y, 963efe556adSCorentin Chary int w, int h, uint32_t bg, uint32_t fg) 964245f7b51SCorentin Chary { 9652116eff9SJes Sorensen ssize_t bytes; 966245f7b51SCorentin Chary int stream = 1; 9676bf21f3dSLi Qiang int level = tight_conf[vs->tight->compression].mono_zlib_level; 968245f7b51SCorentin Chary 969*95f8510eSKshitij Suri #ifdef CONFIG_PNG 970efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) { 971efe556adSCorentin Chary int ret; 9729f64916dSGerd Hoffmann int bpp = vs->client_pf.bytes_per_pixel * 8; 9735136a052SCorentin Chary VncPalette *palette = palette_new(2, bpp); 974efe556adSCorentin Chary 9755136a052SCorentin Chary palette_put(palette, bg); 9765136a052SCorentin Chary palette_put(palette, fg); 977efe556adSCorentin Chary ret = send_png_rect(vs, x, y, w, h, palette); 9785136a052SCorentin Chary palette_destroy(palette); 979efe556adSCorentin Chary return ret; 980efe556adSCorentin Chary } 981efe556adSCorentin Chary #endif 982efe556adSCorentin Chary 983949ed4c2SPhilippe Mathieu-Daudé bytes = DIV_ROUND_UP(w, 8) * h; 984245f7b51SCorentin Chary 985245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4); 986245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE); 987245f7b51SCorentin Chary vnc_write_u8(vs, 1); 988245f7b51SCorentin Chary 9899f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) { 990245f7b51SCorentin Chary case 4: 991245f7b51SCorentin Chary { 992245f7b51SCorentin Chary uint32_t buf[2] = {bg, fg}; 993245f7b51SCorentin Chary size_t ret = sizeof (buf); 994245f7b51SCorentin Chary 9956bf21f3dSLi Qiang if (vs->tight->pixel24) { 996245f7b51SCorentin Chary tight_pack24(vs, (unsigned char*)buf, 2, &ret); 997245f7b51SCorentin Chary } 998245f7b51SCorentin Chary vnc_write(vs, buf, ret); 999245f7b51SCorentin Chary 10006bf21f3dSLi Qiang tight_encode_mono_rect32(vs->tight->tight.buffer, w, h, bg, fg); 1001245f7b51SCorentin Chary break; 1002245f7b51SCorentin Chary } 1003245f7b51SCorentin Chary case 2: 1004245f7b51SCorentin Chary vnc_write(vs, &bg, 2); 1005245f7b51SCorentin Chary vnc_write(vs, &fg, 2); 10066bf21f3dSLi Qiang tight_encode_mono_rect16(vs->tight->tight.buffer, w, h, bg, fg); 1007245f7b51SCorentin Chary break; 1008245f7b51SCorentin Chary default: 1009245f7b51SCorentin Chary vnc_write_u8(vs, bg); 1010245f7b51SCorentin Chary vnc_write_u8(vs, fg); 10116bf21f3dSLi Qiang tight_encode_mono_rect8(vs->tight->tight.buffer, w, h, bg, fg); 1012245f7b51SCorentin Chary break; 1013245f7b51SCorentin Chary } 10146bf21f3dSLi Qiang vs->tight->tight.offset = bytes; 1015245f7b51SCorentin Chary 1016245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes, level, Z_DEFAULT_STRATEGY); 1017245f7b51SCorentin Chary return (bytes >= 0); 1018245f7b51SCorentin Chary } 1019245f7b51SCorentin Chary 1020245f7b51SCorentin Chary struct palette_cb_priv { 1021245f7b51SCorentin Chary VncState *vs; 1022245f7b51SCorentin Chary uint8_t *header; 1023*95f8510eSKshitij Suri #ifdef CONFIG_PNG 1024efe556adSCorentin Chary png_colorp png_palette; 1025efe556adSCorentin Chary #endif 1026245f7b51SCorentin Chary }; 1027245f7b51SCorentin Chary 10285136a052SCorentin Chary static void write_palette(int idx, uint32_t color, void *opaque) 1029245f7b51SCorentin Chary { 1030245f7b51SCorentin Chary struct palette_cb_priv *priv = opaque; 1031245f7b51SCorentin Chary VncState *vs = priv->vs; 10329f64916dSGerd Hoffmann uint32_t bytes = vs->client_pf.bytes_per_pixel; 1033245f7b51SCorentin Chary 1034245f7b51SCorentin Chary if (bytes == 4) { 1035245f7b51SCorentin Chary ((uint32_t*)priv->header)[idx] = color; 1036245f7b51SCorentin Chary } else { 1037245f7b51SCorentin Chary ((uint16_t*)priv->header)[idx] = color; 1038245f7b51SCorentin Chary } 1039245f7b51SCorentin Chary } 1040245f7b51SCorentin Chary 1041efe556adSCorentin Chary static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h) 1042245f7b51SCorentin Chary { 1043245f7b51SCorentin Chary int stream = 3; 10446bf21f3dSLi Qiang int level = tight_conf[vs->tight->compression].gradient_zlib_level; 10452116eff9SJes Sorensen ssize_t bytes; 1046245f7b51SCorentin Chary 10479f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 1) { 1048efe556adSCorentin Chary return send_full_color_rect(vs, x, y, w, h); 10499f64916dSGerd Hoffmann } 1050245f7b51SCorentin Chary 1051245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4); 1052245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_GRADIENT); 1053245f7b51SCorentin Chary 10546bf21f3dSLi Qiang buffer_reserve(&vs->tight->gradient, w * 3 * sizeof(int)); 1055245f7b51SCorentin Chary 10566bf21f3dSLi Qiang if (vs->tight->pixel24) { 10576bf21f3dSLi Qiang tight_filter_gradient24(vs, vs->tight->tight.buffer, w, h); 1058245f7b51SCorentin Chary bytes = 3; 10599f64916dSGerd Hoffmann } else if (vs->client_pf.bytes_per_pixel == 4) { 10606bf21f3dSLi Qiang tight_filter_gradient32(vs, (uint32_t *)vs->tight->tight.buffer, w, h); 1061245f7b51SCorentin Chary bytes = 4; 1062245f7b51SCorentin Chary } else { 10636bf21f3dSLi Qiang tight_filter_gradient16(vs, (uint16_t *)vs->tight->tight.buffer, w, h); 1064245f7b51SCorentin Chary bytes = 2; 1065245f7b51SCorentin Chary } 1066245f7b51SCorentin Chary 10676bf21f3dSLi Qiang buffer_reset(&vs->tight->gradient); 1068245f7b51SCorentin Chary 1069245f7b51SCorentin Chary bytes = w * h * bytes; 10706bf21f3dSLi Qiang vs->tight->tight.offset = bytes; 1071245f7b51SCorentin Chary 1072245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes, 1073245f7b51SCorentin Chary level, Z_FILTERED); 1074245f7b51SCorentin Chary return (bytes >= 0); 1075245f7b51SCorentin Chary } 1076245f7b51SCorentin Chary 1077efe556adSCorentin Chary static int send_palette_rect(VncState *vs, int x, int y, 10785136a052SCorentin Chary int w, int h, VncPalette *palette) 1079245f7b51SCorentin Chary { 1080245f7b51SCorentin Chary int stream = 2; 10816bf21f3dSLi Qiang int level = tight_conf[vs->tight->compression].idx_zlib_level; 1082245f7b51SCorentin Chary int colors; 10832116eff9SJes Sorensen ssize_t bytes; 1084245f7b51SCorentin Chary 1085*95f8510eSKshitij Suri #ifdef CONFIG_PNG 1086efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) { 1087efe556adSCorentin Chary return send_png_rect(vs, x, y, w, h, palette); 1088efe556adSCorentin Chary } 1089efe556adSCorentin Chary #endif 1090efe556adSCorentin Chary 10915136a052SCorentin Chary colors = palette_size(palette); 1092245f7b51SCorentin Chary 1093245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4); 1094245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE); 1095245f7b51SCorentin Chary vnc_write_u8(vs, colors - 1); 1096245f7b51SCorentin Chary 10979f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) { 1098245f7b51SCorentin Chary case 4: 1099245f7b51SCorentin Chary { 1100245f7b51SCorentin Chary size_t old_offset, offset; 11015136a052SCorentin Chary uint32_t header[palette_size(palette)]; 1102245f7b51SCorentin Chary struct palette_cb_priv priv = { vs, (uint8_t *)header }; 1103245f7b51SCorentin Chary 1104245f7b51SCorentin Chary old_offset = vs->output.offset; 11055136a052SCorentin Chary palette_iter(palette, write_palette, &priv); 1106245f7b51SCorentin Chary vnc_write(vs, header, sizeof(header)); 1107245f7b51SCorentin Chary 11086bf21f3dSLi Qiang if (vs->tight->pixel24) { 1109245f7b51SCorentin Chary tight_pack24(vs, vs->output.buffer + old_offset, colors, &offset); 1110245f7b51SCorentin Chary vs->output.offset = old_offset + offset; 1111245f7b51SCorentin Chary } 1112245f7b51SCorentin Chary 11136bf21f3dSLi Qiang tight_encode_indexed_rect32(vs->tight->tight.buffer, w * h, palette); 1114245f7b51SCorentin Chary break; 1115245f7b51SCorentin Chary } 1116245f7b51SCorentin Chary case 2: 1117245f7b51SCorentin Chary { 11185136a052SCorentin Chary uint16_t header[palette_size(palette)]; 1119245f7b51SCorentin Chary struct palette_cb_priv priv = { vs, (uint8_t *)header }; 1120245f7b51SCorentin Chary 11215136a052SCorentin Chary palette_iter(palette, write_palette, &priv); 1122245f7b51SCorentin Chary vnc_write(vs, header, sizeof(header)); 11236bf21f3dSLi Qiang tight_encode_indexed_rect16(vs->tight->tight.buffer, w * h, palette); 1124245f7b51SCorentin Chary break; 1125245f7b51SCorentin Chary } 1126245f7b51SCorentin Chary default: 1127245f7b51SCorentin Chary return -1; /* No palette for 8bits colors */ 1128245f7b51SCorentin Chary } 1129245f7b51SCorentin Chary bytes = w * h; 11306bf21f3dSLi Qiang vs->tight->tight.offset = bytes; 1131245f7b51SCorentin Chary 1132245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes, 1133245f7b51SCorentin Chary level, Z_DEFAULT_STRATEGY); 1134245f7b51SCorentin Chary return (bytes >= 0); 1135245f7b51SCorentin Chary } 1136245f7b51SCorentin Chary 1137245f7b51SCorentin Chary /* 1138efe556adSCorentin Chary * JPEG compression stuff. 1139efe556adSCorentin Chary */ 1140efe556adSCorentin Chary #ifdef CONFIG_VNC_JPEG 1141efe556adSCorentin Chary /* 1142245f7b51SCorentin Chary * Destination manager implementation for JPEG library. 1143245f7b51SCorentin Chary */ 1144245f7b51SCorentin Chary 1145245f7b51SCorentin Chary /* This is called once per encoding */ 1146245f7b51SCorentin Chary static void jpeg_init_destination(j_compress_ptr cinfo) 1147245f7b51SCorentin Chary { 1148245f7b51SCorentin Chary VncState *vs = cinfo->client_data; 11496bf21f3dSLi Qiang Buffer *buffer = &vs->tight->jpeg; 1150245f7b51SCorentin Chary 1151245f7b51SCorentin Chary cinfo->dest->next_output_byte = (JOCTET *)buffer->buffer + buffer->offset; 1152245f7b51SCorentin Chary cinfo->dest->free_in_buffer = (size_t)(buffer->capacity - buffer->offset); 1153245f7b51SCorentin Chary } 1154245f7b51SCorentin Chary 1155245f7b51SCorentin Chary /* This is called when we ran out of buffer (shouldn't happen!) */ 1156245f7b51SCorentin Chary static boolean jpeg_empty_output_buffer(j_compress_ptr cinfo) 1157245f7b51SCorentin Chary { 1158245f7b51SCorentin Chary VncState *vs = cinfo->client_data; 11596bf21f3dSLi Qiang Buffer *buffer = &vs->tight->jpeg; 1160245f7b51SCorentin Chary 1161245f7b51SCorentin Chary buffer->offset = buffer->capacity; 1162245f7b51SCorentin Chary buffer_reserve(buffer, 2048); 1163245f7b51SCorentin Chary jpeg_init_destination(cinfo); 1164245f7b51SCorentin Chary return TRUE; 1165245f7b51SCorentin Chary } 1166245f7b51SCorentin Chary 1167245f7b51SCorentin Chary /* This is called when we are done processing data */ 1168245f7b51SCorentin Chary static void jpeg_term_destination(j_compress_ptr cinfo) 1169245f7b51SCorentin Chary { 1170245f7b51SCorentin Chary VncState *vs = cinfo->client_data; 11716bf21f3dSLi Qiang Buffer *buffer = &vs->tight->jpeg; 1172245f7b51SCorentin Chary 1173245f7b51SCorentin Chary buffer->offset = buffer->capacity - cinfo->dest->free_in_buffer; 1174245f7b51SCorentin Chary } 1175245f7b51SCorentin Chary 1176245f7b51SCorentin Chary static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality) 1177245f7b51SCorentin Chary { 1178245f7b51SCorentin Chary struct jpeg_compress_struct cinfo; 1179245f7b51SCorentin Chary struct jpeg_error_mgr jerr; 1180245f7b51SCorentin Chary struct jpeg_destination_mgr manager; 118147683d66SGerd Hoffmann pixman_image_t *linebuf; 1182245f7b51SCorentin Chary JSAMPROW row[1]; 1183245f7b51SCorentin Chary uint8_t *buf; 1184245f7b51SCorentin Chary int dy; 1185245f7b51SCorentin Chary 1186d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1) { 1187efe556adSCorentin Chary return send_full_color_rect(vs, x, y, w, h); 1188d39fa6d8SGerd Hoffmann } 1189245f7b51SCorentin Chary 11906bf21f3dSLi Qiang buffer_reserve(&vs->tight->jpeg, 2048); 1191245f7b51SCorentin Chary 1192245f7b51SCorentin Chary cinfo.err = jpeg_std_error(&jerr); 1193245f7b51SCorentin Chary jpeg_create_compress(&cinfo); 1194245f7b51SCorentin Chary 1195245f7b51SCorentin Chary cinfo.client_data = vs; 1196245f7b51SCorentin Chary cinfo.image_width = w; 1197245f7b51SCorentin Chary cinfo.image_height = h; 1198245f7b51SCorentin Chary cinfo.input_components = 3; 1199245f7b51SCorentin Chary cinfo.in_color_space = JCS_RGB; 1200245f7b51SCorentin Chary 1201245f7b51SCorentin Chary jpeg_set_defaults(&cinfo); 1202245f7b51SCorentin Chary jpeg_set_quality(&cinfo, quality, true); 1203245f7b51SCorentin Chary 1204245f7b51SCorentin Chary manager.init_destination = jpeg_init_destination; 1205245f7b51SCorentin Chary manager.empty_output_buffer = jpeg_empty_output_buffer; 1206245f7b51SCorentin Chary manager.term_destination = jpeg_term_destination; 1207245f7b51SCorentin Chary cinfo.dest = &manager; 1208245f7b51SCorentin Chary 1209245f7b51SCorentin Chary jpeg_start_compress(&cinfo, true); 1210245f7b51SCorentin Chary 121147683d66SGerd Hoffmann linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w); 121247683d66SGerd Hoffmann buf = (uint8_t *)pixman_image_get_data(linebuf); 1213d9c18c24SCorentin Chary row[0] = buf; 1214245f7b51SCorentin Chary for (dy = 0; dy < h; dy++) { 1215bc210eb1SGerd Hoffmann qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy); 1216245f7b51SCorentin Chary jpeg_write_scanlines(&cinfo, row, 1); 1217245f7b51SCorentin Chary } 121847683d66SGerd Hoffmann qemu_pixman_image_unref(linebuf); 1219245f7b51SCorentin Chary 1220245f7b51SCorentin Chary jpeg_finish_compress(&cinfo); 1221245f7b51SCorentin Chary jpeg_destroy_compress(&cinfo); 1222245f7b51SCorentin Chary 1223245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_JPEG << 4); 1224245f7b51SCorentin Chary 12256bf21f3dSLi Qiang tight_send_compact_size(vs, vs->tight->jpeg.offset); 12266bf21f3dSLi Qiang vnc_write(vs, vs->tight->jpeg.buffer, vs->tight->jpeg.offset); 12276bf21f3dSLi Qiang buffer_reset(&vs->tight->jpeg); 1228245f7b51SCorentin Chary 1229245f7b51SCorentin Chary return 1; 1230245f7b51SCorentin Chary } 1231245f7b51SCorentin Chary #endif /* CONFIG_VNC_JPEG */ 1232245f7b51SCorentin Chary 1233efe556adSCorentin Chary /* 1234efe556adSCorentin Chary * PNG compression stuff. 1235efe556adSCorentin Chary */ 1236*95f8510eSKshitij Suri #ifdef CONFIG_PNG 12375136a052SCorentin Chary static void write_png_palette(int idx, uint32_t pix, void *opaque) 1238efe556adSCorentin Chary { 1239efe556adSCorentin Chary struct palette_cb_priv *priv = opaque; 1240efe556adSCorentin Chary VncState *vs = priv->vs; 1241efe556adSCorentin Chary png_colorp color = &priv->png_palette[idx]; 1242efe556adSCorentin Chary 12436bf21f3dSLi Qiang if (vs->tight->pixel24) 1244efe556adSCorentin Chary { 12459f64916dSGerd Hoffmann color->red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax; 12469f64916dSGerd Hoffmann color->green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax; 12479f64916dSGerd Hoffmann color->blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax; 1248efe556adSCorentin Chary } 1249efe556adSCorentin Chary else 1250efe556adSCorentin Chary { 1251efe556adSCorentin Chary int red, green, blue; 1252efe556adSCorentin Chary 12539f64916dSGerd Hoffmann red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax; 12549f64916dSGerd Hoffmann green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax; 12559f64916dSGerd Hoffmann blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax; 12569f64916dSGerd Hoffmann color->red = ((red * 255 + vs->client_pf.rmax / 2) / 12579f64916dSGerd Hoffmann vs->client_pf.rmax); 12589f64916dSGerd Hoffmann color->green = ((green * 255 + vs->client_pf.gmax / 2) / 12599f64916dSGerd Hoffmann vs->client_pf.gmax); 12609f64916dSGerd Hoffmann color->blue = ((blue * 255 + vs->client_pf.bmax / 2) / 12619f64916dSGerd Hoffmann vs->client_pf.bmax); 1262efe556adSCorentin Chary } 1263efe556adSCorentin Chary } 1264efe556adSCorentin Chary 1265efe556adSCorentin Chary static void png_write_data(png_structp png_ptr, png_bytep data, 1266efe556adSCorentin Chary png_size_t length) 1267efe556adSCorentin Chary { 1268efe556adSCorentin Chary VncState *vs = png_get_io_ptr(png_ptr); 1269efe556adSCorentin Chary 12706bf21f3dSLi Qiang buffer_reserve(&vs->tight->png, vs->tight->png.offset + length); 12716bf21f3dSLi Qiang memcpy(vs->tight->png.buffer + vs->tight->png.offset, data, length); 1272efe556adSCorentin Chary 12736bf21f3dSLi Qiang vs->tight->png.offset += length; 1274efe556adSCorentin Chary } 1275efe556adSCorentin Chary 1276efe556adSCorentin Chary static void png_flush_data(png_structp png_ptr) 1277efe556adSCorentin Chary { 1278efe556adSCorentin Chary } 1279efe556adSCorentin Chary 1280efe556adSCorentin Chary static void *vnc_png_malloc(png_structp png_ptr, png_size_t size) 1281efe556adSCorentin Chary { 12827267c094SAnthony Liguori return g_malloc(size); 1283efe556adSCorentin Chary } 1284efe556adSCorentin Chary 1285efe556adSCorentin Chary static void vnc_png_free(png_structp png_ptr, png_voidp ptr) 1286efe556adSCorentin Chary { 12877267c094SAnthony Liguori g_free(ptr); 1288efe556adSCorentin Chary } 1289efe556adSCorentin Chary 1290efe556adSCorentin Chary static int send_png_rect(VncState *vs, int x, int y, int w, int h, 12915136a052SCorentin Chary VncPalette *palette) 1292efe556adSCorentin Chary { 1293efe556adSCorentin Chary png_byte color_type; 1294efe556adSCorentin Chary png_structp png_ptr; 1295efe556adSCorentin Chary png_infop info_ptr; 1296efe556adSCorentin Chary png_colorp png_palette = NULL; 129747683d66SGerd Hoffmann pixman_image_t *linebuf; 12986bf21f3dSLi Qiang int level = tight_png_conf[vs->tight->compression].png_zlib_level; 12996bf21f3dSLi Qiang int filters = tight_png_conf[vs->tight->compression].png_filters; 1300efe556adSCorentin Chary uint8_t *buf; 1301efe556adSCorentin Chary int dy; 1302efe556adSCorentin Chary 1303efe556adSCorentin Chary png_ptr = png_create_write_struct_2(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL, 1304efe556adSCorentin Chary NULL, vnc_png_malloc, vnc_png_free); 1305efe556adSCorentin Chary 1306efe556adSCorentin Chary if (png_ptr == NULL) 1307efe556adSCorentin Chary return -1; 1308efe556adSCorentin Chary 1309efe556adSCorentin Chary info_ptr = png_create_info_struct(png_ptr); 1310efe556adSCorentin Chary 1311efe556adSCorentin Chary if (info_ptr == NULL) { 1312efe556adSCorentin Chary png_destroy_write_struct(&png_ptr, NULL); 1313efe556adSCorentin Chary return -1; 1314efe556adSCorentin Chary } 1315efe556adSCorentin Chary 1316efe556adSCorentin Chary png_set_write_fn(png_ptr, (void *) vs, png_write_data, png_flush_data); 1317efe556adSCorentin Chary png_set_compression_level(png_ptr, level); 13183941bf6fSCorentin Chary png_set_filter(png_ptr, PNG_FILTER_TYPE_DEFAULT, filters); 1319efe556adSCorentin Chary 1320efe556adSCorentin Chary if (palette) { 1321efe556adSCorentin Chary color_type = PNG_COLOR_TYPE_PALETTE; 1322efe556adSCorentin Chary } else { 1323efe556adSCorentin Chary color_type = PNG_COLOR_TYPE_RGB; 1324efe556adSCorentin Chary } 1325efe556adSCorentin Chary 1326efe556adSCorentin Chary png_set_IHDR(png_ptr, info_ptr, w, h, 1327efe556adSCorentin Chary 8, color_type, PNG_INTERLACE_NONE, 1328efe556adSCorentin Chary PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT); 1329efe556adSCorentin Chary 1330efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) { 1331efe556adSCorentin Chary struct palette_cb_priv priv; 1332efe556adSCorentin Chary 1333efe556adSCorentin Chary png_palette = png_malloc(png_ptr, sizeof(*png_palette) * 13345136a052SCorentin Chary palette_size(palette)); 1335efe556adSCorentin Chary 1336efe556adSCorentin Chary priv.vs = vs; 1337efe556adSCorentin Chary priv.png_palette = png_palette; 13385136a052SCorentin Chary palette_iter(palette, write_png_palette, &priv); 1339efe556adSCorentin Chary 13405136a052SCorentin Chary png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette)); 1341efe556adSCorentin Chary 13429f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4) { 13436bf21f3dSLi Qiang tight_encode_indexed_rect32(vs->tight->tight.buffer, w * h, 13446bf21f3dSLi Qiang palette); 1345efe556adSCorentin Chary } else { 13466bf21f3dSLi Qiang tight_encode_indexed_rect16(vs->tight->tight.buffer, w * h, 13476bf21f3dSLi Qiang palette); 1348efe556adSCorentin Chary } 1349efe556adSCorentin Chary } 1350efe556adSCorentin Chary 1351efe556adSCorentin Chary png_write_info(png_ptr, info_ptr); 1352efe556adSCorentin Chary 13536bf21f3dSLi Qiang buffer_reserve(&vs->tight->png, 2048); 135447683d66SGerd Hoffmann linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w); 135547683d66SGerd Hoffmann buf = (uint8_t *)pixman_image_get_data(linebuf); 1356efe556adSCorentin Chary for (dy = 0; dy < h; dy++) 1357efe556adSCorentin Chary { 1358efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) { 13596bf21f3dSLi Qiang memcpy(buf, vs->tight->tight.buffer + (dy * w), w); 1360efe556adSCorentin Chary } else { 1361bc210eb1SGerd Hoffmann qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy); 1362efe556adSCorentin Chary } 1363efe556adSCorentin Chary png_write_row(png_ptr, buf); 1364efe556adSCorentin Chary } 136547683d66SGerd Hoffmann qemu_pixman_image_unref(linebuf); 1366efe556adSCorentin Chary 1367efe556adSCorentin Chary png_write_end(png_ptr, NULL); 1368efe556adSCorentin Chary 1369efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) { 1370efe556adSCorentin Chary png_free(png_ptr, png_palette); 1371efe556adSCorentin Chary } 1372efe556adSCorentin Chary 1373efe556adSCorentin Chary png_destroy_write_struct(&png_ptr, &info_ptr); 1374efe556adSCorentin Chary 1375efe556adSCorentin Chary vnc_write_u8(vs, VNC_TIGHT_PNG << 4); 1376efe556adSCorentin Chary 13776bf21f3dSLi Qiang tight_send_compact_size(vs, vs->tight->png.offset); 13786bf21f3dSLi Qiang vnc_write(vs, vs->tight->png.buffer, vs->tight->png.offset); 13796bf21f3dSLi Qiang buffer_reset(&vs->tight->png); 1380efe556adSCorentin Chary return 1; 1381efe556adSCorentin Chary } 1382*95f8510eSKshitij Suri #endif /* CONFIG_PNG */ 1383efe556adSCorentin Chary 1384245f7b51SCorentin Chary static void vnc_tight_start(VncState *vs) 1385245f7b51SCorentin Chary { 13866bf21f3dSLi Qiang buffer_reset(&vs->tight->tight); 1387245f7b51SCorentin Chary 1388245f7b51SCorentin Chary // make the output buffer be the zlib buffer, so we can compress it later 13896bf21f3dSLi Qiang vs->tight->tmp = vs->output; 13906bf21f3dSLi Qiang vs->output = vs->tight->tight; 1391245f7b51SCorentin Chary } 1392245f7b51SCorentin Chary 1393245f7b51SCorentin Chary static void vnc_tight_stop(VncState *vs) 1394245f7b51SCorentin Chary { 1395245f7b51SCorentin Chary // switch back to normal output/zlib buffers 13966bf21f3dSLi Qiang vs->tight->tight = vs->output; 13976bf21f3dSLi Qiang vs->output = vs->tight->tmp; 1398245f7b51SCorentin Chary } 1399245f7b51SCorentin Chary 140003817eb8SCorentin Chary static int send_sub_rect_nojpeg(VncState *vs, int x, int y, int w, int h, 140103817eb8SCorentin Chary int bg, int fg, int colors, VncPalette *palette) 140203817eb8SCorentin Chary { 140303817eb8SCorentin Chary int ret; 140403817eb8SCorentin Chary 140503817eb8SCorentin Chary if (colors == 0) { 140603817eb8SCorentin Chary if (tight_detect_smooth_image(vs, w, h)) { 140703817eb8SCorentin Chary ret = send_gradient_rect(vs, x, y, w, h); 140803817eb8SCorentin Chary } else { 140903817eb8SCorentin Chary ret = send_full_color_rect(vs, x, y, w, h); 141003817eb8SCorentin Chary } 141103817eb8SCorentin Chary } else if (colors == 1) { 141203817eb8SCorentin Chary ret = send_solid_rect(vs); 141303817eb8SCorentin Chary } else if (colors == 2) { 141403817eb8SCorentin Chary ret = send_mono_rect(vs, x, y, w, h, bg, fg); 141503817eb8SCorentin Chary } else if (colors <= 256) { 141603817eb8SCorentin Chary ret = send_palette_rect(vs, x, y, w, h, palette); 1417d167f9bcSBlue Swirl } else { 1418d167f9bcSBlue Swirl ret = 0; 141903817eb8SCorentin Chary } 142003817eb8SCorentin Chary return ret; 142103817eb8SCorentin Chary } 142203817eb8SCorentin Chary 142303817eb8SCorentin Chary #ifdef CONFIG_VNC_JPEG 142403817eb8SCorentin Chary static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h, 142503817eb8SCorentin Chary int bg, int fg, int colors, 1426ce702e93SCorentin Chary VncPalette *palette, bool force) 142703817eb8SCorentin Chary { 142803817eb8SCorentin Chary int ret; 142903817eb8SCorentin Chary 143003817eb8SCorentin Chary if (colors == 0) { 14316bf21f3dSLi Qiang if (force || (tight_jpeg_conf[vs->tight->quality].jpeg_full && 1432ce702e93SCorentin Chary tight_detect_smooth_image(vs, w, h))) { 14336bf21f3dSLi Qiang int quality = tight_conf[vs->tight->quality].jpeg_quality; 143403817eb8SCorentin Chary 143503817eb8SCorentin Chary ret = send_jpeg_rect(vs, x, y, w, h, quality); 143603817eb8SCorentin Chary } else { 143703817eb8SCorentin Chary ret = send_full_color_rect(vs, x, y, w, h); 143803817eb8SCorentin Chary } 143903817eb8SCorentin Chary } else if (colors == 1) { 144003817eb8SCorentin Chary ret = send_solid_rect(vs); 144103817eb8SCorentin Chary } else if (colors == 2) { 144203817eb8SCorentin Chary ret = send_mono_rect(vs, x, y, w, h, bg, fg); 144303817eb8SCorentin Chary } else if (colors <= 256) { 1444ce702e93SCorentin Chary if (force || (colors > 96 && 14456bf21f3dSLi Qiang tight_jpeg_conf[vs->tight->quality].jpeg_idx && 1446ce702e93SCorentin Chary tight_detect_smooth_image(vs, w, h))) { 14476bf21f3dSLi Qiang int quality = tight_conf[vs->tight->quality].jpeg_quality; 144803817eb8SCorentin Chary 144903817eb8SCorentin Chary ret = send_jpeg_rect(vs, x, y, w, h, quality); 145003817eb8SCorentin Chary } else { 145103817eb8SCorentin Chary ret = send_palette_rect(vs, x, y, w, h, palette); 145203817eb8SCorentin Chary } 1453ad7ee4adSBlue Swirl } else { 1454ad7ee4adSBlue Swirl ret = 0; 145503817eb8SCorentin Chary } 145603817eb8SCorentin Chary return ret; 145703817eb8SCorentin Chary } 145803817eb8SCorentin Chary #endif 145903817eb8SCorentin Chary 146066668d19SPeter Lieven static __thread VncPalette *color_count_palette; 146166668d19SPeter Lieven static __thread Notifier vnc_tight_cleanup_notifier; 146266668d19SPeter Lieven 146366668d19SPeter Lieven static void vnc_tight_cleanup(Notifier *n, void *value) 146466668d19SPeter Lieven { 146566668d19SPeter Lieven g_free(color_count_palette); 146666668d19SPeter Lieven color_count_palette = NULL; 146766668d19SPeter Lieven } 1468095497ffSPeter Lieven 1469245f7b51SCorentin Chary static int send_sub_rect(VncState *vs, int x, int y, int w, int h) 1470245f7b51SCorentin Chary { 1471245f7b51SCorentin Chary uint32_t bg = 0, fg = 0; 1472245f7b51SCorentin Chary int colors; 1473245f7b51SCorentin Chary int ret = 0; 1474cf76a1ceSPeter Maydell #ifdef CONFIG_VNC_JPEG 1475ce702e93SCorentin Chary bool force_jpeg = false; 1476ce702e93SCorentin Chary bool allow_jpeg = true; 1477cf76a1ceSPeter Maydell #endif 1478245f7b51SCorentin Chary 147966668d19SPeter Lieven if (!color_count_palette) { 1480b21e2380SMarkus Armbruster color_count_palette = g_new(VncPalette, 1); 148166668d19SPeter Lieven vnc_tight_cleanup_notifier.notify = vnc_tight_cleanup; 148266668d19SPeter Lieven qemu_thread_atexit_add(&vnc_tight_cleanup_notifier); 148366668d19SPeter Lieven } 148466668d19SPeter Lieven 14856bf21f3dSLi Qiang vnc_framebuffer_update(vs, x, y, w, h, vs->tight->type); 1486245f7b51SCorentin Chary 1487245f7b51SCorentin Chary vnc_tight_start(vs); 1488245f7b51SCorentin Chary vnc_raw_send_framebuffer_update(vs, x, y, w, h); 1489245f7b51SCorentin Chary vnc_tight_stop(vs); 1490245f7b51SCorentin Chary 1491ce702e93SCorentin Chary #ifdef CONFIG_VNC_JPEG 14926bf21f3dSLi Qiang if (!vs->vd->non_adaptive && vs->tight->quality != (uint8_t)-1) { 1493ce702e93SCorentin Chary double freq = vnc_update_freq(vs, x, y, w, h); 1494ce702e93SCorentin Chary 14956bf21f3dSLi Qiang if (freq < tight_jpeg_conf[vs->tight->quality].jpeg_freq_min) { 1496ce702e93SCorentin Chary allow_jpeg = false; 1497ce702e93SCorentin Chary } 14986bf21f3dSLi Qiang if (freq >= tight_jpeg_conf[vs->tight->quality].jpeg_freq_threshold) { 1499ce702e93SCorentin Chary force_jpeg = true; 1500ce702e93SCorentin Chary vnc_sent_lossy_rect(vs, x, y, w, h); 1501ce702e93SCorentin Chary } 1502ce702e93SCorentin Chary } 1503ce702e93SCorentin Chary #endif 1504ce702e93SCorentin Chary 150566668d19SPeter Lieven colors = tight_fill_palette(vs, x, y, w * h, &bg, &fg, color_count_palette); 1506245f7b51SCorentin Chary 1507245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG 15086bf21f3dSLi Qiang if (allow_jpeg && vs->tight->quality != (uint8_t)-1) { 150966668d19SPeter Lieven ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, 151066668d19SPeter Lieven color_count_palette, force_jpeg); 1511245f7b51SCorentin Chary } else { 151266668d19SPeter Lieven ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, 151366668d19SPeter Lieven color_count_palette); 1514245f7b51SCorentin Chary } 1515245f7b51SCorentin Chary #else 151666668d19SPeter Lieven ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, 151766668d19SPeter Lieven color_count_palette); 1518245f7b51SCorentin Chary #endif 151903817eb8SCorentin Chary 1520245f7b51SCorentin Chary return ret; 1521245f7b51SCorentin Chary } 1522245f7b51SCorentin Chary 1523245f7b51SCorentin Chary static int send_sub_rect_solid(VncState *vs, int x, int y, int w, int h) 1524245f7b51SCorentin Chary { 15256bf21f3dSLi Qiang vnc_framebuffer_update(vs, x, y, w, h, vs->tight->type); 1526245f7b51SCorentin Chary 1527245f7b51SCorentin Chary vnc_tight_start(vs); 1528245f7b51SCorentin Chary vnc_raw_send_framebuffer_update(vs, x, y, w, h); 1529245f7b51SCorentin Chary vnc_tight_stop(vs); 1530245f7b51SCorentin Chary 1531245f7b51SCorentin Chary return send_solid_rect(vs); 1532245f7b51SCorentin Chary } 1533245f7b51SCorentin Chary 1534ce702e93SCorentin Chary static int send_rect_simple(VncState *vs, int x, int y, int w, int h, 1535ce702e93SCorentin Chary bool split) 1536245f7b51SCorentin Chary { 1537245f7b51SCorentin Chary int max_size, max_width; 1538245f7b51SCorentin Chary int max_sub_width, max_sub_height; 1539245f7b51SCorentin Chary int dx, dy; 1540245f7b51SCorentin Chary int rw, rh; 1541245f7b51SCorentin Chary int n = 0; 1542245f7b51SCorentin Chary 15436bf21f3dSLi Qiang max_size = tight_conf[vs->tight->compression].max_rect_size; 15446bf21f3dSLi Qiang max_width = tight_conf[vs->tight->compression].max_rect_width; 1545245f7b51SCorentin Chary 1546ce702e93SCorentin Chary if (split && (w > max_width || w * h > max_size)) { 1547245f7b51SCorentin Chary max_sub_width = (w > max_width) ? max_width : w; 1548245f7b51SCorentin Chary max_sub_height = max_size / max_sub_width; 1549245f7b51SCorentin Chary 1550245f7b51SCorentin Chary for (dy = 0; dy < h; dy += max_sub_height) { 1551245f7b51SCorentin Chary for (dx = 0; dx < w; dx += max_width) { 1552245f7b51SCorentin Chary rw = MIN(max_sub_width, w - dx); 1553245f7b51SCorentin Chary rh = MIN(max_sub_height, h - dy); 1554245f7b51SCorentin Chary n += send_sub_rect(vs, x+dx, y+dy, rw, rh); 1555245f7b51SCorentin Chary } 1556245f7b51SCorentin Chary } 1557245f7b51SCorentin Chary } else { 1558245f7b51SCorentin Chary n += send_sub_rect(vs, x, y, w, h); 1559245f7b51SCorentin Chary } 1560245f7b51SCorentin Chary 1561245f7b51SCorentin Chary return n; 1562245f7b51SCorentin Chary } 1563245f7b51SCorentin Chary 1564245f7b51SCorentin Chary static int find_large_solid_color_rect(VncState *vs, int x, int y, 1565245f7b51SCorentin Chary int w, int h, int max_rows) 1566245f7b51SCorentin Chary { 1567245f7b51SCorentin Chary int dx, dy, dw, dh; 1568245f7b51SCorentin Chary int n = 0; 1569245f7b51SCorentin Chary 1570245f7b51SCorentin Chary /* Try to find large solid-color areas and send them separately. */ 1571245f7b51SCorentin Chary 1572245f7b51SCorentin Chary for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) { 1573245f7b51SCorentin Chary 1574245f7b51SCorentin Chary /* If a rectangle becomes too large, send its upper part now. */ 1575245f7b51SCorentin Chary 1576245f7b51SCorentin Chary if (dy - y >= max_rows) { 1577ce702e93SCorentin Chary n += send_rect_simple(vs, x, y, w, max_rows, true); 1578245f7b51SCorentin Chary y += max_rows; 1579245f7b51SCorentin Chary h -= max_rows; 1580245f7b51SCorentin Chary } 1581245f7b51SCorentin Chary 1582245f7b51SCorentin Chary dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (y + h - dy)); 1583245f7b51SCorentin Chary 1584245f7b51SCorentin Chary for (dx = x; dx < x + w; dx += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) { 1585245f7b51SCorentin Chary uint32_t color_value; 1586245f7b51SCorentin Chary int x_best, y_best, w_best, h_best; 1587245f7b51SCorentin Chary 1588245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (x + w - dx)); 1589245f7b51SCorentin Chary 1590245f7b51SCorentin Chary if (!check_solid_tile(vs, dx, dy, dw, dh, &color_value, false)) { 1591245f7b51SCorentin Chary continue ; 1592245f7b51SCorentin Chary } 1593245f7b51SCorentin Chary 1594245f7b51SCorentin Chary /* Get dimensions of solid-color area. */ 1595245f7b51SCorentin Chary 1596245f7b51SCorentin Chary find_best_solid_area(vs, dx, dy, w - (dx - x), h - (dy - y), 1597245f7b51SCorentin Chary color_value, &w_best, &h_best); 1598245f7b51SCorentin Chary 1599245f7b51SCorentin Chary /* Make sure a solid rectangle is large enough 1600245f7b51SCorentin Chary (or the whole rectangle is of the same color). */ 1601245f7b51SCorentin Chary 1602245f7b51SCorentin Chary if (w_best * h_best != w * h && 1603245f7b51SCorentin Chary w_best * h_best < VNC_TIGHT_MIN_SOLID_SUBRECT_SIZE) { 1604245f7b51SCorentin Chary continue; 1605245f7b51SCorentin Chary } 1606245f7b51SCorentin Chary 1607245f7b51SCorentin Chary /* Try to extend solid rectangle to maximum size. */ 1608245f7b51SCorentin Chary 1609245f7b51SCorentin Chary x_best = dx; y_best = dy; 1610245f7b51SCorentin Chary extend_solid_area(vs, x, y, w, h, color_value, 1611245f7b51SCorentin Chary &x_best, &y_best, &w_best, &h_best); 1612245f7b51SCorentin Chary 1613245f7b51SCorentin Chary /* Send rectangles at top and left to solid-color area. */ 1614245f7b51SCorentin Chary 1615245f7b51SCorentin Chary if (y_best != y) { 1616ce702e93SCorentin Chary n += send_rect_simple(vs, x, y, w, y_best-y, true); 1617245f7b51SCorentin Chary } 1618245f7b51SCorentin Chary if (x_best != x) { 1619efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x, y_best, 1620245f7b51SCorentin Chary x_best-x, h_best); 1621245f7b51SCorentin Chary } 1622245f7b51SCorentin Chary 1623245f7b51SCorentin Chary /* Send solid-color rectangle. */ 1624245f7b51SCorentin Chary n += send_sub_rect_solid(vs, x_best, y_best, w_best, h_best); 1625245f7b51SCorentin Chary 1626245f7b51SCorentin Chary /* Send remaining rectangles (at right and bottom). */ 1627245f7b51SCorentin Chary 1628245f7b51SCorentin Chary if (x_best + w_best != x + w) { 1629efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x_best+w_best, 1630245f7b51SCorentin Chary y_best, 1631245f7b51SCorentin Chary w-(x_best-x)-w_best, 1632245f7b51SCorentin Chary h_best); 1633245f7b51SCorentin Chary } 1634245f7b51SCorentin Chary if (y_best + h_best != y + h) { 1635efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x, y_best+h_best, 1636245f7b51SCorentin Chary w, h-(y_best-y)-h_best); 1637245f7b51SCorentin Chary } 1638245f7b51SCorentin Chary 1639245f7b51SCorentin Chary /* Return after all recursive calls are done. */ 1640245f7b51SCorentin Chary return n; 1641245f7b51SCorentin Chary } 1642245f7b51SCorentin Chary } 1643ce702e93SCorentin Chary return n + send_rect_simple(vs, x, y, w, h, true); 1644245f7b51SCorentin Chary } 1645245f7b51SCorentin Chary 1646efe556adSCorentin Chary static int tight_send_framebuffer_update(VncState *vs, int x, int y, 1647245f7b51SCorentin Chary int w, int h) 1648245f7b51SCorentin Chary { 1649245f7b51SCorentin Chary int max_rows; 1650245f7b51SCorentin Chary 16519f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4 && vs->client_pf.rmax == 0xFF && 16529f64916dSGerd Hoffmann vs->client_pf.bmax == 0xFF && vs->client_pf.gmax == 0xFF) { 16536bf21f3dSLi Qiang vs->tight->pixel24 = true; 1654245f7b51SCorentin Chary } else { 16556bf21f3dSLi Qiang vs->tight->pixel24 = false; 1656245f7b51SCorentin Chary } 1657245f7b51SCorentin Chary 1658cf76a1ceSPeter Maydell #ifdef CONFIG_VNC_JPEG 16596bf21f3dSLi Qiang if (vs->tight->quality != (uint8_t)-1) { 1660ce702e93SCorentin Chary double freq = vnc_update_freq(vs, x, y, w, h); 1661ce702e93SCorentin Chary 16626bf21f3dSLi Qiang if (freq > tight_jpeg_conf[vs->tight->quality].jpeg_freq_threshold) { 1663ce702e93SCorentin Chary return send_rect_simple(vs, x, y, w, h, false); 1664ce702e93SCorentin Chary } 1665ce702e93SCorentin Chary } 1666cf76a1ceSPeter Maydell #endif 1667ce702e93SCorentin Chary 1668ce702e93SCorentin Chary if (w * h < VNC_TIGHT_MIN_SPLIT_RECT_SIZE) { 1669ce702e93SCorentin Chary return send_rect_simple(vs, x, y, w, h, true); 1670ce702e93SCorentin Chary } 1671245f7b51SCorentin Chary 1672245f7b51SCorentin Chary /* Calculate maximum number of rows in one non-solid rectangle. */ 1673245f7b51SCorentin Chary 16746bf21f3dSLi Qiang max_rows = tight_conf[vs->tight->compression].max_rect_size; 16756bf21f3dSLi Qiang max_rows /= MIN(tight_conf[vs->tight->compression].max_rect_width, w); 1676245f7b51SCorentin Chary 1677245f7b51SCorentin Chary return find_large_solid_color_rect(vs, x, y, w, h, max_rows); 1678245f7b51SCorentin Chary } 1679245f7b51SCorentin Chary 1680efe556adSCorentin Chary int vnc_tight_send_framebuffer_update(VncState *vs, int x, int y, 1681efe556adSCorentin Chary int w, int h) 1682efe556adSCorentin Chary { 16836bf21f3dSLi Qiang vs->tight->type = VNC_ENCODING_TIGHT; 1684efe556adSCorentin Chary return tight_send_framebuffer_update(vs, x, y, w, h); 1685efe556adSCorentin Chary } 1686efe556adSCorentin Chary 1687efe556adSCorentin Chary int vnc_tight_png_send_framebuffer_update(VncState *vs, int x, int y, 1688efe556adSCorentin Chary int w, int h) 1689efe556adSCorentin Chary { 16906bf21f3dSLi Qiang vs->tight->type = VNC_ENCODING_TIGHT_PNG; 1691efe556adSCorentin Chary return tight_send_framebuffer_update(vs, x, y, w, h); 1692efe556adSCorentin Chary } 1693efe556adSCorentin Chary 1694245f7b51SCorentin Chary void vnc_tight_clear(VncState *vs) 1695245f7b51SCorentin Chary { 1696245f7b51SCorentin Chary int i; 16976bf21f3dSLi Qiang for (i = 0; i < ARRAY_SIZE(vs->tight->stream); i++) { 16986bf21f3dSLi Qiang if (vs->tight->stream[i].opaque) { 16996bf21f3dSLi Qiang deflateEnd(&vs->tight->stream[i]); 1700245f7b51SCorentin Chary } 1701245f7b51SCorentin Chary } 1702245f7b51SCorentin Chary 17036bf21f3dSLi Qiang buffer_free(&vs->tight->tight); 17046bf21f3dSLi Qiang buffer_free(&vs->tight->zlib); 17056bf21f3dSLi Qiang buffer_free(&vs->tight->gradient); 1706245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG 17076bf21f3dSLi Qiang buffer_free(&vs->tight->jpeg); 1708245f7b51SCorentin Chary #endif 1709*95f8510eSKshitij Suri #ifdef CONFIG_PNG 17106bf21f3dSLi Qiang buffer_free(&vs->tight->png); 1711b5469b11SCorentin Chary #endif 1712245f7b51SCorentin Chary } 1713