1245f7b51SCorentin Chary /* 2245f7b51SCorentin Chary * QEMU VNC display driver: tight encoding 3245f7b51SCorentin Chary * 4245f7b51SCorentin Chary * From libvncserver/libvncserver/tight.c 5245f7b51SCorentin Chary * Copyright (C) 2000, 2001 Const Kaplinsky. All Rights Reserved. 6245f7b51SCorentin Chary * Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved. 7245f7b51SCorentin Chary * 8245f7b51SCorentin Chary * Copyright (C) 2010 Corentin Chary <corentin.chary@gmail.com> 9245f7b51SCorentin Chary * 10245f7b51SCorentin Chary * Permission is hereby granted, free of charge, to any person obtaining a copy 11245f7b51SCorentin Chary * of this software and associated documentation files (the "Software"), to deal 12245f7b51SCorentin Chary * in the Software without restriction, including without limitation the rights 13245f7b51SCorentin Chary * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 14245f7b51SCorentin Chary * copies of the Software, and to permit persons to whom the Software is 15245f7b51SCorentin Chary * furnished to do so, subject to the following conditions: 16245f7b51SCorentin Chary * 17245f7b51SCorentin Chary * The above copyright notice and this permission notice shall be included in 18245f7b51SCorentin Chary * all copies or substantial portions of the Software. 19245f7b51SCorentin Chary * 20245f7b51SCorentin Chary * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21245f7b51SCorentin Chary * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22245f7b51SCorentin Chary * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 23245f7b51SCorentin Chary * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24245f7b51SCorentin Chary * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 25245f7b51SCorentin Chary * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 26245f7b51SCorentin Chary * THE SOFTWARE. 27245f7b51SCorentin Chary */ 28245f7b51SCorentin Chary 29efe556adSCorentin Chary #include "config-host.h" 30245f7b51SCorentin Chary 31f26e428dSRoy Tam /* This needs to be before jpeglib.h line because of conflict with 32f26e428dSRoy Tam INT32 definitions between jmorecfg.h (included by jpeglib.h) and 33f26e428dSRoy Tam Win32 basetsd.h (included by windows.h). */ 34f26e428dSRoy Tam #include "qemu-common.h" 35f26e428dSRoy Tam 36efe556adSCorentin Chary #ifdef CONFIG_VNC_PNG 372fb0c09fSStefan Weil /* The following define is needed by pngconf.h. Otherwise it won't compile, 382fb0c09fSStefan Weil because setjmp.h was already included by qemu-common.h. */ 392fb0c09fSStefan Weil #define PNG_SKIP_SETJMP_CHECK 40efe556adSCorentin Chary #include <png.h> 41efe556adSCorentin Chary #endif 42245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG 43245f7b51SCorentin Chary #include <stdio.h> 44245f7b51SCorentin Chary #include <jpeglib.h> 45245f7b51SCorentin Chary #endif 46245f7b51SCorentin Chary 471de7afc9SPaolo Bonzini #include "qemu/bswap.h" 487b1b5d19SPaolo Bonzini #include "qapi/qmp/qint.h" 49245f7b51SCorentin Chary #include "vnc.h" 50245f7b51SCorentin Chary #include "vnc-enc-tight.h" 515136a052SCorentin Chary #include "vnc-palette.h" 52245f7b51SCorentin Chary 53245f7b51SCorentin Chary /* Compression level stuff. The following array contains various 54245f7b51SCorentin Chary encoder parameters for each of 10 compression levels (0..9). 55245f7b51SCorentin Chary Last three parameters correspond to JPEG quality levels (0..9). */ 56245f7b51SCorentin Chary 57245f7b51SCorentin Chary static const struct { 58245f7b51SCorentin Chary int max_rect_size, max_rect_width; 59245f7b51SCorentin Chary int mono_min_rect_size, gradient_min_rect_size; 60245f7b51SCorentin Chary int idx_zlib_level, mono_zlib_level, raw_zlib_level, gradient_zlib_level; 61245f7b51SCorentin Chary int gradient_threshold, gradient_threshold24; 62245f7b51SCorentin Chary int idx_max_colors_divisor; 63245f7b51SCorentin Chary int jpeg_quality, jpeg_threshold, jpeg_threshold24; 64245f7b51SCorentin Chary } tight_conf[] = { 65245f7b51SCorentin Chary { 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 5, 10000, 23000 }, 66245f7b51SCorentin Chary { 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 10, 8000, 18000 }, 67245f7b51SCorentin Chary { 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 15, 6500, 15000 }, 68245f7b51SCorentin Chary { 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 25, 5000, 12000 }, 69245f7b51SCorentin Chary { 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 37, 4000, 10000 }, 70245f7b51SCorentin Chary { 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 50, 3000, 8000 }, 71245f7b51SCorentin Chary { 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 60, 2000, 5000 }, 72245f7b51SCorentin Chary { 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 }, 73245f7b51SCorentin Chary { 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 }, 74245f7b51SCorentin Chary { 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 } 75245f7b51SCorentin Chary }; 76245f7b51SCorentin Chary 77efe556adSCorentin Chary 78efe556adSCorentin Chary static int tight_send_framebuffer_update(VncState *vs, int x, int y, 79efe556adSCorentin Chary int w, int h); 80efe556adSCorentin Chary 81ce702e93SCorentin Chary #ifdef CONFIG_VNC_JPEG 82ce702e93SCorentin Chary static const struct { 83ce702e93SCorentin Chary double jpeg_freq_min; /* Don't send JPEG if the freq is bellow */ 84ce702e93SCorentin Chary double jpeg_freq_threshold; /* Always send JPEG if the freq is above */ 85ce702e93SCorentin Chary int jpeg_idx; /* Allow indexed JPEG */ 86ce702e93SCorentin Chary int jpeg_full; /* Allow full color JPEG */ 87ce702e93SCorentin Chary } tight_jpeg_conf[] = { 888cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 898cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 908cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 918cb4a6b7SCorentin Chary { 0, 8, 1, 1 }, 928cb4a6b7SCorentin Chary { 0, 10, 1, 1 }, 938cb4a6b7SCorentin Chary { 0.1, 10, 1, 1 }, 948cb4a6b7SCorentin Chary { 0.2, 10, 1, 1 }, 958cb4a6b7SCorentin Chary { 0.3, 12, 0, 0 }, 968cb4a6b7SCorentin Chary { 0.4, 14, 0, 0 }, 978cb4a6b7SCorentin Chary { 0.5, 16, 0, 0 }, 98ce702e93SCorentin Chary }; 99ce702e93SCorentin Chary #endif 100ce702e93SCorentin Chary 101efe556adSCorentin Chary #ifdef CONFIG_VNC_PNG 1023941bf6fSCorentin Chary static const struct { 1033941bf6fSCorentin Chary int png_zlib_level, png_filters; 1043941bf6fSCorentin Chary } tight_png_conf[] = { 1053941bf6fSCorentin Chary { 0, PNG_NO_FILTERS }, 1063941bf6fSCorentin Chary { 1, PNG_NO_FILTERS }, 1073941bf6fSCorentin Chary { 2, PNG_NO_FILTERS }, 1083941bf6fSCorentin Chary { 3, PNG_NO_FILTERS }, 1093941bf6fSCorentin Chary { 4, PNG_NO_FILTERS }, 1103941bf6fSCorentin Chary { 5, PNG_ALL_FILTERS }, 1113941bf6fSCorentin Chary { 6, PNG_ALL_FILTERS }, 1123941bf6fSCorentin Chary { 7, PNG_ALL_FILTERS }, 1133941bf6fSCorentin Chary { 8, PNG_ALL_FILTERS }, 1143941bf6fSCorentin Chary { 9, PNG_ALL_FILTERS }, 1153941bf6fSCorentin Chary }; 1163941bf6fSCorentin Chary 117efe556adSCorentin Chary static int send_png_rect(VncState *vs, int x, int y, int w, int h, 1185136a052SCorentin Chary VncPalette *palette); 119efe556adSCorentin Chary 120efe556adSCorentin Chary static bool tight_can_send_png_rect(VncState *vs, int w, int h) 121efe556adSCorentin Chary { 122d1af0e05SCorentin Chary if (vs->tight.type != VNC_ENCODING_TIGHT_PNG) { 123efe556adSCorentin Chary return false; 124efe556adSCorentin Chary } 125efe556adSCorentin Chary 126d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1 || 1279f64916dSGerd Hoffmann vs->client_pf.bytes_per_pixel == 1) { 128efe556adSCorentin Chary return false; 129efe556adSCorentin Chary } 130efe556adSCorentin Chary 131efe556adSCorentin Chary return true; 132efe556adSCorentin Chary } 133efe556adSCorentin Chary #endif 134efe556adSCorentin Chary 135245f7b51SCorentin Chary /* 136245f7b51SCorentin Chary * Code to guess if given rectangle is suitable for smooth image 137245f7b51SCorentin Chary * compression (by applying "gradient" filter or JPEG coder). 138245f7b51SCorentin Chary */ 139245f7b51SCorentin Chary 140249cdb42SBlue Swirl static unsigned int 141245f7b51SCorentin Chary tight_detect_smooth_image24(VncState *vs, int w, int h) 142245f7b51SCorentin Chary { 143245f7b51SCorentin Chary int off; 144245f7b51SCorentin Chary int x, y, d, dx; 145249cdb42SBlue Swirl unsigned int c; 146249cdb42SBlue Swirl unsigned int stats[256]; 147245f7b51SCorentin Chary int pixels = 0; 148245f7b51SCorentin Chary int pix, left[3]; 149249cdb42SBlue Swirl unsigned int errors; 150d1af0e05SCorentin Chary unsigned char *buf = vs->tight.tight.buffer; 151245f7b51SCorentin Chary 152245f7b51SCorentin Chary /* 153245f7b51SCorentin Chary * If client is big-endian, color samples begin from the second 154245f7b51SCorentin Chary * byte (offset 1) of a 32-bit pixel value. 155245f7b51SCorentin Chary */ 1569f64916dSGerd Hoffmann off = vs->client_be; 157245f7b51SCorentin Chary 158245f7b51SCorentin Chary memset(stats, 0, sizeof (stats)); 159245f7b51SCorentin Chary 160245f7b51SCorentin Chary for (y = 0, x = 0; y < h && x < w;) { 161245f7b51SCorentin Chary for (d = 0; d < h - y && d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; 162245f7b51SCorentin Chary d++) { 163245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 164245f7b51SCorentin Chary left[c] = buf[((y+d)*w+x+d)*4+off+c] & 0xFF; 165245f7b51SCorentin Chary } 166245f7b51SCorentin Chary for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; dx++) { 167245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 168245f7b51SCorentin Chary pix = buf[((y+d)*w+x+d+dx)*4+off+c] & 0xFF; 169245f7b51SCorentin Chary stats[abs(pix - left[c])]++; 170245f7b51SCorentin Chary left[c] = pix; 171245f7b51SCorentin Chary } 172245f7b51SCorentin Chary pixels++; 173245f7b51SCorentin Chary } 174245f7b51SCorentin Chary } 175245f7b51SCorentin Chary if (w > h) { 176245f7b51SCorentin Chary x += h; 177245f7b51SCorentin Chary y = 0; 178245f7b51SCorentin Chary } else { 179245f7b51SCorentin Chary x = 0; 180245f7b51SCorentin Chary y += w; 181245f7b51SCorentin Chary } 182245f7b51SCorentin Chary } 183245f7b51SCorentin Chary 184b5299153SGonglei if (pixels == 0) { 185b5299153SGonglei return 0; 186b5299153SGonglei } 187b5299153SGonglei 188245f7b51SCorentin Chary /* 95% smooth or more ... */ 189245f7b51SCorentin Chary if (stats[0] * 33 / pixels >= 95) { 190245f7b51SCorentin Chary return 0; 191245f7b51SCorentin Chary } 192245f7b51SCorentin Chary 193245f7b51SCorentin Chary errors = 0; 194245f7b51SCorentin Chary for (c = 1; c < 8; c++) { 195245f7b51SCorentin Chary errors += stats[c] * (c * c); 196245f7b51SCorentin Chary if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { 197245f7b51SCorentin Chary return 0; 198245f7b51SCorentin Chary } 199245f7b51SCorentin Chary } 200245f7b51SCorentin Chary for (; c < 256; c++) { 201245f7b51SCorentin Chary errors += stats[c] * (c * c); 202245f7b51SCorentin Chary } 203245f7b51SCorentin Chary errors /= (pixels * 3 - stats[0]); 204245f7b51SCorentin Chary 205245f7b51SCorentin Chary return errors; 206245f7b51SCorentin Chary } 207245f7b51SCorentin Chary 208245f7b51SCorentin Chary #define DEFINE_DETECT_FUNCTION(bpp) \ 209245f7b51SCorentin Chary \ 210249cdb42SBlue Swirl static unsigned int \ 211245f7b51SCorentin Chary tight_detect_smooth_image##bpp(VncState *vs, int w, int h) { \ 212245f7b51SCorentin Chary bool endian; \ 213245f7b51SCorentin Chary uint##bpp##_t pix; \ 214245f7b51SCorentin Chary int max[3], shift[3]; \ 215245f7b51SCorentin Chary int x, y, d, dx; \ 216249cdb42SBlue Swirl unsigned int c; \ 217249cdb42SBlue Swirl unsigned int stats[256]; \ 218245f7b51SCorentin Chary int pixels = 0; \ 219245f7b51SCorentin Chary int sample, sum, left[3]; \ 220249cdb42SBlue Swirl unsigned int errors; \ 221d1af0e05SCorentin Chary unsigned char *buf = vs->tight.tight.buffer; \ 222245f7b51SCorentin Chary \ 223*77bfcf28SBenjamin Herrenschmidt endian = 0; /* FIXME */ \ 224245f7b51SCorentin Chary \ 225245f7b51SCorentin Chary \ 2269f64916dSGerd Hoffmann max[0] = vs->client_pf.rmax; \ 2279f64916dSGerd Hoffmann max[1] = vs->client_pf.gmax; \ 2289f64916dSGerd Hoffmann max[2] = vs->client_pf.bmax; \ 2299f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; \ 2309f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; \ 2319f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; \ 232245f7b51SCorentin Chary \ 233245f7b51SCorentin Chary memset(stats, 0, sizeof(stats)); \ 234245f7b51SCorentin Chary \ 235245f7b51SCorentin Chary y = 0, x = 0; \ 236245f7b51SCorentin Chary while (y < h && x < w) { \ 237245f7b51SCorentin Chary for (d = 0; d < h - y && \ 238245f7b51SCorentin Chary d < w - x - VNC_TIGHT_DETECT_SUBROW_WIDTH; d++) { \ 239245f7b51SCorentin Chary pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d]; \ 240245f7b51SCorentin Chary if (endian) { \ 241ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \ 242245f7b51SCorentin Chary } \ 243245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 244245f7b51SCorentin Chary left[c] = (int)(pix >> shift[c] & max[c]); \ 245245f7b51SCorentin Chary } \ 246245f7b51SCorentin Chary for (dx = 1; dx <= VNC_TIGHT_DETECT_SUBROW_WIDTH; \ 247245f7b51SCorentin Chary dx++) { \ 248245f7b51SCorentin Chary pix = ((uint##bpp##_t *)buf)[(y+d)*w+x+d+dx]; \ 249245f7b51SCorentin Chary if (endian) { \ 250ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \ 251245f7b51SCorentin Chary } \ 252245f7b51SCorentin Chary sum = 0; \ 253245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 254245f7b51SCorentin Chary sample = (int)(pix >> shift[c] & max[c]); \ 255245f7b51SCorentin Chary sum += abs(sample - left[c]); \ 256245f7b51SCorentin Chary left[c] = sample; \ 257245f7b51SCorentin Chary } \ 258245f7b51SCorentin Chary if (sum > 255) { \ 259245f7b51SCorentin Chary sum = 255; \ 260245f7b51SCorentin Chary } \ 261245f7b51SCorentin Chary stats[sum]++; \ 262245f7b51SCorentin Chary pixels++; \ 263245f7b51SCorentin Chary } \ 264245f7b51SCorentin Chary } \ 265245f7b51SCorentin Chary if (w > h) { \ 266245f7b51SCorentin Chary x += h; \ 267245f7b51SCorentin Chary y = 0; \ 268245f7b51SCorentin Chary } else { \ 269245f7b51SCorentin Chary x = 0; \ 270245f7b51SCorentin Chary y += w; \ 271245f7b51SCorentin Chary } \ 272245f7b51SCorentin Chary } \ 273b5299153SGonglei if (pixels == 0) { \ 274b5299153SGonglei return 0; \ 275b5299153SGonglei } \ 276245f7b51SCorentin Chary if ((stats[0] + stats[1]) * 100 / pixels >= 90) { \ 277245f7b51SCorentin Chary return 0; \ 278245f7b51SCorentin Chary } \ 279245f7b51SCorentin Chary \ 280245f7b51SCorentin Chary errors = 0; \ 281245f7b51SCorentin Chary for (c = 1; c < 8; c++) { \ 282245f7b51SCorentin Chary errors += stats[c] * (c * c); \ 283245f7b51SCorentin Chary if (stats[c] == 0 || stats[c] > stats[c-1] * 2) { \ 284245f7b51SCorentin Chary return 0; \ 285245f7b51SCorentin Chary } \ 286245f7b51SCorentin Chary } \ 287245f7b51SCorentin Chary for (; c < 256; c++) { \ 288245f7b51SCorentin Chary errors += stats[c] * (c * c); \ 289245f7b51SCorentin Chary } \ 290245f7b51SCorentin Chary errors /= (pixels - stats[0]); \ 291245f7b51SCorentin Chary \ 292245f7b51SCorentin Chary return errors; \ 293245f7b51SCorentin Chary } 294245f7b51SCorentin Chary 295245f7b51SCorentin Chary DEFINE_DETECT_FUNCTION(16) 296245f7b51SCorentin Chary DEFINE_DETECT_FUNCTION(32) 297245f7b51SCorentin Chary 298245f7b51SCorentin Chary static int 299245f7b51SCorentin Chary tight_detect_smooth_image(VncState *vs, int w, int h) 300245f7b51SCorentin Chary { 301249cdb42SBlue Swirl unsigned int errors; 302d1af0e05SCorentin Chary int compression = vs->tight.compression; 303d1af0e05SCorentin Chary int quality = vs->tight.quality; 304245f7b51SCorentin Chary 305245f7b51SCorentin Chary if (!vs->vd->lossy) { 306245f7b51SCorentin Chary return 0; 307245f7b51SCorentin Chary } 308245f7b51SCorentin Chary 309d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1 || 3109f64916dSGerd Hoffmann vs->client_pf.bytes_per_pixel == 1 || 311245f7b51SCorentin Chary w < VNC_TIGHT_DETECT_MIN_WIDTH || h < VNC_TIGHT_DETECT_MIN_HEIGHT) { 312245f7b51SCorentin Chary return 0; 313245f7b51SCorentin Chary } 314245f7b51SCorentin Chary 3157bccf573SBlue Swirl if (vs->tight.quality != (uint8_t)-1) { 316245f7b51SCorentin Chary if (w * h < VNC_TIGHT_JPEG_MIN_RECT_SIZE) { 317245f7b51SCorentin Chary return 0; 318245f7b51SCorentin Chary } 319245f7b51SCorentin Chary } else { 320245f7b51SCorentin Chary if (w * h < tight_conf[compression].gradient_min_rect_size) { 321245f7b51SCorentin Chary return 0; 322245f7b51SCorentin Chary } 323245f7b51SCorentin Chary } 324245f7b51SCorentin Chary 3259f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4) { 326d1af0e05SCorentin Chary if (vs->tight.pixel24) { 327245f7b51SCorentin Chary errors = tight_detect_smooth_image24(vs, w, h); 3287bccf573SBlue Swirl if (vs->tight.quality != (uint8_t)-1) { 329245f7b51SCorentin Chary return (errors < tight_conf[quality].jpeg_threshold24); 330245f7b51SCorentin Chary } 331245f7b51SCorentin Chary return (errors < tight_conf[compression].gradient_threshold24); 332245f7b51SCorentin Chary } else { 333245f7b51SCorentin Chary errors = tight_detect_smooth_image32(vs, w, h); 334245f7b51SCorentin Chary } 335245f7b51SCorentin Chary } else { 336245f7b51SCorentin Chary errors = tight_detect_smooth_image16(vs, w, h); 337245f7b51SCorentin Chary } 3382e7bcdb9SMarkus Armbruster if (quality != (uint8_t)-1) { 339245f7b51SCorentin Chary return (errors < tight_conf[quality].jpeg_threshold); 340245f7b51SCorentin Chary } 341245f7b51SCorentin Chary return (errors < tight_conf[compression].gradient_threshold); 342245f7b51SCorentin Chary } 343245f7b51SCorentin Chary 344245f7b51SCorentin Chary /* 345245f7b51SCorentin Chary * Code to determine how many different colors used in rectangle. 346245f7b51SCorentin Chary */ 347245f7b51SCorentin Chary #define DEFINE_FILL_PALETTE_FUNCTION(bpp) \ 348245f7b51SCorentin Chary \ 349245f7b51SCorentin Chary static int \ 350245f7b51SCorentin Chary tight_fill_palette##bpp(VncState *vs, int x, int y, \ 351245f7b51SCorentin Chary int max, size_t count, \ 352245f7b51SCorentin Chary uint32_t *bg, uint32_t *fg, \ 3535136a052SCorentin Chary VncPalette **palette) { \ 354245f7b51SCorentin Chary uint##bpp##_t *data; \ 355245f7b51SCorentin Chary uint##bpp##_t c0, c1, ci; \ 356245f7b51SCorentin Chary int i, n0, n1; \ 357245f7b51SCorentin Chary \ 358d1af0e05SCorentin Chary data = (uint##bpp##_t *)vs->tight.tight.buffer; \ 359245f7b51SCorentin Chary \ 360245f7b51SCorentin Chary c0 = data[0]; \ 361245f7b51SCorentin Chary i = 1; \ 362245f7b51SCorentin Chary while (i < count && data[i] == c0) \ 363245f7b51SCorentin Chary i++; \ 364245f7b51SCorentin Chary if (i >= count) { \ 365245f7b51SCorentin Chary *bg = *fg = c0; \ 366245f7b51SCorentin Chary return 1; \ 367245f7b51SCorentin Chary } \ 368245f7b51SCorentin Chary \ 369245f7b51SCorentin Chary if (max < 2) { \ 370245f7b51SCorentin Chary return 0; \ 371245f7b51SCorentin Chary } \ 372245f7b51SCorentin Chary \ 373245f7b51SCorentin Chary n0 = i; \ 374245f7b51SCorentin Chary c1 = data[i]; \ 375245f7b51SCorentin Chary n1 = 0; \ 376245f7b51SCorentin Chary for (i++; i < count; i++) { \ 377245f7b51SCorentin Chary ci = data[i]; \ 378245f7b51SCorentin Chary if (ci == c0) { \ 379245f7b51SCorentin Chary n0++; \ 380245f7b51SCorentin Chary } else if (ci == c1) { \ 381245f7b51SCorentin Chary n1++; \ 382245f7b51SCorentin Chary } else \ 383245f7b51SCorentin Chary break; \ 384245f7b51SCorentin Chary } \ 385245f7b51SCorentin Chary if (i >= count) { \ 386245f7b51SCorentin Chary if (n0 > n1) { \ 387245f7b51SCorentin Chary *bg = (uint32_t)c0; \ 388245f7b51SCorentin Chary *fg = (uint32_t)c1; \ 389245f7b51SCorentin Chary } else { \ 390245f7b51SCorentin Chary *bg = (uint32_t)c1; \ 391245f7b51SCorentin Chary *fg = (uint32_t)c0; \ 392245f7b51SCorentin Chary } \ 393245f7b51SCorentin Chary return 2; \ 394245f7b51SCorentin Chary } \ 395245f7b51SCorentin Chary \ 396245f7b51SCorentin Chary if (max == 2) { \ 397245f7b51SCorentin Chary return 0; \ 398245f7b51SCorentin Chary } \ 399245f7b51SCorentin Chary \ 4005136a052SCorentin Chary *palette = palette_new(max, bpp); \ 4015136a052SCorentin Chary palette_put(*palette, c0); \ 4025136a052SCorentin Chary palette_put(*palette, c1); \ 4035136a052SCorentin Chary palette_put(*palette, ci); \ 404245f7b51SCorentin Chary \ 405245f7b51SCorentin Chary for (i++; i < count; i++) { \ 406245f7b51SCorentin Chary if (data[i] == ci) { \ 407245f7b51SCorentin Chary continue; \ 408245f7b51SCorentin Chary } else { \ 4095d8efe39SCorentin Chary ci = data[i]; \ 4105136a052SCorentin Chary if (!palette_put(*palette, (uint32_t)ci)) { \ 411245f7b51SCorentin Chary return 0; \ 412245f7b51SCorentin Chary } \ 413245f7b51SCorentin Chary } \ 414245f7b51SCorentin Chary } \ 415245f7b51SCorentin Chary \ 4165136a052SCorentin Chary return palette_size(*palette); \ 417245f7b51SCorentin Chary } 418245f7b51SCorentin Chary 419245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(8) 420245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(16) 421245f7b51SCorentin Chary DEFINE_FILL_PALETTE_FUNCTION(32) 422245f7b51SCorentin Chary 423245f7b51SCorentin Chary static int tight_fill_palette(VncState *vs, int x, int y, 424245f7b51SCorentin Chary size_t count, uint32_t *bg, uint32_t *fg, 4255136a052SCorentin Chary VncPalette **palette) 426245f7b51SCorentin Chary { 427245f7b51SCorentin Chary int max; 428245f7b51SCorentin Chary 429d1af0e05SCorentin Chary max = count / tight_conf[vs->tight.compression].idx_max_colors_divisor; 430245f7b51SCorentin Chary if (max < 2 && 431d1af0e05SCorentin Chary count >= tight_conf[vs->tight.compression].mono_min_rect_size) { 432245f7b51SCorentin Chary max = 2; 433245f7b51SCorentin Chary } 434245f7b51SCorentin Chary if (max >= 256) { 435245f7b51SCorentin Chary max = 256; 436245f7b51SCorentin Chary } 437245f7b51SCorentin Chary 4389f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) { 439245f7b51SCorentin Chary case 4: 440245f7b51SCorentin Chary return tight_fill_palette32(vs, x, y, max, count, bg, fg, palette); 441245f7b51SCorentin Chary case 2: 442245f7b51SCorentin Chary return tight_fill_palette16(vs, x, y, max, count, bg, fg, palette); 443245f7b51SCorentin Chary default: 444245f7b51SCorentin Chary max = 2; 445245f7b51SCorentin Chary return tight_fill_palette8(vs, x, y, max, count, bg, fg, palette); 446245f7b51SCorentin Chary } 447245f7b51SCorentin Chary return 0; 448245f7b51SCorentin Chary } 449245f7b51SCorentin Chary 450245f7b51SCorentin Chary /* 451245f7b51SCorentin Chary * Converting truecolor samples into palette indices. 452245f7b51SCorentin Chary */ 453245f7b51SCorentin Chary #define DEFINE_IDX_ENCODE_FUNCTION(bpp) \ 454245f7b51SCorentin Chary \ 455245f7b51SCorentin Chary static void \ 456245f7b51SCorentin Chary tight_encode_indexed_rect##bpp(uint8_t *buf, int count, \ 4575136a052SCorentin Chary VncPalette *palette) { \ 458245f7b51SCorentin Chary uint##bpp##_t *src; \ 459245f7b51SCorentin Chary uint##bpp##_t rgb; \ 460245f7b51SCorentin Chary int i, rep; \ 461245f7b51SCorentin Chary uint8_t idx; \ 462245f7b51SCorentin Chary \ 463245f7b51SCorentin Chary src = (uint##bpp##_t *) buf; \ 464245f7b51SCorentin Chary \ 465245f7b51SCorentin Chary for (i = 0; i < count; i++) { \ 466efe556adSCorentin Chary \ 467245f7b51SCorentin Chary rgb = *src++; \ 468245f7b51SCorentin Chary rep = 0; \ 469245f7b51SCorentin Chary while (i < count && *src == rgb) { \ 470245f7b51SCorentin Chary rep++, src++, i++; \ 471245f7b51SCorentin Chary } \ 4725136a052SCorentin Chary idx = palette_idx(palette, rgb); \ 473245f7b51SCorentin Chary /* \ 474245f7b51SCorentin Chary * Should never happen, but don't break everything \ 475245f7b51SCorentin Chary * if it does, use the first color instead \ 476245f7b51SCorentin Chary */ \ 4777bccf573SBlue Swirl if (idx == (uint8_t)-1) { \ 478245f7b51SCorentin Chary idx = 0; \ 479245f7b51SCorentin Chary } \ 480245f7b51SCorentin Chary while (rep >= 0) { \ 481245f7b51SCorentin Chary *buf++ = idx; \ 482245f7b51SCorentin Chary rep--; \ 483245f7b51SCorentin Chary } \ 484245f7b51SCorentin Chary } \ 485245f7b51SCorentin Chary } 486245f7b51SCorentin Chary 487245f7b51SCorentin Chary DEFINE_IDX_ENCODE_FUNCTION(16) 488245f7b51SCorentin Chary DEFINE_IDX_ENCODE_FUNCTION(32) 489245f7b51SCorentin Chary 490245f7b51SCorentin Chary #define DEFINE_MONO_ENCODE_FUNCTION(bpp) \ 491245f7b51SCorentin Chary \ 492245f7b51SCorentin Chary static void \ 493245f7b51SCorentin Chary tight_encode_mono_rect##bpp(uint8_t *buf, int w, int h, \ 494245f7b51SCorentin Chary uint##bpp##_t bg, uint##bpp##_t fg) { \ 495245f7b51SCorentin Chary uint##bpp##_t *ptr; \ 496245f7b51SCorentin Chary unsigned int value, mask; \ 497245f7b51SCorentin Chary int aligned_width; \ 498245f7b51SCorentin Chary int x, y, bg_bits; \ 499245f7b51SCorentin Chary \ 500245f7b51SCorentin Chary ptr = (uint##bpp##_t *) buf; \ 501245f7b51SCorentin Chary aligned_width = w - w % 8; \ 502245f7b51SCorentin Chary \ 503245f7b51SCorentin Chary for (y = 0; y < h; y++) { \ 504245f7b51SCorentin Chary for (x = 0; x < aligned_width; x += 8) { \ 505245f7b51SCorentin Chary for (bg_bits = 0; bg_bits < 8; bg_bits++) { \ 506245f7b51SCorentin Chary if (*ptr++ != bg) { \ 507245f7b51SCorentin Chary break; \ 508245f7b51SCorentin Chary } \ 509245f7b51SCorentin Chary } \ 510245f7b51SCorentin Chary if (bg_bits == 8) { \ 511245f7b51SCorentin Chary *buf++ = 0; \ 512245f7b51SCorentin Chary continue; \ 513245f7b51SCorentin Chary } \ 514245f7b51SCorentin Chary mask = 0x80 >> bg_bits; \ 515245f7b51SCorentin Chary value = mask; \ 516245f7b51SCorentin Chary for (bg_bits++; bg_bits < 8; bg_bits++) { \ 517245f7b51SCorentin Chary mask >>= 1; \ 518245f7b51SCorentin Chary if (*ptr++ != bg) { \ 519245f7b51SCorentin Chary value |= mask; \ 520245f7b51SCorentin Chary } \ 521245f7b51SCorentin Chary } \ 522245f7b51SCorentin Chary *buf++ = (uint8_t)value; \ 523245f7b51SCorentin Chary } \ 524245f7b51SCorentin Chary \ 525245f7b51SCorentin Chary mask = 0x80; \ 526245f7b51SCorentin Chary value = 0; \ 527245f7b51SCorentin Chary if (x >= w) { \ 528245f7b51SCorentin Chary continue; \ 529245f7b51SCorentin Chary } \ 530245f7b51SCorentin Chary \ 531245f7b51SCorentin Chary for (; x < w; x++) { \ 532245f7b51SCorentin Chary if (*ptr++ != bg) { \ 533245f7b51SCorentin Chary value |= mask; \ 534245f7b51SCorentin Chary } \ 535245f7b51SCorentin Chary mask >>= 1; \ 536245f7b51SCorentin Chary } \ 537245f7b51SCorentin Chary *buf++ = (uint8_t)value; \ 538245f7b51SCorentin Chary } \ 539245f7b51SCorentin Chary } 540245f7b51SCorentin Chary 541245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(8) 542245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(16) 543245f7b51SCorentin Chary DEFINE_MONO_ENCODE_FUNCTION(32) 544245f7b51SCorentin Chary 545245f7b51SCorentin Chary /* 546245f7b51SCorentin Chary * ``Gradient'' filter for 24-bit color samples. 547245f7b51SCorentin Chary * Should be called only when redMax, greenMax and blueMax are 255. 548245f7b51SCorentin Chary * Color components assumed to be byte-aligned. 549245f7b51SCorentin Chary */ 550245f7b51SCorentin Chary 551245f7b51SCorentin Chary static void 552245f7b51SCorentin Chary tight_filter_gradient24(VncState *vs, uint8_t *buf, int w, int h) 553245f7b51SCorentin Chary { 554245f7b51SCorentin Chary uint32_t *buf32; 555245f7b51SCorentin Chary uint32_t pix32; 556245f7b51SCorentin Chary int shift[3]; 557245f7b51SCorentin Chary int *prev; 558245f7b51SCorentin Chary int here[3], upper[3], left[3], upperleft[3]; 559245f7b51SCorentin Chary int prediction; 560245f7b51SCorentin Chary int x, y, c; 561245f7b51SCorentin Chary 562245f7b51SCorentin Chary buf32 = (uint32_t *)buf; 563d1af0e05SCorentin Chary memset(vs->tight.gradient.buffer, 0, w * 3 * sizeof(int)); 564245f7b51SCorentin Chary 565*77bfcf28SBenjamin Herrenschmidt if (1 /* FIXME */) { 5669f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; 5679f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; 5689f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; 569245f7b51SCorentin Chary } else { 5709f64916dSGerd Hoffmann shift[0] = 24 - vs->client_pf.rshift; 5719f64916dSGerd Hoffmann shift[1] = 24 - vs->client_pf.gshift; 5729f64916dSGerd Hoffmann shift[2] = 24 - vs->client_pf.bshift; 573245f7b51SCorentin Chary } 574245f7b51SCorentin Chary 575245f7b51SCorentin Chary for (y = 0; y < h; y++) { 576245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 577245f7b51SCorentin Chary upper[c] = 0; 578245f7b51SCorentin Chary here[c] = 0; 579245f7b51SCorentin Chary } 580d1af0e05SCorentin Chary prev = (int *)vs->tight.gradient.buffer; 581245f7b51SCorentin Chary for (x = 0; x < w; x++) { 582245f7b51SCorentin Chary pix32 = *buf32++; 583245f7b51SCorentin Chary for (c = 0; c < 3; c++) { 584245f7b51SCorentin Chary upperleft[c] = upper[c]; 585245f7b51SCorentin Chary left[c] = here[c]; 586245f7b51SCorentin Chary upper[c] = *prev; 587245f7b51SCorentin Chary here[c] = (int)(pix32 >> shift[c] & 0xFF); 588245f7b51SCorentin Chary *prev++ = here[c]; 589245f7b51SCorentin Chary 590245f7b51SCorentin Chary prediction = left[c] + upper[c] - upperleft[c]; 591245f7b51SCorentin Chary if (prediction < 0) { 592245f7b51SCorentin Chary prediction = 0; 593245f7b51SCorentin Chary } else if (prediction > 0xFF) { 594245f7b51SCorentin Chary prediction = 0xFF; 595245f7b51SCorentin Chary } 596245f7b51SCorentin Chary *buf++ = (char)(here[c] - prediction); 597245f7b51SCorentin Chary } 598245f7b51SCorentin Chary } 599245f7b51SCorentin Chary } 600245f7b51SCorentin Chary } 601245f7b51SCorentin Chary 602245f7b51SCorentin Chary 603245f7b51SCorentin Chary /* 604245f7b51SCorentin Chary * ``Gradient'' filter for other color depths. 605245f7b51SCorentin Chary */ 606245f7b51SCorentin Chary 607245f7b51SCorentin Chary #define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \ 608245f7b51SCorentin Chary \ 609245f7b51SCorentin Chary static void \ 610245f7b51SCorentin Chary tight_filter_gradient##bpp(VncState *vs, uint##bpp##_t *buf, \ 611245f7b51SCorentin Chary int w, int h) { \ 612245f7b51SCorentin Chary uint##bpp##_t pix, diff; \ 613245f7b51SCorentin Chary bool endian; \ 614245f7b51SCorentin Chary int *prev; \ 615245f7b51SCorentin Chary int max[3], shift[3]; \ 616245f7b51SCorentin Chary int here[3], upper[3], left[3], upperleft[3]; \ 617245f7b51SCorentin Chary int prediction; \ 618245f7b51SCorentin Chary int x, y, c; \ 619245f7b51SCorentin Chary \ 620d1af0e05SCorentin Chary memset (vs->tight.gradient.buffer, 0, w * 3 * sizeof(int)); \ 621245f7b51SCorentin Chary \ 622*77bfcf28SBenjamin Herrenschmidt endian = 0; /* FIXME */ \ 623245f7b51SCorentin Chary \ 6249f64916dSGerd Hoffmann max[0] = vs->client_pf.rmax; \ 6259f64916dSGerd Hoffmann max[1] = vs->client_pf.gmax; \ 6269f64916dSGerd Hoffmann max[2] = vs->client_pf.bmax; \ 6279f64916dSGerd Hoffmann shift[0] = vs->client_pf.rshift; \ 6289f64916dSGerd Hoffmann shift[1] = vs->client_pf.gshift; \ 6299f64916dSGerd Hoffmann shift[2] = vs->client_pf.bshift; \ 630245f7b51SCorentin Chary \ 631245f7b51SCorentin Chary for (y = 0; y < h; y++) { \ 632245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 633245f7b51SCorentin Chary upper[c] = 0; \ 634245f7b51SCorentin Chary here[c] = 0; \ 635245f7b51SCorentin Chary } \ 636d1af0e05SCorentin Chary prev = (int *)vs->tight.gradient.buffer; \ 637245f7b51SCorentin Chary for (x = 0; x < w; x++) { \ 638245f7b51SCorentin Chary pix = *buf; \ 639245f7b51SCorentin Chary if (endian) { \ 640ba5e7f82SIzumi Tsutsui pix = bswap##bpp(pix); \ 641245f7b51SCorentin Chary } \ 642245f7b51SCorentin Chary diff = 0; \ 643245f7b51SCorentin Chary for (c = 0; c < 3; c++) { \ 644245f7b51SCorentin Chary upperleft[c] = upper[c]; \ 645245f7b51SCorentin Chary left[c] = here[c]; \ 646245f7b51SCorentin Chary upper[c] = *prev; \ 647245f7b51SCorentin Chary here[c] = (int)(pix >> shift[c] & max[c]); \ 648245f7b51SCorentin Chary *prev++ = here[c]; \ 649245f7b51SCorentin Chary \ 650245f7b51SCorentin Chary prediction = left[c] + upper[c] - upperleft[c]; \ 651245f7b51SCorentin Chary if (prediction < 0) { \ 652245f7b51SCorentin Chary prediction = 0; \ 653245f7b51SCorentin Chary } else if (prediction > max[c]) { \ 654245f7b51SCorentin Chary prediction = max[c]; \ 655245f7b51SCorentin Chary } \ 656245f7b51SCorentin Chary diff |= ((here[c] - prediction) & max[c]) \ 657245f7b51SCorentin Chary << shift[c]; \ 658245f7b51SCorentin Chary } \ 659245f7b51SCorentin Chary if (endian) { \ 660ba5e7f82SIzumi Tsutsui diff = bswap##bpp(diff); \ 661245f7b51SCorentin Chary } \ 662245f7b51SCorentin Chary *buf++ = diff; \ 663245f7b51SCorentin Chary } \ 664245f7b51SCorentin Chary } \ 665245f7b51SCorentin Chary } 666245f7b51SCorentin Chary 667245f7b51SCorentin Chary DEFINE_GRADIENT_FILTER_FUNCTION(16) 668245f7b51SCorentin Chary DEFINE_GRADIENT_FILTER_FUNCTION(32) 669245f7b51SCorentin Chary 670245f7b51SCorentin Chary /* 671245f7b51SCorentin Chary * Check if a rectangle is all of the same color. If needSameColor is 672245f7b51SCorentin Chary * set to non-zero, then also check that its color equals to the 673b0cd712cSStefan Weil * *colorPtr value. The result is 1 if the test is successful, and in 674245f7b51SCorentin Chary * that case new color will be stored in *colorPtr. 675245f7b51SCorentin Chary */ 676245f7b51SCorentin Chary 67794362682SGerd Hoffmann static bool 67894362682SGerd Hoffmann check_solid_tile32(VncState *vs, int x, int y, int w, int h, 67994362682SGerd Hoffmann uint32_t *color, bool samecolor) 68094362682SGerd Hoffmann { 68194362682SGerd Hoffmann VncDisplay *vd = vs->vd; 68294362682SGerd Hoffmann uint32_t *fbptr; 68394362682SGerd Hoffmann uint32_t c; 68494362682SGerd Hoffmann int dx, dy; 68594362682SGerd Hoffmann 68694362682SGerd Hoffmann fbptr = vnc_server_fb_ptr(vd, x, y); 68794362682SGerd Hoffmann 68894362682SGerd Hoffmann c = *fbptr; 68994362682SGerd Hoffmann if (samecolor && (uint32_t)c != *color) { 69094362682SGerd Hoffmann return false; 691245f7b51SCorentin Chary } 692245f7b51SCorentin Chary 69394362682SGerd Hoffmann for (dy = 0; dy < h; dy++) { 69494362682SGerd Hoffmann for (dx = 0; dx < w; dx++) { 69594362682SGerd Hoffmann if (c != fbptr[dx]) { 69694362682SGerd Hoffmann return false; 69794362682SGerd Hoffmann } 69894362682SGerd Hoffmann } 69994362682SGerd Hoffmann fbptr = (uint32_t *) 70094362682SGerd Hoffmann ((uint8_t *)fbptr + vnc_server_fb_stride(vd)); 70194362682SGerd Hoffmann } 70294362682SGerd Hoffmann 70394362682SGerd Hoffmann *color = (uint32_t)c; 70494362682SGerd Hoffmann return true; 70594362682SGerd Hoffmann } 706245f7b51SCorentin Chary 707245f7b51SCorentin Chary static bool check_solid_tile(VncState *vs, int x, int y, int w, int h, 708245f7b51SCorentin Chary uint32_t* color, bool samecolor) 709245f7b51SCorentin Chary { 7109f64916dSGerd Hoffmann switch (VNC_SERVER_FB_BYTES) { 711245f7b51SCorentin Chary case 4: 712245f7b51SCorentin Chary return check_solid_tile32(vs, x, y, w, h, color, samecolor); 713245f7b51SCorentin Chary } 714245f7b51SCorentin Chary } 715245f7b51SCorentin Chary 716245f7b51SCorentin Chary static void find_best_solid_area(VncState *vs, int x, int y, int w, int h, 717245f7b51SCorentin Chary uint32_t color, int *w_ptr, int *h_ptr) 718245f7b51SCorentin Chary { 719245f7b51SCorentin Chary int dx, dy, dw, dh; 720245f7b51SCorentin Chary int w_prev; 721245f7b51SCorentin Chary int w_best = 0, h_best = 0; 722245f7b51SCorentin Chary 723245f7b51SCorentin Chary w_prev = w; 724245f7b51SCorentin Chary 725245f7b51SCorentin Chary for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) { 726245f7b51SCorentin Chary 727245f7b51SCorentin Chary dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, y + h - dy); 728245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, w_prev); 729245f7b51SCorentin Chary 730245f7b51SCorentin Chary if (!check_solid_tile(vs, x, dy, dw, dh, &color, true)) { 731245f7b51SCorentin Chary break; 732245f7b51SCorentin Chary } 733245f7b51SCorentin Chary 734245f7b51SCorentin Chary for (dx = x + dw; dx < x + w_prev;) { 735245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, x + w_prev - dx); 736245f7b51SCorentin Chary 737245f7b51SCorentin Chary if (!check_solid_tile(vs, dx, dy, dw, dh, &color, true)) { 738245f7b51SCorentin Chary break; 739245f7b51SCorentin Chary } 740245f7b51SCorentin Chary dx += dw; 741245f7b51SCorentin Chary } 742245f7b51SCorentin Chary 743245f7b51SCorentin Chary w_prev = dx - x; 744245f7b51SCorentin Chary if (w_prev * (dy + dh - y) > w_best * h_best) { 745245f7b51SCorentin Chary w_best = w_prev; 746245f7b51SCorentin Chary h_best = dy + dh - y; 747245f7b51SCorentin Chary } 748245f7b51SCorentin Chary } 749245f7b51SCorentin Chary 750245f7b51SCorentin Chary *w_ptr = w_best; 751245f7b51SCorentin Chary *h_ptr = h_best; 752245f7b51SCorentin Chary } 753245f7b51SCorentin Chary 754245f7b51SCorentin Chary static void extend_solid_area(VncState *vs, int x, int y, int w, int h, 755245f7b51SCorentin Chary uint32_t color, int *x_ptr, int *y_ptr, 756245f7b51SCorentin Chary int *w_ptr, int *h_ptr) 757245f7b51SCorentin Chary { 758245f7b51SCorentin Chary int cx, cy; 759245f7b51SCorentin Chary 760245f7b51SCorentin Chary /* Try to extend the area upwards. */ 761245f7b51SCorentin Chary for ( cy = *y_ptr - 1; 762245f7b51SCorentin Chary cy >= y && check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true); 763245f7b51SCorentin Chary cy-- ); 764245f7b51SCorentin Chary *h_ptr += *y_ptr - (cy + 1); 765245f7b51SCorentin Chary *y_ptr = cy + 1; 766245f7b51SCorentin Chary 767245f7b51SCorentin Chary /* ... downwards. */ 768245f7b51SCorentin Chary for ( cy = *y_ptr + *h_ptr; 769245f7b51SCorentin Chary cy < y + h && 770245f7b51SCorentin Chary check_solid_tile(vs, *x_ptr, cy, *w_ptr, 1, &color, true); 771245f7b51SCorentin Chary cy++ ); 772245f7b51SCorentin Chary *h_ptr += cy - (*y_ptr + *h_ptr); 773245f7b51SCorentin Chary 774245f7b51SCorentin Chary /* ... to the left. */ 775245f7b51SCorentin Chary for ( cx = *x_ptr - 1; 776245f7b51SCorentin Chary cx >= x && check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true); 777245f7b51SCorentin Chary cx-- ); 778245f7b51SCorentin Chary *w_ptr += *x_ptr - (cx + 1); 779245f7b51SCorentin Chary *x_ptr = cx + 1; 780245f7b51SCorentin Chary 781245f7b51SCorentin Chary /* ... to the right. */ 782245f7b51SCorentin Chary for ( cx = *x_ptr + *w_ptr; 783245f7b51SCorentin Chary cx < x + w && 784245f7b51SCorentin Chary check_solid_tile(vs, cx, *y_ptr, 1, *h_ptr, &color, true); 785245f7b51SCorentin Chary cx++ ); 786245f7b51SCorentin Chary *w_ptr += cx - (*x_ptr + *w_ptr); 787245f7b51SCorentin Chary } 788245f7b51SCorentin Chary 789245f7b51SCorentin Chary static int tight_init_stream(VncState *vs, int stream_id, 790245f7b51SCorentin Chary int level, int strategy) 791245f7b51SCorentin Chary { 792d1af0e05SCorentin Chary z_streamp zstream = &vs->tight.stream[stream_id]; 793245f7b51SCorentin Chary 794245f7b51SCorentin Chary if (zstream->opaque == NULL) { 795245f7b51SCorentin Chary int err; 796245f7b51SCorentin Chary 797245f7b51SCorentin Chary VNC_DEBUG("VNC: TIGHT: initializing zlib stream %d\n", stream_id); 798245f7b51SCorentin Chary VNC_DEBUG("VNC: TIGHT: opaque = %p | vs = %p\n", zstream->opaque, vs); 799245f7b51SCorentin Chary zstream->zalloc = vnc_zlib_zalloc; 800245f7b51SCorentin Chary zstream->zfree = vnc_zlib_zfree; 801245f7b51SCorentin Chary 802245f7b51SCorentin Chary err = deflateInit2(zstream, level, Z_DEFLATED, MAX_WBITS, 803245f7b51SCorentin Chary MAX_MEM_LEVEL, strategy); 804245f7b51SCorentin Chary 805245f7b51SCorentin Chary if (err != Z_OK) { 806245f7b51SCorentin Chary fprintf(stderr, "VNC: error initializing zlib\n"); 807245f7b51SCorentin Chary return -1; 808245f7b51SCorentin Chary } 809245f7b51SCorentin Chary 810d1af0e05SCorentin Chary vs->tight.levels[stream_id] = level; 811245f7b51SCorentin Chary zstream->opaque = vs; 812245f7b51SCorentin Chary } 813245f7b51SCorentin Chary 814d1af0e05SCorentin Chary if (vs->tight.levels[stream_id] != level) { 815245f7b51SCorentin Chary if (deflateParams(zstream, level, strategy) != Z_OK) { 816245f7b51SCorentin Chary return -1; 817245f7b51SCorentin Chary } 818d1af0e05SCorentin Chary vs->tight.levels[stream_id] = level; 819245f7b51SCorentin Chary } 820245f7b51SCorentin Chary return 0; 821245f7b51SCorentin Chary } 822245f7b51SCorentin Chary 823245f7b51SCorentin Chary static void tight_send_compact_size(VncState *vs, size_t len) 824245f7b51SCorentin Chary { 825245f7b51SCorentin Chary int lpc = 0; 826245f7b51SCorentin Chary int bytes = 0; 827245f7b51SCorentin Chary char buf[3] = {0, 0, 0}; 828245f7b51SCorentin Chary 829245f7b51SCorentin Chary buf[bytes++] = len & 0x7F; 830245f7b51SCorentin Chary if (len > 0x7F) { 831245f7b51SCorentin Chary buf[bytes-1] |= 0x80; 832245f7b51SCorentin Chary buf[bytes++] = (len >> 7) & 0x7F; 833245f7b51SCorentin Chary if (len > 0x3FFF) { 834245f7b51SCorentin Chary buf[bytes-1] |= 0x80; 835245f7b51SCorentin Chary buf[bytes++] = (len >> 14) & 0xFF; 836245f7b51SCorentin Chary } 837245f7b51SCorentin Chary } 838245f7b51SCorentin Chary for (lpc = 0; lpc < bytes; lpc++) { 839245f7b51SCorentin Chary vnc_write_u8(vs, buf[lpc]); 840245f7b51SCorentin Chary } 841245f7b51SCorentin Chary } 842245f7b51SCorentin Chary 843245f7b51SCorentin Chary static int tight_compress_data(VncState *vs, int stream_id, size_t bytes, 844245f7b51SCorentin Chary int level, int strategy) 845245f7b51SCorentin Chary { 846d1af0e05SCorentin Chary z_streamp zstream = &vs->tight.stream[stream_id]; 847245f7b51SCorentin Chary int previous_out; 848245f7b51SCorentin Chary 849245f7b51SCorentin Chary if (bytes < VNC_TIGHT_MIN_TO_COMPRESS) { 850d1af0e05SCorentin Chary vnc_write(vs, vs->tight.tight.buffer, vs->tight.tight.offset); 851245f7b51SCorentin Chary return bytes; 852245f7b51SCorentin Chary } 853245f7b51SCorentin Chary 854245f7b51SCorentin Chary if (tight_init_stream(vs, stream_id, level, strategy)) { 855245f7b51SCorentin Chary return -1; 856245f7b51SCorentin Chary } 857245f7b51SCorentin Chary 858245f7b51SCorentin Chary /* reserve memory in output buffer */ 859d1af0e05SCorentin Chary buffer_reserve(&vs->tight.zlib, bytes + 64); 860245f7b51SCorentin Chary 861245f7b51SCorentin Chary /* set pointers */ 862d1af0e05SCorentin Chary zstream->next_in = vs->tight.tight.buffer; 863d1af0e05SCorentin Chary zstream->avail_in = vs->tight.tight.offset; 864d1af0e05SCorentin Chary zstream->next_out = vs->tight.zlib.buffer + vs->tight.zlib.offset; 865d1af0e05SCorentin Chary zstream->avail_out = vs->tight.zlib.capacity - vs->tight.zlib.offset; 8662caa9e9dSMichael Tokarev previous_out = zstream->avail_out; 867245f7b51SCorentin Chary zstream->data_type = Z_BINARY; 868245f7b51SCorentin Chary 869245f7b51SCorentin Chary /* start encoding */ 870245f7b51SCorentin Chary if (deflate(zstream, Z_SYNC_FLUSH) != Z_OK) { 871245f7b51SCorentin Chary fprintf(stderr, "VNC: error during tight compression\n"); 872245f7b51SCorentin Chary return -1; 873245f7b51SCorentin Chary } 874245f7b51SCorentin Chary 875d1af0e05SCorentin Chary vs->tight.zlib.offset = vs->tight.zlib.capacity - zstream->avail_out; 8762caa9e9dSMichael Tokarev /* ...how much data has actually been produced by deflate() */ 8772caa9e9dSMichael Tokarev bytes = previous_out - zstream->avail_out; 878245f7b51SCorentin Chary 879245f7b51SCorentin Chary tight_send_compact_size(vs, bytes); 880d1af0e05SCorentin Chary vnc_write(vs, vs->tight.zlib.buffer, bytes); 881245f7b51SCorentin Chary 882d1af0e05SCorentin Chary buffer_reset(&vs->tight.zlib); 883245f7b51SCorentin Chary 884245f7b51SCorentin Chary return bytes; 885245f7b51SCorentin Chary } 886245f7b51SCorentin Chary 887245f7b51SCorentin Chary /* 888245f7b51SCorentin Chary * Subencoding implementations. 889245f7b51SCorentin Chary */ 890245f7b51SCorentin Chary static void tight_pack24(VncState *vs, uint8_t *buf, size_t count, size_t *ret) 891245f7b51SCorentin Chary { 892245f7b51SCorentin Chary uint32_t *buf32; 893245f7b51SCorentin Chary uint32_t pix; 894245f7b51SCorentin Chary int rshift, gshift, bshift; 895245f7b51SCorentin Chary 896245f7b51SCorentin Chary buf32 = (uint32_t *)buf; 897245f7b51SCorentin Chary 898*77bfcf28SBenjamin Herrenschmidt if (1 /* FIXME */) { 8999f64916dSGerd Hoffmann rshift = vs->client_pf.rshift; 9009f64916dSGerd Hoffmann gshift = vs->client_pf.gshift; 9019f64916dSGerd Hoffmann bshift = vs->client_pf.bshift; 902245f7b51SCorentin Chary } else { 9039f64916dSGerd Hoffmann rshift = 24 - vs->client_pf.rshift; 9049f64916dSGerd Hoffmann gshift = 24 - vs->client_pf.gshift; 9059f64916dSGerd Hoffmann bshift = 24 - vs->client_pf.bshift; 906245f7b51SCorentin Chary } 907245f7b51SCorentin Chary 908245f7b51SCorentin Chary if (ret) { 909245f7b51SCorentin Chary *ret = count * 3; 910245f7b51SCorentin Chary } 911245f7b51SCorentin Chary 912245f7b51SCorentin Chary while (count--) { 913245f7b51SCorentin Chary pix = *buf32++; 914245f7b51SCorentin Chary *buf++ = (char)(pix >> rshift); 915245f7b51SCorentin Chary *buf++ = (char)(pix >> gshift); 916245f7b51SCorentin Chary *buf++ = (char)(pix >> bshift); 917245f7b51SCorentin Chary } 918245f7b51SCorentin Chary } 919245f7b51SCorentin Chary 920efe556adSCorentin Chary static int send_full_color_rect(VncState *vs, int x, int y, int w, int h) 921245f7b51SCorentin Chary { 922245f7b51SCorentin Chary int stream = 0; 9232116eff9SJes Sorensen ssize_t bytes; 924245f7b51SCorentin Chary 925efe556adSCorentin Chary #ifdef CONFIG_VNC_PNG 926efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) { 927efe556adSCorentin Chary return send_png_rect(vs, x, y, w, h, NULL); 928efe556adSCorentin Chary } 929efe556adSCorentin Chary #endif 930efe556adSCorentin Chary 931245f7b51SCorentin Chary vnc_write_u8(vs, stream << 4); /* no flushing, no filter */ 932245f7b51SCorentin Chary 933d1af0e05SCorentin Chary if (vs->tight.pixel24) { 934d1af0e05SCorentin Chary tight_pack24(vs, vs->tight.tight.buffer, w * h, &vs->tight.tight.offset); 935245f7b51SCorentin Chary bytes = 3; 936245f7b51SCorentin Chary } else { 9379f64916dSGerd Hoffmann bytes = vs->client_pf.bytes_per_pixel; 938245f7b51SCorentin Chary } 939245f7b51SCorentin Chary 940245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, w * h * bytes, 941d1af0e05SCorentin Chary tight_conf[vs->tight.compression].raw_zlib_level, 942245f7b51SCorentin Chary Z_DEFAULT_STRATEGY); 943245f7b51SCorentin Chary 944245f7b51SCorentin Chary return (bytes >= 0); 945245f7b51SCorentin Chary } 946245f7b51SCorentin Chary 947245f7b51SCorentin Chary static int send_solid_rect(VncState *vs) 948245f7b51SCorentin Chary { 949245f7b51SCorentin Chary size_t bytes; 950245f7b51SCorentin Chary 951245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILL << 4); /* no flushing, no filter */ 952245f7b51SCorentin Chary 953d1af0e05SCorentin Chary if (vs->tight.pixel24) { 954d1af0e05SCorentin Chary tight_pack24(vs, vs->tight.tight.buffer, 1, &vs->tight.tight.offset); 955245f7b51SCorentin Chary bytes = 3; 956245f7b51SCorentin Chary } else { 9579f64916dSGerd Hoffmann bytes = vs->client_pf.bytes_per_pixel; 958245f7b51SCorentin Chary } 959245f7b51SCorentin Chary 960d1af0e05SCorentin Chary vnc_write(vs, vs->tight.tight.buffer, bytes); 961245f7b51SCorentin Chary return 1; 962245f7b51SCorentin Chary } 963245f7b51SCorentin Chary 964efe556adSCorentin Chary static int send_mono_rect(VncState *vs, int x, int y, 965efe556adSCorentin Chary int w, int h, uint32_t bg, uint32_t fg) 966245f7b51SCorentin Chary { 9672116eff9SJes Sorensen ssize_t bytes; 968245f7b51SCorentin Chary int stream = 1; 969d1af0e05SCorentin Chary int level = tight_conf[vs->tight.compression].mono_zlib_level; 970245f7b51SCorentin Chary 971efe556adSCorentin Chary #ifdef CONFIG_VNC_PNG 972efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) { 973efe556adSCorentin Chary int ret; 9749f64916dSGerd Hoffmann int bpp = vs->client_pf.bytes_per_pixel * 8; 9755136a052SCorentin Chary VncPalette *palette = palette_new(2, bpp); 976efe556adSCorentin Chary 9775136a052SCorentin Chary palette_put(palette, bg); 9785136a052SCorentin Chary palette_put(palette, fg); 979efe556adSCorentin Chary ret = send_png_rect(vs, x, y, w, h, palette); 9805136a052SCorentin Chary palette_destroy(palette); 981efe556adSCorentin Chary return ret; 982efe556adSCorentin Chary } 983efe556adSCorentin Chary #endif 984efe556adSCorentin Chary 985245f7b51SCorentin Chary bytes = ((w + 7) / 8) * h; 986245f7b51SCorentin Chary 987245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4); 988245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE); 989245f7b51SCorentin Chary vnc_write_u8(vs, 1); 990245f7b51SCorentin Chary 9919f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) { 992245f7b51SCorentin Chary case 4: 993245f7b51SCorentin Chary { 994245f7b51SCorentin Chary uint32_t buf[2] = {bg, fg}; 995245f7b51SCorentin Chary size_t ret = sizeof (buf); 996245f7b51SCorentin Chary 997d1af0e05SCorentin Chary if (vs->tight.pixel24) { 998245f7b51SCorentin Chary tight_pack24(vs, (unsigned char*)buf, 2, &ret); 999245f7b51SCorentin Chary } 1000245f7b51SCorentin Chary vnc_write(vs, buf, ret); 1001245f7b51SCorentin Chary 1002d1af0e05SCorentin Chary tight_encode_mono_rect32(vs->tight.tight.buffer, w, h, bg, fg); 1003245f7b51SCorentin Chary break; 1004245f7b51SCorentin Chary } 1005245f7b51SCorentin Chary case 2: 1006245f7b51SCorentin Chary vnc_write(vs, &bg, 2); 1007245f7b51SCorentin Chary vnc_write(vs, &fg, 2); 1008d1af0e05SCorentin Chary tight_encode_mono_rect16(vs->tight.tight.buffer, w, h, bg, fg); 1009245f7b51SCorentin Chary break; 1010245f7b51SCorentin Chary default: 1011245f7b51SCorentin Chary vnc_write_u8(vs, bg); 1012245f7b51SCorentin Chary vnc_write_u8(vs, fg); 1013d1af0e05SCorentin Chary tight_encode_mono_rect8(vs->tight.tight.buffer, w, h, bg, fg); 1014245f7b51SCorentin Chary break; 1015245f7b51SCorentin Chary } 1016d1af0e05SCorentin Chary vs->tight.tight.offset = bytes; 1017245f7b51SCorentin Chary 1018245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes, level, Z_DEFAULT_STRATEGY); 1019245f7b51SCorentin Chary return (bytes >= 0); 1020245f7b51SCorentin Chary } 1021245f7b51SCorentin Chary 1022245f7b51SCorentin Chary struct palette_cb_priv { 1023245f7b51SCorentin Chary VncState *vs; 1024245f7b51SCorentin Chary uint8_t *header; 1025efe556adSCorentin Chary #ifdef CONFIG_VNC_PNG 1026efe556adSCorentin Chary png_colorp png_palette; 1027efe556adSCorentin Chary #endif 1028245f7b51SCorentin Chary }; 1029245f7b51SCorentin Chary 10305136a052SCorentin Chary static void write_palette(int idx, uint32_t color, void *opaque) 1031245f7b51SCorentin Chary { 1032245f7b51SCorentin Chary struct palette_cb_priv *priv = opaque; 1033245f7b51SCorentin Chary VncState *vs = priv->vs; 10349f64916dSGerd Hoffmann uint32_t bytes = vs->client_pf.bytes_per_pixel; 1035245f7b51SCorentin Chary 1036245f7b51SCorentin Chary if (bytes == 4) { 1037245f7b51SCorentin Chary ((uint32_t*)priv->header)[idx] = color; 1038245f7b51SCorentin Chary } else { 1039245f7b51SCorentin Chary ((uint16_t*)priv->header)[idx] = color; 1040245f7b51SCorentin Chary } 1041245f7b51SCorentin Chary } 1042245f7b51SCorentin Chary 1043efe556adSCorentin Chary static bool send_gradient_rect(VncState *vs, int x, int y, int w, int h) 1044245f7b51SCorentin Chary { 1045245f7b51SCorentin Chary int stream = 3; 1046d1af0e05SCorentin Chary int level = tight_conf[vs->tight.compression].gradient_zlib_level; 10472116eff9SJes Sorensen ssize_t bytes; 1048245f7b51SCorentin Chary 10499f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 1) { 1050efe556adSCorentin Chary return send_full_color_rect(vs, x, y, w, h); 10519f64916dSGerd Hoffmann } 1052245f7b51SCorentin Chary 1053245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4); 1054245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_GRADIENT); 1055245f7b51SCorentin Chary 1056d1af0e05SCorentin Chary buffer_reserve(&vs->tight.gradient, w * 3 * sizeof (int)); 1057245f7b51SCorentin Chary 1058d1af0e05SCorentin Chary if (vs->tight.pixel24) { 1059d1af0e05SCorentin Chary tight_filter_gradient24(vs, vs->tight.tight.buffer, w, h); 1060245f7b51SCorentin Chary bytes = 3; 10619f64916dSGerd Hoffmann } else if (vs->client_pf.bytes_per_pixel == 4) { 1062d1af0e05SCorentin Chary tight_filter_gradient32(vs, (uint32_t *)vs->tight.tight.buffer, w, h); 1063245f7b51SCorentin Chary bytes = 4; 1064245f7b51SCorentin Chary } else { 1065d1af0e05SCorentin Chary tight_filter_gradient16(vs, (uint16_t *)vs->tight.tight.buffer, w, h); 1066245f7b51SCorentin Chary bytes = 2; 1067245f7b51SCorentin Chary } 1068245f7b51SCorentin Chary 1069d1af0e05SCorentin Chary buffer_reset(&vs->tight.gradient); 1070245f7b51SCorentin Chary 1071245f7b51SCorentin Chary bytes = w * h * bytes; 1072d1af0e05SCorentin Chary vs->tight.tight.offset = bytes; 1073245f7b51SCorentin Chary 1074245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes, 1075245f7b51SCorentin Chary level, Z_FILTERED); 1076245f7b51SCorentin Chary return (bytes >= 0); 1077245f7b51SCorentin Chary } 1078245f7b51SCorentin Chary 1079efe556adSCorentin Chary static int send_palette_rect(VncState *vs, int x, int y, 10805136a052SCorentin Chary int w, int h, VncPalette *palette) 1081245f7b51SCorentin Chary { 1082245f7b51SCorentin Chary int stream = 2; 1083d1af0e05SCorentin Chary int level = tight_conf[vs->tight.compression].idx_zlib_level; 1084245f7b51SCorentin Chary int colors; 10852116eff9SJes Sorensen ssize_t bytes; 1086245f7b51SCorentin Chary 1087efe556adSCorentin Chary #ifdef CONFIG_VNC_PNG 1088efe556adSCorentin Chary if (tight_can_send_png_rect(vs, w, h)) { 1089efe556adSCorentin Chary return send_png_rect(vs, x, y, w, h, palette); 1090efe556adSCorentin Chary } 1091efe556adSCorentin Chary #endif 1092efe556adSCorentin Chary 10935136a052SCorentin Chary colors = palette_size(palette); 1094245f7b51SCorentin Chary 1095245f7b51SCorentin Chary vnc_write_u8(vs, (stream | VNC_TIGHT_EXPLICIT_FILTER) << 4); 1096245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_FILTER_PALETTE); 1097245f7b51SCorentin Chary vnc_write_u8(vs, colors - 1); 1098245f7b51SCorentin Chary 10999f64916dSGerd Hoffmann switch (vs->client_pf.bytes_per_pixel) { 1100245f7b51SCorentin Chary case 4: 1101245f7b51SCorentin Chary { 1102245f7b51SCorentin Chary size_t old_offset, offset; 11035136a052SCorentin Chary uint32_t header[palette_size(palette)]; 1104245f7b51SCorentin Chary struct palette_cb_priv priv = { vs, (uint8_t *)header }; 1105245f7b51SCorentin Chary 1106245f7b51SCorentin Chary old_offset = vs->output.offset; 11075136a052SCorentin Chary palette_iter(palette, write_palette, &priv); 1108245f7b51SCorentin Chary vnc_write(vs, header, sizeof(header)); 1109245f7b51SCorentin Chary 1110d1af0e05SCorentin Chary if (vs->tight.pixel24) { 1111245f7b51SCorentin Chary tight_pack24(vs, vs->output.buffer + old_offset, colors, &offset); 1112245f7b51SCorentin Chary vs->output.offset = old_offset + offset; 1113245f7b51SCorentin Chary } 1114245f7b51SCorentin Chary 1115d1af0e05SCorentin Chary tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette); 1116245f7b51SCorentin Chary break; 1117245f7b51SCorentin Chary } 1118245f7b51SCorentin Chary case 2: 1119245f7b51SCorentin Chary { 11205136a052SCorentin Chary uint16_t header[palette_size(palette)]; 1121245f7b51SCorentin Chary struct palette_cb_priv priv = { vs, (uint8_t *)header }; 1122245f7b51SCorentin Chary 11235136a052SCorentin Chary palette_iter(palette, write_palette, &priv); 1124245f7b51SCorentin Chary vnc_write(vs, header, sizeof(header)); 1125d1af0e05SCorentin Chary tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette); 1126245f7b51SCorentin Chary break; 1127245f7b51SCorentin Chary } 1128245f7b51SCorentin Chary default: 1129245f7b51SCorentin Chary return -1; /* No palette for 8bits colors */ 1130245f7b51SCorentin Chary break; 1131245f7b51SCorentin Chary } 1132245f7b51SCorentin Chary bytes = w * h; 1133d1af0e05SCorentin Chary vs->tight.tight.offset = bytes; 1134245f7b51SCorentin Chary 1135245f7b51SCorentin Chary bytes = tight_compress_data(vs, stream, bytes, 1136245f7b51SCorentin Chary level, Z_DEFAULT_STRATEGY); 1137245f7b51SCorentin Chary return (bytes >= 0); 1138245f7b51SCorentin Chary } 1139245f7b51SCorentin Chary 1140245f7b51SCorentin Chary /* 1141efe556adSCorentin Chary * JPEG compression stuff. 1142efe556adSCorentin Chary */ 1143efe556adSCorentin Chary #ifdef CONFIG_VNC_JPEG 1144efe556adSCorentin Chary /* 1145245f7b51SCorentin Chary * Destination manager implementation for JPEG library. 1146245f7b51SCorentin Chary */ 1147245f7b51SCorentin Chary 1148245f7b51SCorentin Chary /* This is called once per encoding */ 1149245f7b51SCorentin Chary static void jpeg_init_destination(j_compress_ptr cinfo) 1150245f7b51SCorentin Chary { 1151245f7b51SCorentin Chary VncState *vs = cinfo->client_data; 1152d1af0e05SCorentin Chary Buffer *buffer = &vs->tight.jpeg; 1153245f7b51SCorentin Chary 1154245f7b51SCorentin Chary cinfo->dest->next_output_byte = (JOCTET *)buffer->buffer + buffer->offset; 1155245f7b51SCorentin Chary cinfo->dest->free_in_buffer = (size_t)(buffer->capacity - buffer->offset); 1156245f7b51SCorentin Chary } 1157245f7b51SCorentin Chary 1158245f7b51SCorentin Chary /* This is called when we ran out of buffer (shouldn't happen!) */ 1159245f7b51SCorentin Chary static boolean jpeg_empty_output_buffer(j_compress_ptr cinfo) 1160245f7b51SCorentin Chary { 1161245f7b51SCorentin Chary VncState *vs = cinfo->client_data; 1162d1af0e05SCorentin Chary Buffer *buffer = &vs->tight.jpeg; 1163245f7b51SCorentin Chary 1164245f7b51SCorentin Chary buffer->offset = buffer->capacity; 1165245f7b51SCorentin Chary buffer_reserve(buffer, 2048); 1166245f7b51SCorentin Chary jpeg_init_destination(cinfo); 1167245f7b51SCorentin Chary return TRUE; 1168245f7b51SCorentin Chary } 1169245f7b51SCorentin Chary 1170245f7b51SCorentin Chary /* This is called when we are done processing data */ 1171245f7b51SCorentin Chary static void jpeg_term_destination(j_compress_ptr cinfo) 1172245f7b51SCorentin Chary { 1173245f7b51SCorentin Chary VncState *vs = cinfo->client_data; 1174d1af0e05SCorentin Chary Buffer *buffer = &vs->tight.jpeg; 1175245f7b51SCorentin Chary 1176245f7b51SCorentin Chary buffer->offset = buffer->capacity - cinfo->dest->free_in_buffer; 1177245f7b51SCorentin Chary } 1178245f7b51SCorentin Chary 1179245f7b51SCorentin Chary static int send_jpeg_rect(VncState *vs, int x, int y, int w, int h, int quality) 1180245f7b51SCorentin Chary { 1181245f7b51SCorentin Chary struct jpeg_compress_struct cinfo; 1182245f7b51SCorentin Chary struct jpeg_error_mgr jerr; 1183245f7b51SCorentin Chary struct jpeg_destination_mgr manager; 118447683d66SGerd Hoffmann pixman_image_t *linebuf; 1185245f7b51SCorentin Chary JSAMPROW row[1]; 1186245f7b51SCorentin Chary uint8_t *buf; 1187245f7b51SCorentin Chary int dy; 1188245f7b51SCorentin Chary 1189d39fa6d8SGerd Hoffmann if (surface_bytes_per_pixel(vs->vd->ds) == 1) { 1190efe556adSCorentin Chary return send_full_color_rect(vs, x, y, w, h); 1191d39fa6d8SGerd Hoffmann } 1192245f7b51SCorentin Chary 1193d1af0e05SCorentin Chary buffer_reserve(&vs->tight.jpeg, 2048); 1194245f7b51SCorentin Chary 1195245f7b51SCorentin Chary cinfo.err = jpeg_std_error(&jerr); 1196245f7b51SCorentin Chary jpeg_create_compress(&cinfo); 1197245f7b51SCorentin Chary 1198245f7b51SCorentin Chary cinfo.client_data = vs; 1199245f7b51SCorentin Chary cinfo.image_width = w; 1200245f7b51SCorentin Chary cinfo.image_height = h; 1201245f7b51SCorentin Chary cinfo.input_components = 3; 1202245f7b51SCorentin Chary cinfo.in_color_space = JCS_RGB; 1203245f7b51SCorentin Chary 1204245f7b51SCorentin Chary jpeg_set_defaults(&cinfo); 1205245f7b51SCorentin Chary jpeg_set_quality(&cinfo, quality, true); 1206245f7b51SCorentin Chary 1207245f7b51SCorentin Chary manager.init_destination = jpeg_init_destination; 1208245f7b51SCorentin Chary manager.empty_output_buffer = jpeg_empty_output_buffer; 1209245f7b51SCorentin Chary manager.term_destination = jpeg_term_destination; 1210245f7b51SCorentin Chary cinfo.dest = &manager; 1211245f7b51SCorentin Chary 1212245f7b51SCorentin Chary jpeg_start_compress(&cinfo, true); 1213245f7b51SCorentin Chary 121447683d66SGerd Hoffmann linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w); 121547683d66SGerd Hoffmann buf = (uint8_t *)pixman_image_get_data(linebuf); 1216d9c18c24SCorentin Chary row[0] = buf; 1217245f7b51SCorentin Chary for (dy = 0; dy < h; dy++) { 1218bc210eb1SGerd Hoffmann qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy); 1219245f7b51SCorentin Chary jpeg_write_scanlines(&cinfo, row, 1); 1220245f7b51SCorentin Chary } 122147683d66SGerd Hoffmann qemu_pixman_image_unref(linebuf); 1222245f7b51SCorentin Chary 1223245f7b51SCorentin Chary jpeg_finish_compress(&cinfo); 1224245f7b51SCorentin Chary jpeg_destroy_compress(&cinfo); 1225245f7b51SCorentin Chary 1226245f7b51SCorentin Chary vnc_write_u8(vs, VNC_TIGHT_JPEG << 4); 1227245f7b51SCorentin Chary 1228d1af0e05SCorentin Chary tight_send_compact_size(vs, vs->tight.jpeg.offset); 1229d1af0e05SCorentin Chary vnc_write(vs, vs->tight.jpeg.buffer, vs->tight.jpeg.offset); 1230d1af0e05SCorentin Chary buffer_reset(&vs->tight.jpeg); 1231245f7b51SCorentin Chary 1232245f7b51SCorentin Chary return 1; 1233245f7b51SCorentin Chary } 1234245f7b51SCorentin Chary #endif /* CONFIG_VNC_JPEG */ 1235245f7b51SCorentin Chary 1236efe556adSCorentin Chary /* 1237efe556adSCorentin Chary * PNG compression stuff. 1238efe556adSCorentin Chary */ 1239efe556adSCorentin Chary #ifdef CONFIG_VNC_PNG 12405136a052SCorentin Chary static void write_png_palette(int idx, uint32_t pix, void *opaque) 1241efe556adSCorentin Chary { 1242efe556adSCorentin Chary struct palette_cb_priv *priv = opaque; 1243efe556adSCorentin Chary VncState *vs = priv->vs; 1244efe556adSCorentin Chary png_colorp color = &priv->png_palette[idx]; 1245efe556adSCorentin Chary 1246d1af0e05SCorentin Chary if (vs->tight.pixel24) 1247efe556adSCorentin Chary { 12489f64916dSGerd Hoffmann color->red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax; 12499f64916dSGerd Hoffmann color->green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax; 12509f64916dSGerd Hoffmann color->blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax; 1251efe556adSCorentin Chary } 1252efe556adSCorentin Chary else 1253efe556adSCorentin Chary { 1254efe556adSCorentin Chary int red, green, blue; 1255efe556adSCorentin Chary 12569f64916dSGerd Hoffmann red = (pix >> vs->client_pf.rshift) & vs->client_pf.rmax; 12579f64916dSGerd Hoffmann green = (pix >> vs->client_pf.gshift) & vs->client_pf.gmax; 12589f64916dSGerd Hoffmann blue = (pix >> vs->client_pf.bshift) & vs->client_pf.bmax; 12599f64916dSGerd Hoffmann color->red = ((red * 255 + vs->client_pf.rmax / 2) / 12609f64916dSGerd Hoffmann vs->client_pf.rmax); 12619f64916dSGerd Hoffmann color->green = ((green * 255 + vs->client_pf.gmax / 2) / 12629f64916dSGerd Hoffmann vs->client_pf.gmax); 12639f64916dSGerd Hoffmann color->blue = ((blue * 255 + vs->client_pf.bmax / 2) / 12649f64916dSGerd Hoffmann vs->client_pf.bmax); 1265efe556adSCorentin Chary } 1266efe556adSCorentin Chary } 1267efe556adSCorentin Chary 1268efe556adSCorentin Chary static void png_write_data(png_structp png_ptr, png_bytep data, 1269efe556adSCorentin Chary png_size_t length) 1270efe556adSCorentin Chary { 1271efe556adSCorentin Chary VncState *vs = png_get_io_ptr(png_ptr); 1272efe556adSCorentin Chary 1273d1af0e05SCorentin Chary buffer_reserve(&vs->tight.png, vs->tight.png.offset + length); 1274d1af0e05SCorentin Chary memcpy(vs->tight.png.buffer + vs->tight.png.offset, data, length); 1275efe556adSCorentin Chary 1276d1af0e05SCorentin Chary vs->tight.png.offset += length; 1277efe556adSCorentin Chary } 1278efe556adSCorentin Chary 1279efe556adSCorentin Chary static void png_flush_data(png_structp png_ptr) 1280efe556adSCorentin Chary { 1281efe556adSCorentin Chary } 1282efe556adSCorentin Chary 1283efe556adSCorentin Chary static void *vnc_png_malloc(png_structp png_ptr, png_size_t size) 1284efe556adSCorentin Chary { 12857267c094SAnthony Liguori return g_malloc(size); 1286efe556adSCorentin Chary } 1287efe556adSCorentin Chary 1288efe556adSCorentin Chary static void vnc_png_free(png_structp png_ptr, png_voidp ptr) 1289efe556adSCorentin Chary { 12907267c094SAnthony Liguori g_free(ptr); 1291efe556adSCorentin Chary } 1292efe556adSCorentin Chary 1293efe556adSCorentin Chary static int send_png_rect(VncState *vs, int x, int y, int w, int h, 12945136a052SCorentin Chary VncPalette *palette) 1295efe556adSCorentin Chary { 1296efe556adSCorentin Chary png_byte color_type; 1297efe556adSCorentin Chary png_structp png_ptr; 1298efe556adSCorentin Chary png_infop info_ptr; 1299efe556adSCorentin Chary png_colorp png_palette = NULL; 130047683d66SGerd Hoffmann pixman_image_t *linebuf; 1301d1af0e05SCorentin Chary int level = tight_png_conf[vs->tight.compression].png_zlib_level; 1302d1af0e05SCorentin Chary int filters = tight_png_conf[vs->tight.compression].png_filters; 1303efe556adSCorentin Chary uint8_t *buf; 1304efe556adSCorentin Chary int dy; 1305efe556adSCorentin Chary 1306efe556adSCorentin Chary png_ptr = png_create_write_struct_2(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL, 1307efe556adSCorentin Chary NULL, vnc_png_malloc, vnc_png_free); 1308efe556adSCorentin Chary 1309efe556adSCorentin Chary if (png_ptr == NULL) 1310efe556adSCorentin Chary return -1; 1311efe556adSCorentin Chary 1312efe556adSCorentin Chary info_ptr = png_create_info_struct(png_ptr); 1313efe556adSCorentin Chary 1314efe556adSCorentin Chary if (info_ptr == NULL) { 1315efe556adSCorentin Chary png_destroy_write_struct(&png_ptr, NULL); 1316efe556adSCorentin Chary return -1; 1317efe556adSCorentin Chary } 1318efe556adSCorentin Chary 1319efe556adSCorentin Chary png_set_write_fn(png_ptr, (void *) vs, png_write_data, png_flush_data); 1320efe556adSCorentin Chary png_set_compression_level(png_ptr, level); 13213941bf6fSCorentin Chary png_set_filter(png_ptr, PNG_FILTER_TYPE_DEFAULT, filters); 1322efe556adSCorentin Chary 1323efe556adSCorentin Chary if (palette) { 1324efe556adSCorentin Chary color_type = PNG_COLOR_TYPE_PALETTE; 1325efe556adSCorentin Chary } else { 1326efe556adSCorentin Chary color_type = PNG_COLOR_TYPE_RGB; 1327efe556adSCorentin Chary } 1328efe556adSCorentin Chary 1329efe556adSCorentin Chary png_set_IHDR(png_ptr, info_ptr, w, h, 1330efe556adSCorentin Chary 8, color_type, PNG_INTERLACE_NONE, 1331efe556adSCorentin Chary PNG_COMPRESSION_TYPE_DEFAULT, PNG_FILTER_TYPE_DEFAULT); 1332efe556adSCorentin Chary 1333efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) { 1334efe556adSCorentin Chary struct palette_cb_priv priv; 1335efe556adSCorentin Chary 1336efe556adSCorentin Chary png_palette = png_malloc(png_ptr, sizeof(*png_palette) * 13375136a052SCorentin Chary palette_size(palette)); 1338efe556adSCorentin Chary 1339efe556adSCorentin Chary priv.vs = vs; 1340efe556adSCorentin Chary priv.png_palette = png_palette; 13415136a052SCorentin Chary palette_iter(palette, write_png_palette, &priv); 1342efe556adSCorentin Chary 13435136a052SCorentin Chary png_set_PLTE(png_ptr, info_ptr, png_palette, palette_size(palette)); 1344efe556adSCorentin Chary 13459f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4) { 1346d1af0e05SCorentin Chary tight_encode_indexed_rect32(vs->tight.tight.buffer, w * h, palette); 1347efe556adSCorentin Chary } else { 1348d1af0e05SCorentin Chary tight_encode_indexed_rect16(vs->tight.tight.buffer, w * h, palette); 1349efe556adSCorentin Chary } 1350efe556adSCorentin Chary } 1351efe556adSCorentin Chary 1352efe556adSCorentin Chary png_write_info(png_ptr, info_ptr); 1353efe556adSCorentin Chary 1354d1af0e05SCorentin Chary buffer_reserve(&vs->tight.png, 2048); 135547683d66SGerd Hoffmann linebuf = qemu_pixman_linebuf_create(PIXMAN_BE_r8g8b8, w); 135647683d66SGerd Hoffmann buf = (uint8_t *)pixman_image_get_data(linebuf); 1357efe556adSCorentin Chary for (dy = 0; dy < h; dy++) 1358efe556adSCorentin Chary { 1359efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) { 1360d1af0e05SCorentin Chary memcpy(buf, vs->tight.tight.buffer + (dy * w), w); 1361efe556adSCorentin Chary } else { 1362bc210eb1SGerd Hoffmann qemu_pixman_linebuf_fill(linebuf, vs->vd->server, w, x, y + dy); 1363efe556adSCorentin Chary } 1364efe556adSCorentin Chary png_write_row(png_ptr, buf); 1365efe556adSCorentin Chary } 136647683d66SGerd Hoffmann qemu_pixman_image_unref(linebuf); 1367efe556adSCorentin Chary 1368efe556adSCorentin Chary png_write_end(png_ptr, NULL); 1369efe556adSCorentin Chary 1370efe556adSCorentin Chary if (color_type == PNG_COLOR_TYPE_PALETTE) { 1371efe556adSCorentin Chary png_free(png_ptr, png_palette); 1372efe556adSCorentin Chary } 1373efe556adSCorentin Chary 1374efe556adSCorentin Chary png_destroy_write_struct(&png_ptr, &info_ptr); 1375efe556adSCorentin Chary 1376efe556adSCorentin Chary vnc_write_u8(vs, VNC_TIGHT_PNG << 4); 1377efe556adSCorentin Chary 1378d1af0e05SCorentin Chary tight_send_compact_size(vs, vs->tight.png.offset); 1379d1af0e05SCorentin Chary vnc_write(vs, vs->tight.png.buffer, vs->tight.png.offset); 1380d1af0e05SCorentin Chary buffer_reset(&vs->tight.png); 1381efe556adSCorentin Chary return 1; 1382efe556adSCorentin Chary } 1383efe556adSCorentin Chary #endif /* CONFIG_VNC_PNG */ 1384efe556adSCorentin Chary 1385245f7b51SCorentin Chary static void vnc_tight_start(VncState *vs) 1386245f7b51SCorentin Chary { 1387d1af0e05SCorentin Chary buffer_reset(&vs->tight.tight); 1388245f7b51SCorentin Chary 1389245f7b51SCorentin Chary // make the output buffer be the zlib buffer, so we can compress it later 1390d1af0e05SCorentin Chary vs->tight.tmp = vs->output; 1391d1af0e05SCorentin Chary vs->output = vs->tight.tight; 1392245f7b51SCorentin Chary } 1393245f7b51SCorentin Chary 1394245f7b51SCorentin Chary static void vnc_tight_stop(VncState *vs) 1395245f7b51SCorentin Chary { 1396245f7b51SCorentin Chary // switch back to normal output/zlib buffers 1397d1af0e05SCorentin Chary vs->tight.tight = vs->output; 1398d1af0e05SCorentin Chary vs->output = vs->tight.tmp; 1399245f7b51SCorentin Chary } 1400245f7b51SCorentin Chary 140103817eb8SCorentin Chary static int send_sub_rect_nojpeg(VncState *vs, int x, int y, int w, int h, 140203817eb8SCorentin Chary int bg, int fg, int colors, VncPalette *palette) 140303817eb8SCorentin Chary { 140403817eb8SCorentin Chary int ret; 140503817eb8SCorentin Chary 140603817eb8SCorentin Chary if (colors == 0) { 140703817eb8SCorentin Chary if (tight_detect_smooth_image(vs, w, h)) { 140803817eb8SCorentin Chary ret = send_gradient_rect(vs, x, y, w, h); 140903817eb8SCorentin Chary } else { 141003817eb8SCorentin Chary ret = send_full_color_rect(vs, x, y, w, h); 141103817eb8SCorentin Chary } 141203817eb8SCorentin Chary } else if (colors == 1) { 141303817eb8SCorentin Chary ret = send_solid_rect(vs); 141403817eb8SCorentin Chary } else if (colors == 2) { 141503817eb8SCorentin Chary ret = send_mono_rect(vs, x, y, w, h, bg, fg); 141603817eb8SCorentin Chary } else if (colors <= 256) { 141703817eb8SCorentin Chary ret = send_palette_rect(vs, x, y, w, h, palette); 1418d167f9bcSBlue Swirl } else { 1419d167f9bcSBlue Swirl ret = 0; 142003817eb8SCorentin Chary } 142103817eb8SCorentin Chary return ret; 142203817eb8SCorentin Chary } 142303817eb8SCorentin Chary 142403817eb8SCorentin Chary #ifdef CONFIG_VNC_JPEG 142503817eb8SCorentin Chary static int send_sub_rect_jpeg(VncState *vs, int x, int y, int w, int h, 142603817eb8SCorentin Chary int bg, int fg, int colors, 1427ce702e93SCorentin Chary VncPalette *palette, bool force) 142803817eb8SCorentin Chary { 142903817eb8SCorentin Chary int ret; 143003817eb8SCorentin Chary 143103817eb8SCorentin Chary if (colors == 0) { 1432ce702e93SCorentin Chary if (force || (tight_jpeg_conf[vs->tight.quality].jpeg_full && 1433ce702e93SCorentin Chary tight_detect_smooth_image(vs, w, h))) { 143403817eb8SCorentin Chary int quality = tight_conf[vs->tight.quality].jpeg_quality; 143503817eb8SCorentin Chary 143603817eb8SCorentin Chary ret = send_jpeg_rect(vs, x, y, w, h, quality); 143703817eb8SCorentin Chary } else { 143803817eb8SCorentin Chary ret = send_full_color_rect(vs, x, y, w, h); 143903817eb8SCorentin Chary } 144003817eb8SCorentin Chary } else if (colors == 1) { 144103817eb8SCorentin Chary ret = send_solid_rect(vs); 144203817eb8SCorentin Chary } else if (colors == 2) { 144303817eb8SCorentin Chary ret = send_mono_rect(vs, x, y, w, h, bg, fg); 144403817eb8SCorentin Chary } else if (colors <= 256) { 1445ce702e93SCorentin Chary if (force || (colors > 96 && 1446ce702e93SCorentin Chary tight_jpeg_conf[vs->tight.quality].jpeg_idx && 1447ce702e93SCorentin Chary tight_detect_smooth_image(vs, w, h))) { 144803817eb8SCorentin Chary int quality = tight_conf[vs->tight.quality].jpeg_quality; 144903817eb8SCorentin Chary 145003817eb8SCorentin Chary ret = send_jpeg_rect(vs, x, y, w, h, quality); 145103817eb8SCorentin Chary } else { 145203817eb8SCorentin Chary ret = send_palette_rect(vs, x, y, w, h, palette); 145303817eb8SCorentin Chary } 1454ad7ee4adSBlue Swirl } else { 1455ad7ee4adSBlue Swirl ret = 0; 145603817eb8SCorentin Chary } 145703817eb8SCorentin Chary return ret; 145803817eb8SCorentin Chary } 145903817eb8SCorentin Chary #endif 146003817eb8SCorentin Chary 1461245f7b51SCorentin Chary static int send_sub_rect(VncState *vs, int x, int y, int w, int h) 1462245f7b51SCorentin Chary { 14635136a052SCorentin Chary VncPalette *palette = NULL; 1464245f7b51SCorentin Chary uint32_t bg = 0, fg = 0; 1465245f7b51SCorentin Chary int colors; 1466245f7b51SCorentin Chary int ret = 0; 1467cf76a1ceSPeter Maydell #ifdef CONFIG_VNC_JPEG 1468ce702e93SCorentin Chary bool force_jpeg = false; 1469ce702e93SCorentin Chary bool allow_jpeg = true; 1470cf76a1ceSPeter Maydell #endif 1471245f7b51SCorentin Chary 1472d1af0e05SCorentin Chary vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type); 1473245f7b51SCorentin Chary 1474245f7b51SCorentin Chary vnc_tight_start(vs); 1475245f7b51SCorentin Chary vnc_raw_send_framebuffer_update(vs, x, y, w, h); 1476245f7b51SCorentin Chary vnc_tight_stop(vs); 1477245f7b51SCorentin Chary 1478ce702e93SCorentin Chary #ifdef CONFIG_VNC_JPEG 147980e0c8c3SCorentin Chary if (!vs->vd->non_adaptive && vs->tight.quality != (uint8_t)-1) { 1480ce702e93SCorentin Chary double freq = vnc_update_freq(vs, x, y, w, h); 1481ce702e93SCorentin Chary 1482ce702e93SCorentin Chary if (freq < tight_jpeg_conf[vs->tight.quality].jpeg_freq_min) { 1483ce702e93SCorentin Chary allow_jpeg = false; 1484ce702e93SCorentin Chary } 1485ce702e93SCorentin Chary if (freq >= tight_jpeg_conf[vs->tight.quality].jpeg_freq_threshold) { 1486ce702e93SCorentin Chary force_jpeg = true; 1487ce702e93SCorentin Chary vnc_sent_lossy_rect(vs, x, y, w, h); 1488ce702e93SCorentin Chary } 1489ce702e93SCorentin Chary } 1490ce702e93SCorentin Chary #endif 1491ce702e93SCorentin Chary 1492245f7b51SCorentin Chary colors = tight_fill_palette(vs, x, y, w * h, &fg, &bg, &palette); 1493245f7b51SCorentin Chary 1494245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG 1495ce702e93SCorentin Chary if (allow_jpeg && vs->tight.quality != (uint8_t)-1) { 1496ce702e93SCorentin Chary ret = send_sub_rect_jpeg(vs, x, y, w, h, bg, fg, colors, palette, 1497ce702e93SCorentin Chary force_jpeg); 1498245f7b51SCorentin Chary } else { 149903817eb8SCorentin Chary ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); 1500245f7b51SCorentin Chary } 1501245f7b51SCorentin Chary #else 150203817eb8SCorentin Chary ret = send_sub_rect_nojpeg(vs, x, y, w, h, bg, fg, colors, palette); 1503245f7b51SCorentin Chary #endif 150403817eb8SCorentin Chary 15055136a052SCorentin Chary palette_destroy(palette); 1506245f7b51SCorentin Chary return ret; 1507245f7b51SCorentin Chary } 1508245f7b51SCorentin Chary 1509245f7b51SCorentin Chary static int send_sub_rect_solid(VncState *vs, int x, int y, int w, int h) 1510245f7b51SCorentin Chary { 1511d1af0e05SCorentin Chary vnc_framebuffer_update(vs, x, y, w, h, vs->tight.type); 1512245f7b51SCorentin Chary 1513245f7b51SCorentin Chary vnc_tight_start(vs); 1514245f7b51SCorentin Chary vnc_raw_send_framebuffer_update(vs, x, y, w, h); 1515245f7b51SCorentin Chary vnc_tight_stop(vs); 1516245f7b51SCorentin Chary 1517245f7b51SCorentin Chary return send_solid_rect(vs); 1518245f7b51SCorentin Chary } 1519245f7b51SCorentin Chary 1520ce702e93SCorentin Chary static int send_rect_simple(VncState *vs, int x, int y, int w, int h, 1521ce702e93SCorentin Chary bool split) 1522245f7b51SCorentin Chary { 1523245f7b51SCorentin Chary int max_size, max_width; 1524245f7b51SCorentin Chary int max_sub_width, max_sub_height; 1525245f7b51SCorentin Chary int dx, dy; 1526245f7b51SCorentin Chary int rw, rh; 1527245f7b51SCorentin Chary int n = 0; 1528245f7b51SCorentin Chary 1529d1af0e05SCorentin Chary max_size = tight_conf[vs->tight.compression].max_rect_size; 1530d1af0e05SCorentin Chary max_width = tight_conf[vs->tight.compression].max_rect_width; 1531245f7b51SCorentin Chary 1532ce702e93SCorentin Chary if (split && (w > max_width || w * h > max_size)) { 1533245f7b51SCorentin Chary max_sub_width = (w > max_width) ? max_width : w; 1534245f7b51SCorentin Chary max_sub_height = max_size / max_sub_width; 1535245f7b51SCorentin Chary 1536245f7b51SCorentin Chary for (dy = 0; dy < h; dy += max_sub_height) { 1537245f7b51SCorentin Chary for (dx = 0; dx < w; dx += max_width) { 1538245f7b51SCorentin Chary rw = MIN(max_sub_width, w - dx); 1539245f7b51SCorentin Chary rh = MIN(max_sub_height, h - dy); 1540245f7b51SCorentin Chary n += send_sub_rect(vs, x+dx, y+dy, rw, rh); 1541245f7b51SCorentin Chary } 1542245f7b51SCorentin Chary } 1543245f7b51SCorentin Chary } else { 1544245f7b51SCorentin Chary n += send_sub_rect(vs, x, y, w, h); 1545245f7b51SCorentin Chary } 1546245f7b51SCorentin Chary 1547245f7b51SCorentin Chary return n; 1548245f7b51SCorentin Chary } 1549245f7b51SCorentin Chary 1550245f7b51SCorentin Chary static int find_large_solid_color_rect(VncState *vs, int x, int y, 1551245f7b51SCorentin Chary int w, int h, int max_rows) 1552245f7b51SCorentin Chary { 1553245f7b51SCorentin Chary int dx, dy, dw, dh; 1554245f7b51SCorentin Chary int n = 0; 1555245f7b51SCorentin Chary 1556245f7b51SCorentin Chary /* Try to find large solid-color areas and send them separately. */ 1557245f7b51SCorentin Chary 1558245f7b51SCorentin Chary for (dy = y; dy < y + h; dy += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) { 1559245f7b51SCorentin Chary 1560245f7b51SCorentin Chary /* If a rectangle becomes too large, send its upper part now. */ 1561245f7b51SCorentin Chary 1562245f7b51SCorentin Chary if (dy - y >= max_rows) { 1563ce702e93SCorentin Chary n += send_rect_simple(vs, x, y, w, max_rows, true); 1564245f7b51SCorentin Chary y += max_rows; 1565245f7b51SCorentin Chary h -= max_rows; 1566245f7b51SCorentin Chary } 1567245f7b51SCorentin Chary 1568245f7b51SCorentin Chary dh = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (y + h - dy)); 1569245f7b51SCorentin Chary 1570245f7b51SCorentin Chary for (dx = x; dx < x + w; dx += VNC_TIGHT_MAX_SPLIT_TILE_SIZE) { 1571245f7b51SCorentin Chary uint32_t color_value; 1572245f7b51SCorentin Chary int x_best, y_best, w_best, h_best; 1573245f7b51SCorentin Chary 1574245f7b51SCorentin Chary dw = MIN(VNC_TIGHT_MAX_SPLIT_TILE_SIZE, (x + w - dx)); 1575245f7b51SCorentin Chary 1576245f7b51SCorentin Chary if (!check_solid_tile(vs, dx, dy, dw, dh, &color_value, false)) { 1577245f7b51SCorentin Chary continue ; 1578245f7b51SCorentin Chary } 1579245f7b51SCorentin Chary 1580245f7b51SCorentin Chary /* Get dimensions of solid-color area. */ 1581245f7b51SCorentin Chary 1582245f7b51SCorentin Chary find_best_solid_area(vs, dx, dy, w - (dx - x), h - (dy - y), 1583245f7b51SCorentin Chary color_value, &w_best, &h_best); 1584245f7b51SCorentin Chary 1585245f7b51SCorentin Chary /* Make sure a solid rectangle is large enough 1586245f7b51SCorentin Chary (or the whole rectangle is of the same color). */ 1587245f7b51SCorentin Chary 1588245f7b51SCorentin Chary if (w_best * h_best != w * h && 1589245f7b51SCorentin Chary w_best * h_best < VNC_TIGHT_MIN_SOLID_SUBRECT_SIZE) { 1590245f7b51SCorentin Chary continue; 1591245f7b51SCorentin Chary } 1592245f7b51SCorentin Chary 1593245f7b51SCorentin Chary /* Try to extend solid rectangle to maximum size. */ 1594245f7b51SCorentin Chary 1595245f7b51SCorentin Chary x_best = dx; y_best = dy; 1596245f7b51SCorentin Chary extend_solid_area(vs, x, y, w, h, color_value, 1597245f7b51SCorentin Chary &x_best, &y_best, &w_best, &h_best); 1598245f7b51SCorentin Chary 1599245f7b51SCorentin Chary /* Send rectangles at top and left to solid-color area. */ 1600245f7b51SCorentin Chary 1601245f7b51SCorentin Chary if (y_best != y) { 1602ce702e93SCorentin Chary n += send_rect_simple(vs, x, y, w, y_best-y, true); 1603245f7b51SCorentin Chary } 1604245f7b51SCorentin Chary if (x_best != x) { 1605efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x, y_best, 1606245f7b51SCorentin Chary x_best-x, h_best); 1607245f7b51SCorentin Chary } 1608245f7b51SCorentin Chary 1609245f7b51SCorentin Chary /* Send solid-color rectangle. */ 1610245f7b51SCorentin Chary n += send_sub_rect_solid(vs, x_best, y_best, w_best, h_best); 1611245f7b51SCorentin Chary 1612245f7b51SCorentin Chary /* Send remaining rectangles (at right and bottom). */ 1613245f7b51SCorentin Chary 1614245f7b51SCorentin Chary if (x_best + w_best != x + w) { 1615efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x_best+w_best, 1616245f7b51SCorentin Chary y_best, 1617245f7b51SCorentin Chary w-(x_best-x)-w_best, 1618245f7b51SCorentin Chary h_best); 1619245f7b51SCorentin Chary } 1620245f7b51SCorentin Chary if (y_best + h_best != y + h) { 1621efe556adSCorentin Chary n += tight_send_framebuffer_update(vs, x, y_best+h_best, 1622245f7b51SCorentin Chary w, h-(y_best-y)-h_best); 1623245f7b51SCorentin Chary } 1624245f7b51SCorentin Chary 1625245f7b51SCorentin Chary /* Return after all recursive calls are done. */ 1626245f7b51SCorentin Chary return n; 1627245f7b51SCorentin Chary } 1628245f7b51SCorentin Chary } 1629ce702e93SCorentin Chary return n + send_rect_simple(vs, x, y, w, h, true); 1630245f7b51SCorentin Chary } 1631245f7b51SCorentin Chary 1632efe556adSCorentin Chary static int tight_send_framebuffer_update(VncState *vs, int x, int y, 1633245f7b51SCorentin Chary int w, int h) 1634245f7b51SCorentin Chary { 1635245f7b51SCorentin Chary int max_rows; 1636245f7b51SCorentin Chary 16379f64916dSGerd Hoffmann if (vs->client_pf.bytes_per_pixel == 4 && vs->client_pf.rmax == 0xFF && 16389f64916dSGerd Hoffmann vs->client_pf.bmax == 0xFF && vs->client_pf.gmax == 0xFF) { 1639d1af0e05SCorentin Chary vs->tight.pixel24 = true; 1640245f7b51SCorentin Chary } else { 1641d1af0e05SCorentin Chary vs->tight.pixel24 = false; 1642245f7b51SCorentin Chary } 1643245f7b51SCorentin Chary 1644cf76a1ceSPeter Maydell #ifdef CONFIG_VNC_JPEG 1645368d2588SCorentin Chary if (vs->tight.quality != (uint8_t)-1) { 1646ce702e93SCorentin Chary double freq = vnc_update_freq(vs, x, y, w, h); 1647ce702e93SCorentin Chary 1648ce702e93SCorentin Chary if (freq > tight_jpeg_conf[vs->tight.quality].jpeg_freq_threshold) { 1649ce702e93SCorentin Chary return send_rect_simple(vs, x, y, w, h, false); 1650ce702e93SCorentin Chary } 1651ce702e93SCorentin Chary } 1652cf76a1ceSPeter Maydell #endif 1653ce702e93SCorentin Chary 1654ce702e93SCorentin Chary if (w * h < VNC_TIGHT_MIN_SPLIT_RECT_SIZE) { 1655ce702e93SCorentin Chary return send_rect_simple(vs, x, y, w, h, true); 1656ce702e93SCorentin Chary } 1657245f7b51SCorentin Chary 1658245f7b51SCorentin Chary /* Calculate maximum number of rows in one non-solid rectangle. */ 1659245f7b51SCorentin Chary 1660d1af0e05SCorentin Chary max_rows = tight_conf[vs->tight.compression].max_rect_size; 1661d1af0e05SCorentin Chary max_rows /= MIN(tight_conf[vs->tight.compression].max_rect_width, w); 1662245f7b51SCorentin Chary 1663245f7b51SCorentin Chary return find_large_solid_color_rect(vs, x, y, w, h, max_rows); 1664245f7b51SCorentin Chary } 1665245f7b51SCorentin Chary 1666efe556adSCorentin Chary int vnc_tight_send_framebuffer_update(VncState *vs, int x, int y, 1667efe556adSCorentin Chary int w, int h) 1668efe556adSCorentin Chary { 1669d1af0e05SCorentin Chary vs->tight.type = VNC_ENCODING_TIGHT; 1670efe556adSCorentin Chary return tight_send_framebuffer_update(vs, x, y, w, h); 1671efe556adSCorentin Chary } 1672efe556adSCorentin Chary 1673efe556adSCorentin Chary int vnc_tight_png_send_framebuffer_update(VncState *vs, int x, int y, 1674efe556adSCorentin Chary int w, int h) 1675efe556adSCorentin Chary { 1676d1af0e05SCorentin Chary vs->tight.type = VNC_ENCODING_TIGHT_PNG; 1677efe556adSCorentin Chary return tight_send_framebuffer_update(vs, x, y, w, h); 1678efe556adSCorentin Chary } 1679efe556adSCorentin Chary 1680245f7b51SCorentin Chary void vnc_tight_clear(VncState *vs) 1681245f7b51SCorentin Chary { 1682245f7b51SCorentin Chary int i; 1683d1af0e05SCorentin Chary for (i=0; i<ARRAY_SIZE(vs->tight.stream); i++) { 1684d1af0e05SCorentin Chary if (vs->tight.stream[i].opaque) { 1685d1af0e05SCorentin Chary deflateEnd(&vs->tight.stream[i]); 1686245f7b51SCorentin Chary } 1687245f7b51SCorentin Chary } 1688245f7b51SCorentin Chary 1689d1af0e05SCorentin Chary buffer_free(&vs->tight.tight); 1690d1af0e05SCorentin Chary buffer_free(&vs->tight.zlib); 1691d1af0e05SCorentin Chary buffer_free(&vs->tight.gradient); 1692245f7b51SCorentin Chary #ifdef CONFIG_VNC_JPEG 1693d1af0e05SCorentin Chary buffer_free(&vs->tight.jpeg); 1694245f7b51SCorentin Chary #endif 1695b5469b11SCorentin Chary #ifdef CONFIG_VNC_PNG 1696b5469b11SCorentin Chary buffer_free(&vs->tight.png); 1697b5469b11SCorentin Chary #endif 1698245f7b51SCorentin Chary } 1699