xref: /qemu/ui/vnc-enc-hextile-template.h (revision e12acaf7)
1 #define CONCAT_I(a, b) a ## b
2 #define CONCAT(a, b) CONCAT_I(a, b)
3 #define pixel_t CONCAT(uint, CONCAT(BPP, _t))
4 #ifdef GENERIC
5 #define NAME CONCAT(generic_, BPP)
6 #else
7 #define NAME BPP
8 #endif
9 
10 #define MAX_BYTES_PER_PIXEL 4
11 
CONCAT(send_hextile_tile_,NAME)12 static void CONCAT(send_hextile_tile_, NAME)(VncState *vs,
13                                              int x, int y, int w, int h,
14                                              void *last_bg_,
15                                              void *last_fg_,
16                                              int *has_bg, int *has_fg)
17 {
18     VncDisplay *vd = vs->vd;
19     uint8_t *row = vnc_server_fb_ptr(vd, x, y);
20     pixel_t *irow = (pixel_t *)row;
21     int j, i;
22     pixel_t *last_bg = (pixel_t *)last_bg_;
23     pixel_t *last_fg = (pixel_t *)last_fg_;
24     pixel_t bg = 0;
25     pixel_t fg = 0;
26     int n_colors = 0;
27     int bg_count = 0;
28     int fg_count = 0;
29     int flags = 0;
30     uint8_t data[(MAX_BYTES_PER_PIXEL + 2) * 16 * 16];
31     int n_data = 0;
32     int n_subtiles = 0;
33 
34     /* Enforced by set_pixel_format() */
35     assert(vs->client_pf.bytes_per_pixel <= MAX_BYTES_PER_PIXEL);
36 
37     for (j = 0; j < h; j++) {
38         for (i = 0; i < w; i++) {
39             switch (n_colors) {
40             case 0:
41                 bg = irow[i];
42                 n_colors = 1;
43                 break;
44             case 1:
45                 if (irow[i] != bg) {
46                     fg = irow[i];
47                     n_colors = 2;
48                 }
49                 break;
50             case 2:
51                 if (irow[i] != bg && irow[i] != fg) {
52                     n_colors = 3;
53                 } else {
54                     if (irow[i] == bg)
55                         bg_count++;
56                     else if (irow[i] == fg)
57                         fg_count++;
58                 }
59                 break;
60             default:
61                 break;
62             }
63         }
64         if (n_colors > 2)
65             break;
66         irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
67     }
68 
69     if (n_colors > 1 && fg_count > bg_count) {
70         pixel_t tmp = fg;
71         fg = bg;
72         bg = tmp;
73     }
74 
75     if (!*has_bg || *last_bg != bg) {
76         flags |= 0x02;
77         *has_bg = 1;
78         *last_bg = bg;
79     }
80 
81     if (n_colors < 3 && (!*has_fg || *last_fg != fg)) {
82         flags |= 0x04;
83         *has_fg = 1;
84         *last_fg = fg;
85     }
86 
87     switch (n_colors) {
88     case 1:
89         n_data = 0;
90         break;
91     case 2:
92         flags |= 0x08;
93 
94         irow = (pixel_t *)row;
95 
96         for (j = 0; j < h; j++) {
97             int min_x = -1;
98             for (i = 0; i < w; i++) {
99                 if (irow[i] == fg) {
100                     if (min_x == -1)
101                         min_x = i;
102                 } else if (min_x != -1) {
103                     hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
104                     n_data += 2;
105                     n_subtiles++;
106                     min_x = -1;
107                 }
108             }
109             if (min_x != -1) {
110                 hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
111                 n_data += 2;
112                 n_subtiles++;
113             }
114             irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
115         }
116         break;
117     case 3:
118         flags |= 0x18;
119 
120         irow = (pixel_t *)row;
121 
122         if (!*has_bg || *last_bg != bg)
123             flags |= 0x02;
124 
125         for (j = 0; j < h; j++) {
126             int has_color = 0;
127             int min_x = -1;
128             pixel_t color = 0; /* shut up gcc */
129 
130             for (i = 0; i < w; i++) {
131                 if (!has_color) {
132                     if (irow[i] == bg)
133                         continue;
134                     color = irow[i];
135                     min_x = i;
136                     has_color = 1;
137                 } else if (irow[i] != color) {
138                     has_color = 0;
139 #ifdef GENERIC
140                     vnc_convert_pixel(vs, data + n_data, color);
141                     n_data += vs->client_pf.bytes_per_pixel;
142 #else
143                     memcpy(data + n_data, &color, sizeof(color));
144                     n_data += sizeof(pixel_t);
145 #endif
146                     hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
147                     n_data += 2;
148                     n_subtiles++;
149 
150                     min_x = -1;
151                     if (irow[i] != bg) {
152                         color = irow[i];
153                         min_x = i;
154                         has_color = 1;
155                     }
156                 }
157             }
158             if (has_color) {
159 #ifdef GENERIC
160                 vnc_convert_pixel(vs, data + n_data, color);
161                 n_data += vs->client_pf.bytes_per_pixel;
162 #else
163                 memcpy(data + n_data, &color, sizeof(color));
164                 n_data += sizeof(pixel_t);
165 #endif
166                 hextile_enc_cord(data + n_data, min_x, j, i - min_x, 1);
167                 n_data += 2;
168                 n_subtiles++;
169             }
170             irow += vnc_server_fb_stride(vd) / sizeof(pixel_t);
171         }
172 
173         /* A SubrectsColoured subtile invalidates the foreground color */
174         *has_fg = 0;
175         if (n_data > (w * h * sizeof(pixel_t))) {
176             n_colors = 4;
177             flags = 0x01;
178             *has_bg = 0;
179 
180             /* we really don't have to invalidate either the bg or fg
181                but we've lost the old values.  oh well. */
182         }
183         break;
184     default:
185         break;
186     }
187 
188     if (n_colors > 3) {
189         flags = 0x01;
190         *has_fg = 0;
191         *has_bg = 0;
192         n_colors = 4;
193     }
194 
195     vnc_write_u8(vs, flags);
196     if (n_colors < 4) {
197         if (flags & 0x02)
198             vs->write_pixels(vs, last_bg, sizeof(pixel_t));
199         if (flags & 0x04)
200             vs->write_pixels(vs, last_fg, sizeof(pixel_t));
201         if (n_subtiles) {
202             vnc_write_u8(vs, n_subtiles);
203             vnc_write(vs, data, n_data);
204         }
205     } else {
206         for (j = 0; j < h; j++) {
207             vs->write_pixels(vs, row, w * 4);
208             row += vnc_server_fb_stride(vd);
209         }
210     }
211 }
212 
213 #undef MAX_BYTES_PER_PIXEL
214 #undef NAME
215 #undef pixel_t
216 #undef CONCAT_I
217 #undef CONCAT
218