1 /**
2 * FreeRDP: A Remote Desktop Protocol Implementation
3 * RDP6 Planar Codec
4 *
5 * Copyright 2013 Marc-Andre Moreau <marcandre.moreau@gmail.com>
6 * Copyright 2016 Armin Novak <armin.novak@thincast.com>
7 * Copyright 2016 Thincast Technologies GmbH
8 *
9 * Licensed under the Apache License, Version 2.0 (the "License");
10 * you may not use this file except in compliance with the License.
11 * You may obtain a copy of the License at
12 *
13 * http://www.apache.org/licenses/LICENSE-2.0
14 *
15 * Unless required by applicable law or agreed to in writing, software
16 * distributed under the License is distributed on an "AS IS" BASIS,
17 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18 * See the License for the specific language governing permissions and
19 * limitations under the License.
20 */
21
22 #ifdef HAVE_CONFIG_H
23 # include "config.h"
24 #endif
25
26 #include <winpr/crt.h>
27 #include <winpr/print.h>
28
29 #include <freerdp/primitives.h>
30 #include <freerdp/log.h>
31 #include <freerdp/codec/bitmap.h>
32 #include <freerdp/codec/planar.h>
33
34 #define TAG FREERDP_TAG("codec")
35
36 #define ALIGN(val, align) ((val) % (align) == 0) ? (val) : ((val) + (align) - (val) % (align))
37
planar_invert_format(BITMAP_PLANAR_CONTEXT * planar,BOOL alpha,UINT32 DstFormat)38 static INLINE UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* planar, BOOL alpha,
39 UINT32 DstFormat)
40 {
41
42 if (planar->bgr && alpha)
43 {
44 switch (DstFormat)
45 {
46 case PIXEL_FORMAT_ARGB32:
47 DstFormat = PIXEL_FORMAT_ABGR32;
48 break;
49 case PIXEL_FORMAT_XRGB32:
50 DstFormat = PIXEL_FORMAT_XBGR32;
51 break;
52 case PIXEL_FORMAT_ABGR32:
53 DstFormat = PIXEL_FORMAT_ARGB32;
54 break;
55 case PIXEL_FORMAT_XBGR32:
56 DstFormat = PIXEL_FORMAT_XRGB32;
57 break;
58 case PIXEL_FORMAT_BGRA32:
59 DstFormat = PIXEL_FORMAT_RGBA32;
60 break;
61 case PIXEL_FORMAT_BGRX32:
62 DstFormat = PIXEL_FORMAT_RGBX32;
63 break;
64 case PIXEL_FORMAT_RGBA32:
65 DstFormat = PIXEL_FORMAT_BGRA32;
66 break;
67 case PIXEL_FORMAT_RGBX32:
68 DstFormat = PIXEL_FORMAT_BGRX32;
69 break;
70 case PIXEL_FORMAT_RGB24:
71 DstFormat = PIXEL_FORMAT_BGR24;
72 break;
73 case PIXEL_FORMAT_BGR24:
74 DstFormat = PIXEL_FORMAT_RGB24;
75 break;
76 case PIXEL_FORMAT_RGB16:
77 DstFormat = PIXEL_FORMAT_BGR16;
78 break;
79 case PIXEL_FORMAT_BGR16:
80 DstFormat = PIXEL_FORMAT_RGB16;
81 break;
82 case PIXEL_FORMAT_ARGB15:
83 DstFormat = PIXEL_FORMAT_ABGR15;
84 break;
85 case PIXEL_FORMAT_RGB15:
86 DstFormat = PIXEL_FORMAT_BGR15;
87 break;
88 case PIXEL_FORMAT_ABGR15:
89 DstFormat = PIXEL_FORMAT_ARGB15;
90 break;
91 case PIXEL_FORMAT_BGR15:
92 DstFormat = PIXEL_FORMAT_RGB15;
93 break;
94 default:
95 break;
96 }
97 }
98 return DstFormat;
99 }
100
101 static INLINE BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* plane, UINT32 width,
102 UINT32 height, BYTE* outPlane,
103 UINT32* dstSize);
104 static INLINE BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* inPlane, UINT32 width,
105 UINT32 height, BYTE* outPlane);
106
planar_skip_plane_rle(const BYTE * pSrcData,UINT32 SrcSize,UINT32 nWidth,UINT32 nHeight)107 static INLINE INT32 planar_skip_plane_rle(const BYTE* pSrcData, UINT32 SrcSize, UINT32 nWidth,
108 UINT32 nHeight)
109 {
110 UINT32 used = 0;
111 UINT32 x, y;
112 BYTE controlByte;
113
114 for (y = 0; y < nHeight; y++)
115 {
116 for (x = 0; x < nWidth;)
117 {
118 int cRawBytes;
119 int nRunLength;
120
121 if (used >= SrcSize)
122 return -1;
123
124 controlByte = pSrcData[used++];
125 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
126 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
127
128 if (nRunLength == 1)
129 {
130 nRunLength = cRawBytes + 16;
131 cRawBytes = 0;
132 }
133 else if (nRunLength == 2)
134 {
135 nRunLength = cRawBytes + 32;
136 cRawBytes = 0;
137 }
138
139 used += cRawBytes;
140 x += cRawBytes;
141 x += nRunLength;
142
143 if (x > nWidth)
144 return -1;
145
146 if (used > SrcSize)
147 return -1;
148 }
149 }
150
151 if (used > INT32_MAX)
152 return -1;
153 return (INT32)used;
154 }
155
planar_decompress_plane_rle_only(const BYTE * pSrcData,UINT32 SrcSize,BYTE * pDstData,UINT32 nWidth,UINT32 nHeight)156 static INLINE INT32 planar_decompress_plane_rle_only(const BYTE* pSrcData, UINT32 SrcSize,
157 BYTE* pDstData, UINT32 nWidth, UINT32 nHeight)
158 {
159 INT32 x, y;
160 UINT32 pixel;
161 UINT32 cRawBytes;
162 UINT32 nRunLength;
163 INT32 deltaValue;
164 BYTE controlByte;
165 BYTE* currentScanline;
166 BYTE* previousScanline;
167 const BYTE* srcp = pSrcData;
168
169 if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX))
170 return -1;
171
172 previousScanline = NULL;
173
174 for (y = 0; y < (INT32)nHeight; y++)
175 {
176 BYTE* dstp = &pDstData[((y) * (INT32)nWidth)];
177 pixel = 0;
178 currentScanline = dstp;
179
180 for (x = 0; x < (INT32)nWidth;)
181 {
182 controlByte = *srcp;
183 srcp++;
184
185 if ((srcp - pSrcData) > SrcSize)
186 {
187 WLog_ERR(TAG, "error reading input buffer");
188 return -1;
189 }
190
191 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
192 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
193
194 if (nRunLength == 1)
195 {
196 nRunLength = cRawBytes + 16;
197 cRawBytes = 0;
198 }
199 else if (nRunLength == 2)
200 {
201 nRunLength = cRawBytes + 32;
202 cRawBytes = 0;
203 }
204
205 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth)
206 {
207 WLog_ERR(TAG, "too many pixels in scanline");
208 return -1;
209 }
210
211 if (!previousScanline)
212 {
213 /* first scanline, absolute values */
214 while (cRawBytes > 0)
215 {
216 pixel = *srcp;
217 srcp++;
218 *dstp = pixel;
219 dstp++;
220 x++;
221 cRawBytes--;
222 }
223
224 while (nRunLength > 0)
225 {
226 *dstp = pixel;
227 dstp++;
228 x++;
229 nRunLength--;
230 }
231 }
232 else
233 {
234 /* delta values relative to previous scanline */
235 while (cRawBytes > 0)
236 {
237 deltaValue = *srcp;
238 srcp++;
239
240 if (deltaValue & 1)
241 {
242 deltaValue = deltaValue >> 1;
243 deltaValue = deltaValue + 1;
244 pixel = -deltaValue;
245 }
246 else
247 {
248 deltaValue = deltaValue >> 1;
249 pixel = deltaValue;
250 }
251
252 deltaValue = previousScanline[x] + pixel;
253 *dstp = deltaValue;
254 dstp++;
255 x++;
256 cRawBytes--;
257 }
258
259 while (nRunLength > 0)
260 {
261 deltaValue = previousScanline[x] + pixel;
262 *dstp = deltaValue;
263 dstp++;
264 x++;
265 nRunLength--;
266 }
267 }
268 }
269
270 previousScanline = currentScanline;
271 }
272
273 return (INT32)(srcp - pSrcData);
274 }
275
planar_decompress_plane_rle(const BYTE * pSrcData,UINT32 SrcSize,BYTE * pDstData,INT32 nDstStep,UINT32 nXDst,UINT32 nYDst,UINT32 nWidth,UINT32 nHeight,UINT32 nChannel,BOOL vFlip)276 static INLINE INT32 planar_decompress_plane_rle(const BYTE* pSrcData, UINT32 SrcSize,
277 BYTE* pDstData, INT32 nDstStep, UINT32 nXDst,
278 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight,
279 UINT32 nChannel, BOOL vFlip)
280 {
281 INT32 x, y;
282 UINT32 pixel;
283 UINT32 cRawBytes;
284 UINT32 nRunLength;
285 INT32 deltaValue;
286 INT32 beg, end, inc;
287 BYTE controlByte;
288 BYTE* currentScanline;
289 BYTE* previousScanline;
290 const BYTE* srcp = pSrcData;
291
292 if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX) || (nDstStep > INT32_MAX))
293 return -1;
294
295 previousScanline = NULL;
296
297 if (vFlip)
298 {
299 beg = (INT32)nHeight - 1;
300 end = -1;
301 inc = -1;
302 }
303 else
304 {
305 beg = 0;
306 end = (INT32)nHeight;
307 inc = 1;
308 }
309
310 for (y = beg; y != end; y += inc)
311 {
312 BYTE* dstp = &pDstData[((nYDst + y) * (INT32)nDstStep) + (nXDst * 4) + nChannel];
313 pixel = 0;
314 currentScanline = dstp;
315
316 for (x = 0; x < (INT32)nWidth;)
317 {
318 controlByte = *srcp;
319 srcp++;
320
321 if ((srcp - pSrcData) > SrcSize)
322 {
323 WLog_ERR(TAG, "error reading input buffer");
324 return -1;
325 }
326
327 nRunLength = PLANAR_CONTROL_BYTE_RUN_LENGTH(controlByte);
328 cRawBytes = PLANAR_CONTROL_BYTE_RAW_BYTES(controlByte);
329
330 if (nRunLength == 1)
331 {
332 nRunLength = cRawBytes + 16;
333 cRawBytes = 0;
334 }
335 else if (nRunLength == 2)
336 {
337 nRunLength = cRawBytes + 32;
338 cRawBytes = 0;
339 }
340
341 if (((dstp + (cRawBytes + nRunLength)) - currentScanline) > nWidth * 4)
342 {
343 WLog_ERR(TAG, "too many pixels in scanline");
344 return -1;
345 }
346
347 if (!previousScanline)
348 {
349 /* first scanline, absolute values */
350 while (cRawBytes > 0)
351 {
352 pixel = *srcp;
353 srcp++;
354 *dstp = pixel;
355 dstp += 4;
356 x++;
357 cRawBytes--;
358 }
359
360 while (nRunLength > 0)
361 {
362 *dstp = pixel;
363 dstp += 4;
364 x++;
365 nRunLength--;
366 }
367 }
368 else
369 {
370 /* delta values relative to previous scanline */
371 while (cRawBytes > 0)
372 {
373 deltaValue = *srcp;
374 srcp++;
375
376 if (deltaValue & 1)
377 {
378 deltaValue = deltaValue >> 1;
379 deltaValue = deltaValue + 1;
380 pixel = -deltaValue;
381 }
382 else
383 {
384 deltaValue = deltaValue >> 1;
385 pixel = deltaValue;
386 }
387
388 deltaValue = previousScanline[x * 4] + pixel;
389 *dstp = deltaValue;
390 dstp += 4;
391 x++;
392 cRawBytes--;
393 }
394
395 while (nRunLength > 0)
396 {
397 deltaValue = previousScanline[x * 4] + pixel;
398 *dstp = deltaValue;
399 dstp += 4;
400 x++;
401 nRunLength--;
402 }
403 }
404 }
405
406 previousScanline = currentScanline;
407 }
408
409 return (INT32)(srcp - pSrcData);
410 }
411
planar_set_plane(BYTE bValue,BYTE * pDstData,INT32 nDstStep,UINT32 nXDst,UINT32 nYDst,UINT32 nWidth,UINT32 nHeight,UINT32 nChannel,BOOL vFlip)412 static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, INT32 nDstStep, UINT32 nXDst,
413 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight, UINT32 nChannel,
414 BOOL vFlip)
415 {
416 INT32 x, y;
417 INT32 beg, end, inc;
418
419 if ((nHeight > INT32_MAX) || (nWidth > INT32_MAX) || (nDstStep > INT32_MAX))
420 return -1;
421
422 if (vFlip)
423 {
424 beg = (INT32)nHeight - 1;
425 end = -1;
426 inc = -1;
427 }
428 else
429 {
430 beg = 0;
431 end = (INT32)nHeight;
432 inc = 1;
433 }
434
435 for (y = beg; y != end; y += inc)
436 {
437 BYTE* dstp = &pDstData[((nYDst + y) * (INT32)nDstStep) + (nXDst * 4) + nChannel];
438
439 for (x = 0; x < (INT32)nWidth; ++x)
440 {
441 *dstp = bValue;
442 dstp += 4;
443 }
444 }
445
446 return 0;
447 }
448
writeLine(BYTE ** ppRgba,UINT32 DstFormat,UINT32 width,const BYTE ** ppR,const BYTE ** ppG,const BYTE ** ppB,const BYTE ** ppA)449 static INLINE BOOL writeLine(BYTE** ppRgba, UINT32 DstFormat, UINT32 width, const BYTE** ppR,
450 const BYTE** ppG, const BYTE** ppB, const BYTE** ppA)
451 {
452 UINT32 x;
453
454 if (!ppRgba || !ppR || !ppG || !ppB)
455 return FALSE;
456
457 switch (DstFormat)
458 {
459 case PIXEL_FORMAT_BGRA32:
460 for (x = 0; x < width; x++)
461 {
462 *(*ppRgba)++ = *(*ppB)++;
463 *(*ppRgba)++ = *(*ppG)++;
464 *(*ppRgba)++ = *(*ppR)++;
465 *(*ppRgba)++ = *(*ppA)++;
466 }
467
468 return TRUE;
469
470 case PIXEL_FORMAT_BGRX32:
471 for (x = 0; x < width; x++)
472 {
473 *(*ppRgba)++ = *(*ppB)++;
474 *(*ppRgba)++ = *(*ppG)++;
475 *(*ppRgba)++ = *(*ppR)++;
476 *(*ppRgba)++ = 0xFF;
477 }
478
479 return TRUE;
480
481 default:
482 if (ppA)
483 {
484 for (x = 0; x < width; x++)
485 {
486 BYTE alpha = *(*ppA)++;
487 UINT32 color =
488 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
489 WriteColor(*ppRgba, DstFormat, color);
490 *ppRgba += GetBytesPerPixel(DstFormat);
491 }
492 }
493 else
494 {
495 const BYTE alpha = 0xFF;
496
497 for (x = 0; x < width; x++)
498 {
499 UINT32 color =
500 FreeRDPGetColor(DstFormat, *(*ppR)++, *(*ppG)++, *(*ppB)++, alpha);
501 WriteColor(*ppRgba, DstFormat, color);
502 *ppRgba += GetBytesPerPixel(DstFormat);
503 }
504 }
505
506 return TRUE;
507 }
508 }
509
planar_decompress_planes_raw(const BYTE * pSrcData[4],BYTE * pDstData,UINT32 DstFormat,UINT32 nDstStep,UINT32 nXDst,UINT32 nYDst,UINT32 nWidth,UINT32 nHeight,BOOL vFlip,UINT32 totalHeight)510 static INLINE BOOL planar_decompress_planes_raw(const BYTE* pSrcData[4], BYTE* pDstData,
511 UINT32 DstFormat, UINT32 nDstStep, UINT32 nXDst,
512 UINT32 nYDst, UINT32 nWidth, UINT32 nHeight,
513 BOOL vFlip, UINT32 totalHeight)
514 {
515 INT32 y;
516 INT32 beg, end, inc;
517 const BYTE* pR = pSrcData[0];
518 const BYTE* pG = pSrcData[1];
519 const BYTE* pB = pSrcData[2];
520 const BYTE* pA = pSrcData[3];
521 const UINT32 bpp = GetBytesPerPixel(DstFormat);
522
523 if (vFlip)
524 {
525 beg = nHeight - 1;
526 end = -1;
527 inc = -1;
528 }
529 else
530 {
531 beg = 0;
532 end = nHeight;
533 inc = 1;
534 }
535
536 if (nYDst + nHeight > totalHeight)
537 return FALSE;
538
539 if ((nXDst + nWidth) * bpp > nDstStep)
540 return FALSE;
541
542 for (y = beg; y != end; y += inc)
543 {
544 BYTE* pRGB;
545
546 if (y > (INT64)nHeight)
547 return FALSE;
548
549 pRGB = &pDstData[((nYDst + y) * nDstStep) + (nXDst * bpp)];
550
551 if (!writeLine(&pRGB, DstFormat, nWidth, &pR, &pG, &pB, &pA))
552 return FALSE;
553 }
554
555 return TRUE;
556 }
557
planar_subsample_expand(const BYTE * plane,size_t planeLength,UINT32 nWidth,UINT32 nHeight,UINT32 nPlaneWidth,UINT32 nPlaneHeight,BYTE * deltaPlane)558 static BOOL planar_subsample_expand(const BYTE* plane, size_t planeLength, UINT32 nWidth,
559 UINT32 nHeight, UINT32 nPlaneWidth, UINT32 nPlaneHeight,
560 BYTE* deltaPlane)
561 {
562 size_t pos = 0;
563 UINT32 y;
564 WINPR_UNUSED(planeLength);
565
566 if (!plane || !deltaPlane)
567 return FALSE;
568
569 if (nWidth > nPlaneWidth * 2)
570 return FALSE;
571
572 if (nHeight > nPlaneHeight * 2)
573 return FALSE;
574
575 for (y = 0; y < nHeight; y++)
576 {
577 const BYTE* src = plane + y / 2 * nPlaneWidth;
578 UINT32 x;
579
580 for (x = 0; x < nWidth; x++)
581 {
582 deltaPlane[pos++] = src[x / 2];
583 }
584 }
585
586 return TRUE;
587 }
588
planar_decompress(BITMAP_PLANAR_CONTEXT * planar,const BYTE * pSrcData,UINT32 SrcSize,UINT32 nSrcWidth,UINT32 nSrcHeight,BYTE * pDstData,UINT32 DstFormat,UINT32 nDstStep,UINT32 nXDst,UINT32 nYDst,UINT32 nDstWidth,UINT32 nDstHeight,BOOL vFlip)589 BOOL planar_decompress(BITMAP_PLANAR_CONTEXT* planar, const BYTE* pSrcData, UINT32 SrcSize,
590 UINT32 nSrcWidth, UINT32 nSrcHeight, BYTE* pDstData, UINT32 DstFormat,
591 UINT32 nDstStep, UINT32 nXDst, UINT32 nYDst, UINT32 nDstWidth,
592 UINT32 nDstHeight, BOOL vFlip)
593 {
594 BOOL cs;
595 BOOL rle;
596 UINT32 cll;
597 BOOL alpha;
598 BOOL useAlpha = FALSE;
599 INT32 status;
600 const BYTE* srcp;
601 UINT32 subSize;
602 UINT32 subWidth;
603 UINT32 subHeight;
604 UINT32 planeSize;
605 INT32 rleSizes[4] = { 0, 0, 0, 0 };
606 UINT32 rawSizes[4];
607 UINT32 rawWidths[4];
608 UINT32 rawHeights[4];
609 BYTE FormatHeader;
610 const BYTE* planes[4] = { 0 };
611 const UINT32 w = MIN(nSrcWidth, nDstWidth);
612 const UINT32 h = MIN(nSrcHeight, nDstHeight);
613 const primitives_t* prims = primitives_get();
614
615 if (nDstStep <= 0)
616 nDstStep = nDstWidth * GetBytesPerPixel(DstFormat);
617
618 srcp = pSrcData;
619
620 if (!pDstData)
621 {
622 WLog_ERR(TAG, "Invalid argument pDstData=NULL");
623 return FALSE;
624 }
625
626 FormatHeader = *srcp++;
627 cll = (FormatHeader & PLANAR_FORMAT_HEADER_CLL_MASK);
628 cs = (FormatHeader & PLANAR_FORMAT_HEADER_CS) ? TRUE : FALSE;
629 rle = (FormatHeader & PLANAR_FORMAT_HEADER_RLE) ? TRUE : FALSE;
630 alpha = (FormatHeader & PLANAR_FORMAT_HEADER_NA) ? FALSE : TRUE;
631
632 DstFormat = planar_invert_format(planar, alpha, DstFormat);
633
634 if (alpha)
635 useAlpha = ColorHasAlpha(DstFormat);
636
637 // WLog_INFO(TAG, "CLL: %"PRIu32" CS: %"PRIu8" RLE: %"PRIu8" ALPHA: %"PRIu8"", cll, cs, rle,
638 // alpha);
639
640 if (!cll && cs)
641 {
642 WLog_ERR(TAG, "Chroma subsampling requires YCoCg and does not work with RGB data");
643 return FALSE; /* Chroma subsampling requires YCoCg */
644 }
645
646 subWidth = (nSrcWidth / 2) + (nSrcWidth % 2);
647 subHeight = (nSrcHeight / 2) + (nSrcHeight % 2);
648 planeSize = nSrcWidth * nSrcHeight;
649 subSize = subWidth * subHeight;
650
651 if (!cs)
652 {
653 rawSizes[0] = planeSize; /* LumaOrRedPlane */
654 rawWidths[0] = nSrcWidth;
655 rawHeights[0] = nSrcHeight;
656 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
657 rawWidths[1] = nSrcWidth;
658 rawHeights[1] = nSrcHeight;
659 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
660 rawWidths[2] = nSrcWidth;
661 rawHeights[2] = nSrcHeight;
662 rawSizes[3] = planeSize; /* AlphaPlane */
663 rawWidths[3] = nSrcWidth;
664 rawHeights[3] = nSrcHeight;
665 }
666 else /* Chroma Subsampling */
667 {
668 rawSizes[0] = planeSize; /* LumaOrRedPlane */
669 rawWidths[0] = nSrcWidth;
670 rawHeights[0] = nSrcHeight;
671 rawSizes[1] = subSize; /* OrangeChromaOrGreenPlane */
672 rawWidths[1] = subWidth;
673 rawHeights[1] = subHeight;
674 rawSizes[2] = subSize; /* GreenChromaOrBluePlane */
675 rawWidths[2] = subWidth;
676 rawHeights[2] = subHeight;
677 rawSizes[3] = planeSize; /* AlphaPlane */
678 rawWidths[3] = nSrcWidth;
679 rawHeights[3] = nSrcHeight;
680 }
681
682 if (!rle) /* RAW */
683 {
684 UINT32 base = planeSize * 3;
685 if (cs)
686 base = planeSize + planeSize / 2;
687
688 if (alpha)
689 {
690 if ((SrcSize - (srcp - pSrcData)) < (planeSize + base))
691 return FALSE;
692
693 planes[3] = srcp; /* AlphaPlane */
694 planes[0] = planes[3] + rawSizes[3]; /* LumaOrRedPlane */
695 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
696 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
697
698 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
699 return FALSE;
700 }
701 else
702 {
703 if ((SrcSize - (srcp - pSrcData)) < base)
704 return FALSE;
705
706 planes[0] = srcp; /* LumaOrRedPlane */
707 planes[1] = planes[0] + rawSizes[0]; /* OrangeChromaOrGreenPlane */
708 planes[2] = planes[1] + rawSizes[1]; /* GreenChromaOrBluePlane */
709
710 if ((planes[2] + rawSizes[2]) > &pSrcData[SrcSize])
711 return FALSE;
712 }
713 }
714 else /* RLE */
715 {
716 if (alpha)
717 {
718 planes[3] = srcp;
719 rleSizes[3] = planar_skip_plane_rle(planes[3], SrcSize - (planes[3] - pSrcData),
720 rawWidths[3], rawHeights[3]); /* AlphaPlane */
721
722 if (rleSizes[3] < 0)
723 return FALSE;
724
725 planes[0] = planes[3] + rleSizes[3];
726 }
727 else
728 planes[0] = srcp;
729
730 rleSizes[0] = planar_skip_plane_rle(planes[0], SrcSize - (planes[0] - pSrcData),
731 rawWidths[0], rawHeights[0]); /* RedPlane */
732
733 if (rleSizes[0] < 0)
734 return FALSE;
735
736 planes[1] = planes[0] + rleSizes[0];
737 rleSizes[1] = planar_skip_plane_rle(planes[1], SrcSize - (planes[1] - pSrcData),
738 rawWidths[1], rawHeights[1]); /* GreenPlane */
739
740 if (rleSizes[1] < 1)
741 return FALSE;
742
743 planes[2] = planes[1] + rleSizes[1];
744 rleSizes[2] = planar_skip_plane_rle(planes[2], SrcSize - (planes[2] - pSrcData),
745 rawWidths[2], rawHeights[2]); /* BluePlane */
746
747 if (rleSizes[2] < 1)
748 return FALSE;
749 }
750
751 if (!cll) /* RGB */
752 {
753 UINT32 TempFormat;
754 BYTE* pTempData = pDstData;
755 UINT32 nTempStep = nDstStep;
756 UINT32 nTotalHeight = nYDst + nDstHeight;
757
758 if (useAlpha)
759 TempFormat = PIXEL_FORMAT_BGRA32;
760 else
761 TempFormat = PIXEL_FORMAT_BGRX32;
762
763 if ((TempFormat != DstFormat) || (nSrcWidth != nDstWidth) || (nSrcHeight != nDstHeight))
764 {
765 pTempData = planar->pTempData;
766 nTempStep = planar->nTempStep;
767 nTotalHeight = planar->maxHeight;
768 }
769
770 if (!rle) /* RAW */
771 {
772 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
773 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
774 return FALSE;
775
776 if (alpha)
777 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
778 else /* NoAlpha */
779 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
780
781 if ((SrcSize - (srcp - pSrcData)) == 1)
782 srcp++; /* pad */
783 }
784 else /* RLE */
785 {
786 status =
787 planar_decompress_plane_rle(planes[0], rleSizes[0], pTempData, nTempStep, nXDst,
788 nYDst, nSrcWidth, nSrcHeight, 2, vFlip); /* RedPlane */
789
790 if (status < 0)
791 return FALSE;
792
793 status = planar_decompress_plane_rle(planes[1], rleSizes[1], pTempData, nTempStep,
794 nXDst, nYDst, nSrcWidth, nSrcHeight, 1,
795 vFlip); /* GreenPlane */
796
797 if (status < 0)
798 return FALSE;
799
800 status =
801 planar_decompress_plane_rle(planes[2], rleSizes[2], pTempData, nTempStep, nXDst,
802 nYDst, nSrcWidth, nSrcHeight, 0, vFlip); /* BluePlane */
803
804 if (status < 0)
805 return FALSE;
806
807 srcp += rleSizes[0] + rleSizes[1] + rleSizes[2];
808
809 if (useAlpha)
810 {
811 status = planar_decompress_plane_rle(planes[3], rleSizes[3], pTempData, nTempStep,
812 nXDst, nYDst, nSrcWidth, nSrcHeight, 3,
813 vFlip); /* AlphaPlane */
814 }
815 else
816 status = planar_set_plane(0xFF, pTempData, nTempStep, nXDst, nYDst, nSrcWidth,
817 nSrcHeight, 3, vFlip);
818
819 if (status < 0)
820 return FALSE;
821
822 if (alpha)
823 srcp += rleSizes[3];
824 }
825
826 if (pTempData != pDstData)
827 {
828 if (!freerdp_image_copy(pDstData, DstFormat, nDstStep, nXDst, nYDst, w, h, pTempData,
829 TempFormat, nTempStep, nXDst, nYDst, NULL, FREERDP_FLIP_NONE))
830 return FALSE;
831 }
832 }
833 else /* YCoCg */
834 {
835 UINT32 TempFormat;
836 BYTE* pTempData = planar->pTempData;
837 UINT32 nTempStep = planar->nTempStep;
838 UINT32 nTotalHeight = planar->maxHeight;
839
840 if (useAlpha)
841 TempFormat = PIXEL_FORMAT_BGRA32;
842 else
843 TempFormat = PIXEL_FORMAT_BGRX32;
844
845 if (!pTempData)
846 return FALSE;
847
848 if (rle) /* RLE encoded data. Decode and handle it like raw data. */
849 {
850 BYTE* rleBuffer[4] = { 0 };
851
852 rleBuffer[3] = planar->rlePlanesBuffer; /* AlphaPlane */
853 rleBuffer[0] = rleBuffer[3] + planeSize; /* LumaOrRedPlane */
854 rleBuffer[1] = rleBuffer[0] + planeSize; /* OrangeChromaOrGreenPlane */
855 rleBuffer[2] = rleBuffer[1] + planeSize; /* GreenChromaOrBluePlane */
856 if (useAlpha)
857 {
858 status =
859 planar_decompress_plane_rle_only(planes[3], rleSizes[3], rleBuffer[3],
860 rawWidths[3], rawHeights[3]); /* AlphaPlane */
861
862 if (status < 0)
863 return FALSE;
864 }
865
866 if (alpha)
867 srcp += rleSizes[3];
868
869 status = planar_decompress_plane_rle_only(planes[0], rleSizes[0], rleBuffer[0],
870 rawWidths[0], rawHeights[0]); /* LumaPlane */
871
872 if (status < 0)
873 return FALSE;
874
875 status =
876 planar_decompress_plane_rle_only(planes[1], rleSizes[1], rleBuffer[1], rawWidths[1],
877 rawHeights[1]); /* OrangeChromaPlane */
878
879 if (status < 0)
880 return FALSE;
881
882 status =
883 planar_decompress_plane_rle_only(planes[2], rleSizes[2], rleBuffer[2], rawWidths[2],
884 rawHeights[2]); /* GreenChromaPlane */
885
886 if (status < 0)
887 return FALSE;
888
889 planes[0] = rleBuffer[0];
890 planes[1] = rleBuffer[1];
891 planes[2] = rleBuffer[2];
892 planes[3] = rleBuffer[3];
893 }
894
895 /* RAW */
896 {
897 if (cs)
898 { /* Chroma subsampling for Co and Cg:
899 * Each pixel contains the value that should be expanded to
900 * [2x,2y;2x+1,2y;2x+1,2y+1;2x;2y+1] */
901 if (!planar_subsample_expand(planes[1], rawSizes[1], nSrcWidth, nSrcHeight,
902 rawWidths[1], rawHeights[1], planar->deltaPlanes[0]))
903 return FALSE;
904
905 planes[1] = planar->deltaPlanes[0];
906 rawSizes[1] = planeSize; /* OrangeChromaOrGreenPlane */
907 rawWidths[1] = nSrcWidth;
908 rawHeights[1] = nSrcHeight;
909
910 if (!planar_subsample_expand(planes[2], rawSizes[2], nSrcWidth, nSrcHeight,
911 rawWidths[2], rawHeights[2], planar->deltaPlanes[1]))
912 return FALSE;
913
914 planes[2] = planar->deltaPlanes[1];
915 rawSizes[2] = planeSize; /* GreenChromaOrBluePlane */
916 rawWidths[2] = nSrcWidth;
917 rawHeights[2] = nSrcHeight;
918 }
919
920 if (!planar_decompress_planes_raw(planes, pTempData, TempFormat, nTempStep, nXDst,
921 nYDst, nSrcWidth, nSrcHeight, vFlip, nTotalHeight))
922 return FALSE;
923
924 if (alpha)
925 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2] + rawSizes[3];
926 else /* NoAlpha */
927 srcp += rawSizes[0] + rawSizes[1] + rawSizes[2];
928
929 if ((SrcSize - (srcp - pSrcData)) == 1)
930 srcp++; /* pad */
931 }
932
933 if (prims->YCoCgToRGB_8u_AC4R(pTempData, nTempStep, pDstData, DstFormat, nDstStep, w, h,
934 cll, useAlpha) != PRIMITIVES_SUCCESS)
935 return FALSE;
936 }
937
938 return TRUE;
939 }
940
freerdp_split_color_planes(const BYTE * data,UINT32 format,UINT32 width,UINT32 height,UINT32 scanline,BYTE * planes[4])941 static INLINE BOOL freerdp_split_color_planes(const BYTE* data, UINT32 format, UINT32 width,
942 UINT32 height, UINT32 scanline, BYTE* planes[4])
943 {
944 INT32 i, j, k;
945 if ((width > INT32_MAX) || (height > INT32_MAX) || (scanline > INT32_MAX))
946 return FALSE;
947
948 k = 0;
949
950 if (scanline == 0)
951 scanline = width * GetBytesPerPixel(format);
952
953 for (i = (INT32)height - 1; i >= 0; i--)
954 {
955 const BYTE* pixel = &data[(INT32)scanline * i];
956
957 for (j = 0; j < (INT32)width; j++)
958 {
959 const UINT32 color = ReadColor(pixel, format);
960 pixel += GetBytesPerPixel(format);
961 SplitColor(color, format, &planes[1][k], &planes[2][k], &planes[3][k], &planes[0][k],
962 NULL);
963 k++;
964 }
965 }
966
967 return TRUE;
968 }
969
freerdp_bitmap_planar_write_rle_bytes(const BYTE * pInBuffer,UINT32 cRawBytes,UINT32 nRunLength,BYTE * pOutBuffer,UINT32 outBufferSize)970 static INLINE UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* pInBuffer, UINT32 cRawBytes,
971 UINT32 nRunLength, BYTE* pOutBuffer,
972 UINT32 outBufferSize)
973 {
974 const BYTE* pInput;
975 BYTE* pOutput;
976 BYTE controlByte;
977 UINT32 nBytesToWrite;
978 pInput = pInBuffer;
979 pOutput = pOutBuffer;
980
981 if (!cRawBytes && !nRunLength)
982 return 0;
983
984 if (nRunLength < 3)
985 {
986 cRawBytes += nRunLength;
987 nRunLength = 0;
988 }
989
990 while (cRawBytes)
991 {
992 if (cRawBytes < 16)
993 {
994 if (nRunLength > 15)
995 {
996 if (nRunLength < 18)
997 {
998 controlByte = PLANAR_CONTROL_BYTE(13, cRawBytes);
999 nRunLength -= 13;
1000 cRawBytes = 0;
1001 }
1002 else
1003 {
1004 controlByte = PLANAR_CONTROL_BYTE(15, cRawBytes);
1005 nRunLength -= 15;
1006 cRawBytes = 0;
1007 }
1008 }
1009 else
1010 {
1011 controlByte = PLANAR_CONTROL_BYTE(nRunLength, cRawBytes);
1012 nRunLength = 0;
1013 cRawBytes = 0;
1014 }
1015 }
1016 else
1017 {
1018 controlByte = PLANAR_CONTROL_BYTE(0, 15);
1019 cRawBytes -= 15;
1020 }
1021
1022 if (outBufferSize < 1)
1023 return 0;
1024
1025 outBufferSize--;
1026 *pOutput = controlByte;
1027 pOutput++;
1028 nBytesToWrite = (int)(controlByte >> 4);
1029
1030 if (nBytesToWrite)
1031 {
1032 if (outBufferSize < nBytesToWrite)
1033 return 0;
1034
1035 outBufferSize -= nBytesToWrite;
1036 CopyMemory(pOutput, pInput, nBytesToWrite);
1037 pOutput += nBytesToWrite;
1038 pInput += nBytesToWrite;
1039 }
1040 }
1041
1042 while (nRunLength)
1043 {
1044 if (nRunLength > 47)
1045 {
1046 if (nRunLength < 50)
1047 {
1048 controlByte = PLANAR_CONTROL_BYTE(2, 13);
1049 nRunLength -= 45;
1050 }
1051 else
1052 {
1053 controlByte = PLANAR_CONTROL_BYTE(2, 15);
1054 nRunLength -= 47;
1055 }
1056 }
1057 else if (nRunLength > 31)
1058 {
1059 controlByte = PLANAR_CONTROL_BYTE(2, (nRunLength - 32));
1060 nRunLength = 0;
1061 }
1062 else if (nRunLength > 15)
1063 {
1064 controlByte = PLANAR_CONTROL_BYTE(1, (nRunLength - 16));
1065 nRunLength = 0;
1066 }
1067 else
1068 {
1069 controlByte = PLANAR_CONTROL_BYTE(nRunLength, 0);
1070 nRunLength = 0;
1071 }
1072
1073 if (outBufferSize < 1)
1074 return 0;
1075
1076 --outBufferSize;
1077 *pOutput = controlByte;
1078 pOutput++;
1079 }
1080
1081 return (pOutput - pOutBuffer);
1082 }
1083
freerdp_bitmap_planar_encode_rle_bytes(const BYTE * pInBuffer,UINT32 inBufferSize,BYTE * pOutBuffer,UINT32 outBufferSize)1084 static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* pInBuffer,
1085 UINT32 inBufferSize, BYTE* pOutBuffer,
1086 UINT32 outBufferSize)
1087 {
1088 BYTE symbol;
1089 const BYTE* pInput;
1090 BYTE* pOutput;
1091 const BYTE* pBytes;
1092 UINT32 cRawBytes;
1093 UINT32 nRunLength;
1094 UINT32 bSymbolMatch;
1095 UINT32 nBytesWritten;
1096 UINT32 nTotalBytesWritten;
1097 symbol = 0;
1098 cRawBytes = 0;
1099 nRunLength = 0;
1100 pInput = pInBuffer;
1101 pOutput = pOutBuffer;
1102 nTotalBytesWritten = 0;
1103
1104 if (!outBufferSize)
1105 return 0;
1106
1107 do
1108 {
1109 if (!inBufferSize)
1110 break;
1111
1112 bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
1113 symbol = *pInput;
1114 pInput++;
1115 inBufferSize--;
1116
1117 if (nRunLength && !bSymbolMatch)
1118 {
1119 if (nRunLength < 3)
1120 {
1121 cRawBytes += nRunLength;
1122 nRunLength = 0;
1123 }
1124 else
1125 {
1126 pBytes = pInput - (cRawBytes + nRunLength + 1);
1127 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1128 pOutput, outBufferSize);
1129 nRunLength = 0;
1130
1131 if (!nBytesWritten || (nBytesWritten > outBufferSize))
1132 return nRunLength;
1133
1134 nTotalBytesWritten += nBytesWritten;
1135 outBufferSize -= nBytesWritten;
1136 pOutput += nBytesWritten;
1137 cRawBytes = 0;
1138 }
1139 }
1140
1141 nRunLength += bSymbolMatch;
1142 cRawBytes += (!bSymbolMatch) ? TRUE : FALSE;
1143 } while (outBufferSize);
1144
1145 if (cRawBytes || nRunLength)
1146 {
1147 pBytes = pInput - (cRawBytes + nRunLength);
1148 nBytesWritten = freerdp_bitmap_planar_write_rle_bytes(pBytes, cRawBytes, nRunLength,
1149 pOutput, outBufferSize);
1150
1151 if (!nBytesWritten)
1152 return 0;
1153
1154 nTotalBytesWritten += nBytesWritten;
1155 }
1156
1157 if (inBufferSize)
1158 return 0;
1159
1160 return nTotalBytesWritten;
1161 }
1162
freerdp_bitmap_planar_compress_plane_rle(const BYTE * inPlane,UINT32 width,UINT32 height,BYTE * outPlane,UINT32 * dstSize)1163 BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* inPlane, UINT32 width, UINT32 height,
1164 BYTE* outPlane, UINT32* dstSize)
1165 {
1166 UINT32 index;
1167 const BYTE* pInput;
1168 BYTE* pOutput;
1169 UINT32 outBufferSize;
1170 UINT32 nBytesWritten;
1171 UINT32 nTotalBytesWritten;
1172
1173 if (!outPlane)
1174 return FALSE;
1175
1176 index = 0;
1177 pInput = inPlane;
1178 pOutput = outPlane;
1179 outBufferSize = *dstSize;
1180 nTotalBytesWritten = 0;
1181
1182 while (outBufferSize)
1183 {
1184 nBytesWritten =
1185 freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
1186
1187 if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
1188 return FALSE;
1189
1190 outBufferSize -= nBytesWritten;
1191 nTotalBytesWritten += nBytesWritten;
1192 pOutput += nBytesWritten;
1193 pInput += width;
1194 index++;
1195
1196 if (index >= height)
1197 break;
1198 }
1199
1200 *dstSize = nTotalBytesWritten;
1201 return TRUE;
1202 }
1203
freerdp_bitmap_planar_compress_planes_rle(BYTE * inPlanes[4],UINT32 width,UINT32 height,BYTE * outPlanes,UINT32 * dstSizes,BOOL skipAlpha)1204 static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* inPlanes[4], UINT32 width,
1205 UINT32 height, BYTE* outPlanes,
1206 UINT32* dstSizes, BOOL skipAlpha)
1207 {
1208 UINT32 outPlanesSize = width * height * 4;
1209
1210 /* AlphaPlane */
1211 if (skipAlpha)
1212 {
1213 dstSizes[0] = 0;
1214 }
1215 else
1216 {
1217 dstSizes[0] = outPlanesSize;
1218
1219 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[0], width, height, outPlanes,
1220 &dstSizes[0]))
1221 return FALSE;
1222
1223 outPlanes += dstSizes[0];
1224 outPlanesSize -= dstSizes[0];
1225 }
1226
1227 /* LumaOrRedPlane */
1228 dstSizes[1] = outPlanesSize;
1229
1230 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[1], width, height, outPlanes,
1231 &dstSizes[1]))
1232 return FALSE;
1233
1234 outPlanes += dstSizes[1];
1235 outPlanesSize -= dstSizes[1];
1236 /* OrangeChromaOrGreenPlane */
1237 dstSizes[2] = outPlanesSize;
1238
1239 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[2], width, height, outPlanes,
1240 &dstSizes[2]))
1241 return FALSE;
1242
1243 outPlanes += dstSizes[2];
1244 outPlanesSize -= dstSizes[2];
1245 /* GreenChromeOrBluePlane */
1246 dstSizes[3] = outPlanesSize;
1247
1248 if (!freerdp_bitmap_planar_compress_plane_rle(inPlanes[3], width, height, outPlanes,
1249 &dstSizes[3]))
1250 return FALSE;
1251
1252 return TRUE;
1253 }
1254
freerdp_bitmap_planar_delta_encode_plane(const BYTE * inPlane,UINT32 width,UINT32 height,BYTE * outPlane)1255 BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* inPlane, UINT32 width, UINT32 height,
1256 BYTE* outPlane)
1257 {
1258 char s2c;
1259 UINT32 y, x;
1260 BYTE* outPtr;
1261 const BYTE *srcPtr, *prevLinePtr;
1262
1263 if (!outPlane)
1264 {
1265 if (width * height == 0)
1266 return NULL;
1267
1268 if (!(outPlane = (BYTE*)calloc(height, width)))
1269 return NULL;
1270 }
1271
1272 // first line is copied as is
1273 CopyMemory(outPlane, inPlane, width);
1274 outPtr = outPlane + width;
1275 srcPtr = inPlane + width;
1276 prevLinePtr = inPlane;
1277
1278 for (y = 1; y < height; y++)
1279 {
1280 for (x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++)
1281 {
1282 INT32 delta = *srcPtr - *prevLinePtr;
1283 s2c = (delta >= 0) ? (char)delta : (char)(~((BYTE)(-delta)) + 1);
1284 s2c = (s2c >= 0) ? (char)((UINT32)s2c << 1)
1285 : (char)(((UINT32)(~((BYTE)s2c) + 1) << 1) - 1);
1286 *outPtr = (BYTE)s2c;
1287 }
1288 }
1289
1290 return outPlane;
1291 }
1292
freerdp_bitmap_planar_delta_encode_planes(BYTE * inPlanes[4],UINT32 width,UINT32 height,BYTE * outPlanes[4])1293 static INLINE BOOL freerdp_bitmap_planar_delta_encode_planes(BYTE* inPlanes[4], UINT32 width,
1294 UINT32 height, BYTE* outPlanes[4])
1295 {
1296 UINT32 i;
1297
1298 for (i = 0; i < 4; i++)
1299 {
1300 outPlanes[i] =
1301 freerdp_bitmap_planar_delta_encode_plane(inPlanes[i], width, height, outPlanes[i]);
1302
1303 if (!outPlanes[i])
1304 return FALSE;
1305 }
1306
1307 return TRUE;
1308 }
1309
freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT * context,const BYTE * data,UINT32 format,UINT32 width,UINT32 height,UINT32 scanline,BYTE * dstData,UINT32 * pDstSize)1310 BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* context, const BYTE* data,
1311 UINT32 format, UINT32 width, UINT32 height, UINT32 scanline,
1312 BYTE* dstData, UINT32* pDstSize)
1313 {
1314 UINT32 size;
1315 BYTE* dstp;
1316 UINT32 planeSize;
1317 UINT32 dstSizes[4] = { 0 };
1318 BYTE FormatHeader = 0;
1319
1320 if (!context || !context->rlePlanesBuffer)
1321 return NULL;
1322
1323 if (context->AllowSkipAlpha)
1324 FormatHeader |= PLANAR_FORMAT_HEADER_NA;
1325
1326 planeSize = width * height;
1327
1328 if (!context->AllowSkipAlpha)
1329 format = planar_invert_format(context, TRUE, format);
1330
1331 if (!freerdp_split_color_planes(data, format, width, height, scanline, context->planes))
1332 return NULL;
1333
1334 if (context->AllowRunLengthEncoding)
1335 {
1336 if (!freerdp_bitmap_planar_delta_encode_planes(context->planes, width, height,
1337 context->deltaPlanes))
1338 return NULL;
1339
1340 if (!freerdp_bitmap_planar_compress_planes_rle(context->deltaPlanes, width, height,
1341 context->rlePlanesBuffer, dstSizes,
1342 context->AllowSkipAlpha))
1343 return NULL;
1344
1345 {
1346 int offset = 0;
1347 FormatHeader |= PLANAR_FORMAT_HEADER_RLE;
1348 context->rlePlanes[0] = &context->rlePlanesBuffer[offset];
1349 offset += dstSizes[0];
1350 context->rlePlanes[1] = &context->rlePlanesBuffer[offset];
1351 offset += dstSizes[1];
1352 context->rlePlanes[2] = &context->rlePlanesBuffer[offset];
1353 offset += dstSizes[2];
1354 context->rlePlanes[3] = &context->rlePlanesBuffer[offset];
1355 // WLog_DBG(TAG, "R: [%"PRIu32"/%"PRIu32"] G: [%"PRIu32"/%"PRIu32"] B:
1356 // [%"PRIu32"/%"PRIu32"]", dstSizes[1], planeSize, dstSizes[2], planeSize,
1357 // dstSizes[3],
1358 // planeSize);
1359 }
1360 }
1361
1362 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1363 {
1364 if (!context->AllowRunLengthEncoding)
1365 return NULL;
1366
1367 if (context->rlePlanes[0] == NULL)
1368 return NULL;
1369
1370 if (context->rlePlanes[1] == NULL)
1371 return NULL;
1372
1373 if (context->rlePlanes[2] == NULL)
1374 return NULL;
1375
1376 if (context->rlePlanes[3] == NULL)
1377 return NULL;
1378 }
1379
1380 if (!dstData)
1381 {
1382 size = 1;
1383
1384 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1385 {
1386 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1387 size += dstSizes[0];
1388 else
1389 size += planeSize;
1390 }
1391
1392 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1393 size += (dstSizes[1] + dstSizes[2] + dstSizes[3]);
1394 else
1395 size += (planeSize * 3);
1396
1397 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1398 size++;
1399
1400 dstData = malloc(size);
1401
1402 if (!dstData)
1403 return NULL;
1404
1405 *pDstSize = size;
1406 }
1407
1408 dstp = dstData;
1409 *dstp = FormatHeader; /* FormatHeader */
1410 dstp++;
1411
1412 /* AlphaPlane */
1413
1414 if (!(FormatHeader & PLANAR_FORMAT_HEADER_NA))
1415 {
1416 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1417 {
1418 CopyMemory(dstp, context->rlePlanes[0], dstSizes[0]); /* Alpha */
1419 dstp += dstSizes[0];
1420 }
1421 else
1422 {
1423 CopyMemory(dstp, context->planes[0], planeSize); /* Alpha */
1424 dstp += planeSize;
1425 }
1426 }
1427
1428 /* LumaOrRedPlane */
1429
1430 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1431 {
1432 CopyMemory(dstp, context->rlePlanes[1], dstSizes[1]); /* Red */
1433 dstp += dstSizes[1];
1434 }
1435 else
1436 {
1437 CopyMemory(dstp, context->planes[1], planeSize); /* Red */
1438 dstp += planeSize;
1439 }
1440
1441 /* OrangeChromaOrGreenPlane */
1442
1443 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1444 {
1445 CopyMemory(dstp, context->rlePlanes[2], dstSizes[2]); /* Green */
1446 dstp += dstSizes[2];
1447 }
1448 else
1449 {
1450 CopyMemory(dstp, context->planes[2], planeSize); /* Green */
1451 dstp += planeSize;
1452 }
1453
1454 /* GreenChromeOrBluePlane */
1455
1456 if (FormatHeader & PLANAR_FORMAT_HEADER_RLE)
1457 {
1458 CopyMemory(dstp, context->rlePlanes[3], dstSizes[3]); /* Blue */
1459 dstp += dstSizes[3];
1460 }
1461 else
1462 {
1463 CopyMemory(dstp, context->planes[3], planeSize); /* Blue */
1464 dstp += planeSize;
1465 }
1466
1467 /* Pad1 (1 byte) */
1468
1469 if (!(FormatHeader & PLANAR_FORMAT_HEADER_RLE))
1470 {
1471 *dstp = 0;
1472 dstp++;
1473 }
1474
1475 size = (dstp - dstData);
1476 *pDstSize = size;
1477 return dstData;
1478 }
1479
freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT * context,UINT32 width,UINT32 height)1480 BOOL freerdp_bitmap_planar_context_reset(BITMAP_PLANAR_CONTEXT* context, UINT32 width,
1481 UINT32 height)
1482 {
1483 if (!context)
1484 return FALSE;
1485
1486 context->bgr = FALSE;
1487 context->maxWidth = ALIGN(width, 4);
1488 context->maxHeight = ALIGN(height, 4);
1489 context->maxPlaneSize = context->maxWidth * context->maxHeight;
1490 context->nTempStep = context->maxWidth * 4;
1491 free(context->planesBuffer);
1492 free(context->pTempData);
1493 free(context->deltaPlanesBuffer);
1494 free(context->rlePlanesBuffer);
1495 context->planesBuffer = calloc(context->maxPlaneSize, 4);
1496 context->pTempData = calloc(context->maxPlaneSize, 6);
1497 context->deltaPlanesBuffer = calloc(context->maxPlaneSize, 4);
1498 context->rlePlanesBuffer = calloc(context->maxPlaneSize, 4);
1499
1500 if (!context->planesBuffer || !context->pTempData || !context->deltaPlanesBuffer ||
1501 !context->rlePlanesBuffer)
1502 return FALSE;
1503
1504 context->planes[0] = &context->planesBuffer[context->maxPlaneSize * 0];
1505 context->planes[1] = &context->planesBuffer[context->maxPlaneSize * 1];
1506 context->planes[2] = &context->planesBuffer[context->maxPlaneSize * 2];
1507 context->planes[3] = &context->planesBuffer[context->maxPlaneSize * 3];
1508 context->deltaPlanes[0] = &context->deltaPlanesBuffer[context->maxPlaneSize * 0];
1509 context->deltaPlanes[1] = &context->deltaPlanesBuffer[context->maxPlaneSize * 1];
1510 context->deltaPlanes[2] = &context->deltaPlanesBuffer[context->maxPlaneSize * 2];
1511 context->deltaPlanes[3] = &context->deltaPlanesBuffer[context->maxPlaneSize * 3];
1512 return TRUE;
1513 }
1514
freerdp_bitmap_planar_context_new(DWORD flags,UINT32 maxWidth,UINT32 maxHeight)1515 BITMAP_PLANAR_CONTEXT* freerdp_bitmap_planar_context_new(DWORD flags, UINT32 maxWidth,
1516 UINT32 maxHeight)
1517 {
1518 BITMAP_PLANAR_CONTEXT* context;
1519 context = (BITMAP_PLANAR_CONTEXT*)calloc(1, sizeof(BITMAP_PLANAR_CONTEXT));
1520
1521 if (!context)
1522 return NULL;
1523
1524 if (flags & PLANAR_FORMAT_HEADER_NA)
1525 context->AllowSkipAlpha = TRUE;
1526
1527 if (flags & PLANAR_FORMAT_HEADER_RLE)
1528 context->AllowRunLengthEncoding = TRUE;
1529
1530 if (flags & PLANAR_FORMAT_HEADER_CS)
1531 context->AllowColorSubsampling = TRUE;
1532
1533 context->ColorLossLevel = flags & PLANAR_FORMAT_HEADER_CLL_MASK;
1534
1535 if (context->ColorLossLevel)
1536 context->AllowDynamicColorFidelity = TRUE;
1537
1538 if (!freerdp_bitmap_planar_context_reset(context, maxWidth, maxHeight))
1539 {
1540 freerdp_bitmap_planar_context_free(context);
1541 return NULL;
1542 }
1543
1544 return context;
1545 }
1546
freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT * context)1547 void freerdp_bitmap_planar_context_free(BITMAP_PLANAR_CONTEXT* context)
1548 {
1549 if (!context)
1550 return;
1551
1552 free(context->pTempData);
1553 free(context->planesBuffer);
1554 free(context->deltaPlanesBuffer);
1555 free(context->rlePlanesBuffer);
1556 free(context);
1557 }
1558
freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT * planar,BOOL bgr)1559 void freerdp_planar_switch_bgr(BITMAP_PLANAR_CONTEXT* planar, BOOL bgr)
1560 {
1561 planar->bgr = bgr;
1562 }
1563