1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
|
From: akallabeth <akallabeth@posteo.net>
Date: Mon, 22 Sep 2025 12:15:33 +0200
Subject: [codec,planar] fix datatypes in plane encode
Forwarded: not-needed
Origin: upstream, https://github.com/FreeRDP/FreeRDP/commit/3ac6c06e17af995f4cb7c65c9cda4d5400d93a15
do not assume char is signed, use int8_t instead.
---
libfreerdp/codec/planar.c | 85 +++++++++++++--------------------------
1 file changed, 29 insertions(+), 56 deletions(-)
diff --git a/libfreerdp/codec/planar.c b/libfreerdp/codec/planar.c
index 2b0ad4ca9..9d4cf078c 100644
--- a/libfreerdp/codec/planar.c
+++ b/libfreerdp/codec/planar.c
@@ -113,7 +113,7 @@ static inline BYTE PLANAR_CONTROL_BYTE_RAW_BYTES(UINT32 controlByte)
static INLINE UINT32 planar_invert_format(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT planar, BOOL alpha,
UINT32 DstFormat)
{
-
+ WINPR_ASSERT(planar);
if (planar->bgr && alpha)
{
switch (DstFormat)
@@ -375,8 +375,6 @@ static INLINE INT32 planar_decompress_plane_rle(const BYTE* WINPR_RESTRICT pSrcD
WINPR_ASSERT(nWidth <= INT32_MAX);
WINPR_ASSERT(nDstStep <= INT32_MAX);
- previousScanline = NULL;
-
if (vFlip)
{
beg = (INT32)nHeight - 1;
@@ -500,8 +498,8 @@ static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstSte
BOOL vFlip)
{
INT32 beg = 0;
- INT32 end = 0;
- INT32 inc = 0;
+ INT32 end = (INT32)nHeight;
+ INT32 inc = 1;
WINPR_ASSERT(nHeight <= INT32_MAX);
WINPR_ASSERT(nWidth <= INT32_MAX);
@@ -513,12 +511,6 @@ static INLINE INT32 planar_set_plane(BYTE bValue, BYTE* pDstData, UINT32 nDstSte
end = -1;
inc = -1;
}
- else
- {
- beg = 0;
- end = (INT32)nHeight;
- inc = 1;
- }
for (INT32 y = beg; y != end; y += inc)
{
@@ -1183,12 +1175,10 @@ static INLINE UINT32 freerdp_bitmap_planar_write_rle_bytes(const BYTE* WINPR_RES
BYTE* WINPR_RESTRICT pOutBuffer,
UINT32 outBufferSize)
{
- const BYTE* pInput = NULL;
- BYTE* pOutput = NULL;
+ const BYTE* pInput = pInBuffer;
+ BYTE* pOutput = pOutBuffer;
BYTE controlByte = 0;
UINT32 nBytesToWrite = 0;
- pInput = pInBuffer;
- pOutput = pOutBuffer;
if (!cRawBytes && !nRunLength)
return 0;
@@ -1302,20 +1292,13 @@ static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RE
UINT32 outBufferSize)
{
BYTE symbol = 0;
- const BYTE* pInput = NULL;
- BYTE* pOutput = NULL;
+ const BYTE* pInput = pInBuffer;
+ BYTE* pOutput = pOutBuffer;
const BYTE* pBytes = NULL;
UINT32 cRawBytes = 0;
UINT32 nRunLength = 0;
- UINT32 bSymbolMatch = 0;
UINT32 nBytesWritten = 0;
UINT32 nTotalBytesWritten = 0;
- symbol = 0;
- cRawBytes = 0;
- nRunLength = 0;
- pInput = pInBuffer;
- pOutput = pOutBuffer;
- nTotalBytesWritten = 0;
if (!outBufferSize)
return 0;
@@ -1325,7 +1308,7 @@ static INLINE UINT32 freerdp_bitmap_planar_encode_rle_bytes(const BYTE* WINPR_RE
if (!inBufferSize)
break;
- bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
+ const UINT32 bSymbolMatch = (symbol == *pInput) ? TRUE : FALSE;
symbol = *pInput;
pInput++;
inBufferSize--;
@@ -1380,25 +1363,18 @@ BOOL freerdp_bitmap_planar_compress_plane_rle(const BYTE* WINPR_RESTRICT inPlane
UINT32 height, BYTE* WINPR_RESTRICT outPlane,
UINT32* WINPR_RESTRICT dstSize)
{
- UINT32 index = 0;
- const BYTE* pInput = NULL;
- BYTE* pOutput = NULL;
- UINT32 outBufferSize = 0;
- UINT32 nBytesWritten = 0;
- UINT32 nTotalBytesWritten = 0;
-
if (!outPlane)
return FALSE;
- index = 0;
- pInput = inPlane;
- pOutput = outPlane;
- outBufferSize = *dstSize;
- nTotalBytesWritten = 0;
+ UINT32 index = 0;
+ const BYTE* pInput = inPlane;
+ BYTE* pOutput = outPlane;
+ UINT32 outBufferSize = *dstSize;
+ UINT32 nTotalBytesWritten = 0;
- while (outBufferSize)
+ while (outBufferSize > 0)
{
- nBytesWritten =
+ const UINT32 nBytesWritten =
freerdp_bitmap_planar_encode_rle_bytes(pInput, width, pOutput, outBufferSize);
if ((!nBytesWritten) || (nBytesWritten > outBufferSize))
@@ -1474,35 +1450,33 @@ static INLINE BOOL freerdp_bitmap_planar_compress_planes_rle(BYTE* WINPR_RESTRIC
BYTE* freerdp_bitmap_planar_delta_encode_plane(const BYTE* WINPR_RESTRICT inPlane, UINT32 width,
UINT32 height, BYTE* WINPR_RESTRICT outPlane)
{
- BYTE* outPtr = NULL;
- const BYTE* srcPtr = NULL;
- const BYTE* prevLinePtr = NULL;
-
if (!outPlane)
{
if (width * height == 0)
return NULL;
- if (!(outPlane = (BYTE*)calloc(height, width)))
+ outPlane = (BYTE*)calloc(height, width);
+ if (!outPlane)
return NULL;
}
// first line is copied as is
CopyMemory(outPlane, inPlane, width);
- outPtr = outPlane + width;
- srcPtr = inPlane + width;
- prevLinePtr = inPlane;
for (UINT32 y = 1; y < height; y++)
{
- for (UINT32 x = 0; x < width; x++, outPtr++, srcPtr++, prevLinePtr++)
+ const size_t off = 1ull * width * y;
+ BYTE* outPtr = &outPlane[off];
+ const BYTE* srcPtr = &inPlane[off];
+ const BYTE* prevLinePtr = &inPlane[off - width];
+ for (UINT32 x = 0; x < width; x++)
{
- const INT32 delta = *srcPtr - *prevLinePtr;
- const int s2c1i = (delta >= 0) ? delta : (~((BYTE)(-delta)) + 1);
- const char s2c1 = WINPR_CXX_COMPAT_CAST(char, s2c1i & 0xff);
- const uint32_t s2c = (s2c1 >= 0) ? ((UINT32)s2c1 << 1)
- : (((UINT32)(~((BYTE)(s2c1 & 0xFF)) + 1) << 1) - 1);
- *outPtr = (BYTE)s2c;
+ const int delta = (int)srcPtr[x] - (int)prevLinePtr[x];
+ const int s2c1i = (delta >= 0) ? delta : (INT_MAX + delta) + 1;
+ const int8_t s2c1 = WINPR_CXX_COMPAT_CAST(int8_t, s2c1i);
+ const uint32_t s2c =
+ (s2c1 >= 0) ? ((UINT32)s2c1 << 1) : (((UINT32)(~(s2c1) + 1) << 1) - 1);
+ outPtr[x] = (BYTE)s2c;
}
}
@@ -1532,7 +1506,6 @@ BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT conte
{
UINT32 size = 0;
BYTE* dstp = NULL;
- UINT32 planeSize = 0;
UINT32 dstSizes[4] = { 0 };
BYTE FormatHeader = 0;
@@ -1542,7 +1515,7 @@ BYTE* freerdp_bitmap_compress_planar(BITMAP_PLANAR_CONTEXT* WINPR_RESTRICT conte
if (context->AllowSkipAlpha)
FormatHeader |= PLANAR_FORMAT_HEADER_NA;
- planeSize = width * height;
+ const UINT32 planeSize = width * height;
if (!context->AllowSkipAlpha)
format = planar_invert_format(context, TRUE, format);
--
2.47.3
|
