You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
xrdp-proprietary/libxrdp/xrdp_bitmap32_compress.c

520 lines
15 KiB

/**
* xrdp: A Remote Desktop Protocol server.
*
* Copyright (C) Jay Sorg 2014
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* planar bitmap compressor
* 32 bpp compression
*/
/*
RDP 6.0 Bitmap Compression
http://msdn.microsoft.com/en-us/library/cc241877.aspx
*/
#include "libxrdp.h"
#define FLAGS_RLE 0x10
#define FLAGS_NOALPHA 0x20
#define LLOG_LEVEL 1
#define LLOGLN(_level, _args) \
do { if (_level < LLOG_LEVEL) { g_writeln _args ; } } while (0)
#define LHEXDUMP(_level, _args) \
do { if (_level < LLOG_LEVEL) { g_hexdump _args ; } } while (0)
/*****************************************************************************/
/* split ARGB */
static int APP_CC
fsplit4(char *in_data, int start_line, int width, int e,
char *alpha_data, char *red_data, char *green_data, char *blue_data)
{
#if defined(L_ENDIAN)
int alpha;
int red;
int green;
int blue;
#endif
int index;
int out_index;
int pixel;
int cy;
int *ptr32;
cy = 0;
out_index = 0;
while (start_line >= 0)
{
ptr32 = (int *) (in_data + start_line * width * 4);
index = 0;
#if defined(L_ENDIAN)
while (index + 4 <= width)
{
pixel = *ptr32;
ptr32++;
alpha = (pixel >> 24) & 0x000000ff;
red = (pixel >> 16) & 0x000000ff;
green = (pixel >> 8) & 0x000000ff;
blue = (pixel >> 0) & 0x000000ff;
pixel = *ptr32;
ptr32++;
alpha |= (pixel >> 16) & 0x0000ff00;
red |= (pixel >> 8) & 0x0000ff00;
green |= (pixel << 0) & 0x0000ff00;
blue |= (pixel << 8) & 0x0000ff00;
pixel = *ptr32;
ptr32++;
alpha |= (pixel >> 8) & 0x00ff0000;
red |= (pixel >> 0) & 0x00ff0000;
green |= (pixel << 8) & 0x00ff0000;
blue |= (pixel << 16) & 0x00ff0000;
pixel = *ptr32;
ptr32++;
alpha |= (pixel << 0) & 0xff000000;
red |= (pixel << 8) & 0xff000000;
green |= (pixel << 16) & 0xff000000;
blue |= (pixel << 24) & 0xff000000;
*((int*)(alpha_data + out_index)) = alpha;
*((int*)(red_data + out_index)) = red;
*((int*)(green_data + out_index)) = green;
*((int*)(blue_data + out_index)) = blue;
out_index += 4;
index += 4;
}
#endif
while (index < width)
{
pixel = *ptr32;
ptr32++;
alpha_data[out_index] = pixel >> 24;
red_data[out_index] = pixel >> 16;
green_data[out_index] = pixel >> 8;
blue_data[out_index] = pixel >> 0;
out_index++;
index++;
}
for (index = 0; index < e; index++)
{
alpha_data[out_index] = alpha_data[out_index - 1];
red_data[out_index] = red_data[out_index - 1];
green_data[out_index] = green_data[out_index - 1];
blue_data[out_index] = blue_data[out_index - 1];
out_index++;
}
start_line--;
cy++;
}
return cy;
}
/*****************************************************************************/
/* split RGB */
static int APP_CC
fsplit3(char *in_data, int start_line, int width, int e,
char *red_data, char *green_data, char *blue_data)
{
#if defined(L_ENDIAN)
int red;
int green;
int blue;
#endif
int index;
int out_index;
int pixel;
int cy;
int *ptr32;
cy = 0;
out_index = 0;
while (start_line >= 0)
{
ptr32 = (int *) (in_data + start_line * width * 4);
index = 0;
#if defined(L_ENDIAN)
while (index + 4 <= width)
{
pixel = *ptr32;
ptr32++;
red = (pixel >> 16) & 0x000000ff;
green = (pixel >> 8) & 0x000000ff;
blue = (pixel >> 0) & 0x000000ff;
pixel = *ptr32;
ptr32++;
red |= (pixel >> 8) & 0x0000ff00;
green |= (pixel << 0) & 0x0000ff00;
blue |= (pixel << 8) & 0x0000ff00;
pixel = *ptr32;
ptr32++;
red |= (pixel >> 0) & 0x00ff0000;
green |= (pixel << 8) & 0x00ff0000;
blue |= (pixel << 16) & 0x00ff0000;
pixel = *ptr32;
ptr32++;
red |= (pixel << 8) & 0xff000000;
green |= (pixel << 16) & 0xff000000;
blue |= (pixel << 24) & 0xff000000;
*((int*)(red_data + out_index)) = red;
*((int*)(green_data + out_index)) = green;
*((int*)(blue_data + out_index)) = blue;
out_index += 4;
index += 4;
}
#endif
while (index < width)
{
pixel = *ptr32;
ptr32++;
red_data[out_index] = pixel >> 16;
green_data[out_index] = pixel >> 8;
blue_data[out_index] = pixel >> 0;
out_index++;
index++;
}
for (index = 0; index < e; index++)
{
red_data[out_index] = red_data[out_index - 1];
green_data[out_index] = green_data[out_index - 1];
blue_data[out_index] = blue_data[out_index - 1];
out_index++;
}
start_line--;
cy++;
}
return cy;
}
/*****************************************************************************/
static int APP_CC
fdelta(char *in_plane, char *out_plane, int cx, int cy)
{
char delta;
char *src8;
char *dst8;
int index;
int jndex;
g_memcpy(out_plane, in_plane, cx);
for (jndex = cy - 2; jndex >= 0; jndex--)
{
src8 = in_plane + jndex * cx;
dst8 = out_plane + jndex * cx;
for (index = 0; index < cx; index++)
{
delta = src8[cx] - src8[0];
if (delta & 0x80)
{
delta = (((~delta) + 1) << 1) - 1;
}
else
{
delta = delta << 1;
}
dst8[cx] = delta;
src8++;
dst8++;
}
}
return 0;
}
/*****************************************************************************/
static int APP_CC
fout(int collen, int replen, char *colptr, struct stream *s)
{
int code;
int lcollen;
int lreplen;
int cont;
LLOGLN(10, ("fout: collen %d replen %d", collen, replen));
cont = collen > 13;
while (cont)
{
lcollen = collen;
if (lcollen > 15)
{
lcollen = 15;
}
code = lcollen << 4;
out_uint8(s, code);
out_uint8a(s, colptr, lcollen);
colptr += lcollen;
collen -= lcollen;
cont = collen > 13;
}
cont = (collen > 0) || (replen > 0);
while (cont)
{
lreplen = replen;
if ((collen == 0) && (lreplen > 15))
{
/* big run */
if (lreplen > 47)
{
lreplen = 47;
}
LLOGLN(10, ("fout: big run lreplen %d", lreplen));
replen -= lreplen;
code = ((lreplen & 0xF) << 4) | ((lreplen & 0xF0) >> 4);
out_uint8(s, code);
colptr += lreplen;
}
else
{
if (lreplen > 15)
{
lreplen = 15;
}
replen -= lreplen;
if (lreplen < 3)
{
collen += lreplen;
lreplen = 0;
}
code = (collen << 4) | lreplen;
out_uint8(s, code);
out_uint8a(s, colptr, collen);
colptr += collen + lreplen;
collen = 0;
}
cont = replen > 0;
}
return 0;
}
/*****************************************************************************/
static int APP_CC
fpack(char *plane, int cx, int cy, struct stream *s)
{
char *ptr8;
char *colptr;
char *lend;
char *holdp;
int jndex;
int collen;
int replen;
LLOGLN(10, ("fpack:"));
holdp = s->p;
for (jndex = 0; jndex < cy; jndex++)
{
LLOGLN(10, ("line start line %d cx %d cy %d", jndex, cx, cy));
ptr8 = plane + jndex * cx;
LHEXDUMP(10, (ptr8, cx));
lend = ptr8 + (cx - 1);
colptr = ptr8;
if (colptr[0] == 0)
{
collen = 0;
replen = 1;
}
else
{
collen = 1;
replen = 0;
}
while (ptr8 < lend)
{
if (ptr8[0] == ptr8[1])
{
replen++;
}
else
{
if (replen > 0)
{
if (replen < 3)
{
collen += replen + 1;
replen = 0;
}
else
{
fout(collen, replen, colptr, s);
colptr = ptr8 + 1;
replen = 0;
collen = 1;
}
}
else
{
collen++;
}
}
ptr8++;
}
/* end of line */
fout(collen, replen, colptr, s);
}
return (int) (s->p - holdp);
}
/*****************************************************************************/
static int APP_CC
foutraw3(struct stream *s, int bytes, int header,
char *red_data, char *green_data, char *blue_data)
{
out_uint8(s, header);
out_uint8a(s, red_data, bytes);
out_uint8a(s, green_data, bytes);
out_uint8a(s, blue_data, bytes);
/* pad if no RLE */
out_uint8(s, 0x00);
return 0;
}
/*****************************************************************************/
static int APP_CC
foutraw4(struct stream *s, int bytes, int header,
char *alpha_data, char *red_data, char *green_data, char *blue_data)
{
out_uint8(s, header);
out_uint8a(s, alpha_data, bytes);
out_uint8a(s, red_data, bytes);
out_uint8a(s, green_data, bytes);
out_uint8a(s, blue_data, bytes);
/* pad if no RLE */
out_uint8(s, 0x00);
return 0;
}
/*****************************************************************************/
/* returns the number of lines compressed */
int APP_CC
xrdp_bitmap32_compress(char *in_data, int width, int height,
struct stream *s, int bpp, int byte_limit,
int start_line, struct stream *temp_s,
int e, int flags)
{
char *alpha_data;
char *red_data;
char *green_data;
char *blue_data;
char *salpha_data;
char *sred_data;
char *sgreen_data;
char *sblue_data;
int alpha_bytes;
int red_bytes;
int green_bytes;
int blue_bytes;
int cx;
int cy;
int max_bytes;
int total_bytes;
int header;
LLOGLN(10, ("xrdp_bitmap32_compress:"));
header = flags & 0xFF;
cx = width + e;
salpha_data = temp_s->data;
sred_data = salpha_data + cx * height;
sgreen_data = sred_data + cx * height;
sblue_data = sgreen_data + cx * height;
alpha_data = sblue_data + cx * height;
red_data = alpha_data + cx * height;
green_data = red_data + cx * height;
blue_data = green_data + cx * height;
if (header & FLAGS_NOALPHA)
{
cy = fsplit3(in_data, start_line, width, e,
sred_data, sgreen_data, sblue_data);
if (header & FLAGS_RLE)
{
fdelta(sred_data, red_data, cx, cy);
fdelta(sgreen_data, green_data, cx, cy);
fdelta(sblue_data, blue_data, cx, cy);
out_uint8(s, header);
red_bytes = fpack(red_data, cx, cy, s);
green_bytes = fpack(green_data, cx, cy, s);
blue_bytes = fpack(blue_data, cx, cy, s);
total_bytes = red_bytes + green_bytes + blue_bytes;
if (1 + total_bytes > byte_limit)
{
/* failed */
LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb "
"bytes %d %d %d total_bytes %d cx %d cy %d byte_limit %d",
red_bytes, green_bytes, blue_bytes,
total_bytes, cx, cy, byte_limit));
return 0;
}
max_bytes = cx * cy * 3;
if (total_bytes > max_bytes)
{
/* raw is better */
LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb "
"bytes %d %d %d total_bytes %d cx %d cy %d max_bytes %d",
red_bytes, green_bytes, blue_bytes,
total_bytes, cx, cy, max_bytes));
init_stream(s, 0);
foutraw3(s, cx * cy, FLAGS_NOALPHA, sred_data,
sgreen_data, sblue_data);
}
}
else
{
foutraw3(s, cx * cy, FLAGS_NOALPHA, sred_data,
sgreen_data, sblue_data);
}
}
else
{
cy = fsplit4(in_data, start_line, width, e,
salpha_data, sred_data, sgreen_data, sblue_data);
if (header & FLAGS_RLE)
{
fdelta(salpha_data, alpha_data, cx, cy);
fdelta(sred_data, red_data, cx, cy);
fdelta(sgreen_data, green_data, cx, cy);
fdelta(sblue_data, blue_data, cx, cy);
out_uint8(s, header);
alpha_bytes = fpack(alpha_data, cx, cy, s);
red_bytes = fpack(red_data, cx, cy, s);
green_bytes = fpack(green_data, cx, cy, s);
blue_bytes = fpack(blue_data, cx, cy, s);
max_bytes = cx * cy * 4;
total_bytes = alpha_bytes + red_bytes + green_bytes + blue_bytes;
if (1 + total_bytes > byte_limit)
{
/* failed */
LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb "
"bytes %d %d %d %d total_bytes %d cx %d cy %d byte_limit %d",
alpha_bytes, red_bytes, green_bytes, blue_bytes,
total_bytes, cx, cy, byte_limit));
return 0;
}
if (total_bytes > max_bytes)
{
/* raw is better */
LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb "
"bytes %d %d %d %d total_bytes %d cx %d cy %d max_bytes %d",
alpha_bytes, red_bytes, green_bytes, blue_bytes,
total_bytes, cx, cy, max_bytes));
init_stream(s, 0);
foutraw4(s, cx * cy, 0, salpha_data, sred_data,
sgreen_data, sblue_data);
}
}
else
{
foutraw4(s, cx * cy, 0, salpha_data, sred_data,
sgreen_data, sblue_data);
}
}
return cy;
}