/** * xrdp: A Remote Desktop Protocol server. * * Copyright (C) Jay Sorg 2014 * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * * planar bitmap compressor * 32 bpp compression */ /* RDP 6.0 Bitmap Compression http://msdn.microsoft.com/en-us/library/cc241877.aspx */ #include "libxrdp.h" #define FLAGS_RLE 0x10 #define FLAGS_NOALPHA 0x20 #define LLOG_LEVEL 1 #define LLOGLN(_level, _args) \ do { if (_level < LLOG_LEVEL) { g_writeln _args ; } } while (0) #define LHEXDUMP(_level, _args) \ do { if (_level < LLOG_LEVEL) { g_hexdump _args ; } } while (0) /*****************************************************************************/ /* split RGB */ static int APP_CC fsplit3(char *in_data, int start_line, int width, int e, char *r_data, char *g_data, char *b_data) { #if defined(L_ENDIAN) int rp; int gp; int bp; #endif int index; int out_index; int pixel; int cy; int *ptr32; cy = 0; out_index = 0; while (start_line >= 0) { ptr32 = (int *) (in_data + start_line * width * 4); index = 0; #if defined(L_ENDIAN) while (index + 4 <= width) { pixel = *ptr32; ptr32++; rp = (pixel >> 16) & 0x000000ff; gp = (pixel >> 8) & 0x000000ff; bp = (pixel >> 0) & 0x000000ff; pixel = *ptr32; ptr32++; rp |= (pixel >> 8) & 0x0000ff00; gp |= (pixel << 0) & 0x0000ff00; bp |= (pixel << 8) & 0x0000ff00; pixel = *ptr32; ptr32++; rp |= (pixel >> 0) & 0x00ff0000; gp |= (pixel << 8) & 0x00ff0000; bp |= (pixel << 16) & 0x00ff0000; pixel = *ptr32; ptr32++; rp |= (pixel << 8) & 0xff000000; gp |= (pixel << 16) & 0xff000000; bp |= (pixel << 24) & 0xff000000; *((int*)(r_data + out_index)) = rp; *((int*)(g_data + out_index)) = gp; *((int*)(b_data + out_index)) = bp; out_index += 4; index += 4; } #endif while (index < width) { pixel = *ptr32; ptr32++; r_data[out_index] = pixel >> 16; g_data[out_index] = pixel >> 8; b_data[out_index] = pixel >> 0; out_index++; index++; } for (index = 0; index < e; index++) { r_data[out_index] = r_data[out_index - 1]; g_data[out_index] = g_data[out_index - 1]; b_data[out_index] = b_data[out_index - 1]; out_index++; } start_line--; cy++; if (out_index > 64 * 64) { break; } } return cy; } /*****************************************************************************/ /* split ARGB */ static int APP_CC fsplit4(char *in_data, int start_line, int width, int e, char *a_data, char *r_data, char *g_data, char *b_data) { #if defined(L_ENDIAN) int ap; int rp; int gp; int bp; #endif int index; int out_index; int pixel; int cy; int *ptr32; cy = 0; out_index = 0; while (start_line >= 0) { ptr32 = (int *) (in_data + start_line * width * 4); index = 0; #if defined(L_ENDIAN) while (index + 4 <= width) { pixel = *ptr32; ptr32++; ap = (pixel >> 24) & 0x000000ff; rp = (pixel >> 16) & 0x000000ff; gp = (pixel >> 8) & 0x000000ff; bp = (pixel >> 0) & 0x000000ff; pixel = *ptr32; ptr32++; ap |= (pixel >> 16) & 0x0000ff00; rp |= (pixel >> 8) & 0x0000ff00; gp |= (pixel << 0) & 0x0000ff00; bp |= (pixel << 8) & 0x0000ff00; pixel = *ptr32; ptr32++; ap |= (pixel >> 8) & 0x00ff0000; rp |= (pixel >> 0) & 0x00ff0000; gp |= (pixel << 8) & 0x00ff0000; bp |= (pixel << 16) & 0x00ff0000; pixel = *ptr32; ptr32++; ap |= (pixel << 0) & 0xff000000; rp |= (pixel << 8) & 0xff000000; gp |= (pixel << 16) & 0xff000000; bp |= (pixel << 24) & 0xff000000; *((int*)(a_data + out_index)) = ap; *((int*)(r_data + out_index)) = rp; *((int*)(g_data + out_index)) = gp; *((int*)(b_data + out_index)) = bp; out_index += 4; index += 4; } #endif while (index < width) { pixel = *ptr32; ptr32++; a_data[out_index] = pixel >> 24; r_data[out_index] = pixel >> 16; g_data[out_index] = pixel >> 8; b_data[out_index] = pixel >> 0; out_index++; index++; } for (index = 0; index < e; index++) { a_data[out_index] = a_data[out_index - 1]; r_data[out_index] = r_data[out_index - 1]; g_data[out_index] = g_data[out_index - 1]; b_data[out_index] = b_data[out_index - 1]; out_index++; } start_line--; cy++; if (out_index > 64 * 64) { break; } } return cy; } /*****************************************************************************/ static int APP_CC fdelta(char *in_plane, char *out_plane, int cx, int cy) { char delta; char *src8; char *dst8; int index; int jndex; g_memcpy(out_plane, in_plane, cx); for (jndex = cy - 2; jndex >= 0; jndex--) { src8 = in_plane + jndex * cx; dst8 = out_plane + jndex * cx; for (index = 0; index < cx; index++) { delta = src8[cx] - src8[0]; if (delta & 0x80) { delta = (((~delta) + 1) << 1) - 1; } else { delta = delta << 1; } dst8[cx] = delta; src8++; dst8++; } } return 0; } /*****************************************************************************/ static int APP_CC fout(int collen, int replen, char *colptr, struct stream *s) { int code; int lcollen; int lreplen; int cont; LLOGLN(10, ("fout: collen %d replen %d", collen, replen)); cont = collen > 13; while (cont) { lcollen = collen; if (lcollen > 15) { lcollen = 15; } code = lcollen << 4; out_uint8(s, code); out_uint8a(s, colptr, lcollen); colptr += lcollen; collen -= lcollen; cont = collen > 13; } cont = (collen > 0) || (replen > 0); while (cont) { lreplen = replen; if ((collen == 0) && (lreplen > 15)) { /* big run */ if (lreplen > 47) { lreplen = 47; } LLOGLN(10, ("fout: big run lreplen %d", lreplen)); replen -= lreplen; code = ((lreplen & 0xF) << 4) | ((lreplen & 0xF0) >> 4); out_uint8(s, code); colptr += lreplen; } else { if (lreplen > 15) { lreplen = 15; } replen -= lreplen; if (lreplen < 3) { collen += lreplen; lreplen = 0; } code = (collen << 4) | lreplen; out_uint8(s, code); out_uint8a(s, colptr, collen); colptr += collen + lreplen; collen = 0; } cont = replen > 0; } return 0; } /*****************************************************************************/ static int APP_CC fpack(char *plane, int cx, int cy, struct stream *s) { char *ptr8; char *colptr; char *lend; char *holdp; int jndex; int collen; int replen; LLOGLN(10, ("fpack:")); holdp = s->p; for (jndex = 0; jndex < cy; jndex++) { LLOGLN(10, ("line start line %d cx %d cy %d", jndex, cx, cy)); ptr8 = plane + jndex * cx; LHEXDUMP(10, (ptr8, cx)); lend = ptr8 + (cx - 1); colptr = ptr8; if (colptr[0] == 0) { collen = 0; replen = 1; } else { collen = 1; replen = 0; } while (ptr8 < lend) { if (ptr8[0] == ptr8[1]) { replen++; } else { if (replen > 0) { if (replen < 3) { collen += replen + 1; replen = 0; } else { fout(collen, replen, colptr, s); colptr = ptr8 + 1; replen = 0; collen = 1; } } else { collen++; } } ptr8++; } /* end of line */ fout(collen, replen, colptr, s); } return (int) (s->p - holdp); } /*****************************************************************************/ static int APP_CC foutraw3(struct stream *s, int bytes, int header, char *r_data, char *g_data, char *b_data) { out_uint8(s, header); out_uint8a(s, r_data, bytes); out_uint8a(s, g_data, bytes); out_uint8a(s, b_data, bytes); /* pad if no RLE */ out_uint8(s, 0x00); return 0; } /*****************************************************************************/ static int APP_CC foutraw4(struct stream *s, int bytes, int header, char *a_data, char *r_data, char *g_data, char *b_data) { out_uint8(s, header); out_uint8a(s, a_data, bytes); out_uint8a(s, r_data, bytes); out_uint8a(s, g_data, bytes); out_uint8a(s, b_data, bytes); /* pad if no RLE */ out_uint8(s, 0x00); return 0; } /*****************************************************************************/ /* returns the number of lines compressed */ int APP_CC xrdp_bitmap32_compress(char *in_data, int width, int height, struct stream *s, int bpp, int byte_limit, int start_line, struct stream *temp_s, int e, int flags) { char *a_data; char *r_data; char *g_data; char *b_data; char *sa_data; char *sr_data; char *sg_data; char *sb_data; int a_bytes; int r_bytes; int g_bytes; int b_bytes; int cx; int cy; int max_bytes; int total_bytes; int header; LLOGLN(10, ("xrdp_bitmap32_compress:")); max_bytes = 4 * 1024; /* need max 8, 4K planes for work */ if (max_bytes * 8 > temp_s->size) { return 0; } header = flags & 0xFF; cx = width + e; sa_data = temp_s->data; sr_data = sa_data + max_bytes; sg_data = sr_data + max_bytes; sb_data = sg_data + max_bytes; a_data = sb_data + max_bytes; r_data = a_data + max_bytes; g_data = r_data + max_bytes; b_data = g_data + max_bytes; if (header & FLAGS_NOALPHA) { cy = fsplit3(in_data, start_line, width, e, sr_data, sg_data, sb_data); if (header & FLAGS_RLE) { fdelta(sr_data, r_data, cx, cy); fdelta(sg_data, g_data, cx, cy); fdelta(sb_data, b_data, cx, cy); out_uint8(s, header); r_bytes = fpack(r_data, cx, cy, s); g_bytes = fpack(g_data, cx, cy, s); b_bytes = fpack(b_data, cx, cy, s); total_bytes = r_bytes + g_bytes + b_bytes; if (1 + total_bytes > byte_limit) { /* failed */ LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb " "bytes %d %d %d total_bytes %d cx %d cy %d " "byte_limit %d", r_bytes, g_bytes, b_bytes, total_bytes, cx, cy, byte_limit)); return 0; } max_bytes = cx * cy * 3; if (total_bytes > max_bytes) { /* raw is better */ LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb " "bytes %d %d %d total_bytes %d cx %d cy %d " "max_bytes %d", r_bytes, g_bytes, b_bytes, total_bytes, cx, cy, max_bytes)); init_stream(s, 0); foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); } } else { foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); } } else { cy = fsplit4(in_data, start_line, width, e, sa_data, sr_data, sg_data, sb_data); if (header & FLAGS_RLE) { fdelta(sa_data, a_data, cx, cy); fdelta(sr_data, r_data, cx, cy); fdelta(sg_data, g_data, cx, cy); fdelta(sb_data, b_data, cx, cy); out_uint8(s, header); a_bytes = fpack(a_data, cx, cy, s); r_bytes = fpack(r_data, cx, cy, s); g_bytes = fpack(g_data, cx, cy, s); b_bytes = fpack(b_data, cx, cy, s); max_bytes = cx * cy * 4; total_bytes = a_bytes + r_bytes + g_bytes + b_bytes; if (1 + total_bytes > byte_limit) { /* failed */ LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb " "bytes %d %d %d %d total_bytes %d cx %d cy %d " "byte_limit %d", a_bytes, r_bytes, g_bytes, b_bytes, total_bytes, cx, cy, byte_limit)); return 0; } if (total_bytes > max_bytes) { /* raw is better */ LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " "bytes %d %d %d %d total_bytes %d cx %d cy %d " "max_bytes %d", a_bytes, r_bytes, g_bytes, b_bytes, total_bytes, cx, cy, max_bytes)); init_stream(s, 0); foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); } } else { foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); } } return cy; }