libtdevnc/libvncclient/tight.c

/*
 *  Copyright (C) 2000, 2001 Const Kaplinsky.  All Rights Reserved.
 *
 *  This is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This software is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this software; if not, write to the Free Software
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
 *  USA.
 */

#ifdef LIBVNCSERVER_HAVE_LIBZ
#ifdef LIBVNCSERVER_HAVE_LIBJPEG

/*
 * tight.c - handle ``tight'' encoding.
 *
 * This file shouldn't be compiled directly. It is included multiple
 * times by rfbproto.c, each time with a different definition of the
 * macro BPP. For each value of BPP, this file defines a function
 * which handles a tight-encoded rectangle with BPP bits per pixel.
 *
 */

#define TIGHT_MIN_TO_COMPRESS 12

#define CARDBPP CONCAT3E(uint,BPP,_t)
#define filterPtrBPP CONCAT2E(filterPtr,BPP)

#define HandleTightBPP CONCAT2E(HandleTight,BPP)
#define InitFilterCopyBPP CONCAT2E(InitFilterCopy,BPP)
#define InitFilterPaletteBPP CONCAT2E(InitFilterPalette,BPP)
#define InitFilterGradientBPP CONCAT2E(InitFilterGradient,BPP)
#define FilterCopyBPP CONCAT2E(FilterCopy,BPP)
#define FilterPaletteBPP CONCAT2E(FilterPalette,BPP)
#define FilterGradientBPP CONCAT2E(FilterGradient,BPP)

#if BPP != 8
#define DecompressJpegRectBPP CONCAT2E(DecompressJpegRect,BPP)
#endif

#ifndef RGB_TO_PIXEL

#define RGB_TO_PIXEL(bpp,r,g,b)						\
  (((CARD##bpp)(r) & client->format.redMax) << client->format.redShift |		\
   ((CARD##bpp)(g) & client->format.greenMax) << client->format.greenShift |	\
   ((CARD##bpp)(b) & client->format.blueMax) << client->format.blueShift)

#define RGB24_TO_PIXEL(bpp,r,g,b)                                       \
   ((((CARD##bpp)(r) & 0xFF) * client->format.redMax + 127) / 255             \
    << client->format.redShift |                                              \
    (((CARD##bpp)(g) & 0xFF) * client->format.greenMax + 127) / 255           \
    << client->format.greenShift |                                            \
    (((CARD##bpp)(b) & 0xFF) * client->format.blueMax + 127) / 255            \
    << client->format.blueShift)

#define RGB24_TO_PIXEL32(r,g,b)						\
  (((uint32_t)(r) & 0xFF) << client->format.redShift |				\
   ((uint32_t)(g) & 0xFF) << client->format.greenShift |			\
   ((uint32_t)(b) & 0xFF) << client->format.blueShift)

#endif

/* Type declarations */

typedef void (*filterPtrBPP)(rfbClient* client, int, CARDBPP *);

/* Prototypes */

static int InitFilterCopyBPP (rfbClient* client, int rw, int rh);
static int InitFilterPaletteBPP (rfbClient* client, int rw, int rh);
static int InitFilterGradientBPP (rfbClient* client, int rw, int rh);
static void FilterCopyBPP (rfbClient* client, int numRows, CARDBPP *destBuffer);
static void FilterPaletteBPP (rfbClient* client, int numRows, CARDBPP *destBuffer);
static void FilterGradientBPP (rfbClient* client, int numRows, CARDBPP *destBuffer);

#if BPP != 8
static rfbBool DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h);
#endif

/* Definitions */

static rfbBool
HandleTightBPP (rfbClient* client, int rx, int ry, int rw, int rh)
{
  CARDBPP fill_colour;
  uint8_t comp_ctl;
  uint8_t filter_id;
  filterPtrBPP filterFn;
  z_streamp zs;
  char *buffer2;
  int err, stream_id, compressedLen, bitsPixel;
  int bufferSize, rowSize, numRows, portionLen, rowsProcessed, extraBytes;

  if (!ReadFromRFBServer(client, (char *)&comp_ctl, 1))
    return FALSE;

  /* Flush zlib streams if we are told by the server to do so. */
  for (stream_id = 0; stream_id < 4; stream_id++) {
    if ((comp_ctl & 1) && client->zlibStreamActive[stream_id]) {
      if (inflateEnd (&client->zlibStream[stream_id]) != Z_OK &&
	  client->zlibStream[stream_id].msg != NULL)
	rfbClientLog("inflateEnd: %s\n", client->zlibStream[stream_id].msg);
      client->zlibStreamActive[stream_id] = FALSE;
    }
    comp_ctl >>= 1;
  }

  /* Handle solid rectangles. */
  if (comp_ctl == rfbTightFill) {
#if BPP == 32
    if (client->format.depth == 24 && client->format.redMax == 0xFF &&
	client->format.greenMax == 0xFF && client->format.blueMax == 0xFF) {
      if (!ReadFromRFBServer(client, client->buffer, 3))
	return FALSE;
      fill_colour = RGB24_TO_PIXEL32(client->buffer[0], client->buffer[1], client->buffer[2]);
    } else {
      if (!ReadFromRFBServer(client, (char*)&fill_colour, sizeof(fill_colour)))
	return FALSE;
    }
#else
    if (!ReadFromRFBServer(client, (char*)&fill_colour, sizeof(fill_colour)))
	return FALSE;
#endif

    FillRectangle(client, rx, ry, rw, rh, fill_colour);

    return TRUE;
  }

#if BPP == 8
  if (comp_ctl == rfbTightJpeg) {
    rfbClientLog("Tight encoding: JPEG is not supported in 8 bpp mode.\n");
    return FALSE;
  }
#else
  if (comp_ctl == rfbTightJpeg) {
    return DecompressJpegRectBPP(client, rx, ry, rw, rh);
  }
#endif

  /* Quit on unsupported subencoding value. */
  if (comp_ctl > rfbTightMaxSubencoding) {
    rfbClientLog("Tight encoding: bad subencoding value received.\n");
    return FALSE;
  }

  /*
   * Here primary compression mode handling begins.
   * Data was processed with optional filter + zlib compression.
   */

  /* First, we should identify a filter to use. */
  if ((comp_ctl & rfbTightExplicitFilter) != 0) {
    if (!ReadFromRFBServer(client, (char*)&filter_id, 1))
      return FALSE;

    switch (filter_id) {
    case rfbTightFilterCopy:
      filterFn = FilterCopyBPP;
      bitsPixel = InitFilterCopyBPP(client, rw, rh);
      break;
    case rfbTightFilterPalette:
      filterFn = FilterPaletteBPP;
      bitsPixel = InitFilterPaletteBPP(client, rw, rh);
      break;
    case rfbTightFilterGradient:
      filterFn = FilterGradientBPP;
      bitsPixel = InitFilterGradientBPP(client, rw, rh);
      break;
    default:
      rfbClientLog("Tight encoding: unknown filter code received.\n");
      return FALSE;
    }
  } else {
    filterFn = FilterCopyBPP;
    bitsPixel = InitFilterCopyBPP(client, rw, rh);
  }
  if (bitsPixel == 0) {
    rfbClientLog("Tight encoding: error receiving palette.\n");
    return FALSE;
  }

  /* Determine if the data should be decompressed or just copied. */
  rowSize = (rw * bitsPixel + 7) / 8;
  if (rh * rowSize < TIGHT_MIN_TO_COMPRESS) {
    if (!ReadFromRFBServer(client, (char*)client->buffer, rh * rowSize))
      return FALSE;

    buffer2 = &client->buffer[TIGHT_MIN_TO_COMPRESS * 4];
    filterFn(client, rh, (CARDBPP *)buffer2);

    CopyRectangle(client, buffer2, rx, ry, rw, rh);

    return TRUE;
  }

  /* Read the length (1..3 bytes) of compressed data following. */
  compressedLen = (int)ReadCompactLen(client);
  if (compressedLen <= 0) {
    rfbClientLog("Incorrect data received from the server.\n");
    return FALSE;
  }

  /* Now let's initialize compression stream if needed. */
  stream_id = comp_ctl & 0x03;
  zs = &client->zlibStream[stream_id];
  if (!client->zlibStreamActive[stream_id]) {
    zs->zalloc = Z_NULL;
    zs->zfree = Z_NULL;
    zs->opaque = Z_NULL;
    err = inflateInit(zs);
    if (err != Z_OK) {
      if (zs->msg != NULL)
	rfbClientLog("InflateInit error: %s.\n", zs->msg);
      return FALSE;
    }
    client->zlibStreamActive[stream_id] = TRUE;
  }

  /* Read, decode and draw actual pixel data in a loop. */

  bufferSize = RFB_BUFFER_SIZE * bitsPixel / (bitsPixel + BPP) & 0xFFFFFFFC;
  buffer2 = &client->buffer[bufferSize];
  if (rowSize > bufferSize) {
    /* Should be impossible when RFB_BUFFER_SIZE >= 16384 */
    rfbClientLog("Internal error: incorrect buffer size.\n");
    return FALSE;
  }

  rowsProcessed = 0;
  extraBytes = 0;

  while (compressedLen > 0) {
    if (compressedLen > ZLIB_BUFFER_SIZE)
      portionLen = ZLIB_BUFFER_SIZE;
    else
      portionLen = compressedLen;

    if (!ReadFromRFBServer(client, (char*)client->zlib_buffer, portionLen))
      return FALSE;

    compressedLen -= portionLen;

    zs->next_in = (Bytef *)client->zlib_buffer;
    zs->avail_in = portionLen;

    do {
      zs->next_out = (Bytef *)&client->buffer[extraBytes];
      zs->avail_out = bufferSize - extraBytes;

      err = inflate(zs, Z_SYNC_FLUSH);
      if (err == Z_BUF_ERROR)   /* Input exhausted -- no problem. */
	break;
      if (err != Z_OK && err != Z_STREAM_END) {
	if (zs->msg != NULL) {
	  rfbClientLog("Inflate error: %s.\n", zs->msg);
	} else {
	  rfbClientLog("Inflate error: %d.\n", err);
	}
	return FALSE;
      }

      numRows = (bufferSize - zs->avail_out) / rowSize;

      filterFn(client, numRows, (CARDBPP *)buffer2);

      extraBytes = bufferSize - zs->avail_out - numRows * rowSize;
      if (extraBytes > 0)
	memcpy(client->buffer, &client->buffer[numRows * rowSize], extraBytes);

      CopyRectangle(client, buffer2, rx, ry+rowsProcessed, rw, numRows);

      rowsProcessed += numRows;
    }
    while (zs->avail_out == 0);
  }

  if (rowsProcessed != rh) {
    rfbClientLog("Incorrect number of scan lines after decompression.\n");
    return FALSE;
  }

  return TRUE;
}

/*----------------------------------------------------------------------------
 *
 * Filter stuff.
 *
 */

static int
InitFilterCopyBPP (rfbClient* client, int rw, int rh)
{
  client->rectWidth = rw;

#if BPP == 32
  if (client->format.depth == 24 && client->format.redMax == 0xFF &&
      client->format.greenMax == 0xFF && client->format.blueMax == 0xFF) {
    client->cutZeros = TRUE;
    return 24;
  } else {
    client->cutZeros = FALSE;
  }
#endif

  return BPP;
}

static void
FilterCopyBPP (rfbClient* client, int numRows, CARDBPP *dst)
{

#if BPP == 32
  int x, y;

  if (client->cutZeros) {
    for (y = 0; y < numRows; y++) {
      for (x = 0; x < client->rectWidth; x++) {
	dst[y*client->rectWidth+x] =
	  RGB24_TO_PIXEL32(client->buffer[(y*client->rectWidth+x)*3],
			   client->buffer[(y*client->rectWidth+x)*3+1],
			   client->buffer[(y*client->rectWidth+x)*3+2]);
      }
    }
    return;
  }
#endif

  memcpy (dst, client->buffer, numRows * client->rectWidth * (BPP / 8));
}

static int
InitFilterGradientBPP (rfbClient* client, int rw, int rh)
{
  int bits;

  bits = InitFilterCopyBPP(client, rw, rh);
  if (client->cutZeros)
    memset(client->tightPrevRow, 0, rw * 3);
  else
    memset(client->tightPrevRow, 0, rw * 3 * sizeof(uint16_t));

  return bits;
}

#if BPP == 32

static void
FilterGradient24 (rfbClient* client, int numRows, uint32_t *dst)
{
  int x, y, c;
  uint8_t thisRow[2048*3];
  uint8_t pix[3];
  int est[3];

  for (y = 0; y < numRows; y++) {

    /* First pixel in a row */
    for (c = 0; c < 3; c++) {
      pix[c] = client->tightPrevRow[c] + client->buffer[y*client->rectWidth*3+c];
      thisRow[c] = pix[c];
    }
    dst[y*client->rectWidth] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);

    /* Remaining pixels of a row */
    for (x = 1; x < client->rectWidth; x++) {
      for (c = 0; c < 3; c++) {
	est[c] = (int)client->tightPrevRow[x*3+c] + (int)pix[c] -
		 (int)client->tightPrevRow[(x-1)*3+c];
	if (est[c] > 0xFF) {
	  est[c] = 0xFF;
	} else if (est[c] < 0x00) {
	  est[c] = 0x00;
	}
	pix[c] = (uint8_t)est[c] + client->buffer[(y*client->rectWidth+x)*3+c];
	thisRow[x*3+c] = pix[c];
      }
      dst[y*client->rectWidth+x] = RGB24_TO_PIXEL32(pix[0], pix[1], pix[2]);
    }

    memcpy(client->tightPrevRow, thisRow, client->rectWidth * 3);
  }
}

#endif

static void
FilterGradientBPP (rfbClient* client, int numRows, CARDBPP *dst)
{
  int x, y, c;
  CARDBPP *src = (CARDBPP *)client->buffer;
  uint16_t *thatRow = (uint16_t *)client->tightPrevRow;
  uint16_t thisRow[2048*3];
  uint16_t pix[3];
  uint16_t max[3];
  int shift[3];
  int est[3];

#if BPP == 32
  if (client->cutZeros) {
    FilterGradient24(client, numRows, dst);
    return;
  }
#endif

  max[0] = client->format.redMax;
  max[1] = client->format.greenMax;
  max[2] = client->format.blueMax;

  shift[0] = client->format.redShift;
  shift[1] = client->format.greenShift;
  shift[2] = client->format.blueShift;

  for (y = 0; y < numRows; y++) {

    /* First pixel in a row */
    for (c = 0; c < 3; c++) {
      pix[c] = (uint16_t)(((src[y*client->rectWidth] >> shift[c]) + thatRow[c]) & max[c]);
      thisRow[c] = pix[c];
    }
    dst[y*client->rectWidth] = RGB_TO_PIXEL(BPP, pix[0], pix[1], pix[2]);

    /* Remaining pixels of a row */
    for (x = 1; x < client->rectWidth; x++) {
      for (c = 0; c < 3; c++) {
	est[c] = (int)thatRow[x*3+c] + (int)pix[c] - (int)thatRow[(x-1)*3+c];
	if (est[c] > (int)max[c]) {
	  est[c] = (int)max[c];
	} else if (est[c] < 0) {
	  est[c] = 0;
	}
	pix[c] = (uint16_t)(((src[y*client->rectWidth+x] >> shift[c]) + est[c]) & max[c]);
	thisRow[x*3+c] = pix[c];
      }
      dst[y*client->rectWidth+x] = RGB_TO_PIXEL(BPP, pix[0], pix[1], pix[2]);
    }
    memcpy(thatRow, thisRow, client->rectWidth * 3 * sizeof(uint16_t));
  }
}

static int
InitFilterPaletteBPP (rfbClient* client, int rw, int rh)
{
  uint8_t numColors;
#if BPP == 32
  int i;
  CARDBPP *palette = (CARDBPP *)client->tightPalette;
#endif

  client->rectWidth = rw;

  if (!ReadFromRFBServer(client, (char*)&numColors, 1))
    return 0;

  client->rectColors = (int)numColors;
  if (++client->rectColors < 2)
    return 0;

#if BPP == 32
  if (client->format.depth == 24 && client->format.redMax == 0xFF &&
      client->format.greenMax == 0xFF && client->format.blueMax == 0xFF) {
    if (!ReadFromRFBServer(client, (char*)&client->tightPalette, client->rectColors * 3))
      return 0;
    for (i = client->rectColors - 1; i >= 0; i--) {
      palette[i] = RGB24_TO_PIXEL32(client->tightPalette[i*3],
				    client->tightPalette[i*3+1],
				    client->tightPalette[i*3+2]);
    }
    return (client->rectColors == 2) ? 1 : 8;
  }
#endif

  if (!ReadFromRFBServer(client, (char*)&client->tightPalette, client->rectColors * (BPP / 8)))
    return 0;

  return (client->rectColors == 2) ? 1 : 8;
}

static void
FilterPaletteBPP (rfbClient* client, int numRows, CARDBPP *dst)
{
  int x, y, b, w;
  uint8_t *src = (uint8_t *)client->buffer;
  CARDBPP *palette = (CARDBPP *)client->tightPalette;

  if (client->rectColors == 2) {
    w = (client->rectWidth + 7) / 8;
    for (y = 0; y < numRows; y++) {
      for (x = 0; x < client->rectWidth / 8; x++) {
	for (b = 7; b >= 0; b--)
	  dst[y*client->rectWidth+x*8+7-b] = palette[src[y*w+x] >> b & 1];
      }
      for (b = 7; b >= 8 - client->rectWidth % 8; b--) {
	dst[y*client->rectWidth+x*8+7-b] = palette[src[y*w+x] >> b & 1];
      }
    }
  } else {
    for (y = 0; y < numRows; y++)
      for (x = 0; x < client->rectWidth; x++)
	dst[y*client->rectWidth+x] = palette[(int)src[y*client->rectWidth+x]];
  }
}

#if BPP != 8

/*----------------------------------------------------------------------------
 *
 * JPEG decompression.
 *
 */

static rfbBool
DecompressJpegRectBPP(rfbClient* client, int x, int y, int w, int h)
{
  struct jpeg_decompress_struct cinfo;
  struct jpeg_error_mgr jerr;
  int compressedLen;
  uint8_t *compressedData;
  CARDBPP *pixelPtr;
  JSAMPROW rowPointer[1];
  int dx, dy;

  compressedLen = (int)ReadCompactLen(client);
  if (compressedLen <= 0) {
    rfbClientLog("Incorrect data received from the server.\n");
    return FALSE;
  }

  compressedData = malloc(compressedLen);
  if (compressedData == NULL) {
    rfbClientLog("Memory allocation error.\n");
    return FALSE;
  }

  if (!ReadFromRFBServer(client, (char*)compressedData, compressedLen)) {
    free(compressedData);
    return FALSE;
  }

  cinfo.err = jpeg_std_error(&jerr);
  cinfo.client_data = client;
  jpeg_create_decompress(&cinfo);

  JpegSetSrcManager(&cinfo, compressedData, compressedLen);

  jpeg_read_header(&cinfo, TRUE);
  cinfo.out_color_space = JCS_RGB;

  jpeg_start_decompress(&cinfo);
  if (cinfo.output_width != w || cinfo.output_height != h ||
      cinfo.output_components != 3) {
    rfbClientLog("Tight Encoding: Wrong JPEG data received.\n");
    jpeg_destroy_decompress(&cinfo);
    free(compressedData);
    return FALSE;
  }

  rowPointer[0] = (JSAMPROW)client->buffer;
  dy = 0;
  while (cinfo.output_scanline < cinfo.output_height) {
    jpeg_read_scanlines(&cinfo, rowPointer, 1);
    if (client->jpegError) {
      break;
    }
    pixelPtr = (CARDBPP *)&client->buffer[RFB_BUFFER_SIZE / 2];
    for (dx = 0; dx < w; dx++) {
      *pixelPtr++ =
	RGB24_TO_PIXEL(BPP, client->buffer[dx*3], client->buffer[dx*3+1], client->buffer[dx*3+2]);
    }
    CopyRectangle(client, &client->buffer[RFB_BUFFER_SIZE / 2], x, y + dy, w, 1);
    dy++;
  }

  if (!client->jpegError)
    jpeg_finish_decompress(&cinfo);

  jpeg_destroy_decompress(&cinfo);
  free(compressedData);

  return !client->jpegError;
}

#else

static long
ReadCompactLen (rfbClient* client)
{
  long len;
  uint8_t b;

  if (!ReadFromRFBServer(client, (char *)&b, 1))
    return -1;
  len = (int)b & 0x7F;
  if (b & 0x80) {
    if (!ReadFromRFBServer(client, (char *)&b, 1))
      return -1;
    len |= ((int)b & 0x7F) << 7;
    if (b & 0x80) {
      if (!ReadFromRFBServer(client, (char *)&b, 1))
	return -1;
      len |= ((int)b & 0xFF) << 14;
    }
  }
  return len;
}

/*
 * JPEG source manager functions for JPEG decompression in Tight decoder.
 */

static void
JpegInitSource(j_decompress_ptr cinfo)
{
  rfbClient* client=(rfbClient*)cinfo->client_data;
  client->jpegError = FALSE;
}

static boolean
JpegFillInputBuffer(j_decompress_ptr cinfo)
{
  rfbClient* client=(rfbClient*)cinfo->client_data;
  client->jpegError = TRUE;
  client->jpegSrcManager->bytes_in_buffer = client->jpegBufferLen;
  client->jpegSrcManager->next_input_byte = (JOCTET *)client->jpegBufferPtr;

  return TRUE;
}

static void
JpegSkipInputData(j_decompress_ptr cinfo, long num_bytes)
{
  rfbClient* client=(rfbClient*)cinfo->client_data;
  if (num_bytes < 0 || num_bytes > client->jpegSrcManager->bytes_in_buffer) {
    client->jpegError = TRUE;
    client->jpegSrcManager->bytes_in_buffer = client->jpegBufferLen;
    client->jpegSrcManager->next_input_byte = (JOCTET *)client->jpegBufferPtr;
  } else {
    client->jpegSrcManager->next_input_byte += (size_t) num_bytes;
    client->jpegSrcManager->bytes_in_buffer -= (size_t) num_bytes;
  }
}

static void
JpegTermSource(j_decompress_ptr cinfo)
{
  /* nothing to do here. */
}

static void
JpegSetSrcManager(j_decompress_ptr cinfo,
		  uint8_t *compressedData,
		  int compressedLen)
{
  rfbClient* client=(rfbClient*)cinfo->client_data;
  client->jpegBufferPtr = compressedData;
  client->jpegBufferLen = (size_t)compressedLen;

  if(client->jpegSrcManager == NULL)
    client->jpegSrcManager = malloc(sizeof(struct jpeg_source_mgr));
  client->jpegSrcManager->init_source = JpegInitSource;
  client->jpegSrcManager->fill_input_buffer = JpegFillInputBuffer;
  client->jpegSrcManager->skip_input_data = JpegSkipInputData;
  client->jpegSrcManager->resync_to_restart = jpeg_resync_to_restart;
  client->jpegSrcManager->term_source = JpegTermSource;
  client->jpegSrcManager->next_input_byte = (JOCTET*)client->jpegBufferPtr;
  client->jpegSrcManager->bytes_in_buffer = client->jpegBufferLen;

  cinfo->src = client->jpegSrcManager;
}

#endif

#undef CARDBPP

/* LIBVNCSERVER_HAVE_LIBZ and LIBVNCSERVER_HAVE_LIBJPEG */
#endif
#endif