/* * Copyright (C) 2005 Johannes E. Schindelin. All Rights Reserved. * * This is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This software is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this software; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, * USA. */ #ifdef LIBVNCSERVER_HAVE_LIBZ /* * zrle.c - handle zrle encoding. * * This file shouldn't be compiled directly. It is included multiple times by * rfbproto.c, each time with a different definition of the macro BPP. For * each value of BPP, this file defines a function which handles an zrle * encoded rectangle with BPP bits per pixel. */ #ifndef REALBPP #define REALBPP BPP #endif #if !defined(UNCOMP) || UNCOMP==0 #define HandleZRLE CONCAT2E(HandleZRLE,REALBPP) #define HandleZRLETile CONCAT2E(HandleZRLETile,REALBPP) #elif UNCOMP>0 #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Down) #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Down) #else #define HandleZRLE CONCAT3E(HandleZRLE,REALBPP,Up) #define HandleZRLETile CONCAT3E(HandleZRLETile,REALBPP,Up) #endif #define CARDBPP CONCAT3E(uint,BPP,_t) #define CARDREALBPP CONCAT3E(uint,REALBPP,_t) #define ENDIAN_LITTLE 0 #define ENDIAN_BIG 1 #define ENDIAN_NO 2 #define ZYWRLE_ENDIAN ENDIAN_LITTLE #undef END_FIX #if ZYWRLE_ENDIAN == ENDIAN_LITTLE # define END_FIX LE #elif ZYWRLE_ENDIAN == ENDIAN_BIG # define END_FIX BE #else # define END_FIX NE #endif #define __RFB_CONCAT3E(a,b,c) CONCAT3E(a,b,c) #define __RFB_CONCAT2E(a,b) CONCAT2E(a,b) #undef CPIXEL #if REALBPP != BPP #if UNCOMP == 0 #define CPIXEL REALBPP #elif UNCOMP>0 #define CPIXEL CONCAT2E(REALBPP,Down) #else #define CPIXEL CONCAT2E(REALBPP,Up) #endif #endif #define PIXEL_T __RFB_CONCAT3E(uint,BPP,_t) #if BPP!=8 #define ZYWRLE_DECODE 1 #include "zywrletemplate.c" #endif #undef CPIXEL static int HandleZRLETile(rfbClient* client, uint8_t* buffer,size_t buffer_length, int x,int y,int w,int h); static rfbBool HandleZRLE (rfbClient* client, int rx, int ry, int rw, int rh) { rfbZRLEHeader header; int remaining; int inflateResult; int toRead; int min_buffer_size = rw * rh * (REALBPP / 8) * 2; /* First make sure we have a large enough raw buffer to hold the * decompressed data. In practice, with a fixed REALBPP, fixed frame * buffer size and the first update containing the entire frame * buffer, this buffer allocation should only happen once, on the * first update. */ if ( client->raw_buffer_size < min_buffer_size) { if ( client->raw_buffer != NULL ) { free( client->raw_buffer ); } client->raw_buffer_size = min_buffer_size; client->raw_buffer = (char*) malloc( client->raw_buffer_size ); } if (!ReadFromRFBServer(client, (char *)&header, sz_rfbZRLEHeader)) return FALSE; remaining = rfbClientSwap32IfLE(header.length); /* Need to initialize the decompressor state. */ client->decompStream.next_in = ( Bytef * )client->buffer; client->decompStream.avail_in = 0; client->decompStream.next_out = ( Bytef * )client->raw_buffer; client->decompStream.avail_out = client->raw_buffer_size; client->decompStream.data_type = Z_BINARY; /* Initialize the decompression stream structures on the first invocation. */ if ( client->decompStreamInited == FALSE ) { inflateResult = inflateInit( &client->decompStream ); if ( inflateResult != Z_OK ) { rfbClientLog( "inflateInit returned error: %d, msg: %s\n", inflateResult, client->decompStream.msg); return FALSE; } client->decompStreamInited = TRUE; } inflateResult = Z_OK; /* Process buffer full of data until no more to process, or * some type of inflater error, or Z_STREAM_END. */ while (( remaining > 0 ) && ( inflateResult == Z_OK )) { if ( remaining > RFB_BUFFER_SIZE ) { toRead = RFB_BUFFER_SIZE; } else { toRead = remaining; } /* Fill the buffer, obtaining data from the server. */ if (!ReadFromRFBServer(client, client->buffer,toRead)) return FALSE; client->decompStream.next_in = ( Bytef * )client->buffer; client->decompStream.avail_in = toRead; /* Need to uncompress buffer full. */ inflateResult = inflate( &client->decompStream, Z_SYNC_FLUSH ); /* We never supply a dictionary for compression. */ if ( inflateResult == Z_NEED_DICT ) { rfbClientLog("zlib inflate needs a dictionary!\n"); return FALSE; } if ( inflateResult < 0 ) { rfbClientLog( "zlib inflate returned error: %d, msg: %s\n", inflateResult, client->decompStream.msg); return FALSE; } /* Result buffer allocated to be at least large enough. We should * never run out of space! */ if (( client->decompStream.avail_in > 0 ) && ( client->decompStream.avail_out <= 0 )) { rfbClientLog("zlib inflate ran out of space!\n"); return FALSE; } remaining -= toRead; } /* while ( remaining > 0 ) */ if ( inflateResult == Z_OK ) { char* buf=client->raw_buffer; int i,j; remaining = client->raw_buffer_size-client->decompStream.avail_out; for(j=0; jrw)?rw-i:rfbZRLETileWidth; int subHeight=(j+rfbZRLETileHeight>rh)?rh-j:rfbZRLETileHeight; int result=HandleZRLETile(client,(uint8_t *)buf,remaining,rx+i,ry+j,subWidth,subHeight); if(result<0) { rfbClientLog("ZRLE decoding failed (%d)\n",result); return TRUE; return FALSE; } buf+=result; remaining-=result; } } else { rfbClientLog( "zlib inflate returned error: %d, msg: %s\n", inflateResult, client->decompStream.msg); return FALSE; } return TRUE; } #if REALBPP!=BPP && defined(UNCOMP) && UNCOMP!=0 #if UNCOMP>0 #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)>>UNCOMP) #else #define UncompressCPixel(pointer) ((*(CARDBPP*)pointer)<<(-(UNCOMP))) #endif #else #define UncompressCPixel(pointer) (*(CARDBPP*)pointer) #endif static int HandleZRLETile(rfbClient* client, uint8_t* buffer,size_t buffer_length, int x,int y,int w,int h) { uint8_t* buffer_copy = buffer; uint8_t* buffer_end = buffer+buffer_length; uint8_t type; #if BPP!=8 uint8_t zywrle_level = (client->appData.qualityLevel & 0x80) ? 0 : (3 - client->appData.qualityLevel / 3); #endif if(buffer_length<1) return -2; type = *buffer; buffer++; { if( type == 0 ) /* raw */ #if BPP!=8 if( zywrle_level > 0 ){ CARDBPP* pFrame = (CARDBPP*)client->frameBuffer + y*client->width+x; int ret; client->appData.qualityLevel |= 0x80; ret = HandleZRLETile(client, buffer, buffer_end-buffer, x, y, w, h); client->appData.qualityLevel &= 0x7F; if( ret < 0 ){ return ret; } ZYWRLE_SYNTHESIZE( pFrame, pFrame, w, h, client->width, zywrle_level, (int*)client->zlib_buffer ); buffer += ret; }else #endif { #if REALBPP!=BPP int i,j; if(1+w*h*REALBPP/8>buffer_length) { rfbClientLog("expected %d bytes, got only %d (%dx%d)\n",1+w*h*REALBPP/8,buffer_length,w,h); return -3; } for(j=y*client->width; j<(y+h)*client->width; j+=client->width) for(i=x; iframeBuffer)[j+i] = UncompressCPixel(buffer); #else client->GotBitmap(client, buffer, x, y, w, h); buffer+=w*h*REALBPP/8; #endif } else if( type == 1 ) /* solid */ { CARDBPP color = UncompressCPixel(buffer); if(1+REALBPP/8>buffer_length) return -4; client->GotFillRect(client, x, y, w, h, color); buffer+=REALBPP/8; } else if( (type >= 2)&&(type <= 127) ) /* packed Palette */ { CARDBPP palette[16]; int i,j,shift, bpp=(type>4?(type>16?8:4):(type>2?2:1)), mask=(1<buffer_length) return -5; /* read palette */ for(i=0; iwidth; j<(y+h)*client->width; j+=client->width) { for(i=x,shift=8-bpp; iframeBuffer)[j+i] = palette[((*buffer)>>shift)&mask]; shift-=bpp; if(shift<0) { shift=8-bpp; buffer++; } } if(shift<8-bpp) buffer++; } } /* case 17 ... 127: not used, but valid */ else if( type == 128 ) /* plain RLE */ { int i=0,j=0; while(jbuffer_end) return -7; color = UncompressCPixel(buffer); buffer+=REALBPP/8; /* read run length */ length=1; while(*buffer==0xff) { if(buffer+1>=buffer_end) return -8; length+=*buffer; buffer++; } length+=*buffer; buffer++; while(j0) { ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color; length--; i++; if(i>=w) { i=0; j++; } } if(length>0) rfbClientLog("Warning: possible ZRLE corruption\n"); } } else if( type == 129 ) /* unused */ { return -8; } else if( type >= 130 ) /* palette RLE */ { CARDBPP palette[128]; int i,j; if(2+(type-128)*REALBPP/8>buffer_length) return -9; /* read palette */ for(i=0; i=buffer_end) return -10; color = palette[(*buffer)&0x7f]; length=1; if(*buffer&0x80) { if(buffer+1>=buffer_end) return -11; buffer++; /* read run length */ while(*buffer==0xff) { if(buffer+1>=buffer_end) return -8; length+=*buffer; buffer++; } length+=*buffer; } buffer++; while(j0) { ((CARDBPP*)client->frameBuffer)[(y+j)*client->width+x+i] = color; length--; i++; if(i>=w) { i=0; j++; } } if(length>0) rfbClientLog("Warning: possible ZRLE corruption\n"); } } } return buffer-buffer_copy; } #undef CARDBPP #undef CARDREALBPP #undef HandleZRLE #undef HandleZRLETile #undef UncompressCPixel #undef REALBPP #endif #undef UNCOMP