You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
libtdevnc/libvncserver/rfbssl_gnutls.c

271 lines
7.1 KiB

/*
* rfbssl_gnutls.c - Secure socket funtions (gnutls version)
*/
/*
* Copyright (C) 2011 Gernot Tenchio
*
* This is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This software is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this software; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
* USA.
*/
#include "rfbssl.h"
#include <gnutls/gnutls.h>
#include <errno.h>
struct rfbssl_ctx {
char peekbuf[2048];
int peeklen;
int peekstart;
gnutls_session_t session;
gnutls_certificate_credentials_t x509_cred;
gnutls_dh_params_t dh_params;
#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
gnutls_rsa_params_t rsa_params;
#endif
};
void rfbssl_log_func(int level, const char *msg)
{
rfbErr("SSL: %s", msg);
}
static void rfbssl_error(const char *msg, int e)
{
rfbErr("%s: %s (%ld)\n", msg, gnutls_strerror(e), e);
}
static int rfbssl_init_session(struct rfbssl_ctx *ctx, int fd)
{
gnutls_session_t session;
int ret;
if (!GNUTLS_E_SUCCESS == (ret = gnutls_init(&session, GNUTLS_SERVER))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_priority_set_direct(session, "EXPORT", NULL))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_credentials_set(session, GNUTLS_CRD_CERTIFICATE, ctx->x509_cred))) {
/* */
} else {
gnutls_session_enable_compatibility_mode(session);
gnutls_transport_set_ptr(session, (gnutls_transport_ptr_t)(uintptr_t)fd);
ctx->session = session;
}
return ret;
}
static int generate_dh_params(struct rfbssl_ctx *ctx)
{
int ret;
if (GNUTLS_E_SUCCESS == (ret = gnutls_dh_params_init(&ctx->dh_params)))
ret = gnutls_dh_params_generate2(ctx->dh_params, 1024);
return ret;
}
#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
static int generate_rsa_params(struct rfbssl_ctx *ctx)
{
int ret;
if (GNUTLS_E_SUCCESS == (ret = gnutls_rsa_params_init(&ctx->rsa_params)))
ret = gnutls_rsa_params_generate2(ctx->rsa_params, 512);
return ret;
}
#endif
struct rfbssl_ctx *rfbssl_init_global(char *key, char *cert)
{
int ret = GNUTLS_E_SUCCESS;
struct rfbssl_ctx *ctx = NULL;
if (NULL == (ctx = malloc(sizeof(struct rfbssl_ctx)))) {
ret = GNUTLS_E_MEMORY_ERROR;
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_global_init())) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_certificate_allocate_credentials(&ctx->x509_cred))) {
/* */
} else if ((ret = gnutls_certificate_set_x509_trust_file(ctx->x509_cred, cert, GNUTLS_X509_FMT_PEM)) < 0) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = gnutls_certificate_set_x509_key_file(ctx->x509_cred, cert, key, GNUTLS_X509_FMT_PEM))) {
/* */
} else if (!GNUTLS_E_SUCCESS == (ret = generate_dh_params(ctx))) {
/* */
#ifdef I_LIKE_RSA_PARAMS_THAT_MUCH
} else if (!GNUTLS_E_SUCCESS == (ret = generate_rsa_params(ctx))) {
/* */
#endif
} else {
gnutls_global_set_log_function(rfbssl_log_func);
gnutls_global_set_log_level(1);
gnutls_certificate_set_dh_params(ctx->x509_cred, ctx->dh_params);
/* newly allocated memory should be initialized, at least where it is important */
ctx->peekstart = ctx->peeklen = 0;
return ctx;
}
free(ctx);
return NULL;
}
int rfbssl_init(rfbClientPtr cl)
{
int ret = -1;
struct rfbssl_ctx *ctx;
char *keyfile;
if (!(keyfile = cl->screen->sslkeyfile))
keyfile = cl->screen->sslcertfile;
if (NULL == (ctx = rfbssl_init_global(keyfile, cl->screen->sslcertfile))) {
/* */
} else if (GNUTLS_E_SUCCESS != (ret = rfbssl_init_session(ctx, cl->sock))) {
/* */
} else {
while (GNUTLS_E_SUCCESS != (ret = gnutls_handshake(ctx->session))) {
if (ret == GNUTLS_E_AGAIN)
continue;
break;
}
}
if (ret != GNUTLS_E_SUCCESS) {
rfbssl_error(__func__, ret);
} else {
cl->sslctx = (rfbSslCtx *)ctx;
rfbLog("%s protocol initialized\n", gnutls_protocol_get_name(gnutls_protocol_get_version(ctx->session)));
}
return ret;
}
static int rfbssl_do_read(rfbClientPtr cl, char *buf, int bufsize)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
int ret;
while ((ret = gnutls_record_recv(ctx->session, buf, bufsize)) < 0) {
if (ret == GNUTLS_E_AGAIN) {
/* continue */
} else if (ret == GNUTLS_E_INTERRUPTED) {
/* continue */
} else {
break;
}
}
if (ret < 0) {
rfbssl_error(__func__, ret);
errno = EIO;
ret = -1;
}
return ret < 0 ? -1 : ret;
}
int rfbssl_write(rfbClientPtr cl, const char *buf, int bufsize)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
int ret;
while ((ret = gnutls_record_send(ctx->session, buf, bufsize)) < 0) {
if (ret == GNUTLS_E_AGAIN) {
/* continue */
} else if (ret == GNUTLS_E_INTERRUPTED) {
/* continue */
} else {
break;
}
}
if (ret < 0)
rfbssl_error(__func__, ret);
return ret;
}
static void rfbssl_gc_peekbuf(struct rfbssl_ctx *ctx, int bufsize)
{
if (ctx->peekstart) {
int spaceleft = sizeof(ctx->peekbuf) - ctx->peeklen - ctx->peekstart;
if (spaceleft < bufsize) {
memmove(ctx->peekbuf, ctx->peekbuf + ctx->peekstart, ctx->peeklen);
ctx->peekstart = 0;
}
}
}
static int __rfbssl_read(rfbClientPtr cl, char *buf, int bufsize, int peek)
{
int ret = 0;
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
rfbssl_gc_peekbuf(ctx, bufsize);
if (ctx->peeklen) {
/* If we have any peek data, simply return that. */
ret = bufsize < ctx->peeklen ? bufsize : ctx->peeklen;
memcpy (buf, ctx->peekbuf + ctx->peekstart, ret);
if (!peek) {
ctx->peeklen -= ret;
if (ctx->peeklen != 0)
ctx->peekstart += ret;
else
ctx->peekstart = 0;
}
}
if (ret < bufsize) {
int n;
/* read the remaining data */
if ((n = rfbssl_do_read(cl, buf + ret, bufsize - ret)) <= 0) {
rfbErr("rfbssl_%s: %s error\n", __func__, peek ? "peek" : "read");
return n;
}
if (peek) {
memcpy(ctx->peekbuf + ctx->peekstart + ctx->peeklen, buf + ret, n);
ctx->peeklen += n;
}
ret += n;
}
return ret;
}
int rfbssl_read(rfbClientPtr cl, char *buf, int bufsize)
{
return __rfbssl_read(cl, buf, bufsize, 0);
}
int rfbssl_peek(rfbClientPtr cl, char *buf, int bufsize)
{
return __rfbssl_read(cl, buf, bufsize, 1);
}
int rfbssl_pending(rfbClientPtr cl)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
int ret = ctx->peeklen;
if (ret <= 0)
ret = gnutls_record_check_pending(ctx->session);
return ret;
}
void rfbssl_destroy(rfbClientPtr cl)
{
struct rfbssl_ctx *ctx = (struct rfbssl_ctx *)cl->sslctx;
gnutls_bye(ctx->session, GNUTLS_SHUT_WR);
gnutls_deinit(ctx->session);
gnutls_certificate_free_credentials(ctx->x509_cred);
}