~ubuntu-branches/ubuntu/oneiric/clamav/oneiric-security

/*
 *  HTML Entity & Encoding normalization.
 *
 *  Copyright (C) 2007-2008 Sourcefire, Inc.
 *
 *  Authors: Török Edvin
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 2 as
 *  published by the Free Software Foundation.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, write to the Free Software
 *  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
 *  MA 02110-1301, USA.
 */
#include "clamav-config.h"


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <errno.h>


#ifdef CL_THREAD_SAFE
#include <pthread.h>
#endif

#include <assert.h>

#include "clamav.h"
#include "others.h"
#include "htmlnorm.h"
#include "hashtab.h"
#include "entconv.h"
#include "entitylist.h"
#include "cltypes.h"

#ifdef HAVE_ICONV
#include <iconv.h>
#endif

#include "encoding_aliases.h"

#define MODULE_NAME "entconv: "

#define MAX_LINE 1024

#ifndef EILSEQ
#define EILSEQ 84
#endif

#ifndef HAVE_ICONV
typedef struct {
	enum encodings encoding;
	size_t size;
} * iconv_t;
#endif

static unsigned char tohex[] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};

/* TODO: gcc refuses to inline because it consider call unlikely and code size grows */
static inline unsigned char* u16_normalize(uint16_t u16, unsigned char* out, const ssize_t limit)
{
	assert(limit > 0 && "u16_normalize must be called with positive limit");
	/* \0 is just ignored */
	if(!u16) {
		return out;
	}

	if(u16 < 0xff) {
		assert((uint8_t)u16 != 0);
		*out++ = (uint8_t)u16;
	} else {
		size_t i;
		/* normalize only >255 to speed up */
		if(limit <=  8) {
			/* not enough space available */
			return NULL;
		}
		/* inline version of
		 * out += snprintf(out, max_num_length, "&#x%x;", u16) */
		out[0] = '&';
		out[1] = '#';
		out[2] = 'x';
		out[7] = ';';
		for(i=6; i >= 3; --i) {
			out[i] = tohex[u16 & 0xf];
			u16 >>= 4;
		}
		out += 8;
	}
	return out;
}

/* buffer must be at least 2 bytes in size */
unsigned char* u16_normalize_tobuffer(uint16_t u16, unsigned char* dst, size_t dst_size)
{
	unsigned char* out = u16_normalize(u16, dst, dst_size-1);
	if(out) {
		*out++ = '\0';
		return out;
	}
	return NULL;
}

const char* entity_norm(struct entity_conv* conv,const unsigned char* entity)
{
	struct cli_element* e = cli_hashtab_find(&entities_htable, (const char*)entity, strlen((const char*)entity));
	if(e && e->key) {
		unsigned char* out = u16_normalize(e->data, conv->entity_buff, sizeof(conv->entity_buff)-1);
		if(out) {
			*out++ = '\0';
			return (const char*)conv->entity_buff;
		}
	}
	return NULL;
}

#ifndef HAVE_ICONV
static size_t encoding_bytes(const char* fromcode, enum encodings* encoding)
{
	/* special case for these unusual byteorders */
	struct cli_element * e = cli_hashtab_find(&aliases_htable,fromcode,strlen(fromcode));
	if(e && e->key) {
		*encoding = e->data;
	} else {
		*encoding = E_OTHER;
	}

	switch(*encoding) {
		case E_UCS4:
		case E_UCS4_1234:
		case E_UCS4_4321:
		case E_UCS4_2143:
		case E_UCS4_3412:
			return 4;
		case E_UTF16:
		case E_UTF16_BE:
		case E_UTF16_LE:
			return 2;
		case E_UTF8:
		case E_UNKNOWN:
		case E_OTHER:
		default:
			return 1;
	}
}

static iconv_t iconv_open(const char *tocode, const char* fromcode)
{
	iconv_t iconv = cli_malloc(sizeof(*iconv));
	if(!iconv)
		return NULL;
	cli_dbgmsg(MODULE_NAME "Internal iconv\n");
	/* TODO: check that tocode is UTF16BE */
	iconv->size = encoding_bytes(fromcode,&iconv->encoding);
	return iconv;
}

static int iconv_close(iconv_t cd)
{
	if(cd)
		free(cd);
	return 0;
}

static int iconv(iconv_t iconv_struct,char **inbuf, size_t *inbytesleft,
		char** outbuf, size_t *outbytesleft)
{
	const uint8_t* input;
	uint8_t* output;
	size_t maxcopy, i;
	if(!inbuf || !outbuf) {
		return 0;
	}
	maxcopy = (*inbytesleft > *outbytesleft ? *outbytesleft  : *inbytesleft) & ~(iconv_struct->size - 1);
	input = (const uint8_t*)*inbuf;
	output = (uint8_t*)*outbuf;

	/*,maxcopy is aligned to data size */
	/* output is always utf16be !*/
	switch(iconv_struct->encoding) {
		case E_UCS4:
		case E_UCS4_1234:
			{
				for(i=0;i < maxcopy; i += 4) {
					if(!input[i+2] && !input[i+3]) {
						output[i/2] = input[i+1]; /* is compiler smart enough to replace /2, with >>1 ? */
						output[i/2+1] = input[i];
					}
					else {
						cli_dbgmsg(MODULE_NAME "Warning: unicode character out of utf16 range!\n");
						output[i/2] = 0xff;
						output[i/2+1] = 0xff;
					}
				}
				break;
			}
		case E_UCS4_4321:
			{
				const uint16_t *in = (const uint16_t*)input;/*UCS4_4321, and UTF16_BE have same endianness, no need for byteswap here*/
				uint16_t *out = (uint16_t*)output;
				for(i=0;i<maxcopy/2; i+=2) {
					if(!in[i]) {
						out[i/2] = in[i+1];
					}
					else {
						out[i/2] = 0xffff;
					}
				}
				break;
			}
		case E_UCS4_2143: 
			{
				const uint16_t *in = (const uint16_t*)input;
				uint16_t* out = (uint16_t*)output;
				for(i=0;i<maxcopy/2;i+=2) {
					if(!in[i+1])
						out[i/2] = in[i];
					else
						out[i/2] = 0xffff;
				}
				break;
			}
		case E_UCS4_3412:
			{
				for(i=0;i < maxcopy;i += 4) {
					if(!input[i] && !input[i+1]) {
						output[i/2] = input[i+3];
						output[i/2+1] = input[i+2];
					}
					else {
						output[i/2] = 0xff;
						output[i/2+1] = 0xff;
					}
				}
				break;
			}
		case E_UTF16:
		case E_UTF16_LE:
			{
				for(i=0;i < maxcopy;i += 2) {
					output[i] = input[i+1];
					output[i+1] = input[i];
				}
				break;
			}
		case E_UTF16_BE:
			memcpy(output,input,maxcopy);
			break;
		case E_UNKNOWN:
		case E_OTHER:
			{
				const size_t max_copy = *inbytesleft > (*outbytesleft/2) ? (*outbytesleft/2) : *inbytesleft;
				for(i=0;i<max_copy;i++) {
					output[i*2]   = 0;
					output[i*2+1] = input[i];
				}
				*outbytesleft -= max_copy*2;
				*inbytesleft  -= max_copy;
				*inbuf += max_copy;
				*outbuf += max_copy*2;
				if(*inbytesleft)
					return E2BIG;
				return 0;
			}
		case E_UTF8:
			{
				const size_t maxread  = *inbytesleft;
				const size_t maxwrite = *outbytesleft;
				size_t j;
				for(i=0,j=0 ; i < maxread && j < maxwrite;) {
					if(input[i] < 0x7F)  {
						output[j++] = 0;
						output[j++] = input[i++];
							}
					else if( (input[i]&0xE0) == 0xC0 ) {
						if ((input[i+1]&0xC0) == 0x80) {
							/* 2 bytes long 110yyyyy zzzzzzzz -> 00000yyy yyzzzzzz*/
							output[j++] = ((input[i] & 0x1F) >> 2) & 0x07;
							output[j++] = ((input[i] & 0x1F) << 6) | (input[i+1] & 0x3F);
						}
						else {
							cli_dbgmsg(MODULE_NAME "invalid UTF8 character encountered\n");
							break;
						}
						i+=2;
					}
					else if( (input[i]&0xE0) == 0xE0) {
						if( (input[i+1]&0xC0) == 0x80 && (input[i+2]&0xC0) == 0x80) {
							/* 3 bytes long 1110xxxx 10yyyyyy 10zzzzzzzz -> xxxxyyyy yyzzzzzz*/
							output[j++] = (input[i] << 4) | ((input[i+1] >> 2) & 0x0F);
							output[j++] = (input[i+1] << 6) | (input[i+2] & 0x3F);
						}
						else {
							cli_dbgmsg(MODULE_NAME "invalid UTF8 character encountered\n");
							break;
						}
						i+=3;
					}
					else if( (input[i]&0xF8) == 0xF0) {
						if((input[i+1]&0xC0) == 0x80 && (input[i+2]&0xC0) == 0x80 && (input[i+3]&0xC0) == 0x80) {
							/* 4 bytes long 11110www 10xxxxxx 10yyyyyy 10zzzzzz -> 000wwwxx xxxxyyyy yyzzzzzz*/
							cli_dbgmsg(MODULE_NAME "UTF8 character out of UTF16 range encountered\n");
							output[j++] = 0xff;
							output[j++] = 0xff;

							/*out[j++] = ((input[i] & 0x07) << 2) | ((input[i+1] >> 4) & 0x3);
							out[j++] = (input[i+1] << 4) | ((input[i+2] >> 2) & 0x0F);
							out[j++] = (input[i+2] << 6) | (input[i+2] & 0x3F);*/
						}
						else {
							cli_dbgmsg(MODULE_NAME "invalid UTF8 character encountered\n");
							break;
						}
						i+=4;
					}
					else {
						cli_dbgmsg(MODULE_NAME "invalid UTF8 character encountered\n");
						break;
					}							
				}
				*inbytesleft -= i;
				*outbytesleft -= j;
				*inbuf += i;
				*outbuf += j;
				if(*inbytesleft && *outbytesleft) {
					errno = EILSEQ;/* we had an early exit */
					return -1;
				}
				if(*inbytesleft) {
					errno = E2BIG;
					return -1;
				}
				return 0;
			}
	}
	
	*outbytesleft -= maxcopy;
	*inbytesleft  -= maxcopy;
	*inbuf += maxcopy;
	*outbuf += maxcopy;
	if(*inbytesleft) {
		errno = E2BIG;
		return -1;
	}
	return  0;
}

#else



#endif

static inline const char* detect_encoding(const unsigned char* bom, uint8_t* bom_found, uint8_t* enc_width)
{
	const char* encoding = NULL;
	int has_bom = 0;
	uint8_t enc_bytes = 1; /* default is UTF8, which has a minimum of 1 bytes */
	/* undecided 32-bit encodings are treated as ucs4, and
	 * 16 bit as utf16*/
	switch(bom[0]) {
		case 0x00:
			if(bom[1] == 0x00) {
				if(bom[2] == 0xFE && bom[3] == 0xFF) {
					encoding = UCS4_1234;/* UCS-4 big-endian*/
					has_bom = 1;
					enc_bytes = 4;
				}
				else if(bom[2] == 0xFF && bom[3] == 0xFE) {
					encoding = UCS4_2143;/* UCS-4 unusual order 2143 */
					has_bom = 1;
					enc_bytes = 4;
				}
				else if(bom[2] == 0x00 && bom[3] == 0x3C) {
					/* undecided, treat as ucs4 */
					encoding = UCS4_1234;
					enc_bytes = 4;
				}
				else if(bom[2] == 0x3C && bom[3] == 0x00) {
					encoding = UCS4_2143;
					enc_bytes = 4;
				}
			}/* 0x00 0x00 */
			else if(bom[1] == 0x3C) {
				if(bom[2] == 0x00) {
					if(bom[3] == 0x00) {
						encoding = UCS4_3412;
						enc_bytes = 4;
					}
					else if(bom[3] == 0x3F) {
						encoding = UTF16_BE;
						enc_bytes = 2;
					}
				}/*0x00 0x3C 0x00*/
			}/*0x00 0x3C*/
			break;
		case 0xFF:
			if(bom[1] == 0xFE) {
				if(bom[2] == 0x00 && bom[3] == 0x00) {
					encoding = UCS4_4321;
					enc_bytes = 4;
					has_bom = 1;
				}
				else {
					encoding = UTF16_LE;
					has_bom = 1;
					enc_bytes = 2;
				}
			}/*0xFF 0xFE*/
			break;
		case 0xFE:
			if(bom[1] == 0xFF) {
					if(bom[2] == 0x00 && bom[3] == 0x00) {
						encoding = UCS4_3412;
						enc_bytes = 4;
						has_bom = 1;
					}
					else {
						encoding = UTF16_BE;
						has_bom = 1;
						enc_bytes = 2;
					}
			}/*0xFE 0xFF*/
			break;
		case 0xEF:
			if(bom[1] == 0xBB && bom[2] == 0xBF)  {
					encoding = UTF8;
					has_bom = 1;
					/*enc_bytes = 4;- default, maximum 4 bytes*/
			}/*0xEF 0xBB 0xBF*/
			break;
		case 0x3C:
				if(bom[1] == 0x00) {
					if(bom[2] == 0x00 && bom[3] == 0x00) {
						encoding = UCS4_4321;
						enc_bytes = 4;
					}
					else if(bom[2] == 0x3F && bom[3] == 0x00) {
						encoding = UTF16_LE;
						enc_bytes = 2;
					}
				}/*0x3C 0x00*/
				else if(bom[1] == 0x3F && bom[2] == 0x78 && bom[3]==0x6D) {
					encoding = NULL;
					enc_bytes = 1;
				}/*0x3C 3F 78 6D*/
				break;
		case 0x4C:
				if(bom[1] == 0x6F && bom[2] == 0xA7 && bom[3] == 0x94) {
					cli_dbgmsg(MODULE_NAME "EBCDIC encoding is not supported in line mode\n");
					encoding = NULL;
					enc_bytes = 1;
				}/*4C 6F A7 94*/
				break;
	}/*switch*/
	*enc_width = enc_bytes;
	*bom_found = has_bom;
	return encoding;
}

/* detects UTF-16(LE/BE), UCS-4(all 4 variants).
 * UTF-8 and simple ASCII are ignored, because we can process those as text */
const char* encoding_detect_bom(const unsigned char* bom, const size_t length)
{
	uint8_t has_bom;
	uint8_t enc_width;
	const char* encoding;

	if(length < 4) {
		return NULL;
	}
	encoding = detect_encoding(bom, &has_bom, &enc_width);
	return enc_width > 1 ? encoding : NULL;
}

/*()-./0123456789:ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz*/
static const uint8_t encname_chars[256] = {
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
        0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,
        0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
};

/* checks that encoding is sane, and normalizes to uppercase */
static char* normalize_encoding(const unsigned char* enc)
{
	char* norm;
	size_t i, len;

	if(!enc)
		return NULL;
	len = strlen((const char*)enc);
	if(len > 32)
		return NULL;
	for(i=0;i<len;i++) {
		if(!encname_chars[enc[i]])
			return NULL;
	}
	norm = cli_malloc( len+1 );
	if(!norm)
		return NULL;
	for(i=0;i < len; i++)
		norm[i] = toupper(enc[i]);
	norm[len]='\0';
	return norm;
}

/* sarge leaks on iconv_open/iconv_close, so lets not open/close so many times,
 * just keep on each thread its own pool of iconvs*/

struct iconv_cache {
	iconv_t* tab;
	size_t     len;
	size_t   last;
	struct   cli_hashtable hashtab;
};

static void iconv_cache_init(struct iconv_cache* cache)
{
/*	cache->tab = NULL;
	cache->len = 0;
	cache->used = 0; - already done by memset*/
	cli_dbgmsg(MODULE_NAME "Initializing iconv pool:%p\n",(void*)cache);
	cli_hashtab_init(&cache->hashtab, 32);
}

static void iconv_cache_destroy(struct iconv_cache* cache)
{
	size_t i;
	cli_dbgmsg(MODULE_NAME "Destroying iconv pool:%p\n",(void*)cache);
	for(i=0;i < cache->last;i++) {
		cli_dbgmsg(MODULE_NAME "closing iconv:%p\n",cache->tab[i]);
		iconv_close(cache->tab[i]);
	}
	cli_hashtab_clear(&cache->hashtab);
	free(cache->hashtab.htable);
	free(cache->tab);
	free(cache);
}


#ifdef CL_THREAD_SAFE
static pthread_key_t iconv_pool_tls_key;
static pthread_once_t iconv_pool_tls_key_once = PTHREAD_ONCE_INIT;

/* destructor called for all threads that exit via pthread_exit, or cancellation. Unfortunately that doesn't include
 * the main thread, so we have to call this manually for the main thread.*/

static int cache_atexit_registered = 0;

static void iconv_pool_tls_instance_destroy(void* ptr)
{
	if(ptr) {
		iconv_cache_destroy(ptr);
	}
}

static void iconv_cache_cleanup_main(void)
{
	struct iconv_cache* cache = pthread_getspecific(iconv_pool_tls_key);
	if(cache) {
		iconv_pool_tls_instance_destroy(cache);
		pthread_setspecific(iconv_pool_tls_key,NULL);
	}
	pthread_key_delete(iconv_pool_tls_key);
}

static void iconv_pool_tls_key_alloc(void)
{
	pthread_key_create(&iconv_pool_tls_key, iconv_pool_tls_instance_destroy);
	if(!cache_atexit_registered) {
		cli_dbgmsg(MODULE_NAME "iconv:registering atexit\n");
		if(atexit(iconv_cache_cleanup_main)) {
			cli_dbgmsg(MODULE_NAME "failed to register atexit\n");
		}
		cache_atexit_registered = 1;
	}
}

static void init_iconv_pool_ifneeded(void)
{
	pthread_once(&iconv_pool_tls_key_once, iconv_pool_tls_key_alloc);
}

static inline struct iconv_cache* cache_get_tls_instance(void)
{
	struct iconv_cache* cache = pthread_getspecific(iconv_pool_tls_key);
	if(!cache) {
		cache = cli_calloc(1,sizeof(*cache));
		if(!cache) {
			cli_dbgmsg(MODULE_NAME "!Out of memory allocating TLS iconv instance\n");
			return NULL;
		}
		iconv_cache_init(cache);
		pthread_setspecific(iconv_pool_tls_key, cache);
	}
	return cache;
}

#else

static struct iconv_cache* global_iconv_cache = NULL;
static int    iconv_global_inited = 0;


static void iconv_cache_cleanup_main(void)
{
	iconv_cache_destroy(global_iconv_cache);
}

static inline void init_iconv_pool_ifneeded() 
{
	if(!iconv_global_inited) {
		global_iconv_cache = cli_calloc(1,sizeof(*global_iconv_cache));
		if(global_iconv_cache) {
			iconv_cache_init(global_iconv_cache);
			atexit(iconv_cache_cleanup_main);
			iconv_global_inited = 1;
		}
	}
}


static inline struct iconv_cache* cache_get_tls_instance(void)
{
	return global_iconv_cache;
}

#endif

static iconv_t iconv_open_cached(const char* fromcode)
{
	struct iconv_cache * cache;
	size_t idx;
	const size_t fromcode_len = strlen((const char*)fromcode);
	struct cli_element * e;
	iconv_t  iconv_struct;

	init_iconv_pool_ifneeded();
	cache = cache_get_tls_instance();/* gets TLS iconv pool */
	if(!cache) {
		cli_dbgmsg(MODULE_NAME "!Unable to get TLS iconv cache!\n");
		errno = EINVAL;
		return (iconv_t)-1;
	}

	e = cli_hashtab_find(&cache->hashtab, fromcode, fromcode_len);
	if(e && (e->data < 0 || (size_t)e->data > cache->len)) {
		e = NULL;
	}
	if(e) {
		size_t dummy_in, dummy_out;
		/* reset state */
		iconv(cache->tab[e->data], NULL, &dummy_in, NULL, &dummy_out);
		return cache->tab[e->data];
	}
	cli_dbgmsg(MODULE_NAME "iconv not found in cache, for encoding:%s\n",fromcode);
	iconv_struct = iconv_open("UTF-16BE",(const char*)fromcode);
	if(iconv_struct != (iconv_t)-1) {
		idx = cache->last++;
		if(idx >= cache->len) {
			cache->len += 16;
			cache->tab = cli_realloc2(cache->tab, cache->len*sizeof(cache->tab[0]));
			if(!cache->tab) {
				cli_dbgmsg(MODULE_NAME "!Out of mem in iconv-pool\n");
				errno = ENOMEM;
				return (iconv_t)-1;
			}
		}

		cli_hashtab_insert(&cache->hashtab, fromcode, fromcode_len, idx);
		cache->tab[idx] = iconv_struct;
		cli_dbgmsg(MODULE_NAME "iconv_open(),for:%s -> %p\n",fromcode,(void*)cache->tab[idx]);
		return cache->tab[idx];
	}
	return (iconv_t)-1;
}

static int in_iconv_u16(const m_area_t* in_m_area, iconv_t* iconv_struct, m_area_t* out_m_area)
{
	char   tmp4[4];
	size_t inleft = in_m_area->length - in_m_area->offset;
	size_t rc, alignfix;
	char*  input   = (char*)in_m_area->buffer + in_m_area->offset;
	size_t outleft = out_m_area->length > 0 ? out_m_area->length : 0;
	char* out      = (char*)out_m_area->buffer;
	char err[128];

	out_m_area->offset = 0;
	if(!inleft) {
		return 0;
	}
	/* convert encoding conv->tmp_area. conv->out_area */
	alignfix = inleft%4;/* iconv gives an error if we give him 3 bytes to convert, 
			       and we are using ucs4, ditto for utf16, and 1 byte*/
	inleft -= alignfix;

	if(!inleft && alignfix) {
		/* EOF, and we have less than 4 bytes to convert */
		memset(tmp4, 0, 4);
		memcpy(tmp4, input, alignfix);
		input = tmp4;
		inleft = 4;
		alignfix = 0;
	}

	while (inleft && (outleft >= 2)) { /* iconv doesn't like inleft to be 0 */
		const size_t outleft_last = outleft;
		assert(*iconv_struct != (iconv_t)-1);
		rc = iconv(*iconv_struct, &input,  &inleft, &out, &outleft);
		if(rc == (size_t)-1) {
			if(errno == E2BIG) {
				/* not enough space in output buffer */
				break;
			}
			/*cli_dbgmsg(MODULE_NAME "iconv error:%s\n", cli_strerror(errno, err, sizeof(err)));*/
		} else if(outleft == outleft_last) {
			cli_dbgmsg(MODULE_NAME "iconv stall (no output)\n");
		} else {
			/* everything ok */
			continue;
		}
		/*cli_dbgmsg(MODULE_NAME "resuming (inleft:%lu, outleft:%lu, inpos:%ld, %ld)\n",
					inleft, outleft, input - (char*)in_m_area->buffer,
					out - (char*)out_m_area->buffer);*/
		/* output raw byte, and resume at next byte */
		if(outleft < 2) break;
		outleft -= 2;
		*out++ = 0;
		*out++ = *input++;
		inleft--;
	}
	cli_dbgmsg("in_iconv_u16: unprocessed bytes: %lu\n", (unsigned long)inleft);
	if(out_m_area->length >= 0 && out_m_area->length >= (off_t)outleft) {
		out_m_area->length -= (off_t)outleft;
	} else {
		cli_dbgmsg(MODULE_NAME "outleft overflown, ignoring\n");
		out_m_area->length = 0;
	}
	out_m_area->offset  = 0;
	return 0;
}

int encoding_normalize_toascii(const m_area_t* in_m_area, const char* initial_encoding, m_area_t* out_m_area)
{
	iconv_t iconv_struct;
	off_t i, j;
	char *encoding;

	if(!initial_encoding || !in_m_area || !out_m_area) {
		return CL_ENULLARG;
	}

	encoding = normalize_encoding((const unsigned char*)initial_encoding);
	if(!encoding) {
		cli_dbgmsg(MODULE_NAME "encoding name is not valid, ignoring\n");
		return -1;
	}

	cli_dbgmsg(MODULE_NAME "Encoding %s\n", encoding);
	iconv_struct = iconv_open_cached( encoding );
	if(iconv_struct == (iconv_t)-1) {
		cli_dbgmsg(MODULE_NAME "Encoding not accepted by iconv_open(): %s\n", encoding);
		free(encoding);
		return -1;
	}
	free(encoding);
	in_iconv_u16(in_m_area, &iconv_struct, out_m_area);
	for(i = 0, j = 0; i < out_m_area->length ; i += 2) {
		const unsigned char c = (out_m_area->buffer[i] << 4) + out_m_area->buffer[i+1];
		if(c) {
			out_m_area->buffer[j++] = c;
		}
	}
	out_m_area->length = j;
	return 0;
}