All pastes #160071 Raw Edit

Unnamed

public text v1 · immutable
·published 2006-09-03 20:45 UTC
rendered paste body
* converts a CP1252/ISO-8859-1(5) hybrid to UTF-8                           */
/* Features: 1. It never fails, all 00-FF chars are converted to valid UTF-8 */
/*           2. Uses CP1252 in the range 80-9f because ISO doesn't have any- */
/*              thing useful in this range and it helps us receive from mIRC */
/*           3. The five undefined chars in CP1252 80-9f are replaced with   */
/*              ISO-8859-15 control codes.                                   */
/*           4. Handles 0xa4 as a Euro symbol ala ISO-8859-15.               */
/*           5. Uses ISO-8859-1 (which matches CP1252) for everything else.  */
/*           6. This routine measured 3x faster than g_convert :)            */

static unsigned char *
iso_8859_1_to_utf8 (unsigned char *text, int len, gsize *bytes_written)
{
	unsigned int idx;
	unsigned char *res, *output;
	static const unsigned short lowtable[] = /* 74 byte table for 80-a4 */
	{
	/* compressed utf-8 table: if the first byte's 0x20 bit is set, it
	   indicates a 2-byte utf-8 sequence, otherwise prepend a 0xe2. */
		0x82ac, /* 80 Euro. CP1252 from here on... */
		0xe281, /* 81 NA */
		0x809a, /* 82 */
		0xe692, /* 83 */
		0x809e, /* 84 */
		0x80a6, /* 85 */
		0x80a0, /* 86 */
		0x80a1, /* 87 */
		0xeb86, /* 88 */
		0x80b0, /* 89 */
		0xe5a0, /* 8a */
		0x80b9, /* 8b */
		0xe592, /* 8c */
		0xe28d, /* 8d NA */
		0xe5bd, /* 8e */
		0xe28f, /* 8f NA */
		0xe290, /* 90 NA */
		0x8098, /* 91 */
		0x8099, /* 92 */
		0x809c, /* 93 */
		0x809d, /* 94 */
		0x80a2, /* 95 */
		0x8093, /* 96 */
		0x8094, /* 97 */
		0xeb9c, /* 98 */
		0x84a2, /* 99 */
		0xe5a1, /* 9a */
		0x80ba, /* 9b */
		0xe593, /* 9c */
		0xe29d, /* 9d NA */
		0xe5be, /* 9e */
		0xe5b8, /* 9f */
		0xe2a0, /* a0 */
		0xe2a1, /* a1 */
		0xe2a2, /* a2 */
		0xe2a3, /* a3 */
		0x82ac  /* a4 ISO-8859-15 Euro. */
	};

	if (len == -1)
		len = strlen (text);

	/* worst case scenario: every byte turns into 3 bytes */
	res = output = g_malloc ((len * 3) + 1);
	if (!output)
		return NULL;

	while (len)
	{
		if (G_LIKELY (*text < 0x80))
		{
			*output = *text;	/* ascii maps directly */
		}
		else if (*text <= 0xa4)	/* 80-a4 use a lookup table */
		{
			idx = *text - 0x80;
			if (lowtable[idx] & 0x2000)
			{
				*output++ = (lowtable[idx] >> 8) & 0xdf; /* 2 byte utf-8 */
				*output = lowtable[idx] & 0xff;
			}
			else
			{
				*output++ = 0xe2;	/* 3 byte utf-8 */
				*output++ = (lowtable[idx] >> 8) & 0xff;
				*output = lowtable[idx] & 0xff;
			}
		}
		else if (*text < 0xc0)
		{
			*output++ = 0xc2;
			*output = *text;
		}
		else
		{
			*output++ = 0xc3;
			*output = *text - 0x40;
		}
		output++;
		text++;
		len--;
	}
	*output = 0;	/* terminate */
	*bytes_written = output - res;

	return res;
}