summaryrefslogtreecommitdiff
path: root/source4/lib/charset/charcnv.c
diff options
context:
space:
mode:
Diffstat (limited to 'source4/lib/charset/charcnv.c')
-rw-r--r--source4/lib/charset/charcnv.c723
1 files changed, 723 insertions, 0 deletions
diff --git a/source4/lib/charset/charcnv.c b/source4/lib/charset/charcnv.c
new file mode 100644
index 0000000000..e632790155
--- /dev/null
+++ b/source4/lib/charset/charcnv.c
@@ -0,0 +1,723 @@
+/*
+ Unix SMB/CIFS implementation.
+ Character set conversion Extensions
+ Copyright (C) Igor Vergeichik <iverg@mail.ru> 2001
+ Copyright (C) Andrew Tridgell 2001
+ Copyright (C) Simo Sorce 2001
+
+ This program is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with this program; if not, write to the Free Software
+ Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+
+*/
+#include "includes.h"
+#include "system/iconv.h"
+#include "pstring.h"
+
+/**
+ * @file
+ *
+ * @brief Character-set conversion routines built on our iconv.
+ *
+ * @note Samba's internal character set (at least in the 3.0 series)
+ * is always the same as the one for the Unix filesystem. It is
+ * <b>not</b> necessarily UTF-8 and may be different on machines that
+ * need i18n filenames to be compatible with Unix software. It does
+ * have to be a superset of ASCII. All multibyte sequences must start
+ * with a byte with the high bit set.
+ *
+ * @sa lib/iconv.c
+ */
+
+/**
+ * Return the name of a charset to give to iconv().
+ **/
+static const char *charset_name(charset_t ch)
+{
+ const char *ret = NULL;
+
+ if (ch == CH_UTF16) ret = "UTF-16LE";
+ else if (ch == CH_UNIX) ret = lp_unix_charset();
+ else if (ch == CH_DOS) ret = lp_dos_charset();
+ else if (ch == CH_DISPLAY) ret = lp_display_charset();
+ else if (ch == CH_UTF8) ret = "UTF8";
+ else if (ch == CH_UTF16BE) ret = "UTF-16BE";
+
+ if (!ret || !*ret) ret = "ASCII";
+ return ret;
+}
+
+static smb_iconv_t conv_handles[NUM_CHARSETS][NUM_CHARSETS];
+
+/**
+ re-initialize iconv conversion descriptors
+**/
+void init_iconv(void)
+{
+ charset_t c1, c2;
+ for (c1=0;c1<NUM_CHARSETS;c1++) {
+ for (c2=0;c2<NUM_CHARSETS;c2++) {
+ if (conv_handles[c1][c2] != NULL) {
+ if (conv_handles[c1][c2] != (smb_iconv_t)-1) {
+ smb_iconv_close(conv_handles[c1][c2]);
+ }
+ conv_handles[c1][c2] = NULL;
+ }
+ }
+ }
+
+}
+
+/*
+ on-demand initialisation of conversion handles
+*/
+static smb_iconv_t get_conv_handle(charset_t from, charset_t to)
+{
+ const char *n1, *n2;
+ static int initialised;
+ /* auto-free iconv memory on exit so valgrind reports are easier
+ to look at */
+ if (initialised == 0) {
+ initialised = 1;
+
+#ifdef LC_ALL
+ /* we set back the locale to C to get ASCII-compatible
+ toupper/lower functions. For now we do not need
+ any other POSIX localisations anyway. When we
+ should really need localized string functions one
+ day we need to write our own ascii_tolower etc.
+ */
+ setlocale(LC_ALL, "C");
+#endif
+
+ atexit(init_iconv);
+ }
+
+ if (conv_handles[from][to]) {
+ return conv_handles[from][to];
+ }
+
+ n1 = charset_name(from);
+ n2 = charset_name(to);
+
+ conv_handles[from][to] = smb_iconv_open(n2,n1);
+
+ if (conv_handles[from][to] == (smb_iconv_t)-1) {
+ if ((from == CH_DOS || to == CH_DOS) &&
+ strcasecmp(charset_name(CH_DOS), "ASCII") != 0) {
+ DEBUG(0,("dos charset '%s' unavailable - using ASCII\n",
+ charset_name(CH_DOS)));
+ lp_set_cmdline("dos charset", "ASCII");
+
+ n1 = charset_name(from);
+ n2 = charset_name(to);
+
+ conv_handles[from][to] = smb_iconv_open(n2,n1);
+ }
+ }
+
+ return conv_handles[from][to];
+}
+
+
+/**
+ * Convert string from one encoding to another, making error checking etc
+ *
+ * @param src pointer to source string (multibyte or singlebyte)
+ * @param srclen length of the source string in bytes
+ * @param dest pointer to destination string (multibyte or singlebyte)
+ * @param destlen maximal length allowed for string
+ * @returns the number of bytes occupied in the destination
+ **/
+ssize_t convert_string(charset_t from, charset_t to,
+ void const *src, size_t srclen,
+ void *dest, size_t destlen)
+{
+ size_t i_len, o_len;
+ size_t retval;
+ const char* inbuf = (const char*)src;
+ char* outbuf = (char*)dest;
+ smb_iconv_t descriptor;
+
+ if (srclen == (size_t)-1)
+ srclen = strlen(src)+1;
+
+ descriptor = get_conv_handle(from, to);
+
+ if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
+ /* conversion not supported, use as is */
+ size_t len = MIN(srclen,destlen);
+ memcpy(dest,src,len);
+ return len;
+ }
+
+ i_len=srclen;
+ o_len=destlen;
+ retval = smb_iconv(descriptor, &inbuf, &i_len, &outbuf, &o_len);
+ if(retval==(size_t)-1) {
+ const char *reason;
+ switch(errno) {
+ case EINVAL:
+ reason="Incomplete multibyte sequence";
+ break;
+ case E2BIG:
+ reason="No more room";
+ if (from == CH_UNIX) {
+ DEBUG(0,("E2BIG: convert_string(%s,%s): srclen=%d destlen=%d - '%s'\n",
+ charset_name(from), charset_name(to),
+ (int)srclen, (int)destlen,
+ (const char *)src));
+ } else {
+ DEBUG(0,("E2BIG: convert_string(%s,%s): srclen=%d destlen=%d\n",
+ charset_name(from), charset_name(to),
+ (int)srclen, (int)destlen));
+ }
+ break;
+ case EILSEQ:
+ reason="Illegal multibyte sequence";
+ break;
+ }
+ /* smb_panic(reason); */
+ }
+ return destlen-o_len;
+}
+
+/**
+ * Convert between character sets, allocating a new buffer using talloc for the result.
+ *
+ * @param srclen length of source buffer.
+ * @param dest always set at least to NULL
+ * @note -1 is not accepted for srclen.
+ *
+ * @returns Size in bytes of the converted string; or -1 in case of error.
+ **/
+
+ssize_t convert_string_talloc(TALLOC_CTX *ctx, charset_t from, charset_t to,
+ void const *src, size_t srclen, void **dest)
+{
+ size_t i_len, o_len, destlen;
+ size_t retval;
+ const char *inbuf = (const char *)src;
+ char *outbuf, *ob;
+ smb_iconv_t descriptor;
+
+ *dest = NULL;
+
+ if (src == NULL || srclen == (size_t)-1 || srclen == 0)
+ return (size_t)-1;
+
+ descriptor = get_conv_handle(from, to);
+
+ if (descriptor == (smb_iconv_t)-1 || descriptor == (smb_iconv_t)0) {
+ /* conversion not supported, return -1*/
+ DEBUG(3, ("convert_string_talloc: conversion from %s to %s not supported!\n",
+ charset_name(from), charset_name(to)));
+ return -1;
+ }
+
+ /* it is _very_ rare that a conversion increases the size by
+ more than 3x */
+ destlen = srclen;
+ outbuf = NULL;
+convert:
+ destlen = 2 + (destlen*3);
+ ob = talloc_realloc(ctx, outbuf, char, destlen);
+ if (!ob) {
+ DEBUG(0, ("convert_string_talloc: realloc failed!\n"));
+ talloc_free(outbuf);
+ return (size_t)-1;
+ } else {
+ outbuf = ob;
+ }
+
+ /* we give iconv 2 less bytes to allow us to terminate at the
+ end */
+ i_len = srclen;
+ o_len = destlen-2;
+ retval = smb_iconv(descriptor,
+ &inbuf, &i_len,
+ &outbuf, &o_len);
+ if(retval == (size_t)-1) {
+ const char *reason="unknown error";
+ switch(errno) {
+ case EINVAL:
+ reason="Incomplete multibyte sequence";
+ break;
+ case E2BIG:
+ goto convert;
+ case EILSEQ:
+ reason="Illegal multibyte sequence";
+ break;
+ }
+ DEBUG(0,("Conversion error: %s(%s)\n",reason,inbuf));
+ talloc_free(ob);
+ return (size_t)-1;
+ }
+
+ destlen = (destlen-2) - o_len;
+
+ /* guarantee null termination in all charsets */
+ SSVAL(ob, destlen, 0);
+
+ *dest = ob;
+
+ return destlen;
+}
+
+/**
+ * Copy a string from a char* unix src to a dos codepage string destination.
+ *
+ * @return the number of bytes occupied by the string in the destination.
+ *
+ * @param flags can include
+ * <dl>
+ * <dt>STR_TERMINATE</dt> <dd>means include the null termination</dd>
+ * <dt>STR_UPPER</dt> <dd>means uppercase in the destination</dd>
+ * </dl>
+ *
+ * @param dest_len the maximum length in bytes allowed in the
+ * destination. If @p dest_len is -1 then no maximum is used.
+ **/
+ssize_t push_ascii(void *dest, const char *src, size_t dest_len, int flags)
+{
+ size_t src_len;
+ ssize_t ret;
+
+ if (flags & STR_UPPER) {
+ char *tmpbuf = strupper_talloc(NULL, src);
+ if (tmpbuf == NULL) {
+ return -1;
+ }
+ ret = push_ascii(dest, tmpbuf, dest_len, flags & ~STR_UPPER);
+ talloc_free(tmpbuf);
+ return ret;
+ }
+
+ /* treat a pstring as "unlimited" length */
+ if (dest_len == (size_t)-1)
+ dest_len = sizeof(pstring);
+
+ src_len = strlen(src);
+
+ if (flags & (STR_TERMINATE | STR_TERMINATE_ASCII))
+ src_len++;
+
+ return convert_string(CH_UNIX, CH_DOS, src, src_len, dest, dest_len);
+}
+
+/**
+ * Copy a string from a unix char* src to an ASCII destination,
+ * allocating a buffer using talloc().
+ *
+ * @param dest always set at least to NULL
+ *
+ * @returns The number of bytes occupied by the string in the destination
+ * or -1 in case of error.
+ **/
+ssize_t push_ascii_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
+{
+ size_t src_len = strlen(src)+1;
+
+ *dest = NULL;
+ return convert_string_talloc(ctx, CH_UNIX, CH_DOS, src, src_len, (void **)dest);
+}
+
+
+/**
+ * Copy a string from a dos codepage source to a unix char* destination.
+ *
+ * The resulting string in "dest" is always null terminated.
+ *
+ * @param flags can have:
+ * <dl>
+ * <dt>STR_TERMINATE</dt>
+ * <dd>STR_TERMINATE means the string in @p src
+ * is null terminated, and src_len is ignored.</dd>
+ * </dl>
+ *
+ * @param src_len is the length of the source area in bytes.
+ * @returns the number of bytes occupied by the string in @p src.
+ **/
+ssize_t pull_ascii(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
+{
+ size_t ret;
+
+ if (dest_len == (size_t)-1)
+ dest_len = sizeof(pstring);
+
+ if (flags & (STR_TERMINATE | STR_TERMINATE_ASCII)) {
+ if (src_len == (size_t)-1) {
+ src_len = strlen(src) + 1;
+ } else {
+ size_t len = strnlen(src, src_len);
+ if (len < src_len)
+ len++;
+ src_len = len;
+ }
+ }
+
+ ret = convert_string(CH_DOS, CH_UNIX, src, src_len, dest, dest_len);
+
+ if (dest_len)
+ dest[MIN(ret, dest_len-1)] = 0;
+
+ return src_len;
+}
+
+/**
+ * Copy a string from a char* src to a unicode destination.
+ *
+ * @returns the number of bytes occupied by the string in the destination.
+ *
+ * @param flags can have:
+ *
+ * <dl>
+ * <dt>STR_TERMINATE <dd>means include the null termination.
+ * <dt>STR_UPPER <dd>means uppercase in the destination.
+ * <dt>STR_NOALIGN <dd>means don't do alignment.
+ * </dl>
+ *
+ * @param dest_len is the maximum length allowed in the
+ * destination. If dest_len is -1 then no maxiumum is used.
+ **/
+ssize_t push_ucs2(void *dest, const char *src, size_t dest_len, int flags)
+{
+ size_t len=0;
+ size_t src_len = strlen(src);
+ size_t ret;
+
+ if (flags & STR_UPPER) {
+ char *tmpbuf = strupper_talloc(NULL, src);
+ if (tmpbuf == NULL) {
+ return -1;
+ }
+ ret = push_ucs2(dest, tmpbuf, dest_len, flags & ~STR_UPPER);
+ talloc_free(tmpbuf);
+ return ret;
+ }
+
+ /* treat a pstring as "unlimited" length */
+ if (dest_len == (size_t)-1)
+ dest_len = sizeof(pstring);
+
+ if (flags & STR_TERMINATE)
+ src_len++;
+
+ if (ucs2_align(NULL, dest, flags)) {
+ *(char *)dest = 0;
+ dest = (void *)((char *)dest + 1);
+ if (dest_len) dest_len--;
+ len++;
+ }
+
+ /* ucs2 is always a multiple of 2 bytes */
+ dest_len &= ~1;
+
+ ret = convert_string(CH_UNIX, CH_UTF16, src, src_len, dest, dest_len);
+ if (ret == (size_t)-1) {
+ return 0;
+ }
+
+ len += ret;
+
+ return len;
+}
+
+
+/**
+ * Copy a string from a unix char* src to a UCS2 destination,
+ * allocating a buffer using talloc().
+ *
+ * @param dest always set at least to NULL
+ *
+ * @returns The number of bytes occupied by the string in the destination
+ * or -1 in case of error.
+ **/
+ssize_t push_ucs2_talloc(TALLOC_CTX *ctx, void **dest, const char *src)
+{
+ size_t src_len = strlen(src)+1;
+ *dest = NULL;
+ return convert_string_talloc(ctx, CH_UNIX, CH_UTF16, src, src_len, dest);
+}
+
+
+/**
+ * Copy a string from a unix char* src to a UTF-8 destination, allocating a buffer using talloc
+ *
+ * @param dest always set at least to NULL
+ *
+ * @returns The number of bytes occupied by the string in the destination
+ **/
+
+ssize_t push_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
+{
+ size_t src_len = strlen(src)+1;
+
+ *dest = NULL;
+ return convert_string_talloc(ctx, CH_UNIX, CH_UTF8, src, src_len, (void **)dest);
+}
+
+/**
+ Copy a string from a ucs2 source to a unix char* destination.
+ Flags can have:
+ STR_TERMINATE means the string in src is null terminated.
+ STR_NOALIGN means don't try to align.
+ if STR_TERMINATE is set then src_len is ignored if it is -1.
+ src_len is the length of the source area in bytes
+ Return the number of bytes occupied by the string in src.
+ The resulting string in "dest" is always null terminated.
+**/
+
+size_t pull_ucs2(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
+{
+ size_t ret;
+
+ if (dest_len == (size_t)-1)
+ dest_len = sizeof(pstring);
+
+ if (ucs2_align(NULL, src, flags)) {
+ src = (const void *)((const char *)src + 1);
+ if (src_len > 0)
+ src_len--;
+ }
+
+ if (flags & STR_TERMINATE) {
+ if (src_len == (size_t)-1) {
+ src_len = utf16_len(src);
+ } else {
+ src_len = utf16_len_n(src, src_len);
+ }
+ }
+
+ /* ucs2 is always a multiple of 2 bytes */
+ if (src_len != (size_t)-1)
+ src_len &= ~1;
+
+ ret = convert_string(CH_UTF16, CH_UNIX, src, src_len, dest, dest_len);
+ if (dest_len)
+ dest[MIN(ret, dest_len-1)] = 0;
+
+ return src_len;
+}
+
+/**
+ * Copy a string from a UCS2 src to a unix char * destination, allocating a buffer using talloc
+ *
+ * @param dest always set at least to NULL
+ *
+ * @returns The number of bytes occupied by the string in the destination
+ **/
+
+ssize_t pull_ucs2_talloc(TALLOC_CTX *ctx, char **dest, const void *src)
+{
+ size_t src_len = utf16_len(src);
+ *dest = NULL;
+ return convert_string_talloc(ctx, CH_UTF16, CH_UNIX, src, src_len, (void **)dest);
+}
+
+/**
+ * Copy a string from a UTF-8 src to a unix char * destination, allocating a buffer using talloc
+ *
+ * @param dest always set at least to NULL
+ *
+ * @returns The number of bytes occupied by the string in the destination
+ **/
+
+ssize_t pull_utf8_talloc(TALLOC_CTX *ctx, char **dest, const char *src)
+{
+ size_t src_len = strlen(src)+1;
+ *dest = NULL;
+ return convert_string_talloc(ctx, CH_UTF8, CH_UNIX, src, src_len, (void **)dest);
+}
+
+/**
+ Copy a string from a char* src to a unicode or ascii
+ dos codepage destination choosing unicode or ascii based on the
+ flags in the SMB buffer starting at base_ptr.
+ Return the number of bytes occupied by the string in the destination.
+ flags can have:
+ STR_TERMINATE means include the null termination.
+ STR_UPPER means uppercase in the destination.
+ STR_ASCII use ascii even with unicode packet.
+ STR_NOALIGN means don't do alignment.
+ dest_len is the maximum length allowed in the destination. If dest_len
+ is -1 then no maxiumum is used.
+**/
+
+ssize_t push_string(void *dest, const char *src, size_t dest_len, int flags)
+{
+ if (flags & STR_ASCII) {
+ return push_ascii(dest, src, dest_len, flags);
+ } else if (flags & STR_UNICODE) {
+ return push_ucs2(dest, src, dest_len, flags);
+ } else {
+ smb_panic("push_string requires either STR_ASCII or STR_UNICODE flag to be set");
+ return -1;
+ }
+}
+
+
+/**
+ Copy a string from a unicode or ascii source (depending on
+ the packet flags) to a char* destination.
+ Flags can have:
+ STR_TERMINATE means the string in src is null terminated.
+ STR_UNICODE means to force as unicode.
+ STR_ASCII use ascii even with unicode packet.
+ STR_NOALIGN means don't do alignment.
+ if STR_TERMINATE is set then src_len is ignored is it is -1
+ src_len is the length of the source area in bytes.
+ Return the number of bytes occupied by the string in src.
+ The resulting string in "dest" is always null terminated.
+**/
+
+ssize_t pull_string(char *dest, const void *src, size_t dest_len, size_t src_len, int flags)
+{
+ if (flags & STR_ASCII) {
+ return pull_ascii(dest, src, dest_len, src_len, flags);
+ } else if (flags & STR_UNICODE) {
+ return pull_ucs2(dest, src, dest_len, src_len, flags);
+ } else {
+ smb_panic("pull_string requires either STR_ASCII or STR_UNICODE flag to be set");
+ return -1;
+ }
+}
+
+
+/*
+ return the unicode codepoint for the next multi-byte CH_UNIX character
+ in the string
+
+ also return the number of bytes consumed (which tells the caller
+ how many bytes to skip to get to the next CH_UNIX character)
+
+ return INVALID_CODEPOINT if the next character cannot be converted
+*/
+codepoint_t next_codepoint(const char *str, size_t *size)
+{
+ /* it cannot occupy more than 4 bytes in UTF16 format */
+ uint8_t buf[4];
+ smb_iconv_t descriptor;
+ size_t ilen_orig;
+ size_t ilen;
+ size_t olen;
+ char *outbuf;
+
+ if ((str[0] & 0x80) == 0) {
+ *size = 1;
+ return (codepoint_t)str[0];
+ }
+
+ /* we assume that no multi-byte character can take
+ more than 5 bytes. This is OK as we only
+ support codepoints up to 1M */
+ ilen_orig = strnlen(str, 5);
+ ilen = ilen_orig;
+
+ descriptor = get_conv_handle(CH_UNIX, CH_UTF16);
+ if (descriptor == (smb_iconv_t)-1) {
+ *size = 1;
+ return INVALID_CODEPOINT;
+ }
+
+ /* this looks a little strange, but it is needed to cope
+ with codepoints above 64k */
+ olen = 2;
+ outbuf = buf;
+ smb_iconv(descriptor, &str, &ilen, &outbuf, &olen);
+ if (olen == 2) {
+ olen = 4;
+ outbuf = buf;
+ smb_iconv(descriptor, &str, &ilen, &outbuf, &olen);
+ if (olen == 4) {
+ /* we didn't convert any bytes */
+ *size = 1;
+ return INVALID_CODEPOINT;
+ }
+ olen = 4 - olen;
+ } else {
+ olen = 2 - olen;
+ }
+
+ *size = ilen_orig - ilen;
+
+ if (olen == 2) {
+ return (codepoint_t)SVAL(buf, 0);
+ }
+ if (olen == 4) {
+ /* decode a 4 byte UTF16 character manually */
+ return (codepoint_t)0x10000 +
+ (buf[2] | ((buf[3] & 0x3)<<8) |
+ (buf[0]<<10) | ((buf[1] & 0x3)<<18));
+ }
+
+ /* no other length is valid */
+ return INVALID_CODEPOINT;
+}
+
+/*
+ push a single codepoint into a CH_UNIX string the target string must
+ be able to hold the full character, which is guaranteed if it is at
+ least 5 bytes in size. The caller may pass less than 5 bytes if they
+ are sure the character will fit (for example, you can assume that
+ uppercase/lowercase of a character will not add more than 1 byte)
+
+ return the number of bytes occupied by the CH_UNIX character, or
+ -1 on failure
+*/
+ssize_t push_codepoint(char *str, codepoint_t c)
+{
+ smb_iconv_t descriptor;
+ uint8_t buf[4];
+ size_t ilen, olen;
+ const char *inbuf;
+
+ if (c < 128) {
+ *str = c;
+ return 1;
+ }
+
+ descriptor = get_conv_handle(CH_UTF16, CH_UNIX);
+ if (descriptor == (smb_iconv_t)-1) {
+ return -1;
+ }
+
+ if (c < 0x10000) {
+ ilen = 2;
+ olen = 5;
+ inbuf = buf;
+ SSVAL(buf, 0, c);
+ smb_iconv(descriptor, &inbuf, &ilen, &str, &olen);
+ if (ilen != 0) {
+ return -1;
+ }
+ return 5 - olen;
+ }
+
+ c -= 0x10000;
+
+ buf[0] = (c>>10) & 0xFF;
+ buf[1] = (c>>18) | 0xd8;
+ buf[2] = c & 0xFF;
+ buf[3] = ((c>>8) & 0x3) | 0xdc;
+
+ ilen = 4;
+ olen = 5;
+ inbuf = buf;
+
+ smb_iconv(descriptor, &inbuf, &ilen, &str, &olen);
+ if (ilen != 0) {
+ return -1;
+ }
+ return 5 - olen;
+}
generated by cgit (git 2.34.1) at 2024-10-30 00:15:59 +0000