Re: Multiple char encodings for site internationalzation

Hello, world, On Tue, Aug 03, 1999 at 12:58:44AM -0700, Hironori Sato wrote: > > I see the utf8_encode and decode as well as the convert_cyr_encoding > > functions, but what about middle eastern and asian languages such as > > arabic etc... In php3 cyrillic has same problems with UTF-8: utf8_encode doesn't know anything about russian (cyrillic) charsets (actually you can't use any charset except ISO-8859-1 in php3's utf8_encode()); and convert_cyr_string() doesn't know anything about UTF-8 :) I've made a little patch (see atatchment) which allow you to use cyrillic charsets in utf8_encode()/utf8_decode(). It supports five cyrillic charsets: iso-8859-5, koi8-r, cp-866, windows-1251, x-mac-cyrillic; it depends on cyr_convert.c (because I didn't want to mainain two copies of encoding tables - in xml.c and cyr_convert.c). NOTE: You'll also have to install James Clark's 'expat'-library (xml support will not be turned on otherwise). Are we really need XML-library just to allow using of utf8-encoding/decoding engine? What do php3 developers think about it? > I really feel i18n is becoming very important issue with PHP. What do > everyone at PHP-DEV think? Since there weren't much response from the last > email, I'm pretty disappointed... :( <aol> me too... </aol> Best regards, Vadim. -- Tver Internet Center NOC

Index: Makefile.in =================================================================== RCS file: /repository/php3/Makefile.in,v retrieving revision 1.343 diff -c -r1.343 Makefile.in *** Makefile.in 1999/08/02 17:48:26 1.343 --- Makefile.in 1999/08/04 07:13:45 *************** *** 627,633 **** functions/xml.o: functions/xml.c php.h php_version.h config.h request_info.h \ php3_hash.h alloc.h safe_mode.h fopen-wrappers.h \ mod_php3.h stack.h operators.h token_cache.h variables.h constants.h \ ! internal_functions.h modules.h functions/php3_xml.h functions/unified_odbc.o: functions/unified_odbc.c config.h build-defs.h php.h \ php_version.h request_info.h php3_hash.h alloc.h \ safe_mode.h fopen-wrappers.h mod_php3.h stack.h operators.h \ --- 627,633 ---- functions/xml.o: functions/xml.c php.h php_version.h config.h request_info.h \ php3_hash.h alloc.h safe_mode.h fopen-wrappers.h \ mod_php3.h stack.h operators.h token_cache.h variables.h constants.h \ ! internal_functions.h modules.h functions/php3_xml.h functions/cyr_convert.c functions/unified_odbc.o: functions/unified_odbc.c config.h build-defs.h php.h \ php_version.h request_info.h php3_hash.h alloc.h \ safe_mode.h fopen-wrappers.h mod_php3.h stack.h operators.h \ Index: functions/cyr_convert.c =================================================================== RCS file: /repository/php3/functions/cyr_convert.c,v retrieving revision 1.6 diff -c -r1.6 cyr_convert.c *** functions/cyr_convert.c 1999/06/16 11:34:17 1.6 --- functions/cyr_convert.c 1999/08/04 07:14:29 *************** *** 300,302 **** --- 300,342 ---- RETVAL_STRING((char *)str, 1) } /* }}} */ + + unsigned short + xml_encode_koi8_r(unsigned char c) { + return xml_encode_iso_8859_5(_cyr_iso88595[c+256]); + } + + char + xml_decode_koi8_r(unsigned short c) { + return (char)(_cyr_iso88595[(unsigned char)xml_decode_iso_8859_5(c)]); + } + + unsigned short + xml_encode_windows_1251(unsigned char c) { + return xml_encode_iso_8859_5(_cyr_iso88595[_cyr_win1251[c]+256]); + } + + char + xml_decode_windows_1251(unsigned short c) { + return (char)(_cyr_win1251[_cyr_iso88595[(unsigned char)xml_decode_iso_8859_5(c)]+256]); + } + + unsigned short + xml_encode_cp_866(unsigned char c) { + return xml_encode_iso_8859_5(_cyr_iso88595[_cyr_cp866[c]+256]); + } + + char + xml_decode_cp_866(unsigned short c) { + return (char)(_cyr_cp866[_cyr_iso88595[(unsigned char)xml_decode_iso_8859_5(c)]+256]); + } + + unsigned short + xml_encode_x_mac_cyrillic(unsigned char c) { + return xml_encode_iso_8859_5(_cyr_iso88595[_cyr_mac[c]+256]); + } + + char + xml_decode_x_mac_cyrillic(unsigned short c) { + return (char)(_cyr_mac[_cyr_iso88595[(unsigned char)xml_decode_iso_8859_5(c)]+256]); + } Index: functions/cyr_convert.h =================================================================== RCS file: /repository/php3/functions/cyr_convert.h,v retrieving revision 1.2 diff -c -r1.2 cyr_convert.h *** functions/cyr_convert.h 1999/01/01 17:59:07 1.2 --- functions/cyr_convert.h 1999/08/04 07:14:29 *************** *** 34,40 **** extern void php3_convert_cyr_string(INTERNAL_FUNCTION_PARAMETERS); ! #endif /* _CYR_CONVERT_H */ --- 34,62 ---- extern void php3_convert_cyr_string(INTERNAL_FUNCTION_PARAMETERS); ! inline static unsigned short ! xml_encode_iso_8859_5(unsigned char c) ! { ! return (c > 0x9f && c < 0x100) ? (c+0x400-0xa0) : c; ! } ! ! inline static char ! xml_decode_iso_8859_5(unsigned short c) ! { ! return (char)(c < 0x80) ? c : ! ((c > 0x3ff && c < 0x460) ? (c-0x400+0xa0) : '?'); ! } ! ! extern unsigned short xml_encode_koi8_r(unsigned char c); ! extern char xml_decode_koi8_r(unsigned short c); + extern unsigned short xml_encode_windows_1251(unsigned char c); + extern char xml_decode_windows_1251(unsigned short c); + extern unsigned short xml_encode_cp_866(unsigned char c); + extern char xml_decode_cp_866(unsigned short c); + extern unsigned short xml_encode_x_mac_cyrillic(unsigned char c); + extern char xml_decode_x_mac_cyrillic(unsigned short c); + + #endif /* _CYR_CONVERT_H */ Index: functions/xml.c =================================================================== RCS file: /repository/php3/functions/xml.c,v retrieving revision 1.27 diff -c -r1.27 xml.c *** functions/xml.c 1999/06/21 00:40:57 1.27 --- functions/xml.c 1999/08/04 07:14:51 *************** *** 54,59 **** --- 54,60 ---- # include "snprintf.h" # include "head.h" # include "php3_string.h" + # include "cyr_convert.h" /* Short-term TODO list: * - Implement XML_ExternalEntityParserCreate() *************** *** 173,178 **** --- 174,184 ---- */ xml_encoding xml_encodings[] = { { "ISO-8859-1", xml_decode_iso_8859_1, xml_encode_iso_8859_1 }, + { "ISO-8859-5", xml_decode_iso_8859_5, xml_encode_iso_8859_5 }, + { "KOI8-R", xml_decode_koi8_r, xml_encode_koi8_r }, + { "WINDOWS-1251", xml_decode_windows_1251, xml_encode_windows_1251 }, + { "CP-866", xml_decode_cp_866, xml_encode_cp_866 }, + { "X-MAC-CYRILLIC", xml_decode_x_mac_cyrillic, xml_encode_x_mac_cyrillic }, { "US-ASCII", xml_decode_us_ascii, xml_encode_us_ascii }, { "UTF-8", NULL, NULL }, { NULL, NULL, NULL } *************** *** 1489,1504 **** Encodes an ISO-8859-1 string to UTF-8 */ PHP_FUNCTION(utf8_encode) { ! pval *arg; XML_Char *encoded; int len; XML_TLS_VARS; ! if (ARG_COUNT(ht) != 1 || getParameters(ht, 1, &arg) == FAILURE) { WRONG_PARAM_COUNT; } ! convert_to_string(arg); ! encoded = xml_utf8_encode(arg->value.str.val, arg->value.str.len, &len, "ISO-8859-1"); if (encoded == NULL) { RETURN_FALSE; } --- 1495,1526 ---- Encodes an ISO-8859-1 string to UTF-8 */ PHP_FUNCTION(utf8_encode) { ! pval *arg1, *arg2 = NULL; XML_Char *encoded; int len; XML_TLS_VARS; ! switch(ARG_COUNT(ht)) { ! case 1: ! if (getParameters(ht, 1, &arg1) == FAILURE) { ! WRONG_PARAM_COUNT; ! } ! convert_to_string(arg1); ! break; ! ! case 2: ! if (getParameters(ht, 2, &arg1, &arg2) == FAILURE) { ! WRONG_PARAM_COUNT; ! } ! convert_to_string(arg1); ! convert_to_string(arg2); ! break; ! ! default: WRONG_PARAM_COUNT; } ! ! encoded = xml_utf8_encode(arg1->value.str.val, arg1->value.str.len, &len, arg2 ? arg2->value.str.val : "ISO-8859-1"); if (encoded == NULL) { RETURN_FALSE; } *************** *** 1510,1525 **** Converts a UTF-8 encoded string to ISO-8859-1 */ PHP_FUNCTION(utf8_decode) { ! pval *arg; XML_Char *decoded; int len; XML_TLS_VARS; ! if (ARG_COUNT(ht) != 1 || getParameters(ht, 1, &arg) == FAILURE) { WRONG_PARAM_COUNT; } ! convert_to_string(arg); ! decoded = xml_utf8_decode(arg->value.str.val, arg->value.str.len, &len, "ISO-8859-1"); if (decoded == NULL) { RETURN_FALSE; } --- 1532,1563 ---- Converts a UTF-8 encoded string to ISO-8859-1 */ PHP_FUNCTION(utf8_decode) { ! pval *arg1, *arg2 = NULL; XML_Char *decoded; int len; XML_TLS_VARS; ! switch(ARG_COUNT(ht)) { ! case 1: ! if (getParameters(ht, 1, &arg1) == FAILURE) { ! WRONG_PARAM_COUNT; ! } ! convert_to_string(arg1); ! break; ! ! case 2: ! if (getParameters(ht, 2, &arg1, &arg2) == FAILURE) { ! WRONG_PARAM_COUNT; ! } ! convert_to_string(arg1); ! convert_to_string(arg2); ! break; ! ! default: WRONG_PARAM_COUNT; } ! ! decoded = xml_utf8_decode(arg1->value.str.val, arg1->value.str.len, &len, arg2 ? arg2->value.str.val : "ISO-8859-1"); if (decoded == NULL) { RETURN_FALSE; }