cvs: php4 /ext/mbstring mbfilter.c mbfilter.h mbfilter_cn.c mbfilter_cn.h mbfilter_kr.c

hirokawa Sun May 12 09:06:13 2002 EDT Modified files: /php4/ext/mbstring mbfilter.c mbfilter.h mbfilter_cn.c mbfilter_cn.h mbfilter_kr.c Log: added chinese HZ encoding support. fixed ascii area character conversion was not work in euc-cn and euc-kr.

Index: php4/ext/mbstring/mbfilter.c diff -u php4/ext/mbstring/mbfilter.c:1.27 php4/ext/mbstring/mbfilter.c:1.28 --- php4/ext/mbstring/mbfilter.c:1.27 Wed May 8 08:30:46 2002 +++ php4/ext/mbstring/mbfilter.c Sun May 12 09:06:13 2002 @@ -79,7 +79,7 @@ * */ -/* $Id: mbfilter.c,v 1.27 2002/05/08 12:30:46 hirokawa Exp $ */ +/* $Id: mbfilter.c,v 1.28 2002/05/12 13:06:13 hirokawa Exp $ */ #ifdef HAVE_CONFIG_H @@ -145,6 +145,16 @@ mbfl_no_encoding_7bit }; +static mbfl_language mbfl_language_korean = { + mbfl_no_language_korean, + "Korean", + "ko", + NULL, + mbfl_no_encoding_2022kr, + mbfl_no_encoding_base64, + mbfl_no_encoding_7bit +}; + static mbfl_language mbfl_language_english = { mbfl_no_language_english, "English", @@ -155,12 +165,22 @@ mbfl_no_encoding_8bit }; -static mbfl_language mbfl_language_chinese = { - mbfl_no_language_chinese, - "Chinese", - "zh", +static mbfl_language mbfl_language_simplified_chinese = { + mbfl_no_language_simplified_chinese, + "Simplified Chinese", + "zh-cn", NULL, - mbfl_no_encoding_2022jp, + mbfl_no_encoding_hz, + mbfl_no_encoding_base64, + mbfl_no_encoding_7bit +}; + +static mbfl_language mbfl_language_traditional_chinese = { + mbfl_no_language_traditional_chinese, + "Traditional Chinese", + "zh-tw", + NULL, + mbfl_no_encoding_hz, mbfl_no_encoding_base64, mbfl_no_encoding_7bit }; @@ -168,7 +188,9 @@ static mbfl_language *mbfl_language_ptr_table[] = { &mbfl_language_uni, &mbfl_language_japanese, - &mbfl_language_chinese, + &mbfl_language_korean, + &mbfl_language_simplified_chinese, + &mbfl_language_traditional_chinese, &mbfl_language_english, NULL }; @@ -707,6 +729,15 @@ MBFL_ENCTYPE_MBCS }; +static mbfl_encoding mbfl_encoding_hz = { + mbfl_no_encoding_hz, + "HZ", + "HZ-GB-2312", + NULL, + NULL, + MBFL_ENCTYPE_MBCS | MBFL_ENCTYPE_SHFTCODE +}; + #endif /* HAVE_MBSTR_CN */ #if defined(HAVE_MBSTR_TW) @@ -967,6 +998,7 @@ #if defined(HAVE_MBSTR_CN) &mbfl_encoding_euc_cn, &mbfl_encoding_cp936, + &mbfl_encoding_hz, #endif #if defined(HAVE_MBSTR_TW) &mbfl_encoding_euc_tw, @@ -1072,6 +1104,7 @@ #if defined(HAVE_MBSTR_CN) static int mbfl_filt_ident_euccn(int c, mbfl_identify_filter *filter TSRMLS_DC); static int mbfl_filt_ident_cp936(int c, mbfl_identify_filter *filter TSRMLS_DC); +static int mbfl_filt_ident_hz(int c, mbfl_identify_filter *filter TSRMLS_DC); #endif /* HAVE_MBSTR_CN */ #if defined(HAVE_MBSTR_TW) @@ -1605,6 +1638,23 @@ mbfl_filt_conv_common_dtor, mbfl_filt_conv_wchar_cp936, mbfl_filt_conv_common_flush }; + +static struct mbfl_convert_vtbl vtbl_hz_wchar = { + mbfl_no_encoding_hz, + mbfl_no_encoding_wchar, + mbfl_filt_conv_common_ctor, + mbfl_filt_conv_common_dtor, + mbfl_filt_conv_hz_wchar, + mbfl_filt_conv_common_flush }; + +static struct mbfl_convert_vtbl vtbl_wchar_hz = { + mbfl_no_encoding_wchar, + mbfl_no_encoding_hz, + mbfl_filt_conv_common_ctor, + mbfl_filt_conv_common_dtor, + mbfl_filt_conv_wchar_hz, + mbfl_filt_conv_any_hz_flush }; + #endif /* HAVE_MBSTR_CN */ #if defined(HAVE_MBSTR_TW) @@ -1923,6 +1973,8 @@ &vtbl_wchar_euccn, &vtbl_cp936_wchar, &vtbl_wchar_cp936, + &vtbl_hz_wchar, + &vtbl_wchar_hz, #endif #if defined(HAVE_MBSTR_TW) &vtbl_euctw_wchar, @@ -2083,6 +2135,13 @@ mbfl_filt_ident_common_ctor, mbfl_filt_ident_common_dtor, mbfl_filt_ident_cp936 }; + +static struct mbfl_identify_vtbl vtbl_identify_hz = { + mbfl_no_encoding_hz, + mbfl_filt_ident_common_ctor, + mbfl_filt_ident_common_dtor, + mbfl_filt_ident_hz }; + #endif /* HAVE_MBSTR_CN */ #if defined(HAVE_MBSTR_TW) @@ -2218,6 +2277,7 @@ #if defined(HAVE_MBSTR_CN) &vtbl_identify_euccn, &vtbl_identify_cp936, + &vtbl_identify_hz, #endif #if defined(HAVE_MBSTR_TW) &vtbl_identify_euctw, @@ -5751,6 +5811,53 @@ filter->status = 1; } else { /* bad */ filter->flag = 1; + } + + return c; +} + +static int +mbfl_filt_ident_hz(int c, mbfl_identify_filter *filter TSRMLS_DC) +{ + switch (filter->status & 0xf) { +/* case 0x00: ASCII */ +/* case 0x10: GB2312 */ + case 0: + if (c == 0x7e) { + filter->status += 2; + } else if (filter->status == 0x10 && c > 0x20 && c < 0x7f) { /* DBCS first char */ + filter->status += 1; + } else if (c >= 0 && c < 0x80) { /* latin, CTLs */ + ; + } else { + filter->flag = 1; /* bad */ + } + break; + +/* case 0x11: GB2312 second char */ + case 1: + filter->status &= ~0xf; + if (c < 0x21 || c > 0x7e) { /* bad */ + filter->flag = 1; + } + break; + + case 2: + if (c == 0x7d) { /* '}' */ + filter->status = 0; + } else if (c == 0x7b) { /* '{' */ + filter->status = 0x10; + } else if (c == 0x7e) { /* '~' */ + filter->status = 0; + } else { + filter->flag = 1; /* bad */ + filter->status &= ~0xf; + } + break; + + default: + filter->status = 0; + break; } return c; Index: php4/ext/mbstring/mbfilter.h diff -u php4/ext/mbstring/mbfilter.h:1.9 php4/ext/mbstring/mbfilter.h:1.10 --- php4/ext/mbstring/mbfilter.h:1.9 Wed May 8 08:30:47 2002 +++ php4/ext/mbstring/mbfilter.h Sun May 12 09:06:13 2002 @@ -86,7 +86,7 @@ * */ -/* $Id: mbfilter.h,v 1.9 2002/05/08 12:30:47 hirokawa Exp $ */ +/* $Id: mbfilter.h,v 1.10 2002/05/12 13:06:13 hirokawa Exp $ */ #ifndef MBFL_MBFILTER_H @@ -111,7 +111,8 @@ mbfl_no_language_polish, /* pl */ mbfl_no_language_portuguese, /* pt */ mbfl_no_language_swedish, /* sv */ - mbfl_no_language_chinese, /* zh */ + mbfl_no_language_simplified_chinese, /* zh-cn */ + mbfl_no_language_traditional_chinese, /* zh-tw */ mbfl_no_language_max }; @@ -172,7 +173,9 @@ mbfl_no_encoding_euc_tw, mbfl_no_encoding_big5, mbfl_no_encoding_euc_kr, + mbfl_no_encoding_2022kr, mbfl_no_encoding_uhc, + mbfl_no_encoding_hz, mbfl_no_encoding_charset_max }; Index: php4/ext/mbstring/mbfilter_cn.c diff -u php4/ext/mbstring/mbfilter_cn.c:1.3 php4/ext/mbstring/mbfilter_cn.c:1.4 --- php4/ext/mbstring/mbfilter_cn.c:1.3 Sat May 4 22:22:36 2002 +++ php4/ext/mbstring/mbfilter_cn.c Sun May 12 09:06:13 2002 @@ -20,7 +20,7 @@ * "streamable simplified chinese code filter and converter" */ -/* $Id: mbfilter_cn.c,v 1.3 2002/05/05 02:22:36 hirokawa Exp $ */ +/* $Id: mbfilter_cn.c,v 1.4 2002/05/12 13:06:13 hirokawa Exp $ */ #ifdef HAVE_CONFIG_H #include "config.h" @@ -117,8 +117,8 @@ c1 = (s >> 8) & 0xff; c2 = s & 0xff; - if (c1 < 0xa1 || c2 < 0xa1) { /* exclude CP932 extension */ - s = 0; + if (c1 < 0xa1 || c2 < 0xa1) { /* exclude CP936 extension */ + s = c; } if (s <= 0) { @@ -257,6 +257,154 @@ } return c; +} + + +/* + * HZ => wchar + */ +int +mbfl_filt_conv_hz_wchar(int c, mbfl_convert_filter *filter TSRMLS_DC) +{ + int c1, s, w; + + switch (filter->status & 0xf) { +/* case 0x00: ASCII */ +/* case 0x10: GB2312 */ + case 0: + if (c == 0x7e) { + filter->status += 2; + } else if (filter->status == 0x10 && c > 0x20 && c < 0x7f) { /* DBCS first char */ + filter->cache = c; + filter->status += 1; + } else if (c >= 0 && c < 0x80) { /* latin, CTLs */ + CK((*filter->output_function)(c, filter->data TSRMLS_CC)); + } else { + w = c & MBFL_WCSGROUP_MASK; + w |= MBFL_WCSGROUP_THROUGH; + CK((*filter->output_function)(w, filter->data TSRMLS_CC)); + } + break; + +/* case 0x11: GB2312 second char */ + case 1: + filter->status &= ~0xf; + c1 = filter->cache; + if (c1 > 0x20 && c1 < 0x7f && c > 0x20 && c < 0x7f) { + s = (c1 - 1)*192 + c + 0x40; /* GB2312 */ + if (s >= 0 && s < cp936_ucs_table_size) { + w = cp936_ucs_table[s]; + } else { + w = 0; + } + if (w <= 0) { + w = (c1 << 8) | c; + w &= MBFL_WCSPLANE_MASK; + w |= MBFL_WCSPLANE_GB2312; + } + CK((*filter->output_function)(w, filter->data TSRMLS_CC)); + } else if ((c >= 0 && c < 0x21) || c == 0x7f) { /* CTLs */ + CK((*filter->output_function)(c, filter->data TSRMLS_CC)); + } else { + w = (c1 << 8) | c; + w &= MBFL_WCSGROUP_MASK; + w |= MBFL_WCSGROUP_THROUGH; + CK((*filter->output_function)(w, filter->data TSRMLS_CC)); + } + break; + + /* '~' */ + case 2: + if (c == 0x7d) { /* '}' */ + filter->status = 0x0; + } else if (c == 0x7b) { /* '{' */ + filter->status = 0x10; + } else if (c == 0x7e) { /* '~' */ + filter->status = 0x0; + CK((*filter->output_function)(0x007e, filter->data TSRMLS_CC)); + } + break; + + default: + filter->status = 0; + break; + } + + return c; +} + +/* + * wchar => HZ + */ +int +mbfl_filt_conv_wchar_hz(int c, mbfl_convert_filter *filter TSRMLS_DC) +{ + int s; + + s = 0; + if (c >= ucs_a1_cp936_table_min && c < ucs_a1_cp936_table_max) { + s = ucs_a1_cp936_table[c - ucs_a1_cp936_table_min]; + } else if (c >= ucs_a2_cp936_table_min && c < ucs_a2_cp936_table_max) { + s = ucs_a2_cp936_table[c - ucs_a2_cp936_table_min]; + } else if (c >= ucs_a3_cp936_table_min && c < ucs_a3_cp936_table_max) { + s = ucs_a3_cp936_table[c - ucs_a3_cp936_table_min]; + } else if (c >= ucs_i_cp936_table_min && c < ucs_i_cp936_table_max) { + s = ucs_i_cp936_table[c - ucs_i_cp936_table_min]; + } else if (c >= ucs_hff_cp936_table_min && c < ucs_hff_cp936_table_max) { + s = ucs_hff_cp936_table[c - ucs_hff_cp936_table_min]; + } + if (s >= 0x0080) { + s -= 0x8080; + } + + if (s <= 0) { + if (c == 0) { + s = 0; + } else if (s <= 0) { + s = -1; + } + } else if ((s >= 0x80 && s < 0x2121) || (s > 0x8080)) { + s = -1; + } + if (s >= 0) { + if (s < 0x80) { /* ASCII */ + if ((filter->status & 0xff00) != 0) { + CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC)); /* '~' */ + CK((*filter->output_function)(0x7d, filter->data TSRMLS_CC)); /* '}' */ + } + filter->status = 0; + if (s == 0x7e){ + CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC)); + } + CK((*filter->output_function)(s, filter->data TSRMLS_CC)); + } else { /* GB 2312-80 */ + if ((filter->status & 0xff00) != 0x200) { + CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC)); /* '~' */ + CK((*filter->output_function)(0x7b, filter->data TSRMLS_CC)); /* '{' */ + } + filter->status = 0x200; + CK((*filter->output_function)((s >> 8) & 0x7f, filter->data TSRMLS_CC)); + CK((*filter->output_function)(s & 0x7f, filter->data TSRMLS_CC)); + } + } else { + if (filter->illegal_mode != MBFL_OUTPUTFILTER_ILLEGAL_MODE_NONE) { + CK(mbfl_filt_conv_illegal_output(c, filter TSRMLS_CC)); + } + } + + return c; +} + +int +mbfl_filt_conv_any_hz_flush(mbfl_convert_filter *filter TSRMLS_DC) +{ + /* back to latin */ + if ((filter->status & 0xff00) != 0) { + CK((*filter->output_function)(0x7e, filter->data TSRMLS_CC)); /* ~ */ + CK((*filter->output_function)(0x7d, filter->data TSRMLS_CC)); /* '{' */ + } + filter->status &= 0xff; + return 0; } #endif /* HAVE_MBSTR_CN */ Index: php4/ext/mbstring/mbfilter_cn.h diff -u php4/ext/mbstring/mbfilter_cn.h:1.1 php4/ext/mbstring/mbfilter_cn.h:1.2 --- php4/ext/mbstring/mbfilter_cn.h:1.1 Tue Apr 30 08:00:42 2002 +++ php4/ext/mbstring/mbfilter_cn.h Sun May 12 09:06:13 2002 @@ -17,7 +17,7 @@ */ -/* $Id: mbfilter_cn.h,v 1.1 2002/04/30 12:00:42 hirokawa Exp $ */ +/* $Id: mbfilter_cn.h,v 1.2 2002/05/12 13:06:13 hirokawa Exp $ */ #ifndef MBFL_MBFILTER_CN_H #define MBFL_MBFILTER_CN_H @@ -26,5 +26,8 @@ int mbfl_filt_conv_wchar_euccn(int c, mbfl_convert_filter *filter TSRMLS_DC); int mbfl_filt_conv_cp936_wchar(int c, mbfl_convert_filter *filter TSRMLS_DC); int mbfl_filt_conv_wchar_cp936(int c, mbfl_convert_filter *filter TSRMLS_DC); +int mbfl_filt_conv_hz_wchar(int c, mbfl_convert_filter *filter TSRMLS_DC); +int mbfl_filt_conv_wchar_hz(int c, mbfl_convert_filter *filter TSRMLS_DC); +int mbfl_filt_conv_any_hz_flush(mbfl_convert_filter *filter TSRMLS_DC); #endif /* MBFL_MBFILTER_CN_H */ Index: php4/ext/mbstring/mbfilter_kr.c diff -u php4/ext/mbstring/mbfilter_kr.c:1.2 php4/ext/mbstring/mbfilter_kr.c:1.3 --- php4/ext/mbstring/mbfilter_kr.c:1.2 Tue Apr 30 10:36:43 2002 +++ php4/ext/mbstring/mbfilter_kr.c Sun May 12 09:06:13 2002 @@ -20,7 +20,7 @@ * "streamable korean code filter and converter" */ -/* $Id: mbfilter_kr.c,v 1.2 2002/04/30 14:36:43 sebastian Exp $ */ +/* $Id: mbfilter_kr.c,v 1.3 2002/05/12 13:06:13 hirokawa Exp $ */ #ifdef HAVE_CONFIG_H #include "config.h" @@ -141,8 +141,8 @@ c1 = (s >> 8) & 0xff; c2 = s & 0xff; /* exclude UHC extension area */ - if (c1 < 0xa1 || c1 > 0xfe || c2 < 0xa1 && c2 > 0xfe){ - s = 0; + if (c1 < 0xa1 || c2 < 0xa1){ + s = c; } if (s <= 0) {