cvs: Zend / Zend.dsp Zend.m4 ZendTS.dsp zend_compile.c zend_compile.h zend_globals.h zend_language_parser.y zend_language_scanner.h
zend_language_scanner.l zend_multibyte.c zend_multibyte.h

hirokawa Wed May 8 08:29:52 2002 EDT Added files: /Zend zend_multibyte.c zend_multibyte.h Modified files: /Zend Zend.dsp Zend.m4 ZendTS.dsp zend_compile.c zend_compile.h zend_globals.h zend_language_parser.y zend_language_scanner.h zend_language_scanner.l Log: Added conversion support from script character encoding to internal character encoding. This feature is very useful for japanese who uses Shift_JIS encoding because some of characters in Shift_JIS are including '0x5c' and it causes some troubles on Zend parser. This patch is made by Masaki Fujimoto.

Index: Zend/Zend.dsp diff -u Zend/Zend.dsp:1.19 Zend/Zend.dsp:1.20 --- Zend/Zend.dsp:1.19 Tue Sep 18 22:08:21 2001 +++ Zend/Zend.dsp Wed May 8 08:29:51 2002 @@ -183,6 +183,10 @@ # End Source File # Begin Source File +SOURCE=.\zend_multibyte.c +# End Source File +# Begin Source File + SOURCE=.\zend_opcode.c # End Source File # Begin Source File @@ -300,6 +304,10 @@ # Begin Source File SOURCE=.\zend_modules.h +# End Source File +# Begin Source File + +SOURCE=.\zend_multibyte.h # End Source File # Begin Source File Index: Zend/Zend.m4 diff -u Zend/Zend.m4:1.32 Zend/Zend.m4:1.33 --- Zend/Zend.m4:1.32 Mon Apr 22 04:57:42 2002 +++ Zend/Zend.m4 Wed May 8 08:29:51 2002 @@ -1,5 +1,5 @@ dnl -dnl $Id: Zend.m4,v 1.32 2002/04/22 08:57:42 sas Exp $ +dnl $Id: Zend.m4,v 1.33 2002/05/08 12:29:51 hirokawa Exp $ dnl dnl This file contains Zend specific autoconf functions. dnl @@ -140,6 +140,13 @@ ZEND_MEMORY_LIMIT=no ]) +AC_ARG_ENABLE(zend-multibyte, +[ --enable-zend-multibyte Compile with zend multibyte support. ], [ + ZEND_MULTIBYTE=$enableval +],[ + ZEND_MULTIBYTE=no +]) + AC_MSG_CHECKING(whether to enable experimental ZTS) AC_MSG_RESULT($ZEND_EXPERIMENTAL_ZTS) @@ -151,7 +158,10 @@ AC_MSG_CHECKING(whether to enable Zend debugging) AC_MSG_RESULT($ZEND_DEBUG) - + +AC_MSG_CHECKING(whether to enable Zend multibyte) +AC_MSG_RESULT($ZEND_MULTIBYTE) + if test "$ZEND_DEBUG" = "yes"; then AC_DEFINE(ZEND_DEBUG,1,[ ]) echo " $CFLAGS" | grep ' -g' >/dev/null || DEBUG_CFLAGS="-g" @@ -179,6 +189,9 @@ AC_DEFINE(MEMORY_LIMIT, 0, [Memory limit]) fi +if test "$ZEND_MULTIBYTE" = "yes"; then + AC_DEFINE(ZEND_MULTIBYTE, 1, [ ]) +fi changequote({,}) if test -n "$GCC" && test "$ZEND_INLINE_OPTIMIZATION" != "yes"; then Index: Zend/ZendTS.dsp diff -u Zend/ZendTS.dsp:1.32 Zend/ZendTS.dsp:1.33 --- Zend/ZendTS.dsp:1.32 Wed Sep 19 04:26:38 2001 +++ Zend/ZendTS.dsp Wed May 8 08:29:51 2002 @@ -212,6 +212,10 @@ # End Source File # Begin Source File +SOURCE=.\zend_multibyte.c +# End Source File +# Begin Source File + SOURCE=.\zend_opcode.c # End Source File # Begin Source File @@ -353,6 +357,10 @@ # Begin Source File SOURCE=.\zend_modules.h +# End Source File +# Begin Source File + +SOURCE=.\zend_multibyte.h # End Source File # Begin Source File Index: Zend/zend_compile.c diff -u Zend/zend_compile.c:1.227 Zend/zend_compile.c:1.228 --- Zend/zend_compile.c:1.227 Fri Mar 29 07:29:56 2002 +++ Zend/zend_compile.c Wed May 8 08:29:51 2002 @@ -25,6 +25,9 @@ #include "zend_API.h" #include "zend_fast_cache.h" +#ifdef ZEND_MULTIBYTE +#include "zend_multibyte.h" +#endif /* ZEND_MULTIBYTE */ ZEND_API zend_op_array *(*zend_compile_file)(zend_file_handle *file_handle, int type TSRMLS_DC); @@ -81,6 +84,14 @@ CG(handle_op_arrays) = 1; CG(in_compilation) = 0; init_compiler_declarables(TSRMLS_C); +#ifdef ZEND_MULTIBYTE + CG(script_encoding_list) = NULL; + CG(script_encoding_list_size) = 0; + CG(internal_encoding) = NULL; + CG(encoding_detector) = NULL; + CG(encoding_converter) = NULL; + CG(multibyte_oddlen) = NULL; +#endif /* ZEND_MULTIBYTE */ } @@ -105,6 +116,10 @@ zend_stack_destroy(&CG(list_stack)); zend_hash_destroy(&CG(filenames_table)); zend_llist_destroy(&CG(open_files)); +#ifdef ZEND_MULTIBYTE + if (CG(script_encoding_list)) + efree(CG(script_encoding_list)); +#endif /* ZEND_MULTIBYTE */ } @@ -2388,6 +2403,34 @@ } } +void zend_do_encoding(znode *result, znode *arg TSRMLS_DC) +{ +#ifndef ZEND_MULTIBYTE + zend_error(E_COMPILE_ERROR, "Unsupported directive [encoding]"); +#else /* ZEND_MULTIBYTE */ + zend_encoding *new_encoding, *old_encoding; + zend_multibyte_filter old_input_filter; + + if (!arg->u.constant.value.str.val) + zend_error(E_COMPILE_ERROR, "Cannot figure out encoding string (specified without quotes?)"); + new_encoding = zend_multibyte_fetch_encoding(arg->u.constant.value.str.val); + efree(arg->u.constant.value.str.val); + if (!new_encoding) + { + zend_error(E_COMPILE_WARNING, "Unsupported encoding [%s]\n", arg->u.constant.value.str.val); + return; + } + + old_input_filter = LANG_SCNG(input_filter); + old_encoding = LANG_SCNG(script_encoding); + zend_multibyte_set_filter(new_encoding TSRMLS_CC); + + if (old_input_filter != LANG_SCNG(input_filter) || + ((old_input_filter == zend_multibyte_encoding_filter) && + (new_encoding != old_encoding))) + zend_yyinput_again(old_input_filter, old_encoding TSRMLS_CC); +#endif /* ZEND_MULTIBYTE */ +} int zend_register_auto_global(char *name, uint name_len TSRMLS_DC) { Index: Zend/zend_compile.h diff -u Zend/zend_compile.h:1.141 Zend/zend_compile.h:1.142 --- Zend/zend_compile.h:1.141 Sun Feb 10 07:52:45 2002 +++ Zend/zend_compile.h Wed May 8 08:29:51 2002 @@ -350,6 +350,8 @@ void zend_do_ticks(TSRMLS_D); +void zend_do_encoding(znode *result, znode *arg TSRMLS_DC); + ZEND_API void function_add_ref(zend_function *function); #define INITIAL_OP_ARRAY_SIZE 64 Index: Zend/zend_globals.h diff -u Zend/zend_globals.h:1.82 Zend/zend_globals.h:1.83 --- Zend/zend_globals.h:1.82 Sun Jan 6 10:21:09 2002 +++ Zend/zend_globals.h Wed May 8 08:29:51 2002 @@ -32,6 +32,10 @@ #include "zend_llist.h" #include "zend_fast_cache.h" +#ifdef ZEND_MULTIBYTE +#include "zend_multibyte.h" +#endif /* ZEND_MULTIBYTE */ + /* Define ZTS if you want a thread-safe Zend */ /*#undef ZTS*/ @@ -114,6 +118,17 @@ int interactive; zend_bool increment_lineno; + +#ifdef ZEND_MULTIBYTE + zend_encoding **script_encoding_list; + int script_encoding_list_size; + + zend_encoding *internal_encoding; + + zend_encoding_detector encoding_detector; + zend_encoding_converter encoding_converter; + zend_multibyte_oddlen multibyte_oddlen; +#endif /* ZEND_MULTIBYTE */ }; @@ -235,6 +250,17 @@ char *_yy_last_accepting_cpos; int _yy_more_flag; int _yy_more_len; + +#ifdef ZEND_MULTIBYTE + char *code; /* original */ + int code_size; + char *current_code; /* filtered */ + int current_code_size; + zend_multibyte_filter input_filter; + zend_multibyte_filter output_filter; + zend_encoding *script_encoding; + zend_encoding *internal_encoding; +#endif /* ZEND_MULTIBYTE */ }; #endif /* ZEND_GLOBALS_H */ Index: Zend/zend_language_parser.y diff -u Zend/zend_language_parser.y:1.18 Zend/zend_language_parser.y:1.19 --- Zend/zend_language_parser.y:1.18 Wed Apr 10 17:22:10 2002 +++ Zend/zend_language_parser.y Wed May 8 08:29:51 2002 @@ -73,6 +73,7 @@ %left T_ELSEIF %left T_ELSE %left T_ENDIF +%left T_ENCODING %token T_LNUMBER %token T_DNUMBER %token T_STRING @@ -485,6 +486,7 @@ | T_ARRAY '(' array_pair_list ')' { $$ = $3; } | '`' encaps_list '`' { zend_do_shell_exec(&$$, &$2 TSRMLS_CC); } | T_PRINT expr { zend_do_print(&$$, &$2 TSRMLS_CC); } + | T_ENCODING expr { zend_do_encoding(&$$, &$2 TSRMLS_CC); } ; function_call: Index: Zend/zend_language_scanner.h diff -u Zend/zend_language_scanner.h:1.9 Zend/zend_language_scanner.h:1.10 --- Zend/zend_language_scanner.h:1.9 Sun Jan 6 10:21:09 2002 +++ Zend/zend_language_scanner.h Wed May 8 08:29:51 2002 @@ -27,6 +27,16 @@ FILE *in; uint lineno; char *filename; +#ifdef ZEND_MULTIBYTE + char *code; + int code_size; + char *current_code; + int current_code_size; + zend_multibyte_filter input_filter; + zend_multibyte_filter output_filter; + zend_encoding *script_encoding; + zend_encoding *internal_encoding; +#endif /* ZEND_MULTIBYTE */ } zend_lex_state; Index: Zend/zend_language_scanner.l diff -u Zend/zend_language_scanner.l:1.45 Zend/zend_language_scanner.l:1.46 --- Zend/zend_language_scanner.l:1.45 Wed Apr 10 17:31:54 2002 +++ Zend/zend_language_scanner.l Wed May 8 08:29:51 2002 @@ -145,6 +145,16 @@ lex_state->state = YYSTATE; lex_state->filename = zend_get_compiled_filename(TSRMLS_C); lex_state->lineno = CG(zend_lineno); +#ifdef ZEND_MULTIBYTE + lex_state->code = SCNG(code); + lex_state->code_size = SCNG(code_size); + lex_state->current_code = SCNG(current_code); + lex_state->current_code_size = SCNG(current_code_size); + lex_state->input_filter = SCNG(input_filter); + lex_state->output_filter = SCNG(output_filter); + lex_state->script_encoding = SCNG(script_encoding); + lex_state->internal_encoding = SCNG(internal_encoding); +#endif /* ZEND_MULTIBYTE */ } @@ -163,6 +173,16 @@ BEGIN(lex_state->state); CG(zend_lineno) = lex_state->lineno; zend_restore_compiled_filename(lex_state->filename TSRMLS_CC); +#ifdef ZEND_MULTIBYTE + SCNG(code) = lex_state->code; + SCNG(code_size) = lex_state->code_size; + SCNG(current_code) = lex_state->current_code; + SCNG(current_code_size) = lex_state->current_code_size; + SCNG(input_filter) = lex_state->input_filter; + SCNG(output_filter) = lex_state->output_filter; + SCNG(script_encoding) = lex_state->script_encoding; + SCNG(internal_encoding) = lex_state->internal_encoding; +#endif /* ZEND_MULTIBYTE */ } @@ -235,7 +255,43 @@ } /* Reset the scanner for scanning the new file */ SCNG(yy_in) = file_handle->handle.fp; +#ifdef ZEND_MULTIBYTE + if (zend_read_file(TSRMLS_C) != 0) + return FAILURE; + + /* force flex to use buffer only */ + SCNG(yy_in) = NULL; + SCNG(init) = 0; + SCNG(start) = 1; + + zend_multibyte_set_filter(NULL TSRMLS_CC); + + if (!SCNG(input_filter)) + { + SCNG(current_code) = (char*)emalloc(SCNG(code_size)+1); + if (!SCNG(current_code)) + return FAILURE; + memcpy(SCNG(current_code), SCNG(code), SCNG(code_size)+1); + SCNG(current_code_size) = SCNG(code_size); + } + else + { + SCNG(input_filter)(&SCNG(current_code), &SCNG(current_code_size), + SCNG(code), SCNG(code_size) TSRMLS_CC); + if (!SCNG(current_code) || !SCNG(current_code_size)) + return FAILURE; + } + + /* we need two nulls so that ralloc... */ + SCNG(current_code) = (char*)erealloc(SCNG(current_code), SCNG(current_code_size)+2); + if (!SCNG(current_code)) + return FAILURE; + *(SCNG(current_code)+SCNG(current_code_size)) = (char)NULL; + *(SCNG(current_code)+SCNG(current_code_size)+1) = (char)NULL; + yy_scan_buffer(SCNG(current_code), SCNG(current_code_size)+2 TSRMLS_CC); +#else /* ZEND_MULTIBYTE */ yy_switch_to_buffer(yy_create_buffer(SCNG(yy_in), YY_BUF_SIZE TSRMLS_CC) TSRMLS_CC); +#endif /* ZEND_MULTIBYTE */ BEGIN(INITIAL); if (file_handle->opened_path) { @@ -293,6 +349,12 @@ retval = NULL; } compilation_successful=1; +#ifdef ZEND_MULTIBYTE + if (SCNG(code)) + efree(SCNG(code)); + if (SCNG(current_code)) + efree(SCNG(current_code)); +#endif /* ZEND_MULTIBYTE */ } if (retval) { @@ -350,7 +412,40 @@ str->value.str.val[str->value.str.len+1]=0; SCNG(yy_in)=NULL; +#ifdef ZEND_MULTIBYTE + /* instead of zend_readfile() */ + SCNG(code) = estrdup(str->value.str.val); + if (!SCNG(code)) + return FAILURE; + SCNG(code_size) = str->value.str.len; + + zend_multibyte_set_filter(CG(internal_encoding) TSRMLS_CC); + + if (!SCNG(input_filter)) + { + SCNG(current_code) = (char*)emalloc(SCNG(code_size)+1); + if (!SCNG(current_code)) + return FAILURE; + memcpy(SCNG(current_code), SCNG(code), SCNG(code_size)+1); + SCNG(current_code_size) = SCNG(code_size); + } + else + { + SCNG(input_filter)(&SCNG(current_code), &SCNG(current_code_size), + SCNG(code), SCNG(code_size) TSRMLS_CC); + if (!SCNG(current_code) || !SCNG(current_code_size)) + return FAILURE; + } + + SCNG(current_code) = (char*)erealloc(SCNG(current_code), SCNG(current_code_size)+2); + if (!SCNG(current_code)) + return FAILURE; + *(SCNG(current_code)+SCNG(current_code_size)) = (char)NULL; + *(SCNG(current_code)+SCNG(current_code_size)+1) = (char)NULL; + yy_scan_buffer(SCNG(current_code), SCNG(current_code_size)+2 TSRMLS_CC); +#else /* ZEND_MULTIBYTE */ yy_scan_buffer(str->value.str.val, str->value.str.len+2 TSRMLS_CC); +#endif /* ZEND_MULTIBYTE */ zend_set_compiled_filename(filename TSRMLS_CC); CG(zend_lineno) = 1; @@ -390,6 +485,12 @@ CG(active_op_array) = op_array; BEGIN(ST_IN_SCRIPTING); compiler_result = zendparse(TSRMLS_C); +#ifdef ZEND_MULTIBYTE + if (SCNG(code)) + efree(SCNG(code)); + if (SCNG(current_code)) + efree(SCNG(current_code)); +#endif /* ZEND_MULTIBYTE */ if (compiler_result==1) { CG(active_op_array) = original_active_op_array; @@ -448,6 +549,135 @@ } END_EXTERN_C() +#ifdef ZEND_MULTIBYTE +BEGIN_EXTERN_C() +ZEND_API void zend_yyinput_again(zend_multibyte_filter old_input_filter, zend_encoding *old_encoding TSRMLS_DC) +{ + YY_BUFFER_STATE b = YY_CURRENT_BUFFER; + int offset, original_offset, length, free_flag; + char *p; + zend_encoding *new_encoding; + + /* calculate current position */ + offset = original_offset = yy_c_buf_p - b->yy_ch_buf; + if (old_input_filter && original_offset > 0) + { + /* we need to adjust offset ? (i've gotta improve this...) */ + new_encoding = SCNG(script_encoding); + SCNG(script_encoding) = old_encoding; + do + { + (old_input_filter)(&p, &length, SCNG(code), offset TSRMLS_CC); + if (!p) + { + SCNG(script_encoding) = new_encoding; + return; + } + efree(p); + if (length > original_offset) + offset--; + else if (length < original_offset) + offset++; + } while (original_offset != length); + SCNG(script_encoding) = new_encoding; + } + + /* convert and set */ + if (!SCNG(input_filter)) + { + length = SCNG(code_size)-offset-1; + p = SCNG(code)+offset+1; + free_flag = 0; + } + else + { + SCNG(input_filter)(&p, &length, SCNG(code)+offset+1, SCNG(code_size)-offset-1 TSRMLS_CC); + free_flag = 1; + } + if (original_offset+length+1 > (int)b->yy_buf_size) + { + b->yy_buf_size = original_offset+length+1; + b->yy_ch_buf = (char*)erealloc(b->yy_ch_buf, b->yy_buf_size+2); + SCNG(current_code) = b->yy_ch_buf; + SCNG(current_code_size) = b->yy_buf_size; + } + yy_c_buf_p = b->yy_ch_buf + original_offset; + strncpy(yy_c_buf_p+1, p, length); + b->yy_n_chars = original_offset + length + 1; + SCNG(yy_n_chars) = b->yy_n_chars; + b->yy_ch_buf[SCNG(yy_n_chars)] = YY_END_OF_BUFFER_CHAR; + b->yy_ch_buf[SCNG(yy_n_chars)+1] = YY_END_OF_BUFFER_CHAR; + + if (free_flag) + efree(p); +} + + +ZEND_API int zend_read_file(TSRMLS_D) +{ + char buf[8192]; + int n; + + SCNG(code) = NULL; + SCNG(code_size) = 0; + + do + { +# ifdef __cplusplus + if (SCNG(yy_in)->eof() || SCNG(yy_in)->fail()) + break; + (void)SCNG(yy_in)->read(buf, 8192); + if (SCNG(yy_in)->bad()) + return -1; + n = SCNG(yy_in)->gcount(); +# else + n = fread(buf, 1, 8192, SCNG(yy_in)); +# endif + if (n) + { + SCNG(code_size) += n; + if (SCNG(code)) + { + SCNG(code) = (char*)erealloc(SCNG(code), + SCNG(code_size)+1); + if (!SCNG(code)) + return -1; + } + else + { + SCNG(code) = (char*)emalloc(SCNG(code_size)+1); + if (!SCNG(code)) + return -1; + } + memcpy(SCNG(code)+SCNG(code_size)-n, buf, n); + } + } while(n); +#ifndef __cplusplus + if (ferror(SCNG(yy_in))) + return -1; +#endif + *(SCNG(code)+SCNG(code_size)) = (char)NULL; + return 0; +} + + +/* + * macros for yytext -> zval + */ +# define zend_copy_value(zendlval, yytext, yyleng) \ + if (SCNG(output_filter)) { \ + SCNG(output_filter)(&(zendlval->value.str.val), &(zendlval->value.str.len), \ + yytext, yyleng TSRMLS_CC); \ + } else { \ + zendlval->value.str.val = (char *) estrndup(yytext, yyleng); \ + zendlval->value.str.len = yyleng; \ + } + +#else /* ZEND_MULTIBYTE */ +# define zend_copy_value(zendlval, yytext, yyleng) \ + zendlval->value.str.val = (char *)estrndup(yytext, yyleng); \ + zendlval->value.str.len = yyleng; +#endif /* ZEND_MULTIBYTE */ %} LNUM [0-9]+ @@ -467,6 +697,10 @@ %option noyywrap %% +<ST_IN_SCRIPTING>"encoding" { + return T_ENCODING; +} + <ST_IN_SCRIPTING>"exit" { return T_EXIT; } @@ -594,7 +828,7 @@ <ST_LOOKING_FOR_PROPERTY>{LABEL} { yy_pop_state(TSRMLS_C); - zendlval->value.str.val = (char *)estrndup(yytext, yyleng); + zend_copy_value(zendlval, yytext, yyleng); zendlval->value.str.len = yyleng; zendlval->type = IS_STRING; return T_STRING; @@ -832,8 +1066,7 @@ <ST_LOOKING_FOR_VARNAME>{LABEL} { - zendlval->value.str.val = (char *) estrndup(yytext, yyleng); - zendlval->value.str.len = yyleng; + zend_copy_value(zendlval, yytext, yyleng); zendlval->type = IS_STRING; yy_pop_state(TSRMLS_C); yy_push_state(ST_IN_SCRIPTING TSRMLS_CC); @@ -938,8 +1171,21 @@ <INITIAL>(([^<]|"<"[^?%s<]){1,400})|"<s"|"<" { +#ifdef ZEND_MULTIBYTE + if (SCNG(output_filter)) { + int readsize; + readsize = SCNG(output_filter)(&(zendlval->value.str.val), &(zendlval->value.str.len), yytext, yyleng TSRMLS_CC); + if (readsize < yyleng) { + yyless(readsize); + } + } else { + zendlval->value.str.val = (char *) estrndup(yytext, yyleng); + zendlval->value.str.len = yyleng; + } +#else /* ZEND_MULTIBYTE */ zendlval->value.str.val = (char *) estrndup(yytext, yyleng); zendlval->value.str.len = yyleng; +#endif /* ZEND_MULTIBYTE */ zendlval->type = IS_STRING; HANDLE_NEWLINES(yytext, yyleng); return T_INLINE_HTML; @@ -1014,24 +1260,21 @@ } <ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL} { - zendlval->value.str.val = (char *)estrndup(yytext+1, yyleng-1); - zendlval->value.str.len = yyleng-1; + zend_copy_value(zendlval, (yytext+1), (yyleng-1)); zendlval->type = IS_STRING; return T_VARIABLE; } <ST_IN_SCRIPTING>{LABEL} { - zendlval->value.str.val = (char *)estrndup(yytext, yyleng); - zendlval->value.str.len = yyleng; + zend_copy_value(zendlval, yytext, yyleng); zendlval->type = IS_STRING; return T_STRING; } <ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>{LABEL} { - zendlval->value.str.val = (char *)estrndup(yytext, yyleng); - zendlval->value.str.len = yyleng; + zend_copy_value(zendlval, yytext, yyleng); zendlval->type = IS_STRING; return T_STRING; } @@ -1202,7 +1445,15 @@ } } *t = 0; - + +#ifdef ZEND_MULTIBYTE + if (SCNG(output_filter)) { + s = zendlval->value.str.val; + SCNG(output_filter)(&(zendlval->value.str.val), &(zendlval->value.str.len), s, zendlval->value.str.len TSRMLS_CC); + efree(s); + } +#endif /* ZEND_MULTIBYTE */ + return T_CONSTANT_ENCAPSED_STRING; } @@ -1242,7 +1493,16 @@ } } *t = 0; - + +#ifdef ZEND_MULTIBYTE + if (SCNG(output_filter)) + { + s = zendlval->value.str.val; + SCNG(output_filter)(&(zendlval->value.str.val), &(zendlval->value.str.len), s, zendlval->value.str.len TSRMLS_CC); + efree(s); + } +#endif /* ZEND_MULTIBYTE */ + return T_CONSTANT_ENCAPSED_STRING; } @@ -1310,8 +1570,7 @@ BEGIN(ST_IN_SCRIPTING); return T_END_HEREDOC; } else { - zendlval->value.str.val = (char *)estrndup(yytext, yyleng); - zendlval->value.str.len = yyleng; + zend_copy_value(zendlval, yytext, yyleng); zendlval->type = IS_STRING; return T_STRING; } @@ -1328,24 +1587,21 @@ <ST_SINGLE_QUOTE>([^'\\]|\\[^'\\])+ { HANDLE_NEWLINES(yytext, yyleng); - zendlval->value.str.val = (char *) estrndup(yytext, yyleng); - zendlval->value.str.len = yyleng; + zend_copy_value(zendlval, yytext, yyleng); zendlval->type = IS_STRING; return T_ENCAPSED_AND_WHITESPACE; } <ST_DOUBLE_QUOTES>[`]+ { - zendlval->value.str.val = (char *) estrndup(yytext, yyleng); - zendlval->value.str.len = yyleng; + zend_copy_value(zendlval, yytext, yyleng); zendlval->type = IS_STRING; return T_ENCAPSED_AND_WHITESPACE; } <ST_BACKQUOTE>["]+ { - zendlval->value.str.val = (char *) estrndup(yytext, yyleng); - zendlval->value.str.len = yyleng; + zend_copy_value(zendlval, yytext, yyleng); zendlval->type = IS_STRING; return T_ENCAPSED_AND_WHITESPACE; } Index: Zend/zend_multibyte.c +++ Zend/zend_multibyte.c /* +----------------------------------------------------------------------+ | Zend Engine | +----------------------------------------------------------------------+ | Copyright (c) 1998-2002 Zend Technologies Ltd. (http://www.zend.com) | +----------------------------------------------------------------------+ | This source file is subject to version 2.00 of the Zend license, | | that is bundled with this package in the file LICENSE, and is | | available at through the world-wide-web at | | http://www.zend.com/license/2_00.txt. | | If you did not receive a copy of the Zend license and are unable to | | obtain it through the world-wide-web, please send a note to | | license@zend.com so we can mail you a copy immediately. | +----------------------------------------------------------------------+ | Authors: Masaki Fujimoto <masaki-f@fides.dti.ne.jp> | | Rui Hirokawa <hirokawa@php.net> | +----------------------------------------------------------------------+ */ /* $Id: zend_multibyte.c,v 1.1 2002/05/08 12:29:51 hirokawa Exp $ */ #include "zend.h" #include "zend_compile.h" #include "zend_istdiostream.h" #include "zend_operators.h" #include "zend_multibyte.h" #ifdef ZEND_MULTIBYTE /*-------------------------------------------------------------------------- vars --------------------------------------------------------------------------*/ const char *utf8_aliases[] = {"UTF8", NULL}; zend_encoding encoding_utf8 = { NULL, NULL, "UTF-8", (const char*(*)[])&utf8_aliases, 1 }; const char *euc_jp_aliases[] = {"EUC_JP", "eucJP", "x-euc-jp", NULL}; zend_encoding encoding_euc_jp = { NULL, NULL, "EUC-JP", (const char*(*)[])&euc_jp_aliases, 1 }; const char *sjis_aliases[] = {"sjis", "x-sjis", "SHIFT-JIS", NULL}; zend_encoding encoding_sjis = { sjis_input_filter, sjis_output_filter, "Shift_JIS", (const char*(*)[])&sjis_aliases, 0 }; const char *ascii_aliases[] = {"us-ascii", NULL}; zend_encoding encoding_ascii = { NULL, NULL, "ASCII", (const char*(*)[])&ascii_aliases, 1 }; zend_encoding *zend_encoding_table[] = { &encoding_utf8, &encoding_ascii, &encoding_euc_jp, &encoding_sjis, NULL }; /*-------------------------------------------------------------------------- protos --------------------------------------------------------------------------*/ static char* zend_multibyte_assemble_list(zend_encoding **encoding_list, int encoding_list_size); static int zend_multibyte_parse_encoding_list(const char *encoding_list, int encoding_list_size, zend_encoding ***result, int *result_size); static inline char* zend_memnstr(char *haystack, char *needle, int needle_len, char *end); /*-------------------------------------------------------------------------- interfaces --------------------------------------------------------------------------*/ ZEND_API int zend_multibyte_set_script_encoding(char *encoding_list, int encoding_list_size TSRMLS_DC) { if (CG(script_encoding_list)) { efree(CG(script_encoding_list)); CG(script_encoding_list) = NULL; } CG(script_encoding_list_size) = 0; if (!encoding_list) return 0; zend_multibyte_parse_encoding_list(encoding_list, encoding_list_size, &(CG(script_encoding_list)), &(CG(script_encoding_list_size))); return 0; } ZEND_API int zend_multibyte_set_internal_encoding(char *encoding_name, int encoding_name_size TSRMLS_DC) { CG(internal_encoding) = zend_multibyte_fetch_encoding(encoding_name); return 0; } ZEND_API int zend_multibyte_set_functions(zend_encoding_detector encoding_detector, zend_encoding_converter encoding_converter, zend_multibyte_oddlen multibyte_oddlen TSRMLS_DC) { CG(encoding_detector) = encoding_detector; CG(encoding_converter) = encoding_converter; CG(multibyte_oddlen) = multibyte_oddlen; return 0; } ZEND_API int zend_multibyte_set_filter(zend_encoding *onetime_encoding TSRMLS_DC) { zend_encoding *script_encoding = NULL, *internal_encoding = NULL; char *name, *list; LANG_SCNG(input_filter) = NULL; LANG_SCNG(output_filter) = NULL; LANG_SCNG(script_encoding) = NULL; LANG_SCNG(internal_encoding) = NULL; /* decision making... */ if (!onetime_encoding) { if (!CG(script_encoding_list) || !CG(script_encoding_list_size)) script_encoding = NULL; else if (CG(script_encoding_list_size) > 1 && CG(encoding_detector)) { /* we need to detect automatically */ list = zend_multibyte_assemble_list(CG(script_encoding_list), CG(script_encoding_list_size)); name = CG(encoding_detector)(LANG_SCNG(code), LANG_SCNG(code_size), list TSRMLS_CC); if (list) efree(list); script_encoding = zend_multibyte_fetch_encoding(name); efree(name); } else script_encoding = *(CG(script_encoding_list)); } else script_encoding = onetime_encoding; if (CG(encoding_converter)) internal_encoding = CG(internal_encoding); else internal_encoding = NULL; /* judge input/output filter */ LANG_SCNG(script_encoding) = script_encoding; LANG_SCNG(internal_encoding) = internal_encoding; if (!script_encoding) return 0; if (!internal_encoding) { if (script_encoding->input_filter) LANG_SCNG(input_filter) = script_encoding->input_filter; if (script_encoding->output_filter) LANG_SCNG(output_filter) = script_encoding->output_filter; return 0; } if (script_encoding == internal_encoding) { if (script_encoding->compatible) return 0; if (script_encoding->input_filter) LANG_SCNG(input_filter) = script_encoding->input_filter; if (script_encoding->output_filter) LANG_SCNG(output_filter) = script_encoding->output_filter; return 0; } if (internal_encoding->compatible) { LANG_SCNG(input_filter) = zend_multibyte_encoding_filter; return 0; } if (script_encoding->compatible) { LANG_SCNG(output_filter) = zend_multibyte_encoding_filter; return 0; } /* no way! it's too hard... :( */ if (script_encoding->input_filter && internal_encoding->output_filter) { LANG_SCNG(input_filter) = script_encoding->input_filter; LANG_SCNG(output_filter) = internal_encoding->output_filter; } return 0; } ZEND_API zend_encoding* zend_multibyte_fetch_encoding(char *encoding_name) { int i, j; zend_encoding *encoding; if (!encoding_name) return NULL; for (i = 0; (encoding = zend_encoding_table[i]) != NULL; i++) { if (zend_binary_strcasecmp((char*)encoding->name, strlen(encoding->name), encoding_name, strlen(encoding_name)) == 0) return encoding; } for (i = 0; (encoding = zend_encoding_table[i]) != NULL; i++) { if (encoding->aliases != NULL) { for (j = 0; (*encoding->aliases)[j] != NULL; j++) { if (zend_binary_strcasecmp((char*)(*encoding->aliases)[j], strlen((*encoding->aliases)[j]), encoding_name, strlen(encoding_name)) == 0) return encoding; } } } return NULL; } ZEND_API int zend_multibyte_encoding_filter(char **to, int *to_length, char *from, int from_length TSRMLS_DC) { int oddlen; if (!CG(encoding_converter)) return 0; if (CG(multibyte_oddlen)) { oddlen = CG(multibyte_oddlen)(from, from_length, LANG_SCNG(script_encoding)->name TSRMLS_CC); if (oddlen > 0) from_length -= oddlen; } if (CG(encoding_converter)(to, to_length, from, from_length, LANG_SCNG(internal_encoding)->name, LANG_SCNG(script_encoding)->name TSRMLS_CC) != 0) return 0; return from_length; } /*-------------------------------------------------------------------------- input/output filters --------------------------------------------------------------------------*/ /* * Shift_JIS */ static const unsigned char table_sjis[] = { /* 0x80-0x9f,0xE0-0xEF */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 5, 0, 0, 0 }; int sjis_input_filter(char **buf, int *length, char *sjis, int sjis_length TSRMLS_DC) { unsigned char *p, *q; unsigned char c1, c2; *buf = (char*)emalloc(sjis_length*3/2+1); if (!*buf) return 0; *length = 0; p = (unsigned char*)sjis; q = (unsigned char*)*buf; /* convert [SJIS -> EUC-JP] (for lex scan) -- some other better ways? */ while (*p && (p-(unsigned char*)sjis) < sjis_length) { if (!(*p & 0x80)) { *q++ = *p++; continue; } /* handling 8 bit code */ if (table_sjis[*p] == 1) { /* 1 byte kana */ *q++ = 0x8e; *q++ = *p++; continue; } if (!*(p+1)) { *q++ = *p++; break; } if (table_sjis[*p] == 2) { /* 2 byte kanji code */ c1 = *p++; if (!*p || (p-(unsigned char*)sjis) >= sjis_length) break; c2 = *p++; c1 -= (c1 <= 0x9f) ? 0x71 : 0xb1; c1 = (c1 << 1) + 1; if (c2 >= 0x9e) { c2 -= 0x7e; c1++; } else if (c2 > 0x7f) c2 -= 0x20; else c2 -= 0x1f; c1 |= 0x80; c2 |= 0x80; *q++ = c1; *q++ = c2; } /* * for user defined chars (ATTENTION) * * THESE ARE NOT CODE FOR CONVERSION! :-P * (using *ILLEGALLY* 3byte EUC-JP space) * * we cannot perfectly (== 1 to 1) convert these chars to EUC-JP. * so, these code are for perfect RESTORING in sjis_output_filter() */ else { c1 = *p++; if (!*p || (p-(unsigned char*)sjis) >= sjis_length) break; c2 = *p++; *q++ = (char)0x8f; /* * MAP TO (EUC-JP): * type A: 0xeba1 - 0xf4fe * type B: 0xf5a1 - 0xfefe * type C: 0xa1a1 - 0xa6fe */ c1 -= (c1 > 0xf9) ? (0x79+0x71) : (0x0a+0xb1); c1 = (c1 << 1) + 1; if (c2 >= 0x9e) { c2 -= 0x7e; c1++; } else if (c2 > 0x7f) c2 -= 0x20; else c2 -= 0x1f; c1 |= 0x80; c2 |= 0x80; *q++ = c1; *q++ = c2; } } *q = (char)NULL; *length = (char*)q - *buf; return *length; } static const unsigned char table_eucjp[] = { /* 0xA1-0xFE */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1 }; int sjis_output_filter(char **sjis, int *sjis_length, char *buf, int length TSRMLS_DC) { unsigned char c1, c2; char *p, *q; if (!sjis || !sjis_length) return 0; /* always Shift_JIS <= EUC-JP */ *sjis = (char*)emalloc(length+1); if (!sjis) return 0; p = *sjis; q = buf; /* restore converted strings [EUC-JP -> Shift_JIS] */ while (*q) { if (!(*q & 0x80)) { *p++ = *q++; continue; } /* hankaku kana */ if (*q == (char)0x8e) { q++; if (*q) *p++ = *q++; continue; } /* 2 byte kanji code */ if (table_eucjp[(unsigned char)*q] == 2) { c1 = (*q++ & ~0x80) & 0xff; if (*q) c2 = (*q++ & ~0x80) & 0xff; else { q--; break; } c2 += (c1 & 0x01) ? 0x1f : 0x7d; if (c2 >= 0x7f) c2++; c1 = ((c1 - 0x21) >> 1) + 0x81; if (c1 > 0x9f) c1 += 0x40; *p++ = c1; *p++ = c2; continue; } if (*q == (char)0x8f) { q++; if (*q) c1 = (*q++ & ~0x80) & 0xff; else { q--; break; } if (*q) c2 = (*q++ & ~0x80) & 0xff; else { q -= 2; break; } c2 += (c1 & 0x01) ? 0x1f : 0x7d; if (c2 >= 0x7f) c2++; c1 = ((c1 - 0x21) >> 1) + 0x81; if (c1 > 0x9f) c1 += 0x40; if (c1 >= 0x81 && c1 <= 0x9f) c1 += 0x79; else c1 += 0x0a; *p++ = c1; *p++ = c2; continue; } /* some other chars (may not happen) */ *p++ = *q++; } *p = '\0'; *sjis_length = p - *sjis; return q-buf; /* return length we actually read */ } /*-------------------------------------------------------------------------- static funcs --------------------------------------------------------------------------*/ static char* zend_multibyte_assemble_list(zend_encoding **encoding_list, int encoding_list_size) { int i, list_size = 0; const char *name; char *list = NULL; if (!encoding_list || !encoding_list_size) return NULL; for (i = 0; i < encoding_list_size; i++) { name = (*(encoding_list+i))->name; if (name) { list_size += strlen(name) + 1; if (!list) { list = (char*)emalloc(list_size); if (!list) return NULL; *list = (char)NULL; } else { list = (char*)erealloc(list, list_size); if (!list) return NULL; strcat(list, ","); } strcat(list, name); } } return list; } /* * code stolen from ext/mbstring/mbstring.c */ static int zend_multibyte_parse_encoding_list(const char *encoding_list, int encoding_list_size, zend_encoding ***result, int *result_size) { int n, size; char *p, *p1, *p2, *endp, *tmpstr; zend_encoding **list, **entry, *encoding; list = NULL; if (encoding_list == NULL || encoding_list_size <= 0) { return -1; } else { /* copy the encoding_list string for work */ tmpstr = (char *)estrndup(encoding_list, encoding_list_size); if (tmpstr == NULL) { return -1; } /* count the number of listed encoding names */ endp = tmpstr + encoding_list_size; n = 1; p1 = tmpstr; while ((p2 = zend_memnstr(p1, ",", 1, endp)) != NULL) { p1 = p2 + 1; n++; } size = n; /* make list */ list = (zend_encoding**)ecalloc(size, sizeof(zend_encoding*)); if (list != NULL) { entry = list; n = 0; p1 = tmpstr; do { p2 = p = zend_memnstr(p1, ",", 1, endp); if (p == NULL) { p = endp; } *p = '\0'; /* trim spaces */ while (p1 < p && (*p1 == ' ' || *p1 == '\t')) { p1++; } p--; while (p > p1 && (*p == ' ' || *p == '\t')) { *p = '\0'; p--; } /* convert to the encoding number and check encoding */ encoding = zend_multibyte_fetch_encoding(p1); if (encoding) { *entry++ = encoding; n++; } p1 = p2 + 1; } while (n < size && p2 != NULL); *result = list; *result_size = n; } efree(tmpstr); } if (list == NULL) { return -1; } return 0; } /* * code stolen from ext/standard/php_string.h */ static inline char* zend_memnstr(char *haystack, char *needle, int needle_len, char *end) { char *p = haystack; char first = *needle; /* let end point to the last character where needle may start */ end -= needle_len; while (p <= end) { while (*p != first) if (++p > end) return NULL; if (memcmp(p, needle, needle_len) == 0) return p; p++; } return NULL; } #endif /* ZEND_MULTIBYTE */ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: sw=4 ts=4 tw=78 * vim<600: sw=4 ts=4 tw=78 */ Index: Zend/zend_multibyte.h +++ Zend/zend_multibyte.h /* +----------------------------------------------------------------------+ | Zend Engine | +----------------------------------------------------------------------+ | Copyright (c) 1998-2002 Zend Technologies Ltd. (http://www.zend.com) | +----------------------------------------------------------------------+ | This source file is subject to version 2.00 of the Zend license, | | that is bundled with this package in the file LICENSE, and is | | available at through the world-wide-web at | | http://www.zend.com/license/2_00.txt. | | If you did not receive a copy of the Zend license and are unable to | | obtain it through the world-wide-web, please send a note to | | license@zend.com so we can mail you a copy immediately. | +----------------------------------------------------------------------+ | Authors: Masaki Fujimoto <masaki-f@fides.dti.ne.jp> | | Rui Hirokawa <hirokawa@php.net> | +----------------------------------------------------------------------+ */ /* $Id: zend_multibyte.h,v 1.1 2002/05/08 12:29:51 hirokawa Exp $ */ #ifndef ZEND_MULTIBYTE_H #define ZEND_MULTIBYTE_H #ifdef ZEND_MULTIBYTE /*------------------------------------------------------------------------- defs -------------------------------------------------------------------------*/ /*------------------------------------------------------------------------- types -------------------------------------------------------------------------*/ typedef int (*zend_multibyte_filter)(char **str, int *str_length, char *buf, int length TSRMLS_DC); typedef char* (*zend_encoding_detector)(char *string, int length, char *list TSRMLS_DC); typedef int (*zend_encoding_converter)(char **to, int *to_length, char *from, int from_length, const char *encoding_to, const char *encoding_from TSRMLS_DC); typedef int (*zend_multibyte_oddlen)(char *string, int length, const char *encoding TSRMLS_DC); typedef struct _zend_encoding { zend_multibyte_filter input_filter; /* escape input filter */ zend_multibyte_filter output_filter; /* escape output filter */ const char *name; /* encoding name */ const char *(*aliases)[]; /* encoding name aliases */ int compatible; /* we can parse without any escaping? */ } zend_encoding; /*------------------------------------------------------------------------- protos -------------------------------------------------------------------------*/ BEGIN_EXTERN_C() ZEND_API int zend_multibyte_set_script_encoding(char *encoding_list, int encoding_list_size TSRMLS_DC); ZEND_API int zend_multibyte_set_internal_encoding(char *encoding_name, int encoding_name_size TSRMLS_DC); ZEND_API int zend_multibyte_set_functions(zend_encoding_detector encoding_detector, zend_encoding_converter encoding_converter, zend_multibyte_oddlen multibyte_oddlen TSRMLS_DC); ZEND_API int zend_multibyte_set_filter(zend_encoding *onetime_encoding TSRMLS_DC); ZEND_API zend_encoding* zend_multibyte_fetch_encoding(char *encoding_name); ZEND_API int zend_multibyte_encoding_filter(char **to, int *to_length, char *from, int from_length TSRMLS_DC); /* in zend_language_scanner.l */ ZEND_API void zend_yyinput_again(zend_multibyte_filter old_input_filter, zend_encoding *old_encoding TSRMLS_DC); ZEND_API int zend_read_file(TSRMLS_D); /* input/output filter(s) */ int sjis_input_filter(char **buf, int *length, char *sjis, int sjis_length TSRMLS_DC); int sjis_output_filter(char **sjis, int *sjis_length, char *buf, int length TSRMLS_DC); END_EXTERN_C() #endif /* ZEND_MULTIBYTE */ #endif /* ZEND_MULTIBYTE_H */ /* * Local variables: * tab-width: 4 * c-basic-offset: 4 * End: * vim600: sw=4 ts=4 tw=78 * vim<600: sw=4 ts=4 tw=78 */