Re: ZEND_ADD_STRING patch

Hi Andi, The last patch I submitted was broken as well. Following that, I had the bright idea to run the prospective changes through the unit-tester to ensure correct performance. Here's a patch which achieves that. It does not work for heredocs (i.e. they are tokenized as before, but behave correctly from a language perspective), but otherwise optimizes correctly. Here you go: diff -u -3 -r1.53 zend_language_scanner.l

--- Zend/zend_language_scanner.l    8 Nov 2002 13:40:54 -0000    1.53
+++ Zend/zend_language_scanner.l    12 Nov 2002 22:11:31 -0000

@@ -37,6 +37,7 @@ %x ST_BACKQUOTE %x ST_HEREDOC %x ST_LOOKING_FOR_PROPERTY +%x ST_EXPECTING_OBJECT %x ST_LOOKING_FOR_VARNAME %x ST_COMMENT %x ST_ONE_LINE_COMMENT @@ -692,6 +693,7 @@

HNUM    "0x"[0-9a-fA-F]+
LABEL    [a-zA-Z_\x7f-\xff][a-zA-Z0-9_\x7f-\xff]*

WHITESPACE [ \n\r\t]+ +LABEL_OR_WHITESPACE [a-zA-Z0-9_\x7f-\xff \t\n\r #'.:;,()|^&+-/*=%!~<>?@]+ TABS_AND_SPACES [ \t]* TOKENS [;:,.\[\]()|^&+-/*=%!~$<>?@] ENCAPSED_TOKENS [\[\]{}$] @@ -823,13 +825,22 @@

    return T_EXTENDS;

} -<ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"->" { +<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>"$"{LABEL}"->"{LABEL} {

+        yy_push_state(ST_EXPECTING_OBJECT TSRMLS_CC);
+        yyless(0);

+} + + +<ST_IN_SCRIPTING,ST_EXPECTING_OBJECT>"->" {

    yy_push_state(ST_LOOKING_FOR_PROPERTY TSRMLS_CC);
    return T_OBJECT_OPERATOR;

} <ST_LOOKING_FOR_PROPERTY>{LABEL} {

    yy_pop_state(TSRMLS_C);
+        if(yy_top_state(TSRMLS_C) == ST_EXPECTING_OBJECT) {
+            yy_pop_state(TSRMLS_C);
+        }
     zend_copy_value(zendlval, yytext, yyleng);
    zendlval->value.str.len = yyleng;
    zendlval->type = IS_STRING;

@@ -1265,7 +1276,7 @@

    return T_INLINE_HTML;

} -<ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL} { +<ST_EXPECTING_OBJECT,ST_IN_SCRIPTING,ST_DOUBLE_QUOTES,ST_HEREDOC,ST_BACKQUOTE>"$"{LABEL} {

     zend_copy_value(zendlval, (yytext+1), (yyleng-1));
    zendlval->type = IS_STRING;
    return T_VARIABLE;

@@ -1278,13 +1289,26 @@

    return T_STRING;

} - -<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>{LABEL} { +<ST_DOUBLE_QUOTES,ST_BACKQUOTE>{LABEL_OR_WHITESPACE} {

+        HANDLE_NEWLINES(yytext, yyleng);
     zend_copy_value(zendlval, yytext, yyleng);
    zendlval->type = IS_STRING;
    return T_STRING;

} +<ST_HEREDOC>{LABEL} {

+        zend_copy_value(zendlval, yytext, yyleng);
+        zendlval->type = IS_STRING;
+        return T_STRING;

+} + +<ST_HEREDOC>{ESCAPED_AND_WHITESPACE} {

+        HANDLE_NEWLINES(yytext, yyleng);
+        zendlval->value.str.val = (char *) estrndup(yytext, yyleng);
+        zendlval->value.str.len = yyleng;
+        zendlval->type = IS_STRING;
+        return T_ENCAPSED_AND_WHITESPACE;

+} <ST_IN_SCRIPTING>{WHITESPACE} {

    zendlval->value.str.val = yytext; /* no copying - intentional */

@@ -1581,14 +1605,6 @@

    }

} - -<ST_DOUBLE_QUOTES,ST_BACKQUOTE,ST_HEREDOC>{ESCAPED_AND_WHITESPACE} {

-    HANDLE_NEWLINES(yytext, yyleng);
-    zendlval->value.str.val = (char *) estrndup(yytext, yyleng);
-    zendlval->value.str.len = yyleng;
-    zendlval->type = IS_STRING;
-    return T_ENCAPSED_AND_WHITESPACE;

-} <ST_SINGLE_QUOTE>([^'\\]|\\[^'\\])+ {

    HANDLE_NEWLINES(yytext, yyleng);