cvs: pear /PHP_LexerGenerator package.xml /PHP_LexerGenerator/LexerGenerator Parser.php Parser.y /PHP_LexerGenerator/examples longestfirst.php
longestfirst.plex
| From: | Greg Beaver | Date: | Thu, 12 Oct 2006 18:40:02 +0000 |
| Subject: | cvs: pear /PHP_LexerGenerator package.xml /PHP_LexerGenerator/LexerGenerator Parser.php Parser.y /PHP_LexerGenerator/examples longestfirst.php longestfirst.plex |
||
| Groups: | php.pear.cvs | ||
| Request: | Send a blank email to pear-cvs+get-42505@lists.php.net to get a copy of this message | ||
cellog Thu Oct 12 18:40:02 2006 UTC
Added files:
/pear/PHP_LexerGenerator/examples longestfirst.php
longestfirst.plex
Modified files:
/pear/PHP_LexerGenerator package.xml
/pear/PHP_LexerGenerator/LexerGenerator Parser.php Parser.y
Log:
- fix Bug #9007: yymore does not honor return value "true" to change state
- implement Request #8879: lexer not picking longest match
prepare release of 0.3.0
http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/package.xml?r1=1.5&r2=1.6&diff_format=u Index: pear/PHP_LexerGenerator/package.xml diff -u pear/PHP_LexerGenerator/package.xml:1.5 pear/PHP_LexerGenerator/package.xml:1.6 --- pear/PHP_LexerGenerator/package.xml:1.5 Sat Oct 7 11:40:01 2006 +++ pear/PHP_LexerGenerator/package.xml Thu Oct 12 18:40:01 2006 @@ -13,6 +13,11 @@ Generated lexers are very efficient, more than twice as efficient as other alternatives like csLex (written in C#) because they utilize PHP's built-in Perl-compatible regular expressions to lex for tokens. + + Now in version 0.3.0+, with processing instruction %longestmatch, generated + lexers will always pick the longest string to match, rather than the first. + Generated lexers are slightly slower, but match behavior of legacy lexers like + flex, lex, re2c. </description> <lead> <name>Greg Beaver</name> @@ -20,10 +25,10 @@ <email>cellog@php.net</email> <active>yes</active> </lead> - <date>2006-10-02</date> + <date>2006-10-12</date> <version> - <release>0.2.3</release> - <api>0.1.0</api> + <release>0.3.0</release> + <api>0.3.0</api> </version> <stability> <release>alpha</release> @@ -31,7 +36,9 @@ </stability> <license uri="http://www.opensource.org/licenses/bsd-license.php">New BSD License</license> <notes> - - fix Bug #8618: null causes regex to break + - fix Bug #9006: wrong replacements for plex script (Hartmut Holzgraefe) + - fix Bug #9007: yymore does not honor return value "true" to change state + - implement Request #8879: lexer not picking longest match </notes> <contents> <dir baseinstalldir="PHP" name="/"> @@ -170,5 +177,22 @@ - fix Bug #8618: null causes regex to break </notes> </release> + <release> + <version> + <release>0.3.0</release> + <api>0.3.0</api> + </version> + <stability> + <release>alpha</release> + <api>alpha</api> + </stability> + <date>2006-10-02</date> + <license uri="http://www.opensource.org/licenses/bsd-license.php">New BSD License</license> + <notes> + - fix Bug #9006: wrong replacements for plex script (Hartmut Holzgraefe) + - fix Bug #9007: yymore does not honor return value "true" to change state + - implement Request #8879: lexer not picking longest match + </notes> + </release> </changelog> </package> http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/LexerGenerator/Parser.php?r1=1.2&r2=1.3&diff_format=u Index: pear/PHP_LexerGenerator/LexerGenerator/Parser.php diff -u pear/PHP_LexerGenerator/LexerGenerator/Parser.php:1.2 pear/PHP_LexerGenerator/LexerGenerator/Parser.php:1.3 --- pear/PHP_LexerGenerator/LexerGenerator/Parser.php:1.2 Wed Aug 30 20:10:49 2006 +++ pear/PHP_LexerGenerator/LexerGenerator/Parser.php Thu Oct 12 18:40:02 2006 @@ -140,7 +140,7 @@ * @author Gregory Beaver <cellog@php.net> * @copyright 2006 Gregory Beaver * @license http://www.opensource.org/licenses/bsd-license.php New BSD License - * @version CVS: $Id: Parser.php,v 1.2 2006/08/30 20:10:49 cellog Exp $ + * @version CVS: $Id: Parser.php,v 1.3 2006/10/12 18:40:02 cellog Exp $ * @since File available since Release 0.1.0 */ /** @@ -169,7 +169,7 @@ { /* First off, code is included which follows the "include_class" declaration ** in the input file. */ -#line 82 "Parser.y" +#line 78 "Parser.y" private $patterns; private $out; @@ -179,6 +179,7 @@ private $token; private $value; private $line; + private $matchlongest; private $_regexLexer; private $_regexParser; private $_patternIndex = 0; @@ -205,15 +206,116 @@ $this->_regexParser = new PHP_LexerGenerator_Regex_Parser($this->_regexLexer); } - function outputRules($rules, $statename) + function doLongestMatch($rules, $statename, $ruleindex) + { + fwrite($this->out, ' + do { + $rules = array('); + foreach ($rules as $rule) { + fwrite($this->out, ' + \'/^' . $rule['pattern'] . '/\','); + } + fwrite($this->out, ' + ); + $match = false; + foreach ($rules as $index => $rule) { + if (preg_match($rule, substr(' . $this->input . ', ' . + $this->counter . '), $yymatches)) { + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}($yysubmatches); + if ($r === null) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + // accept this token + return true; + } elseif ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } elseif ($r === false) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + if (' . $this->counter . ' >= strlen(' . $this->input . ')) { + return false; // end of input + } + // skip this token + continue; + } else {'); + fwrite($this->out, ' + $yy_yymore_patterns = array_slice($rules, $this->token, true); + // yymore is needed + do { + if (!isset($yy_yymore_patterns[' . $this->token . '])) { + throw new Exception(\'cannot do yymore for the last token\'); + } + $match = false; + foreach ($yy_yymore_patterns[' . $this->token . '] as $index => $rule) { + if (preg_match($rule, + substr(' . $this->input . ', ' . $this->counter . '), $yymatches)) { + $yymatches = array_filter($yymatches, \'strlen\'); // remove empty sub-patterns + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + ' . $this->line . ' = substr_count("\n", ' . $this->value . '); + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } + } + } while (true); +'); + } + + function doFirstMatch($rules, $statename, $ruleindex) { - static $ruleindex = 1; $patterns = array(); $pattern = '/'; $ruleMap = array(); $tokenindex = array(); - $i = 0; - $actualindex = 1; foreach ($rules as $rule) { $ruleMap[$i++] = $actualindex; $tokenindex[$actualindex] = $rule['subpatterns']; @@ -226,12 +328,7 @@ $tokenindex = implode("\n ", $tokenindex); $pattern .= implode('|', $patterns); $pattern .= '/'; - if (!$statename) { - $statename = $ruleindex; - } fwrite($this->out, ' - function yylex' . $ruleindex . '() - { $tokenMap = ' . $tokenindex . '; if (' . $this->counter . ' >= strlen(' . $this->input . ')) { return false; // end of input @@ -301,11 +398,18 @@ ' . $this->value . ' = current($yymatches); // token value ' . $this->line . ' = substr_count("\n", ' . $this->value . '); } - } while ($this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}() !== null); - // accept - ' . $this->counter . ' += strlen($this->value); - ' . $this->line . ' += substr_count("\n", ' . $this->value . '); - return true; + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } } } else { throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . @@ -313,6 +417,26 @@ } break; } while (true); +'); + } + + function outputRules($rules, $statename) + { + static $ruleindex = 1; + $i = 0; + $actualindex = 1; + if (!$statename) { + $statename = $ruleindex; + } + fwrite($this->out, ' + function yylex' . $ruleindex . '() + {'); + if ($this->matchlongest) { + $this->doLongestMatch($rules, $statename, $ruleindex); + } else { + $this->doFirstMatch($rules, $statename, $ruleindex); + } + fwrite($this->out, ' } // end function '); @@ -322,7 +446,7 @@ '); } foreach ($rules as $i => $rule) { - fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $ruleMap[$i] . '($yy_subpatterns) + fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $i . '($yy_subpatterns) { ' . $rule['code'] . ' } @@ -352,7 +476,7 @@ } return $this->_regexParser->result; } -#line 359 "Parser.php" +#line 483 "Parser.php" /* Next is all token values, as class constants */ @@ -1114,7 +1238,7 @@ ** function yy_r0($yymsp){ ... } // User supplied code ** #line <lineno> <thisfile> */ -#line 268 "Parser.y" +#line 388 "Parser.y" function yy_r1(){ fwrite($this->out, ' private $_yy_state = 1; @@ -1149,8 +1273,8 @@ } } } -#line 1158 "Parser.php" -#line 302 "Parser.y" +#line 1282 "Parser.php" +#line 422 "Parser.y" function yy_r2(){ fwrite($this->out, ' private $_yy_state = 1; @@ -1188,8 +1312,8 @@ } } } -#line 1197 "Parser.php" -#line 339 "Parser.y" +#line 1321 "Parser.php" +#line 459 "Parser.y" function yy_r3(){ if (strlen($this->yystack[$this->yyidx + -2]->minor)) { fwrite($this->out, $this->yystack[$this->yyidx + -2]->minor); @@ -1227,8 +1351,8 @@ } } } -#line 1236 "Parser.php" -#line 376 "Parser.y" +#line 1360 "Parser.php" +#line 496 "Parser.y" function yy_r4(){ if (strlen($this->yystack[$this->yyidx + -3]->minor)) { fwrite($this->out, $this->yystack[$this->yyidx + -3]->minor); @@ -1269,15 +1393,15 @@ } } } -#line 1278 "Parser.php" -#line 417 "Parser.y" +#line 1402 "Parser.php" +#line 537 "Parser.y" function yy_r5(){ $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor; $this->patterns = $this->yystack[$this->yyidx + -1]->minor['patterns']; $this->_patternIndex = 1; } -#line 1285 "Parser.php" -#line 423 "Parser.y" +#line 1409 "Parser.php" +#line 543 "Parser.y" function yy_r6(){ $expected = array( 'counter' => true, @@ -1302,10 +1426,14 @@ 'token' => true, 'value' => true, 'line' => true, + 'matchlongest' => true, ); foreach ($this->yystack[$this->yyidx + -1]->minor as $pi) { if (isset($expected[$pi['pi']])) { $this->{$pi['pi']} = $pi['definition']; + if ($pi['pi'] == 'matchlongest') { + $this->matchlongest = true; + } continue; } $this->error('Unknown processing instruction %' . $pi['pi'] . @@ -1314,26 +1442,26 @@ $this->_retvalue = array('patterns' => $this->yystack[$this->yyidx + 0]->minor, 'pis' => $this->yystack[$this->yyidx + -1]->minor); $this->_patternIndex = 1; } -#line 1323 "Parser.php" -#line 460 "Parser.y" +#line 1451 "Parser.php" +#line 584 "Parser.y" function yy_r7(){ $this->_retvalue = array(array('pi' => $this->yystack[$this->yyidx + -1]->minor, 'definition' => $this->yystack[$this->yyidx + 0]->minor)); } -#line 1328 "Parser.php" -#line 466 "Parser.y" +#line 1456 "Parser.php" +#line 590 "Parser.y" function yy_r9(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; $this->_retvalue[] = array('pi' => $this->yystack[$this->yyidx + -1]->minor, 'definition' => $this->yystack[$this->yyidx + 0]->minor); } -#line 1334 "Parser.php" -#line 475 "Parser.y" +#line 1462 "Parser.php" +#line 599 "Parser.y" function yy_r11(){ $this->_retvalue = array($this->yystack[$this->yyidx + -1]->minor => $this->yystack[$this->yyidx + 0]->minor); // reset internal indicator of where we are in a pattern $this->_patternIndex = 0; } -#line 1341 "Parser.php" -#line 480 "Parser.y" +#line 1469 "Parser.php" +#line 604 "Parser.y" function yy_r12(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; if (isset($this->_retvalue[$this->yystack[$this->yyidx + -1]->minor])) { @@ -1344,13 +1472,13 @@ // reset internal indicator of where we are in a pattern declaration $this->_patternIndex = 0; } -#line 1353 "Parser.php" -#line 491 "Parser.y" +#line 1481 "Parser.php" +#line 615 "Parser.y" function yy_r13(){ $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => '')); } -#line 1358 "Parser.php" -#line 494 "Parser.y" +#line 1486 "Parser.php" +#line 618 "Parser.y" function yy_r14(){ if ($this->yystack[$this->yyidx + -3]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1358,13 +1486,13 @@ } $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => $this->yystack[$this->yyidx + -2]->minor)); } -#line 1367 "Parser.php" -#line 501 "Parser.y" +#line 1495 "Parser.php" +#line 625 "Parser.y" function yy_r15(){ $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => '')); } -#line 1372 "Parser.php" -#line 504 "Parser.y" +#line 1500 "Parser.php" +#line 628 "Parser.y" function yy_r16(){ if ($this->yystack[$this->yyidx + -4]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1373,15 +1501,15 @@ $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => $this->yystack[$this->yyidx + -3]->minor)); $this->_patternIndex = 1; } -#line 1382 "Parser.php" -#line 512 "Parser.y" +#line 1510 "Parser.php" +#line 636 "Parser.y" function yy_r17(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => ''); $this->_patternIndex = 1; } -#line 1389 "Parser.php" -#line 517 "Parser.y" +#line 1517 "Parser.php" +#line 641 "Parser.y" function yy_r18(){ if ($this->yystack[$this->yyidx + -3]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1390,14 +1518,14 @@ $this->_retvalue = $this->yystack[$this->yyidx + -4]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => $this->yystack[$this->yyidx + -2]->minor); } -#line 1399 "Parser.php" -#line 525 "Parser.y" +#line 1527 "Parser.php" +#line 649 "Parser.y" function yy_r19(){ $this->_retvalue = $this->yystack[$this->yyidx + -3]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => ''); } -#line 1405 "Parser.php" -#line 529 "Parser.y" +#line 1533 "Parser.php" +#line 653 "Parser.y" function yy_r20(){ if ($this->yystack[$this->yyidx + -4]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1406,14 +1534,14 @@ $this->_retvalue = $this->yystack[$this->yyidx + -5]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => $this->yystack[$this->yyidx + -3]->minor); } -#line 1415 "Parser.php" -#line 538 "Parser.y" +#line 1543 "Parser.php" +#line 662 "Parser.y" function yy_r21(){ $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor; $this->_patternIndex = 1; } -#line 1421 "Parser.php" -#line 543 "Parser.y" +#line 1549 "Parser.php" +#line 667 "Parser.y" function yy_r22(){ $name = $this->yystack[$this->yyidx + -1]->minor[1]; $this->yystack[$this->yyidx + -1]->minor = $this->yystack[$this->yyidx + -1]->minor[0]; @@ -1424,8 +1552,8 @@ } $this->_retvalue = array(array('pattern' => str_replace('/', '\\/', $this->yystack[$this->yyidx + -1]->minor->string), 'code' => $this->yystack[$this->yyidx + 0]->minor, 'subpatterns' => $this->yystack[$this->yyidx + -1]->minor['subpatterns'])); } -#line 1433 "Parser.php" -#line 553 "Parser.y" +#line 1561 "Parser.php" +#line 677 "Parser.y" function yy_r23(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; $name = $this->yystack[$this->yyidx + -1]->minor[1]; @@ -1437,13 +1565,13 @@ } $this->_retvalue[] = array('pattern' => str_replace('/', '\\/', $this->yystack[$this->yyidx + -1]->minor->string), 'code' => $this->yystack[$this->yyidx + 0]->minor, 'subpatterns' => $this->yystack[$this->yyidx + -1]->minor['subpatterns']); } -#line 1446 "Parser.php" -#line 565 "Parser.y" +#line 1574 "Parser.php" +#line 689 "Parser.y" function yy_r24(){ $this->_retvalue = array(preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'), $this->yystack[$this->yyidx + 0]->minor); } -#line 1451 "Parser.php" -#line 568 "Parser.y" +#line 1579 "Parser.php" +#line 692 "Parser.y" function yy_r25(){ if (!isset($this->patterns[$this->yystack[$this->yyidx + 0]->minor])) { $this->error('Undefined pattern "' . $this->yystack[$this->yyidx + 0]->minor . '" used in rules'); @@ -1451,13 +1579,13 @@ } $this->_retvalue = array($this->patterns[$this->yystack[$this->yyidx + 0]->minor], $this->yystack[$this->yyidx + 0]->minor); } -#line 1460 "Parser.php" -#line 575 "Parser.y" +#line 1588 "Parser.php" +#line 699 "Parser.y" function yy_r26(){ $this->_retvalue = array($this->yystack[$this->yyidx + -1]->minor[0] . preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'), $this->yystack[$this->yyidx + -1]->minor[1] . ' ' . $this->yystack[$this->yyidx + 0]->minor); } -#line 1465 "Parser.php" -#line 578 "Parser.y" +#line 1593 "Parser.php" +#line 702 "Parser.y" function yy_r27(){ if (!isset($this->patterns[$this->yystack[$this->yyidx + 0]->minor])) { $this->error('Undefined pattern "' . $this->yystack[$this->yyidx + 0]->minor . '" used in rules'); @@ -1465,13 +1593,13 @@ } $this->_retvalue = array($this->yystack[$this->yyidx + -1]->minor[0] . $this->patterns[$this->yystack[$this->yyidx + 0]->minor], $this->yystack[$this->yyidx + -1]->minor[1] . ' ' . $this->yystack[$this->yyidx + 0]->minor); } -#line 1474 "Parser.php" -#line 586 "Parser.y" +#line 1602 "Parser.php" +#line 710 "Parser.y" function yy_r28(){ $this->_retvalue = preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'); } -#line 1479 "Parser.php" -#line 589 "Parser.y" +#line 1607 "Parser.php" +#line 713 "Parser.y" function yy_r29(){ // increment internal sub-pattern counter // adjust back-references in pattern based on previous pattern @@ -1479,13 +1607,13 @@ $this->_patternIndex += $test['subpatterns']; $this->_retvalue = $test['pattern']; } -#line 1488 "Parser.php" -#line 596 "Parser.y" +#line 1616 "Parser.php" +#line 720 "Parser.y" function yy_r30(){ $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor . preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'); } -#line 1493 "Parser.php" -#line 599 "Parser.y" +#line 1621 "Parser.php" +#line 723 "Parser.y" function yy_r31(){ // increment internal sub-pattern counter // adjust back-references in pattern based on previous pattern @@ -1493,7 +1621,7 @@ $this->_patternIndex += $test['subpatterns']; $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor . $test['pattern']; } -#line 1502 "Parser.php" +#line 1630 "Parser.php" /** * placeholder for the left hand side in a reduce operation. @@ -1605,7 +1733,7 @@ */ function yy_syntax_error($yymajor, $TOKEN) { -#line 70 "Parser.y" +#line 66 "Parser.y" echo "Syntax Error on line " . $this->lex->line . ": token '" . $this->lex->value . "' while parsing rule:"; @@ -1617,7 +1745,7 @@ } throw new Exception('Unexpected ' . $this->tokenName($yymajor) . '(' . $TOKEN . '), expected one of: ' . implode(',', $expect)); -#line 1627 "Parser.php" +#line 1755 "Parser.php" } /** http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/LexerGenerator/Parser.y?r1=1.2&r2=1.3&diff_format=u Index: pear/PHP_LexerGenerator/LexerGenerator/Parser.y diff -u pear/PHP_LexerGenerator/LexerGenerator/Parser.y:1.2 pear/PHP_LexerGenerator/LexerGenerator/Parser.y:1.3 --- pear/PHP_LexerGenerator/LexerGenerator/Parser.y:1.2 Wed Aug 30 20:10:49 2006 +++ pear/PHP_LexerGenerator/LexerGenerator/Parser.y Thu Oct 12 18:40:02 2006 @@ -45,7 +45,7 @@ * @author Gregory Beaver <cellog@php.net> * @copyright 2006 Gregory Beaver * @license http://www.opensource.org/licenses/bsd-license.php New BSD License - * @version CVS: $Id: Parser.y,v 1.2 2006/08/30 20:10:49 cellog Exp $ + * @version CVS: $Id: Parser.y,v 1.3 2006/10/12 18:40:02 cellog Exp $ * @since File available since Release 0.1.0 */ /** @@ -88,6 +88,7 @@ private $token; private $value; private $line; + private $matchlongest; private $_regexLexer; private $_regexParser; private $_patternIndex = 0; @@ -114,15 +115,116 @@ $this->_regexParser = new PHP_LexerGenerator_Regex_Parser($this->_regexLexer); } - function outputRules($rules, $statename) + function doLongestMatch($rules, $statename, $ruleindex) + { + fwrite($this->out, ' + do { + $rules = array('); + foreach ($rules as $rule) { + fwrite($this->out, ' + \'/^' . $rule['pattern'] . '/\','); + } + fwrite($this->out, ' + ); + $match = false; + foreach ($rules as $index => $rule) { + if (preg_match($rule, substr(' . $this->input . ', ' . + $this->counter . '), $yymatches)) { + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}($yysubmatches); + if ($r === null) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + // accept this token + return true; + } elseif ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } elseif ($r === false) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + if (' . $this->counter . ' >= strlen(' . $this->input . ')) { + return false; // end of input + } + // skip this token + continue; + } else {'); + fwrite($this->out, ' + $yy_yymore_patterns = array_slice($rules, $this->token, true); + // yymore is needed + do { + if (!isset($yy_yymore_patterns[' . $this->token . '])) { + throw new Exception(\'cannot do yymore for the last token\'); + } + $match = false; + foreach ($yy_yymore_patterns[' . $this->token . '] as $index => $rule) { + if (preg_match($rule, + substr(' . $this->input . ', ' . $this->counter . '), $yymatches)) { + $yymatches = array_filter($yymatches, \'strlen\'); // remove empty sub-patterns + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + ' . $this->line . ' = substr_count("\n", ' . $this->value . '); + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } + } + } while (true); +'); + } + + function doFirstMatch($rules, $statename, $ruleindex) { - static $ruleindex = 1; $patterns = array(); $pattern = '/'; $ruleMap = array(); $tokenindex = array(); - $i = 0; - $actualindex = 1; foreach ($rules as $rule) { $ruleMap[$i++] = $actualindex; $tokenindex[$actualindex] = $rule['subpatterns']; @@ -135,12 +237,7 @@ $tokenindex = implode("\n ", $tokenindex); $pattern .= implode('|', $patterns); $pattern .= '/'; - if (!$statename) { - $statename = $ruleindex; - } fwrite($this->out, ' - function yylex' . $ruleindex . '() - { $tokenMap = ' . $tokenindex . '; if (' . $this->counter . ' >= strlen(' . $this->input . ')) { return false; // end of input @@ -210,11 +307,18 @@ ' . $this->value . ' = current($yymatches); // token value ' . $this->line . ' = substr_count("\n", ' . $this->value . '); } - } while ($this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}() !== null); - // accept - ' . $this->counter . ' += strlen($this->value); - ' . $this->line . ' += substr_count("\n", ' . $this->value . '); - return true; + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } } } else { throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . @@ -222,6 +326,26 @@ } break; } while (true); +'); + } + + function outputRules($rules, $statename) + { + static $ruleindex = 1; + $i = 0; + $actualindex = 1; + if (!$statename) { + $statename = $ruleindex; + } + fwrite($this->out, ' + function yylex' . $ruleindex . '() + {'); + if ($this->matchlongest) { + $this->doLongestMatch($rules, $statename, $ruleindex); + } else { + $this->doFirstMatch($rules, $statename, $ruleindex); + } + fwrite($this->out, ' } // end function '); @@ -231,7 +355,7 @@ '); } foreach ($rules as $i => $rule) { - fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $ruleMap[$i] . '($yy_subpatterns) + fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $i . '($yy_subpatterns) { ' . $rule['code'] . ' } @@ -444,10 +568,14 @@ 'token' => true, 'value' => true, 'line' => true, + 'matchlongest' => true, ); foreach (B as $pi) { if (isset($expected[$pi['pi']])) { $this->{$pi['pi']} = $pi['definition']; + if ($pi['pi'] == 'matchlongest') { + $this->matchlongest = true; + } continue; } $this->error('Unknown processing instruction %' . $pi['pi'] . http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/examples/longestfirst.php?view=markup&rev=1.1 Index: pear/PHP_LexerGenerator/examples/longestfirst.php +++ pear/PHP_LexerGenerator/examples/longestfirst.php <?php class CodeGen_PECL_Tools_ProtoParser { const PAR_OPEN = 1; const PAR_CLOSE = 2; const EQ = 3; const COMMA = 4; const SEMICOLON = 5; const VOID = 6; const BOOL = 7; const NUMVAL = 8; const STRVAL = 9; } class CodeGen_PECL_Tools_ProtoLexer { private $data; public $token; public $value; private $line; private $count; function __construct($data) { $this->data = $data; $this->count = 0; $this->line = 1; } private $_yy_state = 1; private $_yy_stack = array(); function yylex() { return $this->{'yylex' . $this->_yy_state}(); } function yypushstate($state) { array_push($this->_yy_stack, $this->_yy_state); $this->_yy_state = $state; } function yypopstate() { $this->_yy_state = array_pop($this->_yy_stack); } function yybegin($state) { $this->_yy_state = $state; } function yylex1() { do { $rules = array( '/^[ \t\n]+/', '/^\\(/', '/^\\)/', '/^=/', '/^,/', '/^;/', '/^void/', '/^bool(ean)?/', '/^[0-9]+|0x[0-9a-fA-F]+/', '/^[_a-zA-Z][_a-zA-Z0-9]+/', ); $match = false; foreach ($rules as $index => $rule) { if (preg_match($rule, substr($this->data, $this->count), $yymatches)) { if ($match) { if (strlen($yymatches[0]) > strlen($match[0][0])) { $match = array($yymatches, $index); // matches, token } } else { $match = array($yymatches, $index); } } } if (!$match) { throw new Exception('Unexpected input at line' . $this->line . ': ' . $this->data[$this->count]); } $this->token = $match[1]; $this->value = $match[0][0]; $yysubmatches = $match[0]; array_shift($yysubmatches); if (!$yysubmatches) { $yysubmatches = array(); } $r = $this->{'yy_r1_' . $this->token}($yysubmatches); if ($r === null) { $this->count += strlen($this->value); $this->line += substr_count("\n", $this->value); // accept this token return true; } elseif ($r === true) { // we have changed state // process this token in the new state return $this->yylex(); } elseif ($r === false) { $this->count += strlen($this->value); $this->line += substr_count("\n", $this->value); if ($this->count >= strlen($this->data)) { return false; // end of input } // skip this token continue; } else { $yy_yymore_patterns = array_slice($rules, $this->token, true); // yymore is needed do { if (!isset($yy_yymore_patterns[$this->token])) { throw new Exception('cannot do yymore for the last token'); } $match = false; foreach ($yy_yymore_patterns[$this->token] as $index => $rule) { if (preg_match($rule, substr($this->data, $this->count), $yymatches)) { $yymatches = array_filter($yymatches, 'strlen'); // remove empty sub-patterns if ($match) { if (strlen($yymatches[0]) > strlen($match[0][0])) { $match = array($yymatches, $index); // matches, token } } else { $match = array($yymatches, $index); } } } if (!$match) { throw new Exception('Unexpected input at line' . $this->line . ': ' . $this->data[$this->count]); } $this->token = $match[1]; $this->value = $match[0][0]; $yysubmatches = $match[0]; array_shift($yysubmatches); if (!$yysubmatches) { $yysubmatches = array(); } $this->line = substr_count("\n", $this->value); $r = $this->{'yy_r1_' . $this->token}(); } while ($r !== null || !$r); if ($r === true) { // we have changed state // process this token in the new state return $this->yylex(); } else { // accept $this->count += strlen($this->value); $this->line += substr_count("\n", $this->value); return true; } } } while (true); } // end function function yy_r1_0($yy_subpatterns) { return false; } function yy_r1_1($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_OPEN; } function yy_r1_2($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_CLOSE; } function yy_r1_3($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::EQ; } function yy_r1_4($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::COMMA; } function yy_r1_5($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::SEMICOLON; } function yy_r1_6($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::VOID; } function yy_r1_7($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::BOOL; } function yy_r1_8($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::NUMVAL; } function yy_r1_9($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::STRVAL; } } $a = new CodeGen_PECL_Tools_ProtoLexer('booler boolboolean boolean bool'); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/examples/longestfirst.plex?view=markup&rev=1.1 Index: pear/PHP_LexerGenerator/examples/longestfirst.plex +++ pear/PHP_LexerGenerator/examples/longestfirst.plex <?php class CodeGen_PECL_Tools_ProtoParser { const PAR_OPEN = 1; const PAR_CLOSE = 2; const EQ = 3; const COMMA = 4; const SEMICOLON = 5; const VOID = 6; const BOOL = 7; const NUMVAL = 8; const STRVAL = 9; } class CodeGen_PECL_Tools_ProtoLexer { private $data; public $token; public $value; private $line; private $count; function __construct($data) { $this->data = $data; $this->count = 0; $this->line = 1; } /*!lex2php %input $this->data %counter $this->count %token $this->token %value $this->value %line $this->line %matchlongest 1 whitespace = /[ \t\n]+/ par_open = /\(/ par_close = /\)/ eq = /=/ comma = /,/ semicolon = /;/ void = /void/ bool = /bool(ean)?/ int = /int(eger)?|long/ float = /float|double/ string = /string/ array = /array/ class = /class|object/ resource = /resource/ mixed = /mixed/ callback = /callback/ stream = /stream/ ellipse = /\.\.\./ true = /true/ false = /false/ null = /null/ numval = /[0-9]+|0x[0-9a-fA-F]+/ strval = /"[^"]*"|'[^']*'/ name = /[_a-zA-Z][_a-zA-Z0-9]+/ */ /*!lex2php whitespace { return false; } par_open { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_OPEN; } par_close { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_CLOSE; } eq { $this->token = CodeGen_PECL_Tools_ProtoParser::EQ; } comma { $this->token = CodeGen_PECL_Tools_ProtoParser::COMMA; } semicolon { $this->token = CodeGen_PECL_Tools_ProtoParser::SEMICOLON; } void { $this->token = CodeGen_PECL_Tools_ProtoParser::VOID; } bool { $this->token = CodeGen_PECL_Tools_ProtoParser::BOOL; } numval { $this->token = CodeGen_PECL_Tools_ProtoParser::NUMVAL; } name { $this->token = CodeGen_PECL_Tools_ProtoParser::STRVAL; } */ } $a = new CodeGen_PECL_Tools_ProtoLexer('booler boolboolean boolean bool'); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value);
http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/package.xml?r1=1.5&r2=1.6&diff_format=u Index: pear/PHP_LexerGenerator/package.xml diff -u pear/PHP_LexerGenerator/package.xml:1.5 pear/PHP_LexerGenerator/package.xml:1.6 --- pear/PHP_LexerGenerator/package.xml:1.5 Sat Oct 7 11:40:01 2006 +++ pear/PHP_LexerGenerator/package.xml Thu Oct 12 18:40:01 2006 @@ -13,6 +13,11 @@ Generated lexers are very efficient, more than twice as efficient as other alternatives like csLex (written in C#) because they utilize PHP's built-in Perl-compatible regular expressions to lex for tokens. + + Now in version 0.3.0+, with processing instruction %longestmatch, generated + lexers will always pick the longest string to match, rather than the first. + Generated lexers are slightly slower, but match behavior of legacy lexers like + flex, lex, re2c. </description> <lead> <name>Greg Beaver</name> @@ -20,10 +25,10 @@ <email>cellog@php.net</email> <active>yes</active> </lead> - <date>2006-10-02</date> + <date>2006-10-12</date> <version> - <release>0.2.3</release> - <api>0.1.0</api> + <release>0.3.0</release> + <api>0.3.0</api> </version> <stability> <release>alpha</release> @@ -31,7 +36,9 @@ </stability> <license uri="http://www.opensource.org/licenses/bsd-license.php">New BSD License</license> <notes> - - fix Bug #8618: null causes regex to break + - fix Bug #9006: wrong replacements for plex script (Hartmut Holzgraefe) + - fix Bug #9007: yymore does not honor return value "true" to change state + - implement Request #8879: lexer not picking longest match </notes> <contents> <dir baseinstalldir="PHP" name="/"> @@ -170,5 +177,22 @@ - fix Bug #8618: null causes regex to break </notes> </release> + <release> + <version> + <release>0.3.0</release> + <api>0.3.0</api> + </version> + <stability> + <release>alpha</release> + <api>alpha</api> + </stability> + <date>2006-10-02</date> + <license uri="http://www.opensource.org/licenses/bsd-license.php">New BSD License</license> + <notes> + - fix Bug #9006: wrong replacements for plex script (Hartmut Holzgraefe) + - fix Bug #9007: yymore does not honor return value "true" to change state + - implement Request #8879: lexer not picking longest match + </notes> + </release> </changelog> </package> http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/LexerGenerator/Parser.php?r1=1.2&r2=1.3&diff_format=u Index: pear/PHP_LexerGenerator/LexerGenerator/Parser.php diff -u pear/PHP_LexerGenerator/LexerGenerator/Parser.php:1.2 pear/PHP_LexerGenerator/LexerGenerator/Parser.php:1.3 --- pear/PHP_LexerGenerator/LexerGenerator/Parser.php:1.2 Wed Aug 30 20:10:49 2006 +++ pear/PHP_LexerGenerator/LexerGenerator/Parser.php Thu Oct 12 18:40:02 2006 @@ -140,7 +140,7 @@ * @author Gregory Beaver <cellog@php.net> * @copyright 2006 Gregory Beaver * @license http://www.opensource.org/licenses/bsd-license.php New BSD License - * @version CVS: $Id: Parser.php,v 1.2 2006/08/30 20:10:49 cellog Exp $ + * @version CVS: $Id: Parser.php,v 1.3 2006/10/12 18:40:02 cellog Exp $ * @since File available since Release 0.1.0 */ /** @@ -169,7 +169,7 @@ { /* First off, code is included which follows the "include_class" declaration ** in the input file. */ -#line 82 "Parser.y" +#line 78 "Parser.y" private $patterns; private $out; @@ -179,6 +179,7 @@ private $token; private $value; private $line; + private $matchlongest; private $_regexLexer; private $_regexParser; private $_patternIndex = 0; @@ -205,15 +206,116 @@ $this->_regexParser = new PHP_LexerGenerator_Regex_Parser($this->_regexLexer); } - function outputRules($rules, $statename) + function doLongestMatch($rules, $statename, $ruleindex) + { + fwrite($this->out, ' + do { + $rules = array('); + foreach ($rules as $rule) { + fwrite($this->out, ' + \'/^' . $rule['pattern'] . '/\','); + } + fwrite($this->out, ' + ); + $match = false; + foreach ($rules as $index => $rule) { + if (preg_match($rule, substr(' . $this->input . ', ' . + $this->counter . '), $yymatches)) { + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}($yysubmatches); + if ($r === null) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + // accept this token + return true; + } elseif ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } elseif ($r === false) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + if (' . $this->counter . ' >= strlen(' . $this->input . ')) { + return false; // end of input + } + // skip this token + continue; + } else {'); + fwrite($this->out, ' + $yy_yymore_patterns = array_slice($rules, $this->token, true); + // yymore is needed + do { + if (!isset($yy_yymore_patterns[' . $this->token . '])) { + throw new Exception(\'cannot do yymore for the last token\'); + } + $match = false; + foreach ($yy_yymore_patterns[' . $this->token . '] as $index => $rule) { + if (preg_match($rule, + substr(' . $this->input . ', ' . $this->counter . '), $yymatches)) { + $yymatches = array_filter($yymatches, \'strlen\'); // remove empty sub-patterns + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + ' . $this->line . ' = substr_count("\n", ' . $this->value . '); + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } + } + } while (true); +'); + } + + function doFirstMatch($rules, $statename, $ruleindex) { - static $ruleindex = 1; $patterns = array(); $pattern = '/'; $ruleMap = array(); $tokenindex = array(); - $i = 0; - $actualindex = 1; foreach ($rules as $rule) { $ruleMap[$i++] = $actualindex; $tokenindex[$actualindex] = $rule['subpatterns']; @@ -226,12 +328,7 @@ $tokenindex = implode("\n ", $tokenindex); $pattern .= implode('|', $patterns); $pattern .= '/'; - if (!$statename) { - $statename = $ruleindex; - } fwrite($this->out, ' - function yylex' . $ruleindex . '() - { $tokenMap = ' . $tokenindex . '; if (' . $this->counter . ' >= strlen(' . $this->input . ')) { return false; // end of input @@ -301,11 +398,18 @@ ' . $this->value . ' = current($yymatches); // token value ' . $this->line . ' = substr_count("\n", ' . $this->value . '); } - } while ($this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}() !== null); - // accept - ' . $this->counter . ' += strlen($this->value); - ' . $this->line . ' += substr_count("\n", ' . $this->value . '); - return true; + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } } } else { throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . @@ -313,6 +417,26 @@ } break; } while (true); +'); + } + + function outputRules($rules, $statename) + { + static $ruleindex = 1; + $i = 0; + $actualindex = 1; + if (!$statename) { + $statename = $ruleindex; + } + fwrite($this->out, ' + function yylex' . $ruleindex . '() + {'); + if ($this->matchlongest) { + $this->doLongestMatch($rules, $statename, $ruleindex); + } else { + $this->doFirstMatch($rules, $statename, $ruleindex); + } + fwrite($this->out, ' } // end function '); @@ -322,7 +446,7 @@ '); } foreach ($rules as $i => $rule) { - fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $ruleMap[$i] . '($yy_subpatterns) + fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $i . '($yy_subpatterns) { ' . $rule['code'] . ' } @@ -352,7 +476,7 @@ } return $this->_regexParser->result; } -#line 359 "Parser.php" +#line 483 "Parser.php" /* Next is all token values, as class constants */ @@ -1114,7 +1238,7 @@ ** function yy_r0($yymsp){ ... } // User supplied code ** #line <lineno> <thisfile> */ -#line 268 "Parser.y" +#line 388 "Parser.y" function yy_r1(){ fwrite($this->out, ' private $_yy_state = 1; @@ -1149,8 +1273,8 @@ } } } -#line 1158 "Parser.php" -#line 302 "Parser.y" +#line 1282 "Parser.php" +#line 422 "Parser.y" function yy_r2(){ fwrite($this->out, ' private $_yy_state = 1; @@ -1188,8 +1312,8 @@ } } } -#line 1197 "Parser.php" -#line 339 "Parser.y" +#line 1321 "Parser.php" +#line 459 "Parser.y" function yy_r3(){ if (strlen($this->yystack[$this->yyidx + -2]->minor)) { fwrite($this->out, $this->yystack[$this->yyidx + -2]->minor); @@ -1227,8 +1351,8 @@ } } } -#line 1236 "Parser.php" -#line 376 "Parser.y" +#line 1360 "Parser.php" +#line 496 "Parser.y" function yy_r4(){ if (strlen($this->yystack[$this->yyidx + -3]->minor)) { fwrite($this->out, $this->yystack[$this->yyidx + -3]->minor); @@ -1269,15 +1393,15 @@ } } } -#line 1278 "Parser.php" -#line 417 "Parser.y" +#line 1402 "Parser.php" +#line 537 "Parser.y" function yy_r5(){ $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor; $this->patterns = $this->yystack[$this->yyidx + -1]->minor['patterns']; $this->_patternIndex = 1; } -#line 1285 "Parser.php" -#line 423 "Parser.y" +#line 1409 "Parser.php" +#line 543 "Parser.y" function yy_r6(){ $expected = array( 'counter' => true, @@ -1302,10 +1426,14 @@ 'token' => true, 'value' => true, 'line' => true, + 'matchlongest' => true, ); foreach ($this->yystack[$this->yyidx + -1]->minor as $pi) { if (isset($expected[$pi['pi']])) { $this->{$pi['pi']} = $pi['definition']; + if ($pi['pi'] == 'matchlongest') { + $this->matchlongest = true; + } continue; } $this->error('Unknown processing instruction %' . $pi['pi'] . @@ -1314,26 +1442,26 @@ $this->_retvalue = array('patterns' => $this->yystack[$this->yyidx + 0]->minor, 'pis' => $this->yystack[$this->yyidx + -1]->minor); $this->_patternIndex = 1; } -#line 1323 "Parser.php" -#line 460 "Parser.y" +#line 1451 "Parser.php" +#line 584 "Parser.y" function yy_r7(){ $this->_retvalue = array(array('pi' => $this->yystack[$this->yyidx + -1]->minor, 'definition' => $this->yystack[$this->yyidx + 0]->minor)); } -#line 1328 "Parser.php" -#line 466 "Parser.y" +#line 1456 "Parser.php" +#line 590 "Parser.y" function yy_r9(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; $this->_retvalue[] = array('pi' => $this->yystack[$this->yyidx + -1]->minor, 'definition' => $this->yystack[$this->yyidx + 0]->minor); } -#line 1334 "Parser.php" -#line 475 "Parser.y" +#line 1462 "Parser.php" +#line 599 "Parser.y" function yy_r11(){ $this->_retvalue = array($this->yystack[$this->yyidx + -1]->minor => $this->yystack[$this->yyidx + 0]->minor); // reset internal indicator of where we are in a pattern $this->_patternIndex = 0; } -#line 1341 "Parser.php" -#line 480 "Parser.y" +#line 1469 "Parser.php" +#line 604 "Parser.y" function yy_r12(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; if (isset($this->_retvalue[$this->yystack[$this->yyidx + -1]->minor])) { @@ -1344,13 +1472,13 @@ // reset internal indicator of where we are in a pattern declaration $this->_patternIndex = 0; } -#line 1353 "Parser.php" -#line 491 "Parser.y" +#line 1481 "Parser.php" +#line 615 "Parser.y" function yy_r13(){ $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => '')); } -#line 1358 "Parser.php" -#line 494 "Parser.y" +#line 1486 "Parser.php" +#line 618 "Parser.y" function yy_r14(){ if ($this->yystack[$this->yyidx + -3]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1358,13 +1486,13 @@ } $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => $this->yystack[$this->yyidx + -2]->minor)); } -#line 1367 "Parser.php" -#line 501 "Parser.y" +#line 1495 "Parser.php" +#line 625 "Parser.y" function yy_r15(){ $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => '')); } -#line 1372 "Parser.php" -#line 504 "Parser.y" +#line 1500 "Parser.php" +#line 628 "Parser.y" function yy_r16(){ if ($this->yystack[$this->yyidx + -4]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1373,15 +1501,15 @@ $this->_retvalue = array(array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => $this->yystack[$this->yyidx + -3]->minor)); $this->_patternIndex = 1; } -#line 1382 "Parser.php" -#line 512 "Parser.y" +#line 1510 "Parser.php" +#line 636 "Parser.y" function yy_r17(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => ''); $this->_patternIndex = 1; } -#line 1389 "Parser.php" -#line 517 "Parser.y" +#line 1517 "Parser.php" +#line 641 "Parser.y" function yy_r18(){ if ($this->yystack[$this->yyidx + -3]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1390,14 +1518,14 @@ $this->_retvalue = $this->yystack[$this->yyidx + -4]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -1]->minor, 'code' => '', 'statename' => $this->yystack[$this->yyidx + -2]->minor); } -#line 1399 "Parser.php" -#line 525 "Parser.y" +#line 1527 "Parser.php" +#line 649 "Parser.y" function yy_r19(){ $this->_retvalue = $this->yystack[$this->yyidx + -3]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => ''); } -#line 1405 "Parser.php" -#line 529 "Parser.y" +#line 1533 "Parser.php" +#line 653 "Parser.y" function yy_r20(){ if ($this->yystack[$this->yyidx + -4]->minor != 'statename') { throw new Exception('Error: only %statename processing instruction ' . @@ -1406,14 +1534,14 @@ $this->_retvalue = $this->yystack[$this->yyidx + -5]->minor; $this->_retvalue[] = array('rules' => $this->yystack[$this->yyidx + -2]->minor, 'code' => $this->yystack[$this->yyidx + 0]->minor, 'statename' => $this->yystack[$this->yyidx + -3]->minor); } -#line 1415 "Parser.php" -#line 538 "Parser.y" +#line 1543 "Parser.php" +#line 662 "Parser.y" function yy_r21(){ $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor; $this->_patternIndex = 1; } -#line 1421 "Parser.php" -#line 543 "Parser.y" +#line 1549 "Parser.php" +#line 667 "Parser.y" function yy_r22(){ $name = $this->yystack[$this->yyidx + -1]->minor[1]; $this->yystack[$this->yyidx + -1]->minor = $this->yystack[$this->yyidx + -1]->minor[0]; @@ -1424,8 +1552,8 @@ } $this->_retvalue = array(array('pattern' => str_replace('/', '\\/', $this->yystack[$this->yyidx + -1]->minor->string), 'code' => $this->yystack[$this->yyidx + 0]->minor, 'subpatterns' => $this->yystack[$this->yyidx + -1]->minor['subpatterns'])); } -#line 1433 "Parser.php" -#line 553 "Parser.y" +#line 1561 "Parser.php" +#line 677 "Parser.y" function yy_r23(){ $this->_retvalue = $this->yystack[$this->yyidx + -2]->minor; $name = $this->yystack[$this->yyidx + -1]->minor[1]; @@ -1437,13 +1565,13 @@ } $this->_retvalue[] = array('pattern' => str_replace('/', '\\/', $this->yystack[$this->yyidx + -1]->minor->string), 'code' => $this->yystack[$this->yyidx + 0]->minor, 'subpatterns' => $this->yystack[$this->yyidx + -1]->minor['subpatterns']); } -#line 1446 "Parser.php" -#line 565 "Parser.y" +#line 1574 "Parser.php" +#line 689 "Parser.y" function yy_r24(){ $this->_retvalue = array(preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'), $this->yystack[$this->yyidx + 0]->minor); } -#line 1451 "Parser.php" -#line 568 "Parser.y" +#line 1579 "Parser.php" +#line 692 "Parser.y" function yy_r25(){ if (!isset($this->patterns[$this->yystack[$this->yyidx + 0]->minor])) { $this->error('Undefined pattern "' . $this->yystack[$this->yyidx + 0]->minor . '" used in rules'); @@ -1451,13 +1579,13 @@ } $this->_retvalue = array($this->patterns[$this->yystack[$this->yyidx + 0]->minor], $this->yystack[$this->yyidx + 0]->minor); } -#line 1460 "Parser.php" -#line 575 "Parser.y" +#line 1588 "Parser.php" +#line 699 "Parser.y" function yy_r26(){ $this->_retvalue = array($this->yystack[$this->yyidx + -1]->minor[0] . preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'), $this->yystack[$this->yyidx + -1]->minor[1] . ' ' . $this->yystack[$this->yyidx + 0]->minor); } -#line 1465 "Parser.php" -#line 578 "Parser.y" +#line 1593 "Parser.php" +#line 702 "Parser.y" function yy_r27(){ if (!isset($this->patterns[$this->yystack[$this->yyidx + 0]->minor])) { $this->error('Undefined pattern "' . $this->yystack[$this->yyidx + 0]->minor . '" used in rules'); @@ -1465,13 +1593,13 @@ } $this->_retvalue = array($this->yystack[$this->yyidx + -1]->minor[0] . $this->patterns[$this->yystack[$this->yyidx + 0]->minor], $this->yystack[$this->yyidx + -1]->minor[1] . ' ' . $this->yystack[$this->yyidx + 0]->minor); } -#line 1474 "Parser.php" -#line 586 "Parser.y" +#line 1602 "Parser.php" +#line 710 "Parser.y" function yy_r28(){ $this->_retvalue = preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'); } -#line 1479 "Parser.php" -#line 589 "Parser.y" +#line 1607 "Parser.php" +#line 713 "Parser.y" function yy_r29(){ // increment internal sub-pattern counter // adjust back-references in pattern based on previous pattern @@ -1479,13 +1607,13 @@ $this->_patternIndex += $test['subpatterns']; $this->_retvalue = $test['pattern']; } -#line 1488 "Parser.php" -#line 596 "Parser.y" +#line 1616 "Parser.php" +#line 720 "Parser.y" function yy_r30(){ $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor . preg_quote($this->yystack[$this->yyidx + 0]->minor, '/'); } -#line 1493 "Parser.php" -#line 599 "Parser.y" +#line 1621 "Parser.php" +#line 723 "Parser.y" function yy_r31(){ // increment internal sub-pattern counter // adjust back-references in pattern based on previous pattern @@ -1493,7 +1621,7 @@ $this->_patternIndex += $test['subpatterns']; $this->_retvalue = $this->yystack[$this->yyidx + -1]->minor . $test['pattern']; } -#line 1502 "Parser.php" +#line 1630 "Parser.php" /** * placeholder for the left hand side in a reduce operation. @@ -1605,7 +1733,7 @@ */ function yy_syntax_error($yymajor, $TOKEN) { -#line 70 "Parser.y" +#line 66 "Parser.y" echo "Syntax Error on line " . $this->lex->line . ": token '" . $this->lex->value . "' while parsing rule:"; @@ -1617,7 +1745,7 @@ } throw new Exception('Unexpected ' . $this->tokenName($yymajor) . '(' . $TOKEN . '), expected one of: ' . implode(',', $expect)); -#line 1627 "Parser.php" +#line 1755 "Parser.php" } /** http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/LexerGenerator/Parser.y?r1=1.2&r2=1.3&diff_format=u Index: pear/PHP_LexerGenerator/LexerGenerator/Parser.y diff -u pear/PHP_LexerGenerator/LexerGenerator/Parser.y:1.2 pear/PHP_LexerGenerator/LexerGenerator/Parser.y:1.3 --- pear/PHP_LexerGenerator/LexerGenerator/Parser.y:1.2 Wed Aug 30 20:10:49 2006 +++ pear/PHP_LexerGenerator/LexerGenerator/Parser.y Thu Oct 12 18:40:02 2006 @@ -45,7 +45,7 @@ * @author Gregory Beaver <cellog@php.net> * @copyright 2006 Gregory Beaver * @license http://www.opensource.org/licenses/bsd-license.php New BSD License - * @version CVS: $Id: Parser.y,v 1.2 2006/08/30 20:10:49 cellog Exp $ + * @version CVS: $Id: Parser.y,v 1.3 2006/10/12 18:40:02 cellog Exp $ * @since File available since Release 0.1.0 */ /** @@ -88,6 +88,7 @@ private $token; private $value; private $line; + private $matchlongest; private $_regexLexer; private $_regexParser; private $_patternIndex = 0; @@ -114,15 +115,116 @@ $this->_regexParser = new PHP_LexerGenerator_Regex_Parser($this->_regexLexer); } - function outputRules($rules, $statename) + function doLongestMatch($rules, $statename, $ruleindex) + { + fwrite($this->out, ' + do { + $rules = array('); + foreach ($rules as $rule) { + fwrite($this->out, ' + \'/^' . $rule['pattern'] . '/\','); + } + fwrite($this->out, ' + ); + $match = false; + foreach ($rules as $index => $rule) { + if (preg_match($rule, substr(' . $this->input . ', ' . + $this->counter . '), $yymatches)) { + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}($yysubmatches); + if ($r === null) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + // accept this token + return true; + } elseif ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } elseif ($r === false) { + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + if (' . $this->counter . ' >= strlen(' . $this->input . ')) { + return false; // end of input + } + // skip this token + continue; + } else {'); + fwrite($this->out, ' + $yy_yymore_patterns = array_slice($rules, $this->token, true); + // yymore is needed + do { + if (!isset($yy_yymore_patterns[' . $this->token . '])) { + throw new Exception(\'cannot do yymore for the last token\'); + } + $match = false; + foreach ($yy_yymore_patterns[' . $this->token . '] as $index => $rule) { + if (preg_match($rule, + substr(' . $this->input . ', ' . $this->counter . '), $yymatches)) { + $yymatches = array_filter($yymatches, \'strlen\'); // remove empty sub-patterns + if ($match) { + if (strlen($yymatches[0]) > strlen($match[0][0])) { + $match = array($yymatches, $index); // matches, token + } + } else { + $match = array($yymatches, $index); + } + } + } + if (!$match) { + throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . + \': \' . ' . $this->input . '[' . $this->counter . ']); + } + ' . $this->token . ' = $match[1]; + ' . $this->value . ' = $match[0][0]; + $yysubmatches = $match[0]; + array_shift($yysubmatches); + if (!$yysubmatches) { + $yysubmatches = array(); + } + ' . $this->line . ' = substr_count("\n", ' . $this->value . '); + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } + } + } while (true); +'); + } + + function doFirstMatch($rules, $statename, $ruleindex) { - static $ruleindex = 1; $patterns = array(); $pattern = '/'; $ruleMap = array(); $tokenindex = array(); - $i = 0; - $actualindex = 1; foreach ($rules as $rule) { $ruleMap[$i++] = $actualindex; $tokenindex[$actualindex] = $rule['subpatterns']; @@ -135,12 +237,7 @@ $tokenindex = implode("\n ", $tokenindex); $pattern .= implode('|', $patterns); $pattern .= '/'; - if (!$statename) { - $statename = $ruleindex; - } fwrite($this->out, ' - function yylex' . $ruleindex . '() - { $tokenMap = ' . $tokenindex . '; if (' . $this->counter . ' >= strlen(' . $this->input . ')) { return false; // end of input @@ -210,11 +307,18 @@ ' . $this->value . ' = current($yymatches); // token value ' . $this->line . ' = substr_count("\n", ' . $this->value . '); } - } while ($this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}() !== null); - // accept - ' . $this->counter . ' += strlen($this->value); - ' . $this->line . ' += substr_count("\n", ' . $this->value . '); - return true; + $r = $this->{\'yy_r' . $ruleindex . '_\' . ' . $this->token . '}(); + } while ($r !== null || !$r); + if ($r === true) { + // we have changed state + // process this token in the new state + return $this->yylex(); + } else { + // accept + ' . $this->counter . ' += strlen($this->value); + ' . $this->line . ' += substr_count("\n", ' . $this->value . '); + return true; + } } } else { throw new Exception(\'Unexpected input at line\' . ' . $this->line . ' . @@ -222,6 +326,26 @@ } break; } while (true); +'); + } + + function outputRules($rules, $statename) + { + static $ruleindex = 1; + $i = 0; + $actualindex = 1; + if (!$statename) { + $statename = $ruleindex; + } + fwrite($this->out, ' + function yylex' . $ruleindex . '() + {'); + if ($this->matchlongest) { + $this->doLongestMatch($rules, $statename, $ruleindex); + } else { + $this->doFirstMatch($rules, $statename, $ruleindex); + } + fwrite($this->out, ' } // end function '); @@ -231,7 +355,7 @@ '); } foreach ($rules as $i => $rule) { - fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $ruleMap[$i] . '($yy_subpatterns) + fwrite($this->out, ' function yy_r' . $ruleindex . '_' . $i . '($yy_subpatterns) { ' . $rule['code'] . ' } @@ -444,10 +568,14 @@ 'token' => true, 'value' => true, 'line' => true, + 'matchlongest' => true, ); foreach (B as $pi) { if (isset($expected[$pi['pi']])) { $this->{$pi['pi']} = $pi['definition']; + if ($pi['pi'] == 'matchlongest') { + $this->matchlongest = true; + } continue; } $this->error('Unknown processing instruction %' . $pi['pi'] . http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/examples/longestfirst.php?view=markup&rev=1.1 Index: pear/PHP_LexerGenerator/examples/longestfirst.php +++ pear/PHP_LexerGenerator/examples/longestfirst.php <?php class CodeGen_PECL_Tools_ProtoParser { const PAR_OPEN = 1; const PAR_CLOSE = 2; const EQ = 3; const COMMA = 4; const SEMICOLON = 5; const VOID = 6; const BOOL = 7; const NUMVAL = 8; const STRVAL = 9; } class CodeGen_PECL_Tools_ProtoLexer { private $data; public $token; public $value; private $line; private $count; function __construct($data) { $this->data = $data; $this->count = 0; $this->line = 1; } private $_yy_state = 1; private $_yy_stack = array(); function yylex() { return $this->{'yylex' . $this->_yy_state}(); } function yypushstate($state) { array_push($this->_yy_stack, $this->_yy_state); $this->_yy_state = $state; } function yypopstate() { $this->_yy_state = array_pop($this->_yy_stack); } function yybegin($state) { $this->_yy_state = $state; } function yylex1() { do { $rules = array( '/^[ \t\n]+/', '/^\\(/', '/^\\)/', '/^=/', '/^,/', '/^;/', '/^void/', '/^bool(ean)?/', '/^[0-9]+|0x[0-9a-fA-F]+/', '/^[_a-zA-Z][_a-zA-Z0-9]+/', ); $match = false; foreach ($rules as $index => $rule) { if (preg_match($rule, substr($this->data, $this->count), $yymatches)) { if ($match) { if (strlen($yymatches[0]) > strlen($match[0][0])) { $match = array($yymatches, $index); // matches, token } } else { $match = array($yymatches, $index); } } } if (!$match) { throw new Exception('Unexpected input at line' . $this->line . ': ' . $this->data[$this->count]); } $this->token = $match[1]; $this->value = $match[0][0]; $yysubmatches = $match[0]; array_shift($yysubmatches); if (!$yysubmatches) { $yysubmatches = array(); } $r = $this->{'yy_r1_' . $this->token}($yysubmatches); if ($r === null) { $this->count += strlen($this->value); $this->line += substr_count("\n", $this->value); // accept this token return true; } elseif ($r === true) { // we have changed state // process this token in the new state return $this->yylex(); } elseif ($r === false) { $this->count += strlen($this->value); $this->line += substr_count("\n", $this->value); if ($this->count >= strlen($this->data)) { return false; // end of input } // skip this token continue; } else { $yy_yymore_patterns = array_slice($rules, $this->token, true); // yymore is needed do { if (!isset($yy_yymore_patterns[$this->token])) { throw new Exception('cannot do yymore for the last token'); } $match = false; foreach ($yy_yymore_patterns[$this->token] as $index => $rule) { if (preg_match($rule, substr($this->data, $this->count), $yymatches)) { $yymatches = array_filter($yymatches, 'strlen'); // remove empty sub-patterns if ($match) { if (strlen($yymatches[0]) > strlen($match[0][0])) { $match = array($yymatches, $index); // matches, token } } else { $match = array($yymatches, $index); } } } if (!$match) { throw new Exception('Unexpected input at line' . $this->line . ': ' . $this->data[$this->count]); } $this->token = $match[1]; $this->value = $match[0][0]; $yysubmatches = $match[0]; array_shift($yysubmatches); if (!$yysubmatches) { $yysubmatches = array(); } $this->line = substr_count("\n", $this->value); $r = $this->{'yy_r1_' . $this->token}(); } while ($r !== null || !$r); if ($r === true) { // we have changed state // process this token in the new state return $this->yylex(); } else { // accept $this->count += strlen($this->value); $this->line += substr_count("\n", $this->value); return true; } } } while (true); } // end function function yy_r1_0($yy_subpatterns) { return false; } function yy_r1_1($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_OPEN; } function yy_r1_2($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_CLOSE; } function yy_r1_3($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::EQ; } function yy_r1_4($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::COMMA; } function yy_r1_5($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::SEMICOLON; } function yy_r1_6($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::VOID; } function yy_r1_7($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::BOOL; } function yy_r1_8($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::NUMVAL; } function yy_r1_9($yy_subpatterns) { $this->token = CodeGen_PECL_Tools_ProtoParser::STRVAL; } } $a = new CodeGen_PECL_Tools_ProtoLexer('booler boolboolean boolean bool'); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); http://cvs.php.net/viewvc.cgi/pear/PHP_LexerGenerator/examples/longestfirst.plex?view=markup&rev=1.1 Index: pear/PHP_LexerGenerator/examples/longestfirst.plex +++ pear/PHP_LexerGenerator/examples/longestfirst.plex <?php class CodeGen_PECL_Tools_ProtoParser { const PAR_OPEN = 1; const PAR_CLOSE = 2; const EQ = 3; const COMMA = 4; const SEMICOLON = 5; const VOID = 6; const BOOL = 7; const NUMVAL = 8; const STRVAL = 9; } class CodeGen_PECL_Tools_ProtoLexer { private $data; public $token; public $value; private $line; private $count; function __construct($data) { $this->data = $data; $this->count = 0; $this->line = 1; } /*!lex2php %input $this->data %counter $this->count %token $this->token %value $this->value %line $this->line %matchlongest 1 whitespace = /[ \t\n]+/ par_open = /\(/ par_close = /\)/ eq = /=/ comma = /,/ semicolon = /;/ void = /void/ bool = /bool(ean)?/ int = /int(eger)?|long/ float = /float|double/ string = /string/ array = /array/ class = /class|object/ resource = /resource/ mixed = /mixed/ callback = /callback/ stream = /stream/ ellipse = /\.\.\./ true = /true/ false = /false/ null = /null/ numval = /[0-9]+|0x[0-9a-fA-F]+/ strval = /"[^"]*"|'[^']*'/ name = /[_a-zA-Z][_a-zA-Z0-9]+/ */ /*!lex2php whitespace { return false; } par_open { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_OPEN; } par_close { $this->token = CodeGen_PECL_Tools_ProtoParser::PAR_CLOSE; } eq { $this->token = CodeGen_PECL_Tools_ProtoParser::EQ; } comma { $this->token = CodeGen_PECL_Tools_ProtoParser::COMMA; } semicolon { $this->token = CodeGen_PECL_Tools_ProtoParser::SEMICOLON; } void { $this->token = CodeGen_PECL_Tools_ProtoParser::VOID; } bool { $this->token = CodeGen_PECL_Tools_ProtoParser::BOOL; } numval { $this->token = CodeGen_PECL_Tools_ProtoParser::NUMVAL; } name { $this->token = CodeGen_PECL_Tools_ProtoParser::STRVAL; } */ } $a = new CodeGen_PECL_Tools_ProtoLexer('booler boolboolean boolean bool'); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value); $a->yylex(); var_dump('advance: ' . $a->value);