cvs: pear /SQL_Parser Lexer.php /SQL_Parser/tests generate_testcases.php update.php

busterb Sat Dec 28 00:40:11 2002 EDT Modified files: /pear/SQL_Parser Lexer.php /pear/SQL_Parser/tests generate_testcases.php update.php Log: Use Jason Pell's modified Lexer for now; I want to make some changes, but it will be easier to start here and make changes from that. This adds a slightly different code for handling Mac/Win newline characters, bug fixes and lookahead capability.

Index: pear/SQL_Parser/Lexer.php diff -u pear/SQL_Parser/Lexer.php:1.5 pear/SQL_Parser/Lexer.php:1.6 --- pear/SQL_Parser/Lexer.php:1.5 Fri Dec 27 16:04:42 2002 +++ pear/SQL_Parser/Lexer.php Sat Dec 28 00:40:11 2002 @@ -18,7 +18,7 @@ // | Foundation, Inc., 59 Temple Place, Suite 330,Boston,MA 02111-1307 USA| // +----------------------------------------------------------------------+ // -// $Id: Lexer.php,v 1.5 2002/12/27 21:04:42 busterb Exp $ +// $Id: Lexer.php,v 1.6 2002/12/28 05:40:11 busterb Exp $ // include 'SQL/ctype.php'; @@ -51,15 +51,27 @@ var $lineBegin = 0; var $string = ''; var $stringLen = 0; + + // Will not be altered by skip() + var $tokAbsStart = 0; + var $skipText = ''; + + // Provide lookahead capability. + var $lookahead = 0; + // Specify how many tokens to save in tokenStack, so the + // token stream can be pushed back. + var $tokenStack = array(); + var $stackPtr = 0; // }}} // {{{ incidental functions - function Lexer($string = '') + function Lexer($string = '', $lookahead=0) { $this->string = $string; $this->stringLen = strlen($string); + $this->lookahead = $lookahead; } - + function get() { ++$this->tokLen; return $this->string{$this->tokPtr++}; @@ -83,48 +95,106 @@ function isCompop($c) { return (($c == '<') || ($c == '>') || ($c == '=') || ($c == '!')); } - - function isWhiteSpace($c) { - return (($c == ' ') || ($c == "\t") || ($c == "\n") || ($c == "\r")); - } - - function skipWhiteSpace() { - $c = $this->get(); - while ($this->isWhiteSpace($c)) { - // Handle Unix line endings - if ($c == "\n") { - ++$this->lineNo; - $this->lineBegin = $this->tokPtr; - // Handle Mac/Windows line endings - } elseif ($c == "\r") { - $c = $this->skip; - if ($c != "\n") - $this->unget(); - ++$this->lineNo; - $this->lineBegin = $this->tokPtr; - } - $c = $this->skip(); - $this->tokLen = 1; - } - return $c; - } - // }}} +/* +* Push back a token, so the very next call to lex() will return that token. +* Calls to this function will be ignored if there is no lookahead specified +* to the constructor, or the pushBack() function has already been called the +* maximum number of token's that can be looked ahead. +*/ +function pushBack() +{ + if($this->lookahead>0 && count($this->tokenStack)>0 && $this->stackPtr>0) + { + $this->stackPtr--; + } +} + // {{{ lex() function lex() { - if ($this->string == '') return; + if($this->lookahead>0) + { + // The stackPtr, should always be the same as the count of + // elements in the tokenStack. The stackPtr, can be thought + // of as pointing to the next token to be added. If however + // a pushBack() call is made, the stackPtr, will be less than the + // count, to indicate that we should take that token from the + // stack, instead of calling nextToken for a new token. + if($this->stackPtr<count($this->tokenStack)) + { + $this->tokText = $this->tokenStack[$this->stackPtr]['tokText']; + $this->skipText = $this->tokenStack[$this->stackPtr]['skipText']; + $token = $this->tokenStack[$this->stackPtr]['token']; + + // We have read the token, so now iterate again. + $this->stackPtr++; + return $token; + } + else + { + // If $tokenStack is full (equal to lookahead), pop the oldest + // element off, to make room for the new one. + if($this->stackPtr == $this->lookahead) + { + // For some reason array_shift,array_pop screw up the indexing, so we do it manually. + for($i=0; $i<(count($this->tokenStack)-1); $i++) + { + $this->tokenStack[$i] = $this->tokenStack[$i+1]; + } + + // Indicate that we should put the element in, at the stackPtr position. + $this->stackPtr--; + } + + $token = $this->nextToken(); + $this->tokenStack[$this->stackPtr] = array('token'=>$token,'tokText'=>$this->tokText,'skipText'=>$this->skipText); + $this->stackPtr++; + return $token; + } + } + else + { + return $this->nextToken(); + } +}// }}} + +// {{{ nextToken() +function nextToken() +{ + if ($this->string == '') return; $state = 0; - while (1) { - //echo "State: $state, Char: $c\n"; + $this->tokAbsStart = $this->tokStart; + + while (true){ + //echo "State: $state, Char: $c\n"; switch($state) { // {{{ State 0 : Start of token case 0: $this->tokPtr = $this->tokStart; $this->tokText = ''; $this->tokLen = 0; - $c = $this->skipWhiteSpace(); + $c = $this->get(); + while (($c == ' ') || ($c == "\t") || ($c == "\n") || ($c == "\r")) { + if ($c == "\n" || $c == "\r") { + // Handle MAC/Unix/Windows line endings. + if($c == "\r") + { + $c = $this->skip(); + + // If not DOS newline + if($c != "\n") + $this->unget(); + } + ++$this->lineNo; + $this->lineBegin = $this->tokPtr; + } + + $c = $this->skip(); + $this->tokLen = 1; + } + if (($c == '\'') || ($c == '"')) { $quote = $c; $state = 12; @@ -186,13 +256,16 @@ /* {{{ State 2 : Complete keyword or ident */ case 2: $this->unget(); - $this->tokText = substr($this->string, $this->tokStart, - $this->tokLen); - $testToken = strtolower($this->tokText); + $this->tokText = substr($this->string, $this->tokStart, $this->tokLen); + + $testToken = strtolower($this->tokText); if (isset($this->symbols[$testToken])) { + + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return $testToken; } else { + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return 'ident'; } @@ -217,8 +290,8 @@ // {{{ State 6: Complete integer number case 6: $this->unget(); - $this->tokText = intval(substr($this->string, $this->tokStart, - $this->tokLen)); + $this->tokText = intval(substr($this->string, $this->tokStart,$this->tokLen)); + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return 'int_val'; break; @@ -248,6 +321,7 @@ $this->unget(); $this->tokText = floatval(substr($this->string, $this->tokStart, $this->tokLen)); + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return 'real_val'; // }}} @@ -282,11 +356,12 @@ // {{{ State 11: Complete comparison operator case 11: $this->unget(); - $tokval = substr($this->string, $this->tokStart, $this->tokLen); - if ($tokval) + $this->tokText = substr($this->string, $this->tokStart, $this->tokLen); + if($this->tokText) { + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; - return $tokval; + return $this->tokText; } $state = 999; break; @@ -325,6 +400,7 @@ // {{{ State 13: Complete text string case 13: + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return 'text_val'; break; @@ -333,22 +409,23 @@ // {{{ State 14: Comment case 14: $c = $this->skip(); - // Handle Unix line endings - if ($c == "\n") { + if ($c == "\n" || $c == "\r"){ + // Handle MAC/Unix/Windows line endings. + if($c == "\r") + { + $c = $this->skip(); + // If not DOS newline + if($c != "\n") + $this->unget(); + } $state = 0; - // Handle Mac/Windows line endings - } elseif ($c == "\r") { - $c = $this->skip(); - if ($c != "\n") { - $this->unget(); - } } else { $state = 14; } break; // }}} - // Analogy Start + // Analogy Start // {{{ State 15: Exponent Sign in Scientific Notation case 15: $c = $this->get(); @@ -381,7 +458,7 @@ $state = 8; // At least 1 exponent digit was required break; // }}} - // Analogy End + // Analogy End // {{{ State 18 : Incomplete System Variable case 18: @@ -397,7 +474,8 @@ // {{{ State 19: Complete Sys Var case 19: $this->unget(); - $this->tokText = substr($this->string, $this->tokStart,$this->tokLen); + $this->tokText = substr($this->string, $this->tokStart, $this->tokLen); + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return 'sys_var'; // }}} @@ -406,6 +484,7 @@ case 999: $this->revert(); $this->tokText = $this->get(); + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return $this->tokText; // }}} @@ -413,10 +492,10 @@ // {{{ State 1000 : End Of Input case 1000: $this->tokText = '*end of input*'; + $this->skipText = substr($this->string, $this->tokAbsStart, $this->tokStart-$this->tokAbsStart); $this->tokStart = $this->tokPtr; return null; // }}} - } } } Index: pear/SQL_Parser/tests/generate_testcases.php diff -u pear/SQL_Parser/tests/generate_testcases.php:1.2 pear/SQL_Parser/tests/generate_testcases.php:1.3 --- pear/SQL_Parser/tests/generate_testcases.php:1.2 Tue Dec 17 22:19:50 2002 +++ pear/SQL_Parser/tests/generate_testcases.php Sat Dec 28 00:40:11 2002 @@ -20,7 +20,7 @@ // | Author: Brent Cook <busterb@mail.utexas.edu> | // +----------------------------------------------------------------------+ // -// $Id: generate_testcases.php,v 1.2 2002/12/18 03:19:50 busterb Exp $ +// $Id: generate_testcases.php,v 1.3 2002/12/28 05:40:11 busterb Exp $ // // test functionality of the sql parser @@ -70,12 +70,12 @@ $progname = basename(array_shift($argv)); echo(count($argv)); -if (count($argv) != 2) { +if (count($argv) != 1) { echo("Usage: generate_testcases.php test_cases.sql\n"); exit(-1); } -$file = $argv[1]; +$file = $argv[0]; if (!$fd = @fopen($file, 'r')) { echo("Could not load the SQL source file: $file\n"); Index: pear/SQL_Parser/tests/update.php diff -u pear/SQL_Parser/tests/update.php:1.1 pear/SQL_Parser/tests/update.php:1.2 --- pear/SQL_Parser/tests/update.php:1.1 Tue Sep 24 13:06:58 2002 +++ pear/SQL_Parser/tests/update.php Sat Dec 28 00:40:11 2002 @@ -1,4 +1,4 @@ -<?php +1<?php $tests = array( array( 'sql' => 'update dogmeat set horse=2 dog=\'forty\' where moose <> \'howdydoo\'', @@ -11,7 +11,7 @@ 'sql' => 'update dogmeat set horse=2, dog=\'forty\' where moose != \'howdydoo\'', 'expect' => 'Parse error: Expected an operator on line 1 update dogmeat set horse=2, dog=\'forty\' where moose != \'howdydoo\' - ^ found: !' + ^ found: !=' ), array(