cvs: phpdoc /en/reference/tokenizer reference.xml /en/reference/tokenizer/functions token-get-all.xml token-name.xml

hholzgra Thu Feb 6 04:38:27 2003 EDT Modified files: /phpdoc/en/reference/tokenizer reference.xml /phpdoc/en/reference/tokenizer/functions token-get-all.xml token-name.xml Log: tokenizer no longer undocumented :) Index: phpdoc/en/reference/tokenizer/reference.xml diff -u phpdoc/en/reference/tokenizer/reference.xml:1.7 phpdoc/en/reference/tokenizer/reference.xml:1.8 --- phpdoc/en/reference/tokenizer/reference.xml:1.7 Thu Jan 16 10:13:15 2003 +++ phpdoc/en/reference/tokenizer/reference.xml Thu Feb 6 04:38:25 2003 @@ -1,5 +1,5 @@ <?xml version="1.0" encoding="iso-8859-1"?> -<!-- $Revision: 1.7 $ --> +<!-- $Revision: 1.8 $ --> <reference id="ref.tokenizer"> <title>Tokenizer functions</title> <titleabbrev>Tokenizer functions</titleabbrev> @@ -7,7 +7,13 @@ <partintro> <section id="tokenizer.intro"> &reftitle.intro; - &warn.experimental; + <para> + The tokenizer functions provide an interface to the + PHP tokenizer embedded in the Zend Engine. Using these + functions you may write your own PHP source analyzation + or modification tools without having to deal with the + language specification at the lexical level. + </para> <para> See also the <link linkend="tokens">appendix about tokens</link>. </para> @@ -25,6 +31,46 @@ --> &reference.tokenizer.constants; + + <section id="tokenizer.examples"> + &reftitle.examples; + <para> + Here is a simple example PHP scripts using the tokenizer that + will read in a PHP file, strip all comments from the source + and print the pure code only. + </para> + <example> + <title>Strip comments</title> + <programlisting role="php"> +<![CDATA[ +<?php + $source = file_get_contents("somefile.php"); + $tokens = token_get_all($source); + foreach ($tokens as $token) { + if (is_string($token)) { + // simple 1-character token + echo $token; + } else { + // token array + list($id, $text) = $token; + switch($id) { + case T_COMMENT: + case T_ML_COMMENT: + // no action on comments + break; + default: + // anything else -> output "as is" + echo $text; + break; + } + } + } +?> +]]> + </programlisting> + </example> + </section> + </partintro> Index: phpdoc/en/reference/tokenizer/functions/token-get-all.xml diff -u phpdoc/en/reference/tokenizer/functions/token-get-all.xml:1.1 phpdoc/en/reference/tokenizer/functions/token-get-all.xml:1.2 --- phpdoc/en/reference/tokenizer/functions/token-get-all.xml:1.1 Thu May 2 09:22:42 2002 +++ phpdoc/en/reference/tokenizer/functions/token-get-all.xml Thu Feb 6 04:38:26 2003 @@ -1,18 +1,34 @@ <?xml version="1.0" encoding="iso-8859-1"?> -<!-- $Revision: 1.1 $ --> +<!-- $Revision: 1.2 $ --> <refentry id="function.token-get-all"> <refnamediv> <refname>token_get_all</refname> - <refpurpose>Split given source in tokens</refpurpose> + <refpurpose>Split given source into PHP tokens</refpurpose> </refnamediv> <refsect1> <title>Description</title> - <methodsynopsis> - <type>array</type><methodname>token_get_all</methodname> - <methodparam><type>string</type><parameter>source</parameter></methodparam> - </methodsynopsis> - &warn.experimental.func; - &warn.undocumented.func; + <methodsynopsis> + <type>array</type><methodname>token_get_all</methodname> + <methodparam><type>string</type><parameter>source</parameter></methodparam> + </methodsynopsis> + <para> + <function>token_get_all</function> parses the given <parameter>source</parameter> + string into PHP language tokens using the Zend engines lexical scanner. + The function returns an array of token descriptions. Each array element itself is + either a one character string or itself an array containing a token id and the + string representation of that token in the source code. + </para> + <informalexample> + <programlisting role="php"> +<![CDATA[ +<?php + $tokens = token_get_all(";"); // => array(";"); + $tokens = token_get_all("foreach") // => array(T_FOREACH => "foreach"); + $tokens = token_get_all("/* comment */") // => array(T_ML_COMMENT => "/* comment */"); +?> +]]> + </programlisting> + </informalexample> </refsect1> </refentry> Index: phpdoc/en/reference/tokenizer/functions/token-name.xml diff -u phpdoc/en/reference/tokenizer/functions/token-name.xml:1.1 phpdoc/en/reference/tokenizer/functions/token-name.xml:1.2 --- phpdoc/en/reference/tokenizer/functions/token-name.xml:1.1 Thu May 2 09:22:42 2002 +++ phpdoc/en/reference/tokenizer/functions/token-name.xml Thu Feb 6 04:38:26 2003 @@ -1,18 +1,34 @@ <?xml version="1.0" encoding="iso-8859-1"?> -<!-- $Revision: 1.1 $ --> +<!-- $Revision: 1.2 $ --> <refentry id="function.token-name"> <refnamediv> <refname>token_name</refname> - <refpurpose>Get the name of a given token</refpurpose> + <refpurpose>Get the symbolic name of a given PHP token</refpurpose> </refnamediv> <refsect1> <title>Description</title> - <methodsynopsis> - <type>string</type><methodname>token_name</methodname> - <methodparam><type>int</type><parameter>type</parameter></methodparam> - </methodsynopsis> - &warn.experimental.func; - &warn.undocumented.func; + <methodsynopsis> + <type>string</type><methodname>token_name</methodname> + <methodparam><type>int</type><parameter>token</parameter></methodparam> + </methodsynopsis> + <para> + This function will returen the symbolic name for a PHP + <parameter>token</parameter> value. The symbolic name + returned matches the name of the matching token constant. + </para> + <informalexample> + <programlisting role="php"> +<![CDATA[ +<?php + // 260 is the token value for the T_REQUIRE token + echo token_name(260); // -> "T_REQUIRE" + + // a token constant maps to its own name + echo token_name(T_FUNCTION); // -> "T_FUNCTION" +?> +]]> + </programlisting> + </informalexample> </refsect1> </refentry>