parser/test/tokenize.php

0001 <?php
0002 //output tokens in a readable form
0003 //uses the token_get_all php function (needs this extension installed)
0004
0005 if (!isset($_SERVER['argv'][1])) {
0006     die("Usage: {$_SERVER['argv'][0]} file.php\n");
0007 }
0008 $c = file_get_contents($_SERVER['argv'][1]);
0009 $c = str_replace("\r", "", $c);
0010 $tokens = token_get_all($c);
0011
0012 $maxLength = 0;
0013 foreach ($tokens as $k=>$i) {
0014     $i[1] = $tokens[$k][1] = str_replace("\n", '\n', $i[1]);
0015     if (ord($i[1]) == 0) {
0016         $i[1] = $tokens[$k][1] = '';
0017     }
0018     if ($maxLength < strlen($i[1].$i[2])) $maxLength = strlen($i[1].$i[2]);
0019 }
0020 if ($maxLength > 50) $maxLength = 50;
0021 foreach ($tokens as $i) {
0022     echo $i[2].'  ';
0023     if (!$i[2]) echo ' ';
0024     echo $i[1].str_repeat(' ', $maxLength-strlen($i[1].$i[2])).': ';
0025     if (is_int($i[0])) {
0026         echo token_name($i[0])."\n";
0027     } else {
0028         echo $i[0]."\n";
0029     }
0030 }