Patch tokenizer_patch.txt for Unknown/Other Function Bug #54089

Patch version 2011-09-15 14:27 UTC

Obsolete patches:

• tokenizer_patch.txt, revision 2011-09-13 15:50 UTC

• 2011-09-15 14:27 UTC [diff to current]
• 2011-09-13 15:50 UTC [diff to current]
• 2011-09-13 07:51 UTC [diff to current]

Developer: nikic@php.net

Index: ext/tokenizer/tests/bug54089.phpt
===================================================================
--- ext/tokenizer/tests/bug54089.phpt	(revision 316815)
+++ ext/tokenizer/tests/bug54089.phpt	(working copy)
@@ -17,7 +17,7 @@
 var_dump($code);
 ?>
 --EXPECTF--
-array(2) {
+array(5) {
   [0]=>
   array(3) {
     [0]=>
@@ -36,5 +36,11 @@
     [2]=>
     int(1)
   }
+  [2]=>
+  string(1) "("
+  [3]=>
+  string(1) ")"
+  [4]=>
+  string(1) ";"
 }
-string(21) "<?php __halt_compiler"
+string(24) "<?php __halt_compiler();"
Index: ext/tokenizer/tests/token_get_all_variation16.phpt
===================================================================
--- ext/tokenizer/tests/token_get_all_variation16.phpt	(revision 316815)
+++ ext/tokenizer/tests/token_get_all_variation16.phpt	(working copy)
@@ -47,17 +47,17 @@
 
 list($value1,$value2) = $c;
 if(empty($value1) && !isset($value1)) {
-  __halt_compiler();
+  myFunction();
 }
 ?>';
 $tokens =  token_get_all($source);
 var_dump($tokens);
 
-echo "Done"
+echo "Done";
 ?>
 --EXPECTF--
 *** Testing token_get_all() : with different function constructs ***
-array(135) {
+array(142) {
   [0]=>
   array(3) {
     [0]=>
@@ -958,9 +958,46 @@
     [0]=>
     int(%d)
     [1]=>
-    string(15) "__halt_compiler"
+    string(10) "myFunction"
     [2]=>
     int(26)
   }
+  [135]=>
+  string(1) "("
+  [136]=>
+  string(1) ")"
+  [137]=>
+  string(1) ";"
+  [138]=>
+  array(3) {
+    [0]=>
+    int(%d)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(26)
+  }
+  [139]=>
+  string(1) "}"
+  [140]=>
+  array(3) {
+    [0]=>
+    int(%d)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(27)
+  }
+  [141]=>
+  array(3) {
+    [0]=>
+    int(%d)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(28)
+  }
 }
 Done
Index: ext/tokenizer/tokenizer.c
===================================================================
--- ext/tokenizer/tokenizer.c	(revision 316815)
+++ ext/tokenizer/tokenizer.c	(working copy)
@@ -106,6 +106,7 @@
 	int token_type;
 	zend_bool destroy;
 	int token_line = 1;
+	int need_tokens = -1; // for __halt_compiler lexing. -1 = disabled
 
 	array_init(return_value);
 
@@ -152,8 +153,16 @@
 
 		token_line = CG(zend_lineno);
 
-		if (token_type == T_HALT_COMPILER) {
-			break;
+		// after T_HALT_COMPILER collect the next three non-dropped tokens
+		if (need_tokens != -1) {
+			if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
+			    && token_type != T_COMMENT && token_type != T_DOC_COMMENT
+			    && --need_tokens == 0
+			) {
+				break;
+			}
+		} else if (token_type == T_HALT_COMPILER) {
+			need_tokens = 3;
 		}
 	}
 }