1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
|
--TEST--
token_get_all() produces T_BAD_CHARACTER for unexpected characters
--EXTENSIONS--
tokenizer
--FILE--
<?php
$codes = [
"<?php \0 foo",
"<?php \1 bar",
"<?php \1\2 bar \3",
];
foreach ($codes as $code) {
foreach (token_get_all($code) as $token) {
if (is_array($token)) {
echo token_name($token[0]), " ", strlen($token[1]), "\n";
} else {
echo $token, "\n";
}
}
echo "\n";
}
?>
--EXPECT--
T_OPEN_TAG 6
T_BAD_CHARACTER 1
T_WHITESPACE 1
T_STRING 3
T_OPEN_TAG 6
T_BAD_CHARACTER 1
T_WHITESPACE 1
T_STRING 3
T_OPEN_TAG 6
T_BAD_CHARACTER 1
T_BAD_CHARACTER 1
T_WHITESPACE 1
T_STRING 3
T_WHITESPACE 1
T_BAD_CHARACTER 1
|
