1 /*
2    +----------------------------------------------------------------------+
3    | Copyright (c) The PHP Group                                          |
4    +----------------------------------------------------------------------+
5    | This source file is subject to version 3.01 of the PHP license,      |
6    | that is bundled with this package in the file LICENSE, and is        |
7    | available through the world-wide-web at the following url:           |
8    | https://www.php.net/license/3_01.txt                                 |
9    | If you did not receive a copy of the PHP license and are unable to   |
10    | obtain it through the world-wide-web, please send a note to          |
11    | license@php.net so we can mail you a copy immediately.               |
12    +----------------------------------------------------------------------+
13    | Author: Andrei Zmievski <andrei@php.net>                             |
14    +----------------------------------------------------------------------+
15 */
16 
17 #ifdef HAVE_CONFIG_H
18 #include "config.h"
19 #endif
20 
21 #include "php.h"
22 #include "php_ini.h"
23 #include "ext/standard/info.h"
24 #include "php_tokenizer.h"
25 #include "tokenizer_arginfo.h"
26 
27 #include "zend.h"
28 #include "zend_exceptions.h"
29 #include "zend_language_scanner.h"
30 #include "zend_language_scanner_defs.h"
31 #include <zend_language_parser.h>
32 #include "zend_interfaces.h"
33 
34 #define zendtext   LANG_SCNG(yy_text)
35 #define zendleng   LANG_SCNG(yy_leng)
36 #define zendcursor LANG_SCNG(yy_cursor)
37 #define zendlimit  LANG_SCNG(yy_limit)
38 
39 #define TOKEN_PARSE (1 << 0)
40 
41 zend_class_entry *php_token_ce;
42 
tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS)43 void tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS) {
44 	REGISTER_LONG_CONSTANT("TOKEN_PARSE", TOKEN_PARSE, CONST_CS|CONST_PERSISTENT);
45 }
46 
47 /* {{{ tokenizer_module_entry */
48 zend_module_entry tokenizer_module_entry = {
49 	STANDARD_MODULE_HEADER,
50 	"tokenizer",
51 	ext_functions,
52 	PHP_MINIT(tokenizer),
53 	NULL,
54 	NULL,
55 	NULL,
56 	PHP_MINFO(tokenizer),
57 	PHP_TOKENIZER_VERSION,
58 	STANDARD_MODULE_PROPERTIES
59 };
60 /* }}} */
61 
62 #ifdef COMPILE_DL_TOKENIZER
ZEND_GET_MODULE(tokenizer)63 ZEND_GET_MODULE(tokenizer)
64 #endif
65 
66 static zval *php_token_get_id(zval *obj) {
67 	zval *id = OBJ_PROP_NUM(Z_OBJ_P(obj), 0);
68 	if (Z_ISUNDEF_P(id)) {
69 		zend_throw_error(NULL,
70 			"Typed property PhpToken::$id must not be accessed before initialization");
71 		return NULL;
72 	}
73 
74 	ZVAL_DEREF(id);
75 	ZEND_ASSERT(Z_TYPE_P(id) == IS_LONG);
76 	return id;
77 }
78 
php_token_get_text(zval * obj)79 static zend_string *php_token_get_text(zval *obj) {
80 	zval *text_zval = OBJ_PROP_NUM(Z_OBJ_P(obj), 1);
81 	if (Z_ISUNDEF_P(text_zval)) {
82 		zend_throw_error(NULL,
83 			"Typed property PhpToken::$text must not be accessed before initialization");
84 		return NULL;
85 	}
86 
87 	ZVAL_DEREF(text_zval);
88 	ZEND_ASSERT(Z_TYPE_P(text_zval) == IS_STRING);
89 	return Z_STR_P(text_zval);
90 }
91 
92 static bool tokenize_common(
93 		zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class);
94 
PHP_METHOD(PhpToken,tokenize)95 PHP_METHOD(PhpToken, tokenize)
96 {
97 	zend_string *source;
98 	zend_long flags = 0;
99 	zend_class_entry *token_class;
100 
101 	ZEND_PARSE_PARAMETERS_START(1, 2)
102 		Z_PARAM_STR(source)
103 		Z_PARAM_OPTIONAL
104 		Z_PARAM_LONG(flags)
105 	ZEND_PARSE_PARAMETERS_END();
106 
107 	token_class = zend_get_called_scope(execute_data);
108 
109 	/* Check construction preconditions in advance, so these are not repeated for each token. */
110 	if (token_class->ce_flags & ZEND_ACC_EXPLICIT_ABSTRACT_CLASS) {
111 		zend_throw_error(NULL, "Cannot instantiate abstract class %s", ZSTR_VAL(token_class->name));
112 		RETURN_THROWS();
113 	}
114 	if (zend_update_class_constants(token_class) == FAILURE) {
115 		RETURN_THROWS();
116 	}
117 
118 	if (!tokenize_common(return_value, source, flags, token_class)) {
119 		RETURN_THROWS();
120 	}
121 }
122 
PHP_METHOD(PhpToken,__construct)123 PHP_METHOD(PhpToken, __construct)
124 {
125 	zend_long id;
126 	zend_string *text;
127 	zend_long line = -1;
128 	zend_long pos = -1;
129 	zend_object *obj = Z_OBJ_P(ZEND_THIS);
130 
131 	ZEND_PARSE_PARAMETERS_START(2, 4)
132 		Z_PARAM_LONG(id)
133 		Z_PARAM_STR(text)
134 		Z_PARAM_OPTIONAL
135 		Z_PARAM_LONG(line)
136 		Z_PARAM_LONG(pos)
137 	ZEND_PARSE_PARAMETERS_END();
138 
139 	ZVAL_LONG(OBJ_PROP_NUM(obj, 0), id);
140 	zval_ptr_dtor(OBJ_PROP_NUM(obj, 1));
141 	ZVAL_STR_COPY(OBJ_PROP_NUM(obj, 1), text);
142 	ZVAL_LONG(OBJ_PROP_NUM(obj, 2), line);
143 	ZVAL_LONG(OBJ_PROP_NUM(obj, 3), pos);
144 }
145 
PHP_METHOD(PhpToken,is)146 PHP_METHOD(PhpToken, is)
147 {
148 	zval *kind;
149 
150 	ZEND_PARSE_PARAMETERS_START(1, 1)
151 		Z_PARAM_ZVAL(kind)
152 	ZEND_PARSE_PARAMETERS_END();
153 
154 	if (Z_TYPE_P(kind) == IS_LONG) {
155 		zval *id_zval = php_token_get_id(ZEND_THIS);
156 		if (!id_zval) {
157 			RETURN_THROWS();
158 		}
159 
160 		RETURN_BOOL(Z_LVAL_P(id_zval) == Z_LVAL_P(kind));
161 	} else if (Z_TYPE_P(kind) == IS_STRING) {
162 		zend_string *text = php_token_get_text(ZEND_THIS);
163 		if (!text) {
164 			RETURN_THROWS();
165 		}
166 
167 		RETURN_BOOL(zend_string_equals(text, Z_STR_P(kind)));
168 	} else if (Z_TYPE_P(kind) == IS_ARRAY) {
169 		zval *id_zval = NULL, *entry;
170 		zend_string *text = NULL;
171 		ZEND_HASH_FOREACH_VAL(Z_ARRVAL_P(kind), entry) {
172 			ZVAL_DEREF(entry);
173 			if (Z_TYPE_P(entry) == IS_LONG) {
174 				if (!id_zval) {
175 					id_zval = php_token_get_id(ZEND_THIS);
176 					if (!id_zval) {
177 						RETURN_THROWS();
178 					}
179 				}
180 				if (Z_LVAL_P(id_zval) == Z_LVAL_P(entry)) {
181 					RETURN_TRUE;
182 				}
183 			} else if (Z_TYPE_P(entry) == IS_STRING) {
184 				if (!text) {
185 					text = php_token_get_text(ZEND_THIS);
186 					if (!text) {
187 						RETURN_THROWS();
188 					}
189 				}
190 				if (zend_string_equals(text, Z_STR_P(entry))) {
191 					RETURN_TRUE;
192 				}
193 			} else {
194 				zend_argument_type_error(1, "must only have elements of type string|int, %s given", zend_zval_type_name(entry));
195 				RETURN_THROWS();
196 			}
197 		} ZEND_HASH_FOREACH_END();
198 		RETURN_FALSE;
199 	} else {
200 		zend_argument_type_error(1, "must be of type string|int|array, %s given", zend_zval_type_name(kind));
201 		RETURN_THROWS();
202 	}
203 }
204 
PHP_METHOD(PhpToken,isIgnorable)205 PHP_METHOD(PhpToken, isIgnorable)
206 {
207 	ZEND_PARSE_PARAMETERS_NONE();
208 
209 	zval *id_zval = php_token_get_id(ZEND_THIS);
210 	if (!id_zval) {
211 		RETURN_THROWS();
212 	}
213 
214 	zend_long id = Z_LVAL_P(id_zval);
215 	RETURN_BOOL(id == T_WHITESPACE || id == T_COMMENT || id == T_DOC_COMMENT || id == T_OPEN_TAG);
216 }
217 
PHP_METHOD(PhpToken,getTokenName)218 PHP_METHOD(PhpToken, getTokenName)
219 {
220 	ZEND_PARSE_PARAMETERS_NONE();
221 
222 	zval *id_zval = php_token_get_id(ZEND_THIS);
223 	if (!id_zval) {
224 		RETURN_THROWS();
225 	}
226 
227 	if (Z_LVAL_P(id_zval) < 256) {
228 		RETURN_CHAR(Z_LVAL_P(id_zval));
229 	} else {
230 		const char *token_name = get_token_type_name(Z_LVAL_P(id_zval));
231 		if (!token_name) {
232 			RETURN_NULL();
233 		}
234 
235 		RETURN_STRING(token_name);
236 	}
237 }
238 
PHP_METHOD(PhpToken,__toString)239 PHP_METHOD(PhpToken, __toString)
240 {
241 	ZEND_PARSE_PARAMETERS_NONE();
242 
243 	zend_string *text = php_token_get_text(ZEND_THIS);
244 	if (!text) {
245 		RETURN_THROWS();
246 	}
247 
248 	RETURN_STR_COPY(text);
249 }
250 
251 /* {{{ PHP_MINIT_FUNCTION */
PHP_MINIT_FUNCTION(tokenizer)252 PHP_MINIT_FUNCTION(tokenizer)
253 {
254 	tokenizer_register_constants(INIT_FUNC_ARGS_PASSTHRU);
255 	tokenizer_token_get_all_register_constants(INIT_FUNC_ARGS_PASSTHRU);
256 	php_token_ce = register_class_PhpToken(zend_ce_stringable);
257 
258 	return SUCCESS;
259 }
260 /* }}} */
261 
262 /* {{{ PHP_MINFO_FUNCTION */
PHP_MINFO_FUNCTION(tokenizer)263 PHP_MINFO_FUNCTION(tokenizer)
264 {
265 	php_info_print_table_start();
266 	php_info_print_table_row(2, "Tokenizer Support", "enabled");
267 	php_info_print_table_end();
268 }
269 /* }}} */
270 
make_str(unsigned char * text,size_t leng,HashTable * interned_strings)271 static zend_string *make_str(unsigned char *text, size_t leng, HashTable *interned_strings) {
272 	if (leng == 1) {
273 		return ZSTR_CHAR(text[0]);
274 	} else if (interned_strings) {
275 		zend_string *interned_str = zend_hash_str_find_ptr(interned_strings, (char *) text, leng);
276 		if (interned_str) {
277 			return zend_string_copy(interned_str);
278 		}
279 		interned_str = zend_string_init((char *) text, leng, 0);
280 		zend_hash_add_new_ptr(interned_strings, interned_str, interned_str);
281 		return interned_str;
282 	} else {
283 		return zend_string_init((char *) text, leng, 0);
284 	}
285 }
286 
add_token(zval * return_value,int token_type,unsigned char * text,size_t leng,int lineno,zend_class_entry * token_class,HashTable * interned_strings)287 static void add_token(
288 		zval *return_value, int token_type, unsigned char *text, size_t leng, int lineno,
289 		zend_class_entry *token_class, HashTable *interned_strings) {
290 	zval token;
291 	if (token_class) {
292 		zend_object *obj = zend_objects_new(token_class);
293 		ZVAL_OBJ(&token, obj);
294 		ZVAL_LONG(OBJ_PROP_NUM(obj, 0), token_type);
295 		ZVAL_STR(OBJ_PROP_NUM(obj, 1), make_str(text, leng, interned_strings));
296 		ZVAL_LONG(OBJ_PROP_NUM(obj, 2), lineno);
297 		ZVAL_LONG(OBJ_PROP_NUM(obj, 3), text - LANG_SCNG(yy_start));
298 
299 		/* If the class is extended with additional properties, initialized them as well. */
300 		if (UNEXPECTED(token_class->default_properties_count > 4)) {
301 			zval *dst = OBJ_PROP_NUM(obj, 4);
302 			zval *src = &token_class->default_properties_table[4];
303 			zval *end = token_class->default_properties_table
304 				+ token_class->default_properties_count;
305 			for (; src < end; src++, dst++) {
306 				ZVAL_COPY_PROP(dst, src);
307 			}
308 		}
309 	} else if (token_type >= 256) {
310 		array_init_size(&token, 3);
311 		zend_hash_real_init_packed(Z_ARRVAL(token));
312 		ZEND_HASH_FILL_PACKED(Z_ARRVAL(token)) {
313 			ZEND_HASH_FILL_SET_LONG(token_type);
314 			ZEND_HASH_FILL_NEXT();
315 			ZEND_HASH_FILL_SET_STR(make_str(text, leng, interned_strings));
316 			ZEND_HASH_FILL_NEXT();
317 			ZEND_HASH_FILL_SET_LONG(lineno);
318 			ZEND_HASH_FILL_NEXT();
319 		} ZEND_HASH_FILL_END();
320 	} else {
321 		ZVAL_STR(&token, make_str(text, leng, interned_strings));
322 	}
323 	zend_hash_next_index_insert_new(Z_ARRVAL_P(return_value), &token);
324 }
325 
tokenize(zval * return_value,zend_string * source,zend_class_entry * token_class)326 static bool tokenize(zval *return_value, zend_string *source, zend_class_entry *token_class)
327 {
328 	zval source_zval;
329 	zend_lex_state original_lex_state;
330 	zval token;
331 	int token_type;
332 	int token_line = 1;
333 	int need_tokens = -1; /* for __halt_compiler lexing. -1 = disabled */
334 	HashTable interned_strings;
335 
336 	ZVAL_STR_COPY(&source_zval, source);
337 	zend_save_lexical_state(&original_lex_state);
338 
339 	zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC());
340 
341 	LANG_SCNG(yy_state) = yycINITIAL;
342 	zend_hash_init(&interned_strings, 0, NULL, NULL, 0);
343 	array_init(return_value);
344 
345 	while ((token_type = lex_scan(&token, NULL))) {
346 		ZEND_ASSERT(token_type != T_ERROR);
347 
348 		add_token(
349 			return_value, token_type, zendtext, zendleng, token_line,
350 			token_class, &interned_strings);
351 
352 		if (Z_TYPE(token) != IS_UNDEF) {
353 			zval_ptr_dtor_nogc(&token);
354 			ZVAL_UNDEF(&token);
355 		}
356 
357 		/* after T_HALT_COMPILER collect the next three non-dropped tokens */
358 		if (need_tokens != -1) {
359 			if (token_type != T_WHITESPACE && token_type != T_OPEN_TAG
360 				&& token_type != T_COMMENT && token_type != T_DOC_COMMENT
361 				&& --need_tokens == 0
362 			) {
363 				/* fetch the rest into a T_INLINE_HTML */
364 				if (zendcursor < zendlimit) {
365 					add_token(
366 						return_value, T_INLINE_HTML, zendcursor, zendlimit - zendcursor,
367 						token_line, token_class, &interned_strings);
368 				}
369 				break;
370 			}
371 		} else if (token_type == T_HALT_COMPILER) {
372 			need_tokens = 3;
373 		}
374 
375 		if (CG(increment_lineno)) {
376 			CG(zend_lineno)++;
377 			CG(increment_lineno) = 0;
378 		}
379 
380 		token_line = CG(zend_lineno);
381 	}
382 
383 	zval_ptr_dtor_str(&source_zval);
384 	zend_restore_lexical_state(&original_lex_state);
385 	zend_hash_destroy(&interned_strings);
386 
387 	return 1;
388 }
389 
390 struct event_context {
391 	zval *tokens;
392 	zend_class_entry *token_class;
393 };
394 
extract_token_id_to_replace(zval * token_zv,const char * text,size_t length)395 static zval *extract_token_id_to_replace(zval *token_zv, const char *text, size_t length) {
396 	zval *id_zv, *text_zv;
397 	ZEND_ASSERT(token_zv);
398 	if (Z_TYPE_P(token_zv) == IS_ARRAY) {
399 		id_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 0);
400 		text_zv = zend_hash_index_find(Z_ARRVAL_P(token_zv), 1);
401 	} else if (Z_TYPE_P(token_zv) == IS_OBJECT) {
402 		id_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 0);
403 		text_zv = OBJ_PROP_NUM(Z_OBJ_P(token_zv), 1);
404 	} else {
405 		return NULL;
406 	}
407 
408 	/* There are multiple candidate tokens to which this feedback may apply,
409 	 * check text to make sure this is the right one. */
410 	ZEND_ASSERT(Z_TYPE_P(text_zv) == IS_STRING);
411 	if (Z_STRLEN_P(text_zv) == length && !memcmp(Z_STRVAL_P(text_zv), text, length)) {
412 		return id_zv;
413 	}
414 	return NULL;
415 }
416 
on_event(zend_php_scanner_event event,int token,int line,const char * text,size_t length,void * context)417 void on_event(
418 		zend_php_scanner_event event, int token, int line,
419 		const char *text, size_t length, void *context)
420 {
421 	struct event_context *ctx = context;
422 
423 	switch (event) {
424 		case ON_TOKEN:
425 			if (token == END) break;
426 			/* Special cases */
427 			if (token == ';' && LANG_SCNG(yy_leng) > 1) { /* ?> or ?>\n or ?>\r\n */
428 				token = T_CLOSE_TAG;
429 			} else if (token == T_ECHO && LANG_SCNG(yy_leng) == sizeof("<?=") - 1) {
430 				token = T_OPEN_TAG_WITH_ECHO;
431 			}
432 			add_token(
433 				ctx->tokens, token, (unsigned char *) text, length, line, ctx->token_class, NULL);
434 			break;
435 		case ON_FEEDBACK: {
436 			HashTable *tokens_ht = Z_ARRVAL_P(ctx->tokens);
437 			zval *token_zv, *id_zv = NULL;
438 			ZEND_HASH_REVERSE_FOREACH_VAL(tokens_ht, token_zv) {
439 				id_zv = extract_token_id_to_replace(token_zv, text, length);
440 				if (id_zv) {
441 					break;
442 				}
443 			} ZEND_HASH_FOREACH_END();
444 			ZEND_ASSERT(id_zv);
445 			ZVAL_LONG(id_zv, token);
446 			break;
447 		}
448 		case ON_STOP:
449 			if (LANG_SCNG(yy_cursor) != LANG_SCNG(yy_limit)) {
450 				add_token(ctx->tokens, T_INLINE_HTML, LANG_SCNG(yy_cursor),
451 					LANG_SCNG(yy_limit) - LANG_SCNG(yy_cursor), CG(zend_lineno),
452 					ctx->token_class, NULL);
453 			}
454 			break;
455 	}
456 }
457 
tokenize_parse(zval * return_value,zend_string * source,zend_class_entry * token_class)458 static bool tokenize_parse(
459 		zval *return_value, zend_string *source, zend_class_entry *token_class)
460 {
461 	zval source_zval;
462 	struct event_context ctx;
463 	zval token_stream;
464 	zend_lex_state original_lex_state;
465 	bool original_in_compilation;
466 	bool success;
467 
468 	ZVAL_STR_COPY(&source_zval, source);
469 
470 	original_in_compilation = CG(in_compilation);
471 	CG(in_compilation) = 1;
472 	zend_save_lexical_state(&original_lex_state);
473 
474 	zend_prepare_string_for_scanning(&source_zval, ZSTR_EMPTY_ALLOC());
475 	array_init(&token_stream);
476 
477 	ctx.tokens = &token_stream;
478 	ctx.token_class = token_class;
479 
480 	CG(ast) = NULL;
481 	CG(ast_arena) = zend_arena_create(1024 * 32);
482 	LANG_SCNG(yy_state) = yycINITIAL;
483 	LANG_SCNG(on_event) = on_event;
484 	LANG_SCNG(on_event_context) = &ctx;
485 
486 	if((success = (zendparse() == SUCCESS))) {
487 		ZVAL_COPY_VALUE(return_value, &token_stream);
488 	} else {
489 		zval_ptr_dtor(&token_stream);
490 	}
491 
492 	zend_ast_destroy(CG(ast));
493 	zend_arena_destroy(CG(ast_arena));
494 
495 	/* restore compiler and scanner global states */
496 	zend_restore_lexical_state(&original_lex_state);
497 	CG(in_compilation) = original_in_compilation;
498 
499 	zval_ptr_dtor_str(&source_zval);
500 
501 	return success;
502 }
503 
tokenize_common(zval * return_value,zend_string * source,zend_long flags,zend_class_entry * token_class)504 static bool tokenize_common(
505 		zval *return_value, zend_string *source, zend_long flags, zend_class_entry *token_class)
506 {
507 	if (flags & TOKEN_PARSE) {
508 		return tokenize_parse(return_value, source, token_class);
509 	} else {
510 		int success = tokenize(return_value, source, token_class);
511 		/* Normal token_get_all() should not throw. */
512 		zend_clear_exception();
513 		return success;
514 	}
515 }
516 
517 /* }}} */
518 
519 /* {{{ */
PHP_FUNCTION(token_get_all)520 PHP_FUNCTION(token_get_all)
521 {
522 	zend_string *source;
523 	zend_long flags = 0;
524 
525 	ZEND_PARSE_PARAMETERS_START(1, 2)
526 		Z_PARAM_STR(source)
527 		Z_PARAM_OPTIONAL
528 		Z_PARAM_LONG(flags)
529 	ZEND_PARSE_PARAMETERS_END();
530 
531 	if (!tokenize_common(return_value, source, flags, /* token_class */ NULL)) {
532 		RETURN_THROWS();
533 	}
534 }
535 /* }}} */
536 
537 /* {{{ */
PHP_FUNCTION(token_name)538 PHP_FUNCTION(token_name)
539 {
540 	zend_long type;
541 
542 	ZEND_PARSE_PARAMETERS_START(1, 1)
543 		Z_PARAM_LONG(type)
544 	ZEND_PARSE_PARAMETERS_END();
545 
546 	const char *token_name = get_token_type_name(type);
547 	if (!token_name) {
548 		token_name = "UNKNOWN";
549 	}
550 	RETURN_STRING(token_name);
551 }
552 /* }}} */
553