1 /* textoken.h 2 3 Copyright 2006-2008 Taco Hoekwater <taco@luatex.org> 4 5 This file is part of LuaTeX. 6 7 LuaTeX is free software; you can redistribute it and/or modify it under 8 the terms of the GNU General Public License as published by the Free 9 Software Foundation; either version 2 of the License, or (at your 10 option) any later version. 11 12 LuaTeX is distributed in the hope that it will be useful, but WITHOUT 13 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public 15 License for more details. 16 17 You should have received a copy of the GNU General Public License along 18 with LuaTeX; if not, see <http://www.gnu.org/licenses/>. */ 19 20 21 #ifndef TEXTOKEN_H 22 # define TEXTOKEN_H 23 24 # define token_list 0 25 # define null 0 26 # define cs_token_flag 0x1FFFFFFF 27 28 # define left_brace_token 0x200000 /* $2^{21}\cdot|left_brace|$ */ 29 # define right_brace_token 0x400000 /* $2^{21}\cdot|right_brace|$ */ 30 # define left_brace_token 0x200000 /* $2^{21}\cdot|left_brace|$ */ 31 # define left_brace_limit 0x400000 /* $2^{21}\cdot(|left_brace|+1)$ */ 32 # define right_brace_token 0x400000 /* $2^{21}\cdot|right_brace|$ */ 33 # define right_brace_limit 0x600000 /* $2^{21}\cdot(|right_brace|+1)$ */ 34 # define math_shift_token 0x600000 /* $2^{21}\cdot|math_shift|$ */ 35 # define tab_token 0x800000 /* $2^{21}\cdot|tab_mark|$ */ 36 # define out_param_token 0xA00000 /* $2^{21}\cdot|out_param|$ */ 37 # define space_token 0x1400020 /* $2^{21}\cdot|spacer|+|" "|$ */ 38 # define letter_token 0x1600000 /* $2^{21}\cdot|letter|$ */ 39 # define other_token 0x1800000 /* $2^{21}\cdot|other_char|$ */ 40 # define match_token 0x1A00000 /* $2^{21}\cdot|match|$ */ 41 # define end_match_token 0x1C00000 /* $2^{21}\cdot|end_match|$ */ 42 # define protected_token 0x1C00001 /* $2^{21}\cdot|end_match|+1$ */ 43 44 # include "tex/stringpool.h" 45 46 typedef struct smemory_word_ { 47 # ifdef WORDS_BIGENDIAN 48 halfword hhrh; 49 halfword hhlh; 50 # else 51 halfword hhlh; 52 halfword hhrh; 53 # endif 54 } smemory_word; 55 56 # define fix_mem_init 10000 57 58 extern smemory_word *fixmem; 59 extern unsigned fix_mem_min; 60 extern unsigned fix_mem_max; 61 62 extern halfword garbage; /* head of a junk list, write only */ 63 extern halfword temp_token_head; /* head of a temporary list of some kind */ 64 extern halfword hold_token_head; /* head of a temporary list of another kind */ 65 extern halfword omit_template; /* a constant token list */ 66 extern halfword null_list; /* permanently empty list */ 67 extern halfword backup_head; /* head of token list built by |scan_keyword| */ 68 69 extern void initialize_tokens(void); 70 71 extern int dyn_used; 72 73 # define token_info(a) fixmem[(a)].hhlh 74 # define token_link(a) fixmem[(a)].hhrh 75 # define set_token_info(a,b) fixmem[(a)].hhlh=(b) 76 # define set_token_link(a,b) fixmem[(a)].hhrh=(b) 77 78 extern halfword avail; /* head of the list of available one-word nodes */ 79 extern unsigned fix_mem_end; /* the last one-word node used in |mem| */ 80 81 extern halfword get_avail(void); 82 83 /* A one-word node is recycled by calling |free_avail|. 84 This routine is part of \TeX's ``inner loop,'' so we want it to be fast. 85 */ 86 87 # define free_avail(A) do { /* single-word node liberation */ \ 88 token_link(A)=avail; avail=(A); decr(dyn_used); \ 89 } while (0) 90 91 /* 92 There's also a |fast_get_avail| routine, which saves the procedure-call 93 overhead at the expense of extra programming. This routine is used in 94 the places that would otherwise account for the most calls of |get_avail|. 95 */ 96 97 # define fast_get_avail(A) do { \ 98 (A)=avail; /* avoid |get_avail| if possible, to save time */ \ 99 if ((A)==null) { (A)=get_avail(); } \ 100 else { avail=token_link((A)); token_link((A))=null; incr(dyn_used); } \ 101 } while (0) 102 103 extern void print_meaning(void); 104 105 extern void flush_list(halfword p); 106 extern void show_token_list(int p, int q, int l); 107 extern void token_show(halfword p); 108 109 # define token_ref_count(a) token_info((a)) /* reference count preceding a token list */ 110 # define set_token_ref_count(a,b) token_info((a))=b 111 # define add_token_ref(a) token_ref_count(a)++ /* new reference to a token list */ 112 113 # define store_new_token(a) do { \ 114 q=get_avail(); token_link(p)=q; token_info(q)=(a); p=q; \ 115 } while (0) 116 117 # define fast_store_new_token(a) do { \ 118 fast_get_avail(q); token_link(p)=q; token_info(q)=(a); p=q; \ 119 } while (0) 120 121 extern void delete_token_ref(halfword p); 122 123 extern void make_token_table(lua_State * L, int cmd, int chr, int cs); 124 125 # define NO_CAT_TABLE -2 126 # define DEFAULT_CAT_TABLE -1 127 128 extern void get_next(void); 129 extern void check_outer_validity(void); 130 extern boolean scan_keyword(const char *); 131 extern halfword active_to_cs(int, int); 132 extern void get_token_lua(void); 133 halfword string_to_toks(char *); 134 extern int get_char_cat_code(int); 135 136 /* 137 |no_expand_flag| is a special character value that is inserted by 138 |get_next| if it wants to suppress expansion. 139 */ 140 141 # define no_expand_flag special_char 142 # define end_line_char int_par(end_line_char_code) 143 # define end_line_char_inactive ((end_line_char < 0) || (end_line_char > 127)) 144 145 extern halfword par_loc; 146 extern halfword par_token; 147 extern boolean force_eof; 148 extern int luacstrings; 149 150 extern void firm_up_the_line(void); 151 extern void get_token(void); 152 153 extern halfword str_toks(lstring b); 154 extern void ins_the_toks(void); 155 156 extern int scan_lua_state(void); 157 extern void conv_toks(void); 158 159 extern boolean in_lua_escape; 160 extern boolean is_convert(halfword c); 161 extern str_number the_convert_string(halfword c, int i); 162 163 # define closed 2 /* not open, or at end of file */ 164 # define just_open 1 /* newly opened, first line not yet read */ 165 166 extern FILE *read_file[16]; /* used for \.{\\read} */ 167 extern int read_open[17]; /* state of |read_file[n]| */ 168 169 extern void initialize_read(void); 170 171 extern void read_toks(int n, halfword r, halfword j); 172 173 extern str_number tokens_to_string(halfword p); /* return a string from tokens list */ 174 175 extern char *tokenlist_to_cstring(int p, int inhibit_par, int *siz); 176 extern lstring *tokenlist_to_lstring(int pp, int inhibit_par); 177 extern void free_lstring(lstring * ls); 178 179 # define token_cmd(A) ((A) >> STRING_OFFSET_BITS) 180 # define token_chr(A) ((A) & (STRING_OFFSET - 1)) 181 # define token_val(A,B) (((A)<<STRING_OFFSET_BITS)+(B)) 182 183 #endif 184