1 /*
2 Copyright (c) 2010 Peter "Corsix" Cawley
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy of
5 this software and associated documentation files (the "Software"), to deal in
6 the Software without restriction, including without limitation the rights to
7 use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
8 of the Software, and to permit persons to whom the Software is furnished to do
9 so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be included in all
12 copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
20 SOFTWARE.
21 */
22
23 #include "persist_lua.h"
24
25 #include <errno.h>
26
27 #include <array>
28 #include <cmath>
29 #include <cstdio>
30 #include <cstdlib>
31 #include <cstring>
32 #include <string>
33 #include <vector>
34
35 #include "th_lua.h"
36 #ifdef _MSC_VER
37 #pragma warning( \
38 disable : 4996) // Disable "std::strcpy unsafe" warnings under MSVC
39 #endif
40
41 namespace {
42
43 enum persist_type {
44 // LUA_TNIL = 0,
45 // LUA_TBOOLEAN, // Used for false
46 // LUA_TLIGHTUSERDATA, // Not currently persisted
47 // LUA_TNUMBER,
48 // LUA_TSTRING,
49 // LUA_TTABLE, // Used for tables without metatables
50 // LUA_TFUNCTION, // Used for Lua closures
51 // LUA_TUSERDATA,
52 // LUA_TTHREAD, // Not currently persisted
53 PERSIST_TPERMANENT = LUA_TTHREAD + 1,
54 PERSIST_TTRUE,
55 PERSIST_TTABLE_WITH_META,
56 PERSIST_TINTEGER,
57 PERSIST_TPROTOTYPE,
58 PERSIST_TRESERVED1, // Not currently used
59 PERSIST_TRESERVED2, // Not currently used
60 PERSIST_TCOUNT, // must equal 16 (for compatibility)
61 };
62
63 template <class T>
l_crude_gc(lua_State * L)64 int l_crude_gc(lua_State* L) {
65 // This __gc metamethod does not verify that the given value is the correct
66 // type of userdata, or that the value is userdata at all.
67 reinterpret_cast<T*>(lua_touserdata(L, 1))->~T();
68 return 0;
69 }
70
71 } // namespace
72
73 //! Structure for loading multiple strings as a Lua chunk, avoiding
74 //! concatenation
75 /*!
76 luaL_loadbuffer() is a good way to load a string as a Lua chunk. If there
77 are several strings which need to be concatenated before being loaded, then
78 it can be more efficient to use this structure, which can load them without
79 concatenating them. Sample usage is:
80
81 ```
82 load_multi_buffer ls;
83 ls.s[0] = lua_tolstring(L, -2, &ls.i[0]);
84 ls.s[1] = lua_tolstring(L, -1, &ls.i[1]);
85 lua_load(L, LoadMultiBuffer_t::load_fn, &ls, "chunk name");
86 ```
87 */
88 class load_multi_buffer {
89 public:
90 const char* s[3];
91 size_t i[3];
92 int n;
93
load_multi_buffer()94 load_multi_buffer() {
95 s[0] = s[1] = s[2] = nullptr;
96 i[0] = i[1] = i[2] = 0;
97 n = 0;
98 }
99
load_fn(lua_State * L,void * ud,size_t * size)100 static const char* load_fn(lua_State* L, void* ud, size_t* size) {
101 load_multi_buffer* pThis = reinterpret_cast<load_multi_buffer*>(ud);
102
103 for (; pThis->n < 3; ++pThis->n) {
104 if (pThis->i[pThis->n] != 0) {
105 *size = pThis->i[pThis->n];
106 return pThis->s[pThis->n++];
107 }
108 }
109
110 *size = 0;
111 return nullptr;
112 }
113 };
114
115 //! Basic implementation of persistence interface
116 /*!
117 self - Instance of lua_persist_basic_writer allocated as a Lua userdata
118 self metatable:
119 `__gc` - ~lua_persist_basic_writer (via l_crude_gc)
120 `<file>:<line>` - index of function prototype in already written data
121 [1] - pre-populated prototype persistence names
122 `<file>:<line>` - `<name>`
123 err - an object which could not be persisted
124 self environment:
125 `<object>` - index of object in already written data
126 [1] - permanents table
127 self environment metatable
128 */
129 class lua_persist_basic_writer : public lua_persist_writer {
130 public:
lua_persist_basic_writer(lua_State * L)131 lua_persist_basic_writer(lua_State* L) : L(L), data() {}
132
133 ~lua_persist_basic_writer() override = default;
134
get_stack()135 lua_State* get_stack() override { return L; }
136
init()137 void init() {
138 lua_createtable(L, 1, 8); // Environment
139 lua_pushvalue(L, 2); // Permanent objects
140 lua_rawseti(L, -2, 1);
141 lua_createtable(L, 1, 0); // Environment metatable
142 lua_setmetatable(L, -2);
143 lua_setfenv(L, 1);
144 lua_createtable(L, 1, 4); // Metatable
145 luaT_pushcclosure(L, l_crude_gc<lua_persist_basic_writer>, 0);
146 lua_setfield(L, -2, "__gc");
147 lua_pushvalue(L, luaT_upvalueindex(1)); // Prototype persistence names
148 lua_rawseti(L, -2, 1);
149 lua_setmetatable(L, 1);
150
151 next_index = 1;
152 data_size = 0;
153 had_error = false;
154 }
155
finish()156 int finish() {
157 if (get_error() != nullptr) {
158 lua_pushnil(L);
159 lua_pushstring(L, get_error());
160 lua_getmetatable(L, 1);
161 lua_getfield(L, -1, "err");
162 lua_replace(L, -2);
163 return 3;
164 } else {
165 lua_pushlstring(L, data.c_str(), data.length());
166 return 1;
167 }
168 }
169
fast_write_stack_object(int iIndex)170 void fast_write_stack_object(int iIndex) override {
171 if (lua_type(L, iIndex) != LUA_TUSERDATA) {
172 write_stack_object(iIndex);
173 return;
174 }
175
176 // Convert index from relative to absolute
177 if (iIndex < 0 && iIndex > LUA_REGISTRYINDEX)
178 iIndex = lua_gettop(L) + 1 + iIndex;
179
180 // Check for no cycle
181 lua_getfenv(L, 1);
182 lua_pushvalue(L, iIndex);
183 lua_rawget(L, -2);
184 lua_rawgeti(L, -2, 1);
185 lua_pushvalue(L, iIndex);
186 lua_gettable(L, -2);
187 lua_replace(L, -2);
188 if (!lua_isnil(L, -1) || !lua_isnil(L, -2)) {
189 lua_pop(L, 3);
190 write_stack_object(iIndex);
191 return;
192 }
193 lua_pop(L, 2);
194
195 // Save the index to the cache
196 lua_pushvalue(L, iIndex);
197 lua_pushnumber(L, (lua_Number)(next_index++));
198 lua_settable(L, -3);
199
200 if (!check_that_userdata_can_be_depersisted(iIndex)) return;
201
202 // Write type, metatable, and then environment
203 uint8_t iType = LUA_TUSERDATA;
204 write_byte_stream(&iType, 1);
205 write_stack_object(-1);
206 lua_getfenv(L, iIndex);
207 write_stack_object(-1);
208 lua_pop(L, 1);
209
210 // Write the raw data
211 if (lua_type(L, -1) == LUA_TTABLE) {
212 lua_getfield(L, -1, "__persist");
213 if (lua_isnil(L, -1))
214 lua_pop(L, 1);
215 else {
216 lua_pushvalue(L, iIndex);
217 lua_checkstack(L, 20);
218 lua_CFunction fn = lua_tocfunction(L, -2);
219 fn(L);
220 lua_pop(L, 2);
221 }
222 }
223 write_uint((uint64_t)0x42); // sync marker
224 lua_pop(L, 1);
225 }
226
write_stack_object(int iIndex)227 void write_stack_object(int iIndex) override {
228 // Convert index from relative to absolute
229 if (iIndex < 0 && iIndex > LUA_REGISTRYINDEX)
230 iIndex = lua_gettop(L) + 1 + iIndex;
231
232 if (lua_type(L, 2) != LUA_TTABLE) {
233 luaL_error(L, "Permanents table was lost!");
234 }
235
236 // Basic types always have their value written
237 int iType = lua_type(L, iIndex);
238 if (iType == LUA_TNIL || iType == LUA_TNONE) {
239 uint8_t iByte = LUA_TNIL;
240 write_byte_stream(&iByte, 1);
241 } else if (iType == LUA_TBOOLEAN) {
242 uint8_t iByte;
243 if (lua_toboolean(L, iIndex))
244 iByte = PERSIST_TTRUE;
245 else
246 iByte = LUA_TBOOLEAN;
247 write_byte_stream(&iByte, 1);
248 } else if (iType == LUA_TNUMBER) {
249 double fValue = lua_tonumber(L, iIndex);
250 if (floor(fValue) == fValue && 0.0 <= fValue && fValue <= 16383.0) {
251 // Small integers are written as just a few bytes
252 // NB: 16383 = 2^14-1, which is the maximum value which
253 // can fit into two bytes of VUInt.
254 uint8_t iByte = PERSIST_TINTEGER;
255 write_byte_stream(&iByte, 1);
256 uint16_t iValue = (uint16_t)fValue;
257 write_uint(iValue);
258 } else {
259 // Other numbers are written as an 8 byte double
260 uint8_t iByte = LUA_TNUMBER;
261 write_byte_stream(&iByte, 1);
262 write_byte_stream(reinterpret_cast<uint8_t*>(&fValue), sizeof(double));
263 }
264 } else {
265 // Complex values are cached, and are only written once (if this
266 // weren't done, then cycles in the object graph would break
267 // things).
268 lua_getfenv(L, 1);
269 lua_pushvalue(L, iIndex);
270
271 lua_rawget(L, -2);
272 uint64_t iValue = static_cast<uint64_t>(lua_tonumber(L, -1));
273 if (iValue != 0) {
274 lua_pop(L, 2);
275 // If the value has not previously been written, then
276 // write_object_raw would have been called, and the appropriate
277 // data written, and 0 would be returned. Otherwise, the index
278 // would be returned, which we offset by the number of types,
279 // and then write.
280 write_uint(iValue + PERSIST_TCOUNT - 1);
281 } else {
282 lua_pop(L, 1);
283
284 // Save the index to the cache
285 lua_pushvalue(L, iIndex);
286 lua_pushnumber(L, static_cast<lua_Number>(next_index++));
287 lua_rawset(L, -3);
288
289 // Remove the fenv and add the item back to the top
290 lua_pop(L, 1);
291 lua_pushvalue(L, iIndex);
292
293 // write the item and restore the stack to the start state
294 write_object_raw();
295 lua_pop(L, 1);
296 }
297 }
298 }
299
write_object_raw()300 void write_object_raw() {
301 int top = lua_gettop(L);
302 int item_index = top; // same position in write_stack_object
303 int self_index = 1; // index 1 in write_stack_object
304 int permanents_index = 2; // also in fenv self [1]
305 uint8_t item_type = lua_type(L, item_index);
306
307 lua_checkstack(L, top + 5);
308
309 // Lookup the object in the permanents table
310 lua_pushvalue(L, item_index);
311 lua_gettable(L, permanents_index);
312 if (lua_type(L, -1) != LUA_TNIL) {
313 // Object is in the permanents table.
314 uint8_t item_type = PERSIST_TPERMANENT;
315 write_byte_stream(&item_type, 1);
316
317 // Write the key corresponding to the permanent object
318 write_stack_object(-1);
319 lua_pop(L, 1);
320 } else {
321 // Object is not in the permanents table.
322 lua_pop(L, 1);
323
324 switch (item_type) {
325 // LUA_TNIL handled in write_stack_object
326 // LUA_TBOOLEAN handled in write_stack_object
327 // LUA_TNUMBER handled in write_stack_object
328
329 case LUA_TSTRING: {
330 write_byte_stream(&item_type, 1);
331 // Strings are simple: length and then bytes (not null
332 // terminated)
333 size_t iLength;
334 const char* sString = lua_tolstring(L, item_index, &iLength);
335 write_uint(iLength);
336 write_byte_stream(reinterpret_cast<const uint8_t*>(sString), iLength);
337 break;
338 }
339
340 case LUA_TTABLE: {
341 // Save env and insert prior to table
342 lua_getfenv(L, self_index);
343 lua_insert(L, item_index);
344 int table_env_index = item_index;
345 item_index += 1;
346
347 // Handle the metatable
348 if (lua_getmetatable(L, item_index)) {
349 item_type = PERSIST_TTABLE_WITH_META;
350 write_byte_stream(&item_type, 1);
351 write_stack_object(-1);
352 lua_pop(L, 1);
353 } else {
354 write_byte_stream(&item_type, 1);
355 }
356
357 // Write the children as key, value pairs
358 lua_pushnil(L);
359 while (lua_next(L, item_index)) {
360 write_stack_object(-2); // write the key
361 write_stack_object(-1); // write the value
362 lua_pop(L, 1); // remove the value
363 // key is passed back to lua_next
364 }
365
366 // Write a nil to mark the end of the children (as nil is
367 // the only value which cannot be used as a key in a table).
368 uint8_t nil_type = LUA_TNIL;
369 write_byte_stream(&nil_type, 1);
370
371 lua_remove(L, table_env_index);
372 } break;
373
374 case LUA_TFUNCTION:
375 if (lua_iscfunction(L, item_index)) {
376 set_error_object(item_index, self_index);
377 set_error("Cannot persist C functions");
378 break;
379 }
380 write_byte_stream(&item_type, 1);
381
382 // Write the prototype (the part of a function which is
383 // common across multiple closures - see LClosure /
384 // Proto in Lua's lobject.h).
385 lua_Debug proto_info;
386 lua_pushvalue(L, item_index);
387 lua_getinfo(L, ">Su", &proto_info);
388 write_prototype(&proto_info, item_index);
389
390 // Write the values of the upvalues
391 // If available, also write the upvalue IDs (so that in
392 // the future, we could hypothetically rejoin shared
393 // upvalues). An ID is just an opaque sequence of bytes.
394 write_uint(proto_info.nups);
395 #if LUA_VERSION_NUM >= 502
396 write_uint(sizeof(void*));
397 #else
398 write_uint(0);
399 #endif
400 for (int i = 1; i <= proto_info.nups; ++i) {
401 lua_getupvalue(L, item_index, i);
402 write_stack_object(-1);
403 #if LUA_VERSION_NUM >= 502
404 void* pUpvalueID = lua_upvalueid(L, item_index, i);
405 write_byte_stream((uint8_t*)&pUpvalueID, sizeof(void*));
406 #endif
407 lua_pop(L, 1);
408 }
409
410 // Write the environment table
411 lua_getfenv(L, item_index);
412 write_stack_object(-1);
413 lua_pop(L, 1);
414 break;
415
416 case LUA_TUSERDATA:
417 if (!check_that_userdata_can_be_depersisted(item_index)) {
418 break;
419 }
420
421 // Write type, metatable, and then environment
422 write_byte_stream(&item_type, 1);
423 write_stack_object(-1);
424 lua_getfenv(L, item_index);
425 write_stack_object(-1);
426 lua_pop(L, 1);
427
428 // Write the raw data
429 if (lua_type(L, -1) == LUA_TTABLE) {
430 lua_getfield(L, -1, "__persist");
431 if (lua_isnil(L, -1)) {
432 lua_pop(L, 1);
433 } else {
434 lua_pushvalue(L, item_index);
435 lua_pushvalue(L, self_index);
436 lua_pushvalue(L, permanents_index);
437 lua_call(L, 3, 0);
438 }
439 }
440 lua_pop(L, 1); // remove userdata metatable
441 write_uint((uint64_t)0x42); // sync marker
442 break;
443
444 default:
445 set_error(lua_pushfstring(L, "Cannot persist %s values",
446 luaL_typename(L, item_type)));
447 }
448 }
449
450 if (had_error) {
451 luaL_error(L, get_error());
452 }
453 }
454
455 // Checks if userdata can be persisted to file, that is if it has a size
456 // of zero, or has a metatable with a __depersist_size, __persist, and
457 // __depersist function. The metatable is pushed to the top of the stack.
check_that_userdata_can_be_depersisted(int iIndex)458 bool check_that_userdata_can_be_depersisted(int iIndex) {
459 if (lua_getmetatable(L, iIndex)) {
460 lua_getfield(L, -1, "__depersist_size");
461 if (lua_isnumber(L, -1)) {
462 if (static_cast<int>(lua_tointeger(L, -1)) !=
463 static_cast<int>(lua_objlen(L, iIndex))) {
464 set_error(lua_pushfstring(L,
465 "__depersist_size is "
466 "incorrect (%d vs. %d)",
467 (int)lua_objlen(L, iIndex),
468 (int)lua_tointeger(L, -1)));
469 return false;
470 }
471 if (lua_objlen(L, iIndex) != 0) {
472 lua_getfield(L, -2, "__persist");
473 lua_getfield(L, -3, "__depersist");
474 if (lua_isnil(L, -1) || lua_isnil(L, -2)) {
475 set_error(
476 "Can only persist non-empty userdata"
477 " if they have __persist and __depersist "
478 "metamethods");
479 return false;
480 }
481 lua_pop(L, 2);
482 }
483 } else {
484 if (lua_objlen(L, iIndex) != 0) {
485 set_error(
486 "Can only persist non-empty userdata if "
487 "they have a __depersist_size metafield");
488 return false;
489 }
490 }
491 lua_pop(L, 1);
492 } else {
493 if (lua_objlen(L, iIndex) != 0) {
494 set_error(
495 "Can only persist userdata without a metatable"
496 " if their size is zero");
497 return false;
498 }
499 lua_pushnil(L);
500 }
501 return true;
502 }
503
write_prototype(lua_Debug * pProtoInfo,int iInstanceIndex)504 void write_prototype(lua_Debug* pProtoInfo, int iInstanceIndex) {
505 // Sanity checks
506 if (pProtoInfo->source[0] != '@') {
507 // @ denotes that the source was a file
508 // (http://www.lua.org/manual/5.1/manual.html#lua_Debug)
509 set_error("Can only persist Lua functions defined in source files");
510 return;
511 }
512 if (std::strcmp(pProtoInfo->what, "Lua") != 0) {
513 // what == "C" should have been caught by write_object_raw().
514 // what == "tail" should be impossible.
515 // Hence "Lua" and "main" should be the only values seen.
516 // NB: Chunks are not functions defined *in* source files, because
517 // chunks *are* source files.
518 set_error(lua_pushfstring(L, "Cannot persist entire Lua chunks (%s)",
519 pProtoInfo->source + 1));
520 lua_pop(L, 1);
521 return;
522 }
523
524 // Attempt cached lookup (prototypes are not publicly visible Lua
525 // objects, and hence cannot be cached in the normal way of self's
526 // environment).
527 lua_getmetatable(L, 1);
528 lua_pushfstring(L, "%s:%d", pProtoInfo->source + 1,
529 pProtoInfo->linedefined);
530 lua_pushvalue(L, -1);
531 lua_rawget(L, -3);
532 if (!lua_isnil(L, -1)) {
533 uint64_t iValue = (uint64_t)lua_tonumber(L, -1);
534 lua_pop(L, 3);
535 write_uint(iValue + PERSIST_TCOUNT - 1);
536 return;
537 }
538 lua_pop(L, 1);
539 lua_pushvalue(L, -1);
540 lua_pushnumber(L, (lua_Number)next_index++);
541 lua_rawset(L, -4);
542
543 uint8_t iType = PERSIST_TPROTOTYPE;
544 write_byte_stream(&iType, 1);
545
546 // Write upvalue names
547 write_uint(pProtoInfo->nups);
548 for (int i = 1; i <= pProtoInfo->nups; ++i) {
549 lua_pushstring(L, lua_getupvalue(L, iInstanceIndex, i));
550 write_stack_object(-1);
551 lua_pop(L, 2);
552 }
553
554 // Write the function's persist name
555 lua_rawgeti(L, -2, 1);
556 lua_replace(L, -3);
557 lua_rawget(L, -2);
558 if (lua_isnil(L, -1)) {
559 set_error(lua_pushfstring(
560 L,
561 "Lua functions must be given a unique "
562 "persistable name in order to be persisted (attempt to "
563 "persist"
564 " %s:%d)",
565 pProtoInfo->source + 1, pProtoInfo->linedefined));
566 lua_pop(L, 2);
567 return;
568 }
569 write_stack_object(-1);
570 lua_pop(L, 2);
571 }
572
write_byte_stream(const uint8_t * pBytes,size_t iCount)573 void write_byte_stream(const uint8_t* pBytes, size_t iCount) override {
574 if (had_error) {
575 // If an error occurred, then silently fail to write any
576 // data.
577 return;
578 }
579
580 data.append(reinterpret_cast<const char*>(pBytes), iCount);
581 }
582
set_error(const char * sError)583 void set_error(const char* sError) override {
584 // If multiple errors occur, only record the first.
585 if (had_error) {
586 return;
587 }
588 had_error = true;
589
590 // Use the written data buffer to store the error message
591 data.assign(sError);
592 }
593
set_error_object(int iStackObject,int self_index)594 void set_error_object(int iStackObject, int self_index) {
595 if (had_error) return;
596
597 lua_pushvalue(L, iStackObject);
598 lua_getmetatable(L, self_index);
599 lua_insert(L, -2);
600 lua_setfield(L, -2, "err");
601 lua_pop(L, 1);
602 }
603
get_error()604 const char* get_error() {
605 if (had_error)
606 return data.c_str();
607 else
608 return nullptr;
609 }
610
611 private:
612 lua_State* L;
613 uint64_t next_index;
614 std::string data;
615 size_t data_size;
616 bool had_error;
617 };
618
619 //! Basic implementation of depersistence interface
620 /*!
621 self - Instance of lua_persist_basic_reader allocated as a Lua userdata
622 self environment:
623 [-3] - self
624 [-2] - pre-populated prototype persistence code
625 `<name>` - `<code>`
626 [-1] - pre-populated prototype persistence filenames
627 `<name>` - `<filename>`
628 [ 0] - permanents table
629 `<index>` - object already depersisted
630 self metatable:
631 `__gc` - ~lua_persist_basic_reader (via l_crude_gc)
632 `<N>` - userdata to have second `__depersist` call
633 */
634 class lua_persist_basic_reader : public lua_persist_reader {
635 public:
lua_persist_basic_reader(lua_State * L)636 lua_persist_basic_reader(lua_State* L) : L(L), string_buffer() {}
637
638 ~lua_persist_basic_reader() override = default;
639
get_stack()640 lua_State* get_stack() override { return L; }
641
set_error(const char * sError)642 void set_error(const char* sError) override {
643 had_error = true;
644 string_buffer.assign(sError);
645 }
646
init(const uint8_t * pData,size_t iLength)647 void init(const uint8_t* pData, size_t iLength) {
648 data = pData;
649 data_buffer_size = iLength;
650 next_index = 1;
651 had_error = false;
652 lua_createtable(L, 32, 0); // Environment
653 lua_pushvalue(L, 2);
654 lua_rawseti(L, -2, 0);
655 lua_pushvalue(L, luaT_upvalueindex(1));
656 lua_rawseti(L, -2, -1);
657 lua_pushvalue(L, luaT_upvalueindex(2));
658 lua_rawseti(L, -2, -2);
659 lua_pushvalue(L, 1);
660 lua_rawseti(L, -2, -3);
661 lua_setfenv(L, 1);
662 lua_createtable(L, 0, 1); // Metatable
663 luaT_pushcclosure(L, l_crude_gc<lua_persist_basic_reader>, 0);
664 lua_setfield(L, -2, "__gc");
665 lua_setmetatable(L, 1);
666 }
667
read_stack_object()668 bool read_stack_object() override {
669 uint64_t iIndex;
670 if (!read_uint(iIndex)) {
671 set_error("Expected stack object");
672 return false;
673 }
674 if (lua_type(L, 1) != LUA_TTABLE) {
675 // Ensure that index #1 is self environment
676 lua_getfenv(L, 1);
677 lua_replace(L, 1);
678 }
679 if (iIndex >= PERSIST_TCOUNT) {
680 iIndex += 1 - PERSIST_TCOUNT;
681 if (iIndex < (uint64_t)INT_MAX)
682 lua_rawgeti(L, 1, (int)iIndex);
683 else {
684 lua_pushnumber(L, (lua_Number)iIndex);
685 lua_rawget(L, 1);
686 }
687 if (lua_isnil(L, -1)) {
688 set_error(
689 "Cycle while depersisting permanent object key or "
690 "userdata metatable");
691 return false;
692 }
693 } else {
694 uint8_t iType = (uint8_t)iIndex;
695 switch (iType) {
696 case LUA_TNIL:
697 lua_pushnil(L);
698 break;
699 case PERSIST_TPERMANENT: {
700 uint64_t iOldIndex = next_index;
701 ++next_index; // Temporary marker
702 lua_rawgeti(L, 1, 0); // Permanents table
703 if (!read_stack_object()) return false;
704 lua_gettable(L, -2);
705 lua_replace(L, -2);
706 // Replace marker with actual object
707 uint64_t iNewIndex = next_index;
708 next_index = iOldIndex;
709 save_stack_object();
710 next_index = iNewIndex;
711 break;
712 }
713 case LUA_TBOOLEAN:
714 lua_pushboolean(L, 0);
715 break;
716 case PERSIST_TTRUE:
717 lua_pushboolean(L, 1);
718 break;
719 case LUA_TSTRING: {
720 size_t iLength;
721 if (!read_uint(iLength)) return false;
722 if (!read_byte_stream(string_buffer, iLength)) return false;
723 lua_pushlstring(L, string_buffer.c_str(), string_buffer.length());
724 save_stack_object();
725 break;
726 }
727 case LUA_TTABLE:
728 lua_newtable(L);
729 save_stack_object();
730 if (!lua_checkstack(L, 8)) return false;
731 if (!read_table_contents()) return false;
732 break;
733 case PERSIST_TTABLE_WITH_META:
734 lua_newtable(L);
735 save_stack_object();
736 if (!lua_checkstack(L, 8)) return false;
737 if (!read_stack_object()) return false;
738 lua_setmetatable(L, -2);
739 if (!read_table_contents()) return false;
740 break;
741 case LUA_TNUMBER: {
742 double fValue;
743 if (!read_byte_stream(reinterpret_cast<uint8_t*>(&fValue),
744 sizeof(double)))
745 return false;
746 lua_pushnumber(L, fValue);
747 break;
748 }
749 case LUA_TFUNCTION: {
750 if (!lua_checkstack(L, 8)) return false;
751 uint64_t iOldIndex = next_index;
752 ++next_index; // Temporary marker
753 if (!read_stack_object()) return false;
754 lua_call(L, 0, 2);
755 // Replace marker with closure
756 uint64_t iNewIndex = next_index;
757 next_index = iOldIndex;
758 save_stack_object();
759 next_index = iNewIndex;
760 // Set upvalues
761 lua_insert(L, -2);
762 int iNups, i;
763 if (!read_uint(iNups)) return false;
764 size_t iIDSize;
765 if (!read_uint(iIDSize)) return false;
766 for (i = 0; i < iNups; ++i) {
767 if (!read_stack_object()) return false;
768 // For now, just skip over the upvalue IDs. In the
769 // future, the ID may be used to rejoin shared upvalues.
770 if (!read_byte_stream(nullptr, iIDSize)) return false;
771 }
772 lua_call(L, iNups, 0);
773 // Read environment
774 if (!read_stack_object()) return false;
775 lua_setfenv(L, -2);
776 break;
777 }
778 case PERSIST_TPROTOTYPE: {
779 if (!lua_checkstack(L, 8)) return false;
780
781 uint64_t iOldIndex = next_index;
782 ++next_index; // Temporary marker
783 int iNups;
784 if (!read_uint(iNups)) return false;
785 if (iNups == 0)
786 lua_pushliteral(L, "return function() end,");
787 else {
788 lua_pushliteral(L, "local ");
789 lua_checkstack(L, (iNups + 1) * 2);
790 for (int i = 0; i < iNups; ++i) {
791 if (i != 0) lua_pushliteral(L, ",");
792 if (!read_stack_object()) return false;
793 if (lua_type(L, -1) != LUA_TSTRING) {
794 set_error("Upvalue name not a string");
795 return false;
796 }
797 }
798 lua_concat(L, iNups * 2 - 1);
799 lua_pushliteral(L, ";return function(...)");
800 lua_pushvalue(L, -2);
801 lua_pushliteral(L, "=...end,");
802 lua_concat(L, 5);
803 }
804 // Fetch name and then lookup filename and code
805 if (!read_stack_object()) return false;
806 lua_pushliteral(L, "@");
807
808 lua_rawgeti(L, 1, -1);
809 lua_pushvalue(L, -3);
810 lua_gettable(L, -2);
811 lua_replace(L, -2);
812
813 if (lua_isnil(L, -1)) {
814 set_error(lua_pushfstring(L,
815 "Unable to depersist prototype"
816 " \'%s\'",
817 lua_tostring(L, -3)));
818 return false;
819 }
820 lua_concat(L, 2); // Prepend the @ to the filename
821 lua_rawgeti(L, 1, -2);
822 lua_pushvalue(L, -3);
823
824 lua_gettable(L, -2);
825 lua_replace(L, -2);
826 lua_remove(L, -3);
827 // Construct the closure factory
828 load_multi_buffer ls;
829 ls.s[0] = lua_tolstring(L, -3, &ls.i[0]);
830 ls.s[1] = lua_tolstring(L, -1, &ls.i[1]);
831 if (luaT_load(L, load_multi_buffer::load_fn, &ls, lua_tostring(L, -2),
832 "bt") != 0) {
833 // Should never happen
834 lua_error(L);
835 return false;
836 }
837 lua_replace(L, -4);
838 lua_pop(L, 2);
839 // Replace marker with closure factory
840 uint64_t iNewIndex = next_index;
841 next_index = iOldIndex;
842 save_stack_object();
843 next_index = iNewIndex;
844 break;
845 }
846 case LUA_TUSERDATA: {
847 bool bHasSetMetatable = false;
848 uint64_t iOldIndex = next_index;
849 ++next_index; // Temporary marker
850 // Read metatable
851 if (!read_stack_object()) return false;
852 lua_getfield(L, -1, "__depersist_size");
853 if (!lua_isnumber(L, -1)) {
854 set_error("Userdata missing __depersist_size metafield");
855 return false;
856 }
857 lua_newuserdata(L, (size_t)lua_tonumber(L, -1));
858 lua_replace(L, -2);
859 // Replace marker with userdata
860 uint64_t iNewIndex = next_index;
861 next_index = iOldIndex;
862 save_stack_object();
863 next_index = iNewIndex;
864 // Perform immediate initialisation
865 lua_getfield(L, -2, "__pre_depersist");
866 if (lua_isnil(L, -1))
867 lua_pop(L, 1);
868 else {
869 // Set metatable now, as pre-depersister may expect it
870 // NB: Setting metatable if there isn't a
871 // pre-depersister is not a good idea, as if there is an
872 // error while the environment table is being
873 // de-persisted, then the __gc handler of the userdata
874 // will eventually be called with the userdata's
875 // contents still being uninitialised.
876 lua_pushvalue(L, -3);
877 lua_setmetatable(L, -3);
878 bHasSetMetatable = true;
879 lua_pushvalue(L, -2);
880 lua_call(L, 1, 0);
881 }
882 // Read environment
883 if (!read_stack_object()) return false;
884 lua_setfenv(L, -2);
885 // Set metatable and read the raw data
886 if (!bHasSetMetatable) {
887 lua_pushvalue(L, -2);
888 lua_setmetatable(L, -2);
889 }
890 lua_getfield(L, -2, "__depersist");
891 if (lua_isnil(L, -1))
892 lua_pop(L, 1);
893 else {
894 lua_pushvalue(L, -2);
895 lua_rawgeti(L, 1, -3);
896 lua_call(L, 2, 1);
897 if (lua_toboolean(L, -1) != 0) {
898 lua_pop(L, 1);
899 lua_rawgeti(L, 1, -3);
900 lua_getmetatable(L, -1);
901 lua_replace(L, -2);
902 lua_pushvalue(L, -2);
903 lua_rawseti(L, -2, (int)lua_objlen(L, -2) + 1);
904 }
905 lua_pop(L, 1);
906 }
907 lua_replace(L, -2);
908 uint64_t iSyncMarker;
909 if (!read_uint(iSyncMarker)) return false;
910 if (iSyncMarker != 0x42) {
911 set_error("sync fail");
912 return false;
913 }
914 break;
915 }
916 case PERSIST_TINTEGER: {
917 uint16_t iValue;
918 if (!read_uint(iValue)) return false;
919 lua_pushinteger(L, iValue);
920 break;
921 }
922 default:
923 lua_pushliteral(L, "Unable to depersist values of type \'");
924 if (iType <= LUA_TTHREAD)
925 lua_pushstring(L, lua_typename(L, iType));
926 else {
927 switch (iType) {
928 case PERSIST_TPERMANENT:
929 lua_pushliteral(L, "permanent");
930 break;
931 case PERSIST_TTRUE:
932 lua_pushliteral(L, "boolean-true");
933 break;
934 case PERSIST_TTABLE_WITH_META:
935 lua_pushliteral(L, "table-with-metatable");
936 break;
937 case PERSIST_TINTEGER:
938 lua_pushliteral(L, "integer");
939 break;
940 case PERSIST_TPROTOTYPE:
941 lua_pushliteral(L, "prototype");
942 break;
943 case PERSIST_TRESERVED1:
944 lua_pushliteral(L, "reserved1");
945 break;
946 case PERSIST_TRESERVED2:
947 lua_pushliteral(L, "reserved2");
948 break;
949 }
950 }
951 lua_pushliteral(L, "\'");
952 lua_concat(L, 3);
953 set_error(lua_tostring(L, -1));
954 lua_pop(L, 1);
955 return false;
956 }
957 }
958 return true;
959 }
960
save_stack_object()961 void save_stack_object() {
962 if (next_index < (uint64_t)INT_MAX) {
963 lua_pushvalue(L, -1);
964 lua_rawseti(L, 1, (int)next_index);
965 } else {
966 lua_pushnumber(L, (lua_Number)next_index);
967 lua_pushvalue(L, -2);
968 lua_rawset(L, 1);
969 }
970 ++next_index;
971 }
972
read_table_contents()973 bool read_table_contents() {
974 while (true) {
975 if (!read_stack_object()) return false;
976 if (lua_type(L, -1) == LUA_TNIL) {
977 lua_pop(L, 1);
978 return true;
979 }
980 if (!read_stack_object()) return false;
981 // NB: lua_rawset used rather than lua_settable to avoid invoking
982 // any metamethods, as they may not have been designed to be called
983 // during depersistence.
984 lua_rawset(L, -3);
985 }
986 }
987
finish()988 bool finish() {
989 // Ensure that all data has been read
990 if (data_buffer_size != 0) {
991 set_error(lua_pushfstring(L, "%d bytes of data remain unpersisted",
992 (int)data_buffer_size));
993 return false;
994 }
995
996 // Ensure that index #1 is self environment
997 if (lua_type(L, 1) != LUA_TTABLE) {
998 lua_getfenv(L, 1);
999 lua_replace(L, 1);
1000 }
1001 // Ensure that index #1 is self metatable
1002 lua_rawgeti(L, 1, -3);
1003 lua_getmetatable(L, -1);
1004 lua_replace(L, 1);
1005 lua_pop(L, 1);
1006
1007 // Call all the __depersist functions which need a 2nd call
1008 int iNumCalls = (int)lua_objlen(L, 1);
1009 for (int i = 1; i <= iNumCalls; ++i) {
1010 lua_rawgeti(L, 1, i);
1011 luaL_getmetafield(L, -1, "__depersist");
1012 lua_insert(L, -2);
1013 lua_call(L, 1, 0);
1014 }
1015 return true;
1016 }
1017
read_byte_stream(uint8_t * pBytes,size_t iCount)1018 bool read_byte_stream(uint8_t* pBytes, size_t iCount) override {
1019 if (iCount > data_buffer_size) {
1020 set_error(lua_pushfstring(
1021 L, "End of input reached while attempting to read %d byte%s",
1022 (int)iCount, iCount == 1 ? "" : "s"));
1023 lua_pop(L, 1);
1024 return false;
1025 }
1026
1027 if (pBytes != nullptr) std::memcpy(pBytes, data, iCount);
1028
1029 data += iCount;
1030 data_buffer_size -= iCount;
1031 return true;
1032 }
1033
read_byte_stream(std::string & bytes,size_t iCount)1034 bool read_byte_stream(std::string& bytes, size_t iCount) {
1035 if (iCount > data_buffer_size) {
1036 set_error(lua_pushfstring(
1037 L, "End of input reached while attempting to read %d byte%s",
1038 (int)iCount, iCount == 1 ? "" : "s"));
1039 lua_pop(L, 1);
1040 return false;
1041 }
1042
1043 bytes.assign(reinterpret_cast<const char*>(data), iCount);
1044
1045 data += iCount;
1046 data_buffer_size -= iCount;
1047 return true;
1048 }
1049
get_pointer()1050 const uint8_t* get_pointer() { return data; }
get_object_count()1051 uint64_t get_object_count() { return next_index; }
1052
get_error()1053 const char* get_error() {
1054 if (had_error)
1055 return string_buffer.c_str();
1056 else
1057 return nullptr;
1058 }
1059
1060 private:
1061 lua_State* L;
1062 uint64_t next_index;
1063 const uint8_t* data;
1064 size_t data_buffer_size;
1065 std::string string_buffer;
1066 bool had_error;
1067 };
1068
1069 namespace {
1070
l_dump_toplevel(lua_State * L)1071 int l_dump_toplevel(lua_State* L) {
1072 luaL_checktype(L, 2, LUA_TTABLE);
1073 lua_settop(L, 2);
1074 lua_pushvalue(L, 1);
1075 lua_persist_basic_writer* pWriter =
1076 new (lua_newuserdata(L, sizeof(lua_persist_basic_writer)))
1077 lua_persist_basic_writer(L);
1078 lua_replace(L, 1);
1079 pWriter->init();
1080 pWriter->write_stack_object(3);
1081 return pWriter->finish();
1082 }
1083
l_load_toplevel(lua_State * L)1084 int l_load_toplevel(lua_State* L) {
1085 size_t iDataLength;
1086 const uint8_t* pData = luaT_checkfile(L, 1, &iDataLength);
1087 luaL_checktype(L, 2, LUA_TTABLE);
1088 lua_settop(L, 2);
1089 lua_pushvalue(L, 1);
1090 lua_persist_basic_reader* pReader =
1091 new (lua_newuserdata(L, sizeof(lua_persist_basic_reader)))
1092 lua_persist_basic_reader(L);
1093 lua_replace(L, 1);
1094 pReader->init(pData, iDataLength);
1095 if (!pReader->read_stack_object() || !pReader->finish()) {
1096 int iNumObjects = (int)pReader->get_object_count();
1097 int iNumBytes = (int)(pReader->get_pointer() - pData);
1098 lua_pushnil(L);
1099 lua_pushfstring(L, "%s after %d objects (%d bytes)",
1100 pReader->get_error() ? pReader->get_error()
1101 : "Error while depersisting",
1102 iNumObjects, iNumBytes);
1103 return 2;
1104 } else {
1105 return 1;
1106 }
1107 }
1108
calculate_line_number(const char * sStart,const char * sPosition)1109 int calculate_line_number(const char* sStart, const char* sPosition) {
1110 int iLine = 1;
1111 for (; sStart != sPosition; ++sStart) {
1112 switch (sStart[0]) {
1113 case '\0':
1114 return -1; // error return value
1115 case '\n':
1116 ++iLine;
1117 if (sStart[1] == '\r') ++sStart;
1118 break;
1119 case '\r':
1120 ++iLine;
1121 if (sStart[1] == '\n') ++sStart;
1122 break;
1123 }
1124 }
1125 return iLine;
1126 }
1127
find_function_end(lua_State * L,const char * sStart)1128 const char* find_function_end(lua_State* L, const char* sStart) {
1129 const char* sEnd = sStart;
1130 while (sEnd) {
1131 sEnd = std::strstr(sEnd, "end");
1132 if (sEnd) {
1133 sEnd += 3;
1134 load_multi_buffer ls;
1135 ls.s[0] = "return function";
1136 ls.i[0] = sizeof("return function") - 1;
1137 ls.s[1] = sStart;
1138 ls.i[1] = sEnd - sStart;
1139 if (luaT_load(L, load_multi_buffer::load_fn, &ls, "", "bt") == 0) {
1140 lua_pop(L, 1);
1141 return sEnd;
1142 }
1143 lua_pop(L, 1);
1144 }
1145 }
1146 return nullptr;
1147 }
1148
l_persist_dofile(lua_State * L)1149 int l_persist_dofile(lua_State* L) {
1150 const char* sFilename = luaL_checkstring(L, 1);
1151 lua_settop(L, 1);
1152
1153 // Read entire file into memory
1154 std::FILE* fFile = std::fopen(sFilename, "r");
1155 if (fFile == nullptr) {
1156 const char* sError = std::strerror(errno);
1157 return luaL_error(L, "cannot open %s: %s", sFilename, sError);
1158 }
1159 size_t iBufferSize = lua_objlen(L, luaT_upvalueindex(1));
1160 size_t iBufferUsed = 0;
1161 while (!std::feof(fFile)) {
1162 iBufferUsed += std::fread(
1163 reinterpret_cast<char*>(lua_touserdata(L, luaT_upvalueindex(1))) +
1164 iBufferUsed,
1165 1, iBufferSize - iBufferUsed, fFile);
1166 if (iBufferUsed == iBufferSize) {
1167 iBufferSize *= 2;
1168 std::memcpy(lua_newuserdata(L, iBufferSize),
1169 lua_touserdata(L, luaT_upvalueindex(1)), iBufferUsed);
1170 lua_replace(L, luaT_upvalueindex(1));
1171 } else
1172 break;
1173 }
1174 int iStatus = std::ferror(fFile);
1175 std::fclose(fFile);
1176 if (iStatus) {
1177 const char* sError = std::strerror(errno);
1178 return luaL_error(L, "cannot read %s: %s", sFilename, sError);
1179 }
1180
1181 // Check file
1182 char* sFile =
1183 reinterpret_cast<char*>(lua_touserdata(L, luaT_upvalueindex(1)));
1184 sFile[iBufferUsed] = 0;
1185 if (sFile[0] == '#') {
1186 do {
1187 ++sFile;
1188 --iBufferUsed;
1189 } while (sFile[0] != 0 && sFile[0] != '\r' && sFile[0] != '\n');
1190 }
1191 if (sFile[0] == LUA_SIGNATURE[0]) {
1192 return luaL_error(L, "cannot load %s: compiled files not permitted",
1193 sFilename);
1194 }
1195
1196 // Load and do file
1197 lua_pushliteral(L, "@");
1198 lua_pushvalue(L, 1);
1199 lua_concat(L, 2);
1200 if (luaL_loadbuffer(L, sFile, iBufferUsed, lua_tostring(L, -1)) != 0)
1201 return lua_error(L);
1202 lua_remove(L, -2);
1203 int iBufferCopyIndex = lua_gettop(L);
1204 std::memcpy(lua_newuserdata(L, iBufferUsed + 1), sFile, iBufferUsed + 1);
1205 lua_insert(L, -2);
1206 lua_call(L, 0, LUA_MULTRET);
1207 sFile = reinterpret_cast<char*>(lua_touserdata(L, luaT_upvalueindex(1)));
1208 std::memcpy(sFile, lua_touserdata(L, iBufferCopyIndex), iBufferUsed + 1);
1209 lua_remove(L, iBufferCopyIndex);
1210
1211 // Extract persistable functions
1212 const char* sPosition = sFile;
1213 while (true) {
1214 sPosition = std::strstr(sPosition, "--[[persistable:");
1215 if (!sPosition) break;
1216 sPosition += 16;
1217 const char* sNameEnd = std::strstr(sPosition, "]]");
1218 if (sNameEnd) {
1219 int iLineNumber = calculate_line_number(sFile, sNameEnd);
1220 const char* sFunctionArgs = std::strchr(sNameEnd + 2, '(');
1221 const char* sFunctionEnd = find_function_end(L, sFunctionArgs);
1222 if ((sNameEnd - sPosition) == 1 && *sPosition == ':') {
1223 // --[[persistable::]] means take the existing name of the
1224 // function
1225 sPosition = std::strstr(sNameEnd, "function") + 8;
1226 sPosition += std::strspn(sPosition, " \t");
1227 sNameEnd = sFunctionArgs;
1228 while (sNameEnd[-1] == ' ') --sNameEnd;
1229 }
1230 if (iLineNumber != -1 && sFunctionArgs && sFunctionEnd) {
1231 // Save <filename>:<line> => <name>
1232 lua_pushfstring(L, "%s:%d", sFilename, iLineNumber);
1233 lua_pushvalue(L, -1);
1234 lua_gettable(L, luaT_upvalueindex(2));
1235 if (lua_isnil(L, -1)) {
1236 lua_pop(L, 1);
1237 lua_pushlstring(L, sPosition, sNameEnd - sPosition);
1238 lua_settable(L, luaT_upvalueindex(2));
1239 } else {
1240 return luaL_error(L,
1241 "Multiple persistable functions defined"
1242 "on the same line (%s:%d)",
1243 sFilename, iLineNumber);
1244 }
1245
1246 // Save <name> => <filename>
1247 lua_pushlstring(L, sPosition, sNameEnd - sPosition);
1248 lua_pushvalue(L, -1);
1249 lua_gettable(L, luaT_upvalueindex(3));
1250 if (lua_isnil(L, -1)) {
1251 lua_pop(L, 1);
1252 lua_pushvalue(L, 1);
1253 lua_settable(L, luaT_upvalueindex(3));
1254 } else {
1255 return luaL_error(L,
1256 "Persistable function name \'%s\' is"
1257 " not unique (defined in both %s and %s)",
1258 lua_tostring(L, -2), lua_tostring(L, -1),
1259 sFilename);
1260 }
1261
1262 // Save <name> => <code>
1263 lua_pushlstring(L, sPosition, sNameEnd - sPosition);
1264 lua_pushliteral(L, "\n");
1265 lua_getfield(L, -1, "rep");
1266 lua_insert(L, -2);
1267 lua_pushinteger(L, iLineNumber - 1);
1268 lua_call(L, 2, 1);
1269 lua_pushliteral(L, "function");
1270 lua_pushlstring(L, sFunctionArgs, sFunctionEnd - sFunctionArgs);
1271 lua_concat(L, 3);
1272 lua_settable(L, luaT_upvalueindex(4));
1273 }
1274 }
1275 }
1276
1277 // Finish
1278 return lua_gettop(L) - 1;
1279 }
1280
l_errcatch(lua_State * L)1281 int l_errcatch(lua_State* L) {
1282 // Dummy function for debugging - place a breakpoint on the following
1283 // return statement to inspect the full C call stack when a Lua error
1284 // occurs (assuming that l_errcatch is used as the error catch handler).
1285 return 1;
1286 }
1287
1288 // Due to the various required upvalues, functions are registered manually, but
1289 // we still need a dummy to pass to luaL_register.
1290 constexpr std::array<luaL_Reg, 2> persist_lib{
1291 {{"errcatch", l_errcatch}, {nullptr, nullptr}}};
1292
1293 } // namespace
1294
luaopen_persist(lua_State * L)1295 int luaopen_persist(lua_State* L) {
1296 luaT_register(L, "persist", persist_lib);
1297 lua_newuserdata(L, 512); // buffer for dofile
1298 lua_newtable(L);
1299 lua_newtable(L);
1300 lua_newtable(L);
1301 lua_pushvalue(L, -3);
1302 luaT_pushcclosure(L, l_dump_toplevel, 1);
1303 lua_setfield(L, -6, "dump");
1304 lua_pushvalue(L, -2);
1305 lua_pushvalue(L, -2);
1306 luaT_pushcclosure(L, l_load_toplevel, 2);
1307 lua_setfield(L, -6, "load");
1308 luaT_pushcclosure(L, l_persist_dofile, 4);
1309 lua_setfield(L, -2, "dofile");
1310 return 1;
1311 }
1312