1 /*
2  * This file was written by Bill Cox.  It is hereby placed into the public domain.
3  */
4 
5 /*--------------------------------------------------------------------------------------------------
6   Functions supporting database persistence.  It would really complicate things if this module
7   depended in any way on itself, so it's coded bare-bones in C.
8 --------------------------------------------------------------------------------------------------*/
9 #include <stdlib.h>
10 #include "ddutil.h"
11 #include "utpersist.h"
12 
13 /* This buffer reduces calls to fwrite */
14 /* #define UT_CHANGE_BUFFER_LENGTH 16384 */
15 /* temp - for debugging */
16 #define UT_CHANGE_BUFFER_LENGTH 16
17 static uint8 *utCommandBuffer;
18 static uint32 utBufferPosition, utBufferSize;
19 static FILE *utRecentChangesFile;
20 /* The utBufferStartPosition is relative to the first change in recentChanges if we are
21  * persistent, and should be equal to 0 otherwise. */
22 static uint32 utLastTransactionPosition, utBufferStartPosition;
23 /* utRedoTransaction is the next transaction that would be redone if we did one.  It's usually
24  * NULL, unless we undo one. */
25 static utTransaction utRedoTransaction;
26 /* These are used while undoing a transaction, since we have to compute the command sizes */
27 static uint32 *utTransactionCommands;
28 static uint32 utTransactionUsedCommands, utTransactionAllocatedCommands;
29 /* This says whether or not we are in the middle of a transaction */
30 static bool utTransactionInProgress;
31 /* These are used while registering classes and fields so we don't have to keep passing it. */
32 static utModule utCurrentModule;
33 static utClass utCurrentClass;
34 /* The size of the database file */
35 static uint64 utDatabaseSize;
36 /* When true, keep a copy of the previous database file */
37 static bool utKeepBackup;
38 
39 /* The command buffer is only used while applying commands.  It holds one transaction at a time. */
40 static uint32 utAllocatedCommands;
41 static uint32 utUsedCommands;
42 static uint8 *utCommands;
43 
44 /* This checksum is to verify a transaction has not been corrupted */
45 static uint32 utChecksum;
46 
47 /* Globals that keep track of the allocated objects */
48 struct utModuleStruct *utModules;
49 uint8 utAllocatedModules, utUsedModules;
50 struct utClassStruct *utClasses;
51 uint16 utAllocatedClasses, utUsedClasses;
52 struct utFieldStruct *utFields;
53 uint16 utAllocatedFields, utUsedFields;
54 struct utTransactionStruct *utTransactions;
55 uint32 utUsedTransactions, utAllocatedTransactions;
56 struct utFieldStruct *utFields;
57 uint16 utAllocatedFields, utUsedFields;
58 struct utEnumStruct *utEnums;
59 uint16 utAllocatedEnums, utUsedEnums;
60 struct utEntryStruct *utEntries;
61 uint16 utAllocatedEntries, utUsedEntries;
62 struct utUnionStruct *utUnions;
63 uint16 utAllocatedUnions, utUsedUnions;
64 struct utUnioncaseStruct *utUnioncases;
65 uint16 utAllocatedUnioncases, utUsedUnioncases;
66 
67 /* This keeps us from writing the the unopened changes file before starting persistence */
68 bool utPersistenceInitialized;
69 char *utDatabaseDirectory;
70 bool utUseTextDatabaseFormat;
71 
72 /*--------------------------------------------------------------------------------------------------
73   Open the recentChanges file with the mode.
74 --------------------------------------------------------------------------------------------------*/
openRecentChanges(char * mode)75 static void openRecentChanges(
76     char *mode)
77 {
78     char *fileName = utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP);
79 
80     utRecentChangesFile = fopen(fileName, mode);
81     if(utRecentChangesFile == NULL) {
82         utExit("Could not open file %s", fileName);
83     }
84 }
85 
86 /*--------------------------------------------------------------------------------------------------
87   Register a module.
88 --------------------------------------------------------------------------------------------------*/
utRegisterModule(char * prefix,bool persistent,uint32 hashValue,uint16 numClasses,uint16 numFields,uint16 numEnums,uint16 globalSize,void * globalData,void (* start)(void),void (* stop)(void))89 uint8 utRegisterModule(
90     char *prefix,
91     bool persistent,
92     uint32 hashValue,
93     uint16 numClasses,
94     uint16 numFields,
95     uint16 numEnums,
96     uint16 globalSize,
97     void *globalData,
98     void (*start)(void),
99     void (*stop)(void))
100 {
101     utModule module = utFindModule(prefix);
102 
103     if(module != NULL) {
104         /* Already registered, so ignore it */
105         utCurrentModule = NULL;
106         utModuleSetInitialized(module,  true);
107         return module - utModules;
108     }
109     if(utUsedModules == utAllocatedModules) {
110         utAllocatedModules += utAllocatedModules >> 1;
111         utResizeArray(utModules, utAllocatedModules);
112     }
113     module = utModules + utUsedModules;
114     utModuleSetPrefix(module,  calloc(strlen(prefix) + 1, sizeof(char)));
115     strcpy(utModuleGetPrefix(module), prefix);
116     utModuleSetPersistent(module, persistent);
117     utModuleSetHashValue(module, hashValue);
118     utModuleSetGlobalSize(module,  globalSize);
119     utModuleSetGlobalData(module, globalData);
120     utModuleSetStart(module, start);
121     utModuleSetStop(module, stop);
122     utModuleSetFirstClassIndex(module, utUsedClasses);
123     utModuleSetNumClasses(module, numClasses);
124     utModuleSetFirstFieldIndex(module, utUsedFields);
125     utModuleSetNumFields(module, numFields);
126     utModuleSetFirstEnumIndex(module, utUsedEnums);
127     utModuleSetNumEnums(module, numEnums);
128     utModuleSetInitialized(module,  true);
129     utCurrentModule = module;
130     return utUsedModules++;
131 }
132 
133 /*--------------------------------------------------------------------------------------------------
134   Set a module to uninitialized.
135 --------------------------------------------------------------------------------------------------*/
utUnregisterModule(uint8 moduleID)136 void utUnregisterModule(
137     uint8 moduleID)
138 {
139     utModule module = utModules + moduleID;
140 
141     utModuleSetInitialized(module,  false);
142 }
143 
144 /*--------------------------------------------------------------------------------------------------
145   Register a class.
146 --------------------------------------------------------------------------------------------------*/
utRegisterClass(char * name,uint16 numFields,void * numUsedPtr,void * numAllocatedPtr,void * firstFreePtr,uint16 nextFreeFieldIndex,uint8 referenceSize,uint64 (* constructor)(void),void (* destructor)(uint64 objectIndex))147 void utRegisterClass(
148     char *name,
149     uint16 numFields,
150     void *numUsedPtr,
151     void *numAllocatedPtr,
152     void *firstFreePtr,
153     uint16 nextFreeFieldIndex,
154     uint8 referenceSize,
155     uint64 (*constructor)(void),
156     void (*destructor)(uint64 objectIndex))
157 {
158     utClass theClass;
159 
160     if(utCurrentModule == NULL) {
161         return;
162     }
163     if(utUsedClasses == utAllocatedClasses) {
164         utAllocatedClasses += utAllocatedClasses >> 1;
165         utResizeArray(utClasses, utAllocatedClasses);
166     }
167     theClass = utClasses + utUsedClasses++;
168     utClassSetName(theClass, calloc(strlen(name) + 1, sizeof(char)));
169     strcpy(utClassGetName(theClass), name);
170     utClassSetFirstFieldIndex(theClass, utUsedFields);
171     utClassSetNumFields(theClass, numFields);
172     utClassSetNumUsedPtr(theClass, numUsedPtr);
173     utClassSetNumAllocatedPtr(theClass, numAllocatedPtr);
174     utClassSetFirstFreePtr(theClass, firstFreePtr);
175     utClassSetNextFreeFieldIndex(theClass, nextFreeFieldIndex);
176     utClassSetReferenceSize(theClass, referenceSize);
177     utClassSetConstructor(theClass, constructor);
178     utClassSetDestructor(theClass, destructor);
179     utClassSetModuleIndex(theClass, utCurrentModule - utModules);
180     utClassSetNumHiddenFields(theClass, 0);
181     utClassSetBaseClassIndex(theClass, UINT16_MAX);
182     utClassSetBaseModuleIndex(theClass, UINT8_MAX);
183     utCurrentClass = theClass;
184 }
185 
186 /*--------------------------------------------------------------------------------------------------
187   This function tells the manager the base class of a class extension.
188 --------------------------------------------------------------------------------------------------*/
utRegisterBaseClass(char * baseModulePrefix,uint16 baseClassIndex)189 void utRegisterBaseClass(
190     char *baseModulePrefix,
191     uint16 baseClassIndex)
192 {
193     utModule baseModule = utFindModule(baseModulePrefix);
194 
195     if(baseModule == utModuleNull) {
196         utExit("Base module %s not registered", baseModulePrefix);
197     }
198     utClassSetBaseModuleIndex(utCurrentClass, baseModule - utModules);
199     utClassSetBaseClassIndex(utCurrentClass, utModuleGetFirstClassIndex(baseModule) + baseClassIndex);
200 }
201 
202 /*--------------------------------------------------------------------------------------------------
203   Register a field.
204 --------------------------------------------------------------------------------------------------*/
utRegisterField(char * name,void * arrayPtr,uint32 size,utFieldType type,char * destName)205 void utRegisterField(
206     char *name,
207     void *arrayPtr,
208     uint32 size,
209     utFieldType type,
210     char *destName)
211 {
212     utField field;
213 
214     if(utCurrentModule == NULL) {
215         return;
216     }
217     if(utUsedFields == utAllocatedFields) {
218         utAllocatedFields += utAllocatedFields >> 1;
219         utResizeArray(utFields, utAllocatedFields);
220     }
221     field = utFields + utUsedFields++;
222     utFieldSetName(field, calloc(strlen(name) + 1, sizeof(char)));
223     strcpy(utFieldGetName(field), name);
224     utFieldSetArrayPtr(field, arrayPtr);
225     utFieldSetSize(field, size);
226     utFieldSetType(field, type);
227     utFieldSetClassIndex(field, utCurrentClass - utClasses);
228     if(destName == NULL) {
229         utFieldSetDestName(field, NULL);
230     } else {
231         utFieldSetDestName(field, calloc(strlen(destName) + 1, sizeof(char)));
232         strcpy(utFieldGetDestName(field), destName);
233     }
234     utFieldSetHidden(field, false);
235     utFieldSetArray(field, false);
236     utFieldSetNumUsedPtr(field, NULL);
237     utFieldSetNumAllocatedPtr(field, NULL);
238     utFieldSetGetValues(field, NULL);
239     utFieldSetAllocValues(field, NULL);
240     utFieldSetUnionIndex(field, 0);
241 }
242 
243 /*--------------------------------------------------------------------------------------------------
244   Set the previously registered field as hidden.
245 --------------------------------------------------------------------------------------------------*/
utSetFieldHidden(void)246 void utSetFieldHidden(void)
247 {
248     utField field = utFields + utUsedFields - 1;
249 
250     if(utCurrentModule == NULL) {
251         return;
252     }
253     utFieldSetHidden(field, true);
254         utClassSetNumHiddenFields(utCurrentClass, utClassGetNumHiddenFields(utCurrentClass) + 1);
255 }
256 
257 /*--------------------------------------------------------------------------------------------------
258   Set the field as an array.
259 --------------------------------------------------------------------------------------------------*/
utRegisterArray(uint32 * numUsedPtr,uint32 * numAllocatedPtr,void * (* getValues)(uint64 objectNumber,uint32 * numValues),void * (* allocValues)(uint64 objectNumber,uint32 numValues),void (* compactArray)(void))260 void utRegisterArray(
261     uint32 *numUsedPtr,
262     uint32 *numAllocatedPtr,
263     void *(*getValues)(uint64 objectNumber, uint32 *numValues),
264     void *(*allocValues)(uint64 objectNumber, uint32 numValues),
265     void (*compactArray)(void))
266 {
267     utField arrayField = utFields + utUsedFields - 1;
268 
269     if(utCurrentModule == NULL) {
270         return;
271     }
272     utFieldSetArray(arrayField, true);
273     utFieldSetNumUsedPtr(arrayField, numUsedPtr);
274     utFieldSetNumAllocatedPtr(arrayField, numAllocatedPtr);
275     utFieldSetGetValues(arrayField, getValues);
276     utFieldSetAllocValues(arrayField, allocValues);
277     utFieldSetCompactArray(arrayField, compactArray);
278 }
279 
280 /*--------------------------------------------------------------------------------------------------
281   Set the field as a fixed size array.
282 --------------------------------------------------------------------------------------------------*/
utRegisterFixedArray(uint32 length,void * (* getValues)(uint64 objectNumber,uint32 * numValues))283 void utRegisterFixedArray(
284     uint32 length,
285     void *(*getValues)(uint64 objectNumber, uint32 *numValues))
286 {
287     utField arrayField = utFields + utUsedFields - 1;
288 
289     if(utCurrentModule == NULL) {
290         return;
291     }
292     utFieldSetArray(arrayField, true);
293     utFieldSetFixedSize(arrayField, true);
294     utFieldSetLength(arrayField, length);
295     utFieldSetGetValues(arrayField, getValues);
296 }
297 
298 /*--------------------------------------------------------------------------------------------------
299   Flush recent changes to disk.
300 --------------------------------------------------------------------------------------------------*/
flushRecentChanges(void)301 static void flushRecentChanges(void)
302 {
303     if(utPersistenceInitialized) {
304         if(utBufferPosition > 0) {
305             fwrite((void *)utCommandBuffer, sizeof(uint8), utBufferPosition, utRecentChangesFile);
306         }
307         utBufferStartPosition += utBufferPosition;
308         utBufferPosition = 0;
309     } else if(utBufferPosition == utBufferSize) {
310         utBufferSize += utBufferSize >> 1;
311         utResizeArray(utCommandBuffer, utBufferSize);
312     }
313 }
314 
315 /*--------------------------------------------------------------------------------------------------
316   Write a byte to the recentChanges buffer.
317 --------------------------------------------------------------------------------------------------*/
writeUint8(uint8 value)318 static void writeUint8(
319     uint8 value)
320 {
321     if(utBufferPosition == utBufferSize) {
322         flushRecentChanges();
323     }
324     utAssert(utTransactionInProgress || utChecksum == 0);
325     if(utRedoTransaction != NULL) {
326         utBufferPosition = utTransactionGetPosition(utRedoTransaction) - utBufferStartPosition;
327         utLastTransactionPosition = utBufferStartPosition + utBufferPosition;
328         utUsedTransactions = utRedoTransaction - utTransactions;
329         utRedoTransaction = NULL;
330     }
331     utCommandBuffer[utBufferPosition++] = value;
332     utTransactionInProgress = true;
333     utChecksum = (utChecksum ^ value)*1103515245 + 12345;
334 }
335 
336 /*--------------------------------------------------------------------------------------------------
337   Write a uint16 to the recentChanges buffer.
338 --------------------------------------------------------------------------------------------------*/
writeUint16(uint16 value)339 static void writeUint16(
340     uint16 value)
341 {
342     uint8 *values = (uint8 *)(void *)&value;
343 
344     writeUint8(*values++);
345     writeUint8(*values);
346 }
347 
348 /*--------------------------------------------------------------------------------------------------
349   Write a uint32 to the recentChanges buffer.
350 --------------------------------------------------------------------------------------------------*/
writeUint32(uint32 value)351 static void writeUint32(
352     uint32 value)
353 {
354     uint8 *values = (uint8 *)(void *)&value;
355 
356     writeUint8(*values++);
357     writeUint8(*values++);
358     writeUint8(*values++);
359     writeUint8(*values);
360 }
361 
362 /*--------------------------------------------------------------------------------------------------
363   Write a uint64 to the recentChanges buffer.
364 --------------------------------------------------------------------------------------------------*/
writeUint64(uint64 value)365 static void writeUint64(
366     uint64 value)
367 {
368     uint8 *values = (uint8 *)(void *)&value;
369 
370     writeUint8(*values++);
371     writeUint8(*values++);
372     writeUint8(*values++);
373     writeUint8(*values++);
374     writeUint8(*values++);
375     writeUint8(*values++);
376     writeUint8(*values++);
377     writeUint8(*values);
378 }
379 
380 /*--------------------------------------------------------------------------------------------------
381   Write a number of bytes to the recentChanges buffer.
382 --------------------------------------------------------------------------------------------------*/
writeValues(uint8 * values,uint32 numBytes)383 static void writeValues(
384     uint8 *values,
385     uint32 numBytes)
386 {
387     while(numBytes-- != 0) {
388         writeUint8(*values++);
389     }
390 }
391 
392 /*--------------------------------------------------------------------------------------------------
393   Create a new transaction object to track a group of commands associated with a transaction.
394 --------------------------------------------------------------------------------------------------*/
utTransactionCreate(uint32 position,uint32 length)395 static utTransaction utTransactionCreate(
396     uint32 position,
397     uint32 length)
398 {
399     utTransaction transaction;
400 
401     if(length == 0) {
402         return utTransactionNull;
403     }
404     if(utPersistenceInitialized && utBufferStartPosition == 0) {
405         utAssert(utCommandBuffer[position] != 0);
406     }
407     if(utUsedTransactions == utAllocatedTransactions) {
408         utAllocatedTransactions += utAllocatedTransactions >> 1;
409         utResizeArray(utTransactions, utAllocatedTransactions);
410     }
411     transaction = utTransactions + utUsedTransactions++;
412         utTransactionSetPosition(transaction, position);
413         utTransactionSetLength(transaction, length);
414     utTransactionInProgress = false;
415     return transaction;
416 }
417 
418 /*--------------------------------------------------------------------------------------------------
419   Empty the command buffer.
420 --------------------------------------------------------------------------------------------------*/
resetCommandBuffer(void)421 static void resetCommandBuffer(void)
422 {
423     utBufferPosition = 0;
424     utLastTransactionPosition = 0;
425     utBufferStartPosition = 0;
426     utChecksum = 0;
427     utTransactionInProgress = false;
428     utUsedTransactions = 0;
429     utRedoTransaction = NULL;
430 }
431 
432 /*--------------------------------------------------------------------------------------------------
433   Resize all our arrays so that numAllocated == numUsed.  This is needed when saving an ASCII
434   database, since next time we load it, we don't know how much extra will be allocated for each
435   class.
436 --------------------------------------------------------------------------------------------------*/
shrinkMemoryToFit(void)437 static void shrinkMemoryToFit(void)
438 {
439     utModule module;
440     utField field;
441     utClass theClass;
442     uint64 numUsed;
443 
444     utForeachModule(module) {
445         if(utModuleInitialized(module)) {
446             utForeachModuleClass(module, theClass) {
447                 numUsed = utFindIntValue(utClassGetNumUsedPtr(theClass), utClassGetReferenceSize(theClass));
448                 numUsed = utMax(numUsed, 2);
449                 utSetInteger(utClassGetNumAllocatedPtr(theClass), numUsed, utClassGetReferenceSize(theClass));
450             } utEndModuleClass;
451             utForeachModuleField(module, field) {
452                 if(utFieldArray(field) && !utFieldFixedSize(field)) {
453                     (utFieldGetCompactArray(field))();
454                     numUsed = utMax(*(utFieldGetNumUsedPtr(field)), 2);
455                     *utFieldGetNumAllocatedPtr(field) = (uint32)numUsed;
456                     *(uint8 **)(utFieldGetArrayPtr(field)) = utRealloc(*(uint8 **)utFieldGetArrayPtr(field), numUsed,
457                         utFieldGetSize(field));
458                 } else {
459                     theClass = utClasses + utFieldGetClassIndex(field);
460                     numUsed = utFindIntValue(utClassGetNumAllocatedPtr(theClass), utClassGetReferenceSize(theClass));
461                     if(!utFieldFixedSize(field)) {
462                         *(uint8 **)(utFieldGetArrayPtr(field)) = utRealloc(*(uint8 **)utFieldGetArrayPtr(field), numUsed,
463                             utFieldGetSize(field));
464                     } else {
465                         *(uint8 **)(utFieldGetArrayPtr(field)) = utRealloc(*(uint8 **)utFieldGetArrayPtr(field),
466                             numUsed*utFieldGetLength(field), utFieldGetSize(field));
467                     }
468                }
469             } utEndModuleField;
470         }
471     } utEndModule;
472 }
473 
474 /*--------------------------------------------------------------------------------------------------
475   Compact the database, and truncate recentChanges.
476 --------------------------------------------------------------------------------------------------*/
utCompactDatabase(void)477 void utCompactDatabase(void)
478 {
479     char *fileName, *backupFileName;
480 
481     if(utKeepBackup) {
482         fileName = utSprintf("%s%cdatabase", utDatabaseDirectory, UTDIRSEP);
483         backupFileName = utSprintf("%s.old", fileName);
484         rename(fileName, backupFileName);
485     }
486     shrinkMemoryToFit();
487     if(utUseTextDatabaseFormat) {
488         utSaveTextDatabase(NULL);
489     } else {
490         utSaveBinaryDatabase(NULL);
491     }
492     utDatabaseSize = utFindFileSize(utSprintf("%s%cdatabase", utDatabaseDirectory, UTDIRSEP));
493     resetCommandBuffer();
494     fclose(utRecentChangesFile);
495     openRecentChanges("wb");
496 }
497 
498 /*--------------------------------------------------------------------------------------------------
499   Mark the transaction complete, and optionally fush to disk.
500 --------------------------------------------------------------------------------------------------*/
utTransactionComplete(bool flushToDisk)501 void utTransactionComplete(
502     bool flushToDisk)
503 {
504     uint32 checksum = utChecksum;
505     uint32 length;
506 
507     writeUint8(UT_TRANSACTION_COMPLETE);
508     writeUint32(checksum);
509     utChecksum = 0;
510     length = utBufferStartPosition + utBufferPosition - utLastTransactionPosition;
511     utTransactionCreate(utLastTransactionPosition, length);
512     utLastTransactionPosition += length;
513     if(utPersistenceInitialized) {
514         if(flushToDisk) {
515             flushRecentChanges();
516             fflush(utRecentChangesFile);
517         }
518 #ifndef UT_USE_UTDATABASEUP_H
519         /* Don't do this if we are a persistent undo/redo database */
520         if((utBufferStartPosition << 2) > utDatabaseSize) {
521             utCompactDatabase();
522         }
523 #endif
524     }
525 }
526 
527 /*--------------------------------------------------------------------------------------------------
528   Register an enumerated type.
529 --------------------------------------------------------------------------------------------------*/
utRegisterEnum(char * name,uint16 numEntries)530 void utRegisterEnum(
531     char *name,
532     uint16 numEntries)
533 {
534     utEnum theEnum;
535 
536     if(utUsedEnums == utAllocatedEnums) {
537         utAllocatedEnums += utAllocatedEnums >> 1;
538         utResizeArray(utEnums, utAllocatedEnums);
539     }
540     theEnum = utEnums + utUsedEnums++;
541     utEnumSetName(theEnum, calloc(strlen(name) + 1, sizeof(char)));
542     strcpy(utEnumGetName(theEnum), name);
543     utEnumSetFirstEntryIndex(theEnum, utUsedEntries);
544     utEnumSetNumEntries(theEnum, numEntries);
545 }
546 
547 /*--------------------------------------------------------------------------------------------------
548   Register an entry in an enumerated type.
549 --------------------------------------------------------------------------------------------------*/
utRegisterEntry(char * name,uint32 value)550 void utRegisterEntry(
551     char *name,
552     uint32 value)
553 {
554     utEntry entry;
555 
556     if(utUsedEntries == utAllocatedEntries) {
557         utAllocatedEntries += utAllocatedEntries >> 1;
558         utResizeArray(utEntries, utAllocatedEntries);
559     }
560     entry = utEntries + utUsedEntries++;
561     utEntrySetName(entry, calloc(strlen(name) + 1, sizeof(char)));
562     utEntrySetValue(entry, value);
563     strcpy(utEntryGetName(entry), name);
564 }
565 
566 /*--------------------------------------------------------------------------------------------------
567   Register the previously registered field as a union.
568 --------------------------------------------------------------------------------------------------*/
utRegisterUnion(char * switchFieldName,uint16 numCases)569 void utRegisterUnion(
570     char *switchFieldName,
571     uint16 numCases)
572 {
573     utField unionField = utFields + utUsedFields - 1;
574     utUnion theUnion;
575     utField switchField = utFindField(utCurrentClass, switchFieldName);
576 
577     if(utUsedUnions == utAllocatedUnions) {
578         utAllocatedUnions += utAllocatedUnions >> 1;
579         utResizeArray(utUnions, utAllocatedUnions);
580     }
581     utFieldSetUnionIndex(unionField, utUsedUnions);
582     theUnion = utUnions + utUsedUnions++;
583     utUnionSetSwitchFieldIndex(theUnion, switchField - utFields);
584     utUnionSetFieldIndex(theUnion, utUsedFields - 1);
585     utUnionSetFirstUnioncaseIndex(theUnion, utUsedUnioncases);
586     utUnionSetNumUnioncases(theUnion, numCases);
587 }
588 
589 /*--------------------------------------------------------------------------------------------------
590   Write the variable sized integer.
591 --------------------------------------------------------------------------------------------------*/
utRegisterUnionCase(uint32 value,utFieldType type,uint32 size)592 void utRegisterUnionCase(
593     uint32 value,
594     utFieldType type,
595     uint32 size)
596 {
597     utUnioncase unionCase;
598 
599     if(utUsedUnioncases == utAllocatedUnioncases) {
600         utAllocatedUnioncases += utAllocatedUnioncases >> 1;
601         utResizeArray(utUnioncases, utAllocatedUnioncases);
602     }
603     unionCase = utUnioncases + utUsedUnioncases++;
604     utUnioncaseSetValue(unionCase, value);
605     if(type == UT_BIT) {
606         type = UT_BOOL; /* We convert bit to bool in unions */
607         size = sizeof(bool);
608     }
609     utUnioncaseSetSize(unionCase, size);
610     utUnioncaseSetType(unionCase, type);
611 }
612 
613 /*--------------------------------------------------------------------------------------------------
614   Write the variable sized integer.
615 --------------------------------------------------------------------------------------------------*/
writeInteger(uint64 value,uint8 size)616 static void writeInteger(
617     uint64 value,
618     uint8 size)
619 {
620     switch(size) {
621     case 1: writeUint8((uint8)value); break;
622     case 2: writeUint16((uint16)value); break;
623     case 4: writeUint32((uint32)value); break;
624     case 8: writeUint64(value); break;
625     default:
626         utExit("Invalid reference size");
627     }
628 }
629 
630 /*--------------------------------------------------------------------------------------------------
631   Record the change to the field.
632 --------------------------------------------------------------------------------------------------*/
utRecordField(uint8 moduleID,uint16 fieldIndex,uint64 objectNumber,bool undo)633 void utRecordField(
634     uint8 moduleID,
635     uint16 fieldIndex,
636     uint64 objectNumber,
637     bool undo)
638 {
639     utModule module = utModules + moduleID;
640     uint16 fieldPosition = utModuleGetFirstFieldIndex(module) + fieldIndex;
641     utField field = utFields + fieldPosition;
642     utClass theClass = utClasses + utFieldGetClassIndex(field);
643     uint8 *values = *(uint8 **)(utFieldGetArrayPtr(field)) + objectNumber*utFieldGetSize(field);
644 
645     writeUint8((uint8)(UT_WRITE_FIELD | (undo? 0x80 : 0)));
646     writeUint16(fieldPosition);
647     writeInteger(objectNumber, utClassGetReferenceSize(theClass));
648     writeValues(values, utFieldGetSize(field));
649 }
650 
651 /*--------------------------------------------------------------------------------------------------
652   Record the change to the field.
653 --------------------------------------------------------------------------------------------------*/
utRecordArray(uint8 moduleID,uint16 fieldIndex,uint32 dataIndex,uint32 length,bool undo)654 void utRecordArray(
655     uint8 moduleID,
656     uint16 fieldIndex,
657     uint32 dataIndex,
658     uint32 length,
659     bool undo)
660 {
661     utModule module = utModules + moduleID;
662     uint16 fieldPosition = utModuleGetFirstFieldIndex(module) + fieldIndex;
663     utField field = utFields + fieldPosition;
664     uint8 *values = *(uint8 **)(utFieldGetArrayPtr(field)) + dataIndex*utFieldGetSize(field);
665 
666     writeUint8((uint8)(UT_WRITE_ARRAY | (undo? 0x80 : 0)));
667     writeUint16(fieldPosition);
668     writeUint32(dataIndex);
669     writeUint32(length);
670     writeValues(values, utFieldGetSize(field)*length);
671 }
672 
673 /*--------------------------------------------------------------------------------------------------
674   Record the change to the global value.
675 --------------------------------------------------------------------------------------------------*/
utRecordGlobal(uint8 moduleID,uint8 numBytes,void * location,bool undo)676 void utRecordGlobal(
677     uint8 moduleID,
678     uint8 numBytes,
679     void *location,
680     bool undo)
681 {
682     utModule module = utModules + moduleID;
683     uint16 offset = ((uint8 *) location) - ((uint8 *) utModuleGetGlobalData(module));
684 
685     writeUint8((uint8)(UT_WRITE_GLOBAL | (undo? 0x80 : 0)));
686     writeUint8(moduleID);
687     writeUint16(offset);
688     writeUint8(numBytes);
689     writeValues(location, numBytes);
690 }
691 
692 /*--------------------------------------------------------------------------------------------------
693   Record when a field is resized.
694 --------------------------------------------------------------------------------------------------*/
utRecordResize(uint8 moduleID,uint16 fieldIndex,uint64 length,bool undo)695 void utRecordResize(
696     uint8 moduleID,
697     uint16 fieldIndex,
698     uint64 length,
699     bool undo)
700 {
701     utModule module = utModules + moduleID;
702     uint16 fieldPosition = utModuleGetFirstFieldIndex(module) + fieldIndex;
703 
704     writeUint8((uint8)(UT_RESIZE_FIELD | (undo? 0x80 : 0)));
705     writeUint16(fieldPosition);
706     writeUint64(length);
707 }
708 
709 #define readUint8(buffer) (*(buffer))
710 
711 /*--------------------------------------------------------------------------------------------------
712   Read a uint16 from the command buffer.
713 --------------------------------------------------------------------------------------------------*/
readUint16(uint8 * buffer)714 static uint16 readUint16(
715     uint8 *buffer)
716 {
717     return *(uint16 *)(void *)(buffer);
718 }
719 
720 /*--------------------------------------------------------------------------------------------------
721   Read a uint32 from a buffer.
722 --------------------------------------------------------------------------------------------------*/
readUint32(uint8 * buffer)723 static uint32 readUint32(
724     uint8 *buffer)
725 {
726     return *(uint32 *)(void *)(buffer);
727 }
728 
729 /*--------------------------------------------------------------------------------------------------
730   Read a uint64 from a buffer.
731 --------------------------------------------------------------------------------------------------*/
readUint64(uint8 * buffer)732 static uint64 readUint64(
733     uint8 *buffer)
734 {
735     return *(uint64 *)(void *)(buffer);
736 }
737 
738 /*--------------------------------------------------------------------------------------------------
739   Apply a field value.
740   WRITE_FIELD <16-bit field #> <32-bit index> <value>
741 --------------------------------------------------------------------------------------------------*/
applyField(uint8 * command)742 static void applyField(
743     uint8 *command)
744 {
745     uint16 xField = readUint16(command);
746     utField field = utFields + xField;
747     utClass theClass = utClasses + utFieldGetClassIndex(field);
748     uint8 *values;
749     uint64 objectNumber;
750 
751     command += 2;
752     objectNumber = utFindIntValue(command, utClassGetReferenceSize(theClass));
753     command += utClassGetReferenceSize(theClass);
754     values = *(uint8 **)(utFieldGetArrayPtr(field)) + objectNumber*utFieldGetSize(field);
755     memcpy(values, command, utFieldGetSize(field));
756 }
757 
758 /*--------------------------------------------------------------------------------------------------
759   Apply an array of values.
760   WRITE_ARRAY <16-bit field #> <32-bit index> <32-bit numValues> <values>
761 --------------------------------------------------------------------------------------------------*/
applyArray(uint8 * command)762 static void applyArray(
763     uint8 *command)
764 {
765     uint16 xField = readUint16(command);
766     utField field = utFields + xField;
767     uint8 *values;
768     uint32 length, bytes;
769     uint32 dataIndex;
770 
771     command += 2;
772     dataIndex = readUint32(command);
773     command += 4;
774     length = readUint32(command);
775     command += 4;
776     bytes = utFieldGetSize(field)*length;
777     values = *(uint8 **)(utFieldGetArrayPtr(field)) + dataIndex*utFieldGetSize(field);
778     memcpy(values, command, bytes);
779 }
780 
781 /*--------------------------------------------------------------------------------------------------
782   Apply a global value change.
783   WRITE_GLOBAL <8-bit moduleID> <16-bit offset> <8-bit numBytes> <values>
784 --------------------------------------------------------------------------------------------------*/
applyGlobal(uint8 * command)785 static void applyGlobal(
786     uint8 *command)
787 {
788     uint8 moduleID = readUint8(command);
789     utModule module = utModules + moduleID;
790     uint16 offset;
791     uint8 numBytes;
792     uint8 *values;
793 
794     command++;
795     offset = readUint16(command);
796     command += 2;
797     numBytes = readUint8(command);
798     command++;
799     values = (uint8 *)(utModuleGetGlobalData(module)) + offset;
800     memcpy(values, command, numBytes);
801 }
802 
803 /*--------------------------------------------------------------------------------------------------
804   Apply a resize field command.
805   RESIZE_FIELD <16-bit field #> <64-bit size>
806 --------------------------------------------------------------------------------------------------*/
applyResize(uint8 * command)807 static void applyResize(
808     uint8 *command)
809 {
810     uint16 xField = readUint16(command);
811     utField field = utFields + xField;
812     uint64 size;
813 
814     command += 2;
815     size = readUint64(command);
816     command += 8;
817     *(uint8 **)(utFieldGetArrayPtr(field)) =utRealloc(*(uint8 **)utFieldGetArrayPtr(field), size, utFieldGetSize(field));
818 }
819 
820 /*--------------------------------------------------------------------------------------------------
821   Find the size of a command.
822 --------------------------------------------------------------------------------------------------*/
findCommandSize(uint8 * command)823 static uint32 findCommandSize(
824     uint8 *command)
825 {
826     utClass theClass;
827     utField field;
828     uint16 xField;
829 
830     switch((*command) & 0x7f) {
831     case UT_TRANSACTION_COMPLETE:
832         /* TRANSACTION_COMPLETE <32-bit checksum> */
833         return  5;
834     case UT_WRITE_FIELD:
835         /* WRITE_FIELD <16-bit field #> <32-bit index> <value> */
836         xField = readUint16(command + 1);
837         field = utFields + xField;
838         theClass = utClasses + utFieldGetClassIndex(field);
839         return 3 + utFieldGetSize(field) + utClassGetReferenceSize(theClass);
840     case UT_WRITE_ARRAY:
841         /* WRITE_ARRAY <16-bit field #> <32-bit index> <32-bit numValues> <values> */
842         xField = readUint16(command + 1);
843         field = utFields + xField;
844         return 11 + utFieldGetSize(field)*readUint32(command + 7);
845     case UT_WRITE_GLOBAL:
846         /* WRITE_GLOBAL <8-bit moduleID> <16-bit offset> <8-bit numBytes> <values> */
847         return 5 + readUint8(command + 4);
848     case UT_RESIZE_FIELD:
849         /* RESIZE_FIELD <16-bit field #> <64-bit size> */
850         return 11;
851     default:
852         utExit("Invalid command");
853     }
854     return 0;
855 }
856 
857 /*--------------------------------------------------------------------------------------------------
858   Apply the command.
859 --------------------------------------------------------------------------------------------------*/
applyCommand(uint8 * command)860 static void applyCommand(
861     uint8 *command)
862 {
863     utIfDebug(3) {
864         utDumpCommand(command);
865     }
866     switch(*command++ & 0x7f) {
867     case UT_WRITE_FIELD:
868         applyField(command);
869         break;
870     case UT_WRITE_ARRAY:
871         applyArray(command);
872         break;
873     case UT_WRITE_GLOBAL:
874         applyGlobal(command);
875         break;
876     case UT_RESIZE_FIELD:
877         applyResize(command);
878         break;
879     default:
880         utExit("Unknown command in recentChanges file");
881     }
882 }
883 
884 /*--------------------------------------------------------------------------------------------------
885   Apply changes from the commands in the buffer.
886 --------------------------------------------------------------------------------------------------*/
applyCommands(uint8 * commands,uint32 length)887 static void applyCommands(
888     uint8 *commands,
889     uint32 length)
890 {
891     uint32 xCommand = 0;
892     uint32 size;
893 
894     while(xCommand < length) {
895         size = findCommandSize(commands + xCommand);
896         if(!(commands[xCommand] & 0x80)) {
897             /* Skip undo commands */
898             utIfDebug(3) {
899                 utMemCheckTrace("foo", 1);
900             }
901             applyCommand(commands + xCommand);
902             utIfDebug(3) {
903                 utMemCheckTrace("foo", 1);
904             }
905         }
906         xCommand += size;
907     }
908 }
909 
910 /*--------------------------------------------------------------------------------------------------
911   Truncate the recentChanges file
912 --------------------------------------------------------------------------------------------------*/
truncateChangesFile(uint32 size)913 static void truncateChangesFile(
914     uint32 size)
915 {
916     utWarning("recentChanges file damaged... truncating back to last complete transaction.");
917     fclose(utRecentChangesFile);
918     utTruncateFile(utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP), size);
919     openRecentChanges("rb");
920 }
921 
922 /*--------------------------------------------------------------------------------------------------
923   Apply recent changes from the commands in the recent changes buffer.  utBufferStartPosition and
924   lastTransactionPosition are used to help truncate the recentChanges file if it is corrupt at the
925   end.  Return true if everything is ok, and false if we had to truncate the file.
926 --------------------------------------------------------------------------------------------------*/
applyCommandsInBuffer(bool endOfInput,uint32 * checksum,uint32 * unparsedCommands)927 static bool applyCommandsInBuffer(
928     bool endOfInput,
929     uint32 *checksum,
930     uint32 *unparsedCommands)
931 {
932     uint32 xCommand = 0;
933     uint32 size;
934     bool done = false;
935 
936     while(!done && (xCommand + UT_COMMAND_MAX_HEADER_SIZE < utBufferPosition ||
937             (endOfInput && xCommand < utBufferPosition))) {
938         if(utCommandBuffer[xCommand] == UT_TRANSACTION_COMPLETE) {
939             xCommand++;
940             if(readUint32(utCommandBuffer + xCommand) != *checksum) {
941                 truncateChangesFile(utLastTransactionPosition);
942                 return false;
943             }
944             xCommand += 4;
945             utTransactionCreate(utLastTransactionPosition, utUsedCommands);
946             applyCommands(utCommands, utUsedCommands);
947             utUsedCommands = 0;
948             utLastTransactionPosition = utBufferStartPosition + xCommand;
949             *checksum = 0;
950         } else {
951             size = findCommandSize(utCommandBuffer + xCommand);
952             if(size == 0) {
953                 truncateChangesFile(utLastTransactionPosition);
954                 return false;
955             }
956             if(utUsedCommands + size >= utAllocatedCommands) {
957                 utAllocatedCommands += (utAllocatedCommands >> 1) + size;
958                 utResizeArray(utCommands, utAllocatedCommands);
959             }
960             if(xCommand + size <= utBufferPosition) {
961                 while(size-- != 0) {
962                     *checksum = (*checksum ^ utCommandBuffer[xCommand])*1103515245 + 12345;
963                     utCommands[utUsedCommands++] = utCommandBuffer[xCommand++];
964                 }
965             } else {
966                 done = true;
967             }
968         }
969     }
970     *unparsedCommands = utBufferPosition - xCommand;
971     if(*unparsedCommands > 0) {
972         memmove(utCommandBuffer, utCommandBuffer + xCommand, *unparsedCommands);
973     }
974     return true;
975 }
976 
977 /*--------------------------------------------------------------------------------------------------
978   Apply recent changes to the database.  The changes will have been recorded in recentChanges.
979 --------------------------------------------------------------------------------------------------*/
applyRecentChanges(void)980 static void applyRecentChanges(void)
981 {
982     uint32 checksum = 0;
983     uint32 unparsedCommands = 0;
984 
985     utAssert(utChecksum == 0);
986     utLastTransactionPosition = 0;
987     utBufferStartPosition = 0;
988     utUsedCommands = 0;
989     fclose(utRecentChangesFile);
990     openRecentChanges("rb");
991     utDo {
992         if(unparsedCommands + UT_CHANGE_BUFFER_LENGTH > utBufferSize) {
993             utBufferSize = unparsedCommands + UT_CHANGE_BUFFER_LENGTH;
994             utResizeArray(utCommandBuffer, utBufferSize);
995         }
996         utBufferPosition = fread(utCommandBuffer + unparsedCommands, sizeof(uint8),
997             UT_CHANGE_BUFFER_LENGTH, utRecentChangesFile) + unparsedCommands;
998     } utWhile(utBufferPosition > 0 &&
999             applyCommandsInBuffer((bool)(utBufferPosition < UT_CHANGE_BUFFER_LENGTH + unparsedCommands),
1000             &checksum, &unparsedCommands)) {
1001         utBufferStartPosition += utBufferPosition - unparsedCommands;
1002     } utRepeat;
1003     fclose(utRecentChangesFile);
1004     openRecentChanges("ab");
1005     utBufferPosition = 0;
1006     utAssert(utChecksum == 0);
1007 }
1008 
1009 /*--------------------------------------------------------------------------------------------------
1010   Load the module from the file.
1011 --------------------------------------------------------------------------------------------------*/
loadModule(utModule module,FILE * file)1012 static bool loadModule(
1013     utModule module,
1014     FILE *file)
1015 {
1016     utClass theClass;
1017     utField field;
1018     uint8 *values;
1019     uint64 numUsed, numAllocated;
1020 
1021     utModuleGetStop(module)();
1022     if(fread(utModuleGetGlobalData(module), utModuleGetGlobalSize(module), 1, file) != 1) {
1023         utWarning("File too short, and does not contain header info");
1024         return false;
1025     }
1026     if(utModuleGetHashValue(module) != *(uint32 *)(utModuleGetGlobalData(module))) {
1027         utWarning("Incompatible database format");
1028         return false;
1029     }
1030     utForeachModuleField(module, field) {
1031         theClass = utClasses + utFieldGetClassIndex(field);
1032         if(utFieldArray(field)) {
1033             numUsed = *(utFieldGetNumUsedPtr(field));
1034             numAllocated =  *(utFieldGetNumAllocatedPtr(field));
1035         } else {
1036             numUsed = utFindIntValue(utClassGetNumUsedPtr(theClass), utClassGetReferenceSize(theClass));
1037             numAllocated = utFindIntValue(utClassGetNumAllocatedPtr(theClass), utClassGetReferenceSize(theClass));
1038         }
1039         values = utCalloc(numAllocated, utFieldGetSize(field));
1040         *(uint8 **)(utFieldGetArrayPtr(field)) = values;
1041         if(fread(values, utFieldGetSize(field), numUsed, file) != numUsed) {
1042             utWarning("Unable to read from file");
1043             return false;
1044         }
1045     } utEndModuleField;
1046     return true;
1047 }
1048 
1049 /*--------------------------------------------------------------------------------------------------
1050   Load the database from memory.
1051 --------------------------------------------------------------------------------------------------*/
utLoadBinaryDatabase(FILE * databaseFile)1052 void utLoadBinaryDatabase(
1053     FILE *databaseFile)
1054 {
1055     char *fileName;
1056     utModule module;
1057     bool useDefault = databaseFile == NULL;
1058 
1059     if(useDefault) {
1060         fileName = utSprintf("%s%cdatabase", utDatabaseDirectory, UTDIRSEP);
1061         databaseFile = fopen(fileName, "rb");
1062         if(databaseFile == NULL) {
1063             utExit("Could not read from %s", fileName);
1064         }
1065     }
1066     utForeachModule(module) {
1067         if(utModuleInitialized(module) && !loadModule(module, databaseFile)) {
1068             utExit("Could not read from %s");
1069         }
1070     } utEndModule;
1071     if(useDefault) {
1072         fclose(databaseFile);
1073     }
1074 }
1075 
1076 /*--------------------------------------------------------------------------------------------------
1077   Save the module to the file.
1078 --------------------------------------------------------------------------------------------------*/
saveModule(utModule module,FILE * file)1079 static bool saveModule(
1080     utModule module,
1081     FILE *file)
1082 {
1083     utClass theClass;
1084     utField field;
1085     uint8 *values;
1086     uint64 numUsed;
1087 
1088     if(fwrite(utModuleGetGlobalData(module), utModuleGetGlobalSize(module), 1, file) != 1) {
1089         utWarning("Unable to write to file");
1090         return false;
1091     }
1092     utForeachModuleField(module, field) {
1093         theClass = utClasses + utFieldGetClassIndex(field);
1094         values = *(uint8 **)(utFieldGetArrayPtr(field));
1095         if(utFieldArray(field)) {
1096             numUsed = *(utFieldGetNumUsedPtr(field));
1097         } else {
1098             numUsed = utFindIntValue(utClassGetNumUsedPtr(theClass), utClassGetReferenceSize(theClass));
1099         }
1100         if(fwrite(values, utFieldGetSize(field), numUsed, file) != numUsed) {
1101             utWarning("Unable to write to file");
1102             return false;
1103         }
1104     } utEndModuleField;
1105     return true;
1106 }
1107 
1108 /*--------------------------------------------------------------------------------------------------
1109   Save the database to disk.
1110 --------------------------------------------------------------------------------------------------*/
utSaveBinaryDatabase(FILE * databaseFile)1111 void utSaveBinaryDatabase(
1112     FILE *databaseFile)
1113 {
1114     char *fileName = NULL;
1115     utModule module;
1116     bool useDefault = databaseFile == NULL;
1117 
1118     if(useDefault) {
1119         fileName = utSprintf("%s%cdatabase", utDatabaseDirectory, UTDIRSEP);
1120         databaseFile = fopen(fileName, "wb");
1121     }
1122     if(databaseFile == NULL) {
1123         utExit("Could not write to %s", fileName);
1124     }
1125     utForeachModule(module) {
1126         if(utModuleInitialized(module) && !saveModule(module, databaseFile)) {
1127             utExit("Could not save database");
1128         }
1129     } utEndModule;
1130     if(useDefault) {
1131         fclose(databaseFile);
1132     }
1133 }
1134 
1135 /*--------------------------------------------------------------------------------------------------
1136   Find the oldest transaction indicated by numChanges, and load it into the database.
1137 --------------------------------------------------------------------------------------------------*/
loadChangesIntoCommandBuffer(uint32 xTransaction)1138 static void loadChangesIntoCommandBuffer(
1139     uint32 xTransaction)
1140 {
1141     utTransaction transaction = utTransactions + xTransaction;
1142     uint32 filePosition, bufferSize;
1143 
1144     if(utTransactionGetPosition(transaction) >= utBufferStartPosition) {
1145         return; /* Already loaded */
1146     }
1147     flushRecentChanges();
1148     if(utBufferStartPosition >= UT_CHANGE_BUFFER_LENGTH) {
1149         filePosition = utBufferStartPosition - UT_CHANGE_BUFFER_LENGTH;
1150     } else {
1151         filePosition = 0;
1152     }
1153     filePosition = utMin(filePosition, utTransactionGetPosition(transaction));
1154     bufferSize = utBufferStartPosition + utBufferPosition - filePosition;
1155     if(bufferSize > utBufferSize) {
1156         utBufferSize = bufferSize + (bufferSize >> 1);
1157         utResizeArray(utCommandBuffer, utBufferSize);
1158     }
1159     fclose(utRecentChangesFile);
1160     openRecentChanges("rb");
1161     fseek(utRecentChangesFile, filePosition, SEEK_SET);
1162     utBufferPosition = fread(utCommandBuffer, sizeof(uint8), bufferSize, utRecentChangesFile);
1163     utAssert(utBufferPosition == bufferSize);
1164     utBufferStartPosition = filePosition;
1165     fclose(utRecentChangesFile);
1166     utTruncateFile(utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP), filePosition);
1167     openRecentChanges("ab");
1168 }
1169 
1170 /*--------------------------------------------------------------------------------------------------
1171   Compute the transaction command positions.
1172 --------------------------------------------------------------------------------------------------*/
computeTransactionCommandPositions(utTransaction transaction)1173 static void computeTransactionCommandPositions(
1174     utTransaction transaction)
1175 {
1176     uint8 *commands = utCommandBuffer + utTransactionGetPosition(transaction) - utBufferStartPosition;
1177     uint32 xCommand = 0;
1178     uint32 size;
1179 
1180     utTransactionUsedCommands = 0;
1181     while(xCommand < utTransactionGetLength(transaction)) {
1182         size = findCommandSize(commands + xCommand);
1183         if(utTransactionUsedCommands == utTransactionAllocatedCommands) {
1184             utTransactionAllocatedCommands += utTransactionAllocatedCommands >> 1;
1185             utResizeArray(utTransactionCommands, utTransactionAllocatedCommands);
1186         }
1187         utTransactionCommands[utTransactionUsedCommands++] = xCommand + utTransactionGetPosition(transaction) -
1188             utBufferStartPosition;
1189         xCommand += size;
1190     }
1191 }
1192 
1193 /*--------------------------------------------------------------------------------------------------
1194   Undo the transaction.
1195 --------------------------------------------------------------------------------------------------*/
undoTransaction(utTransaction transaction)1196 static void undoTransaction(
1197     utTransaction transaction)
1198 {
1199     uint8 *command;
1200     uint32 xCommand;
1201 
1202     computeTransactionCommandPositions(transaction);
1203     xCommand = utTransactionUsedCommands;
1204     while(xCommand-- != 0) {
1205         command = utCommandBuffer + utTransactionCommands[xCommand];
1206         if(*command & 0x80) {
1207             applyCommand(command);
1208         }
1209     }
1210 }
1211 
1212 /*--------------------------------------------------------------------------------------------------
1213   Undo the last transaction, by scanning backwards through the recent changes buffer, executing
1214   undo commands.
1215 --------------------------------------------------------------------------------------------------*/
utUndo(uint32 numChanges)1216 uint32 utUndo(
1217     uint32 numChanges)
1218 {
1219     utTransaction transaction = NULL;
1220     uint32 xTransaction;
1221     uint32 firstTransaction, lastTransaction;
1222 
1223     if(utTransactionInProgress) {
1224         utExit("The current transaction has not yet been completed... can't undo!");
1225     }
1226     utAssert(utChecksum == 0);
1227     if(utRedoTransaction == NULL) {
1228         firstTransaction = utUsedTransactions;
1229     } else {
1230         firstTransaction = utRedoTransaction - utTransactions;
1231     }
1232     lastTransaction = firstTransaction >= numChanges? firstTransaction - numChanges : 0;
1233     if(firstTransaction == lastTransaction) {
1234         /* Nothing to do */
1235         return 0;
1236     }
1237     if(utPersistenceInitialized) {
1238         loadChangesIntoCommandBuffer(lastTransaction);
1239     }
1240     for(xTransaction = firstTransaction; xTransaction > lastTransaction; xTransaction--) {
1241         transaction = utTransactions + xTransaction - 1;
1242         undoTransaction(transaction);
1243     }
1244     utRedoTransaction = transaction;
1245     utAssert(utChecksum == 0);
1246     return firstTransaction - lastTransaction;
1247 }
1248 
1249 /*--------------------------------------------------------------------------------------------------
1250   Redo the transaction.
1251 --------------------------------------------------------------------------------------------------*/
redoTransaction(utTransaction transaction)1252 static void redoTransaction(
1253     utTransaction transaction)
1254 {
1255     uint8 *command = utCommandBuffer + utTransactionGetPosition(transaction) - utBufferStartPosition;
1256     uint32 length = utTransactionGetLength(transaction) - 5; /* Subract out the transaction-complete command */
1257 
1258     applyCommands(command, length);
1259 }
1260 
1261 /*--------------------------------------------------------------------------------------------------
1262   Redo the last transaction, by scanning forwards through the recent changes buffer, executing
1263   non-undo commands.
1264 --------------------------------------------------------------------------------------------------*/
utRedo(uint32 numChanges)1265 uint32 utRedo(
1266     uint32 numChanges)
1267 {
1268     utTransaction transaction;
1269     uint32 xTransaction;
1270     uint32 firstTransaction, lastTransaction;
1271 
1272     if(utRedoTransaction == NULL) {
1273         return 0;
1274     }
1275     utAssert(utChecksum == 0);
1276     firstTransaction = utRedoTransaction - utTransactions;
1277     lastTransaction = utMin(firstTransaction + numChanges, utUsedTransactions);
1278     for(xTransaction = firstTransaction; xTransaction < lastTransaction; xTransaction++) {
1279         transaction = utTransactions + xTransaction;
1280         redoTransaction(transaction);
1281     }
1282     if(lastTransaction == utUsedTransactions) {
1283         utRedoTransaction = NULL;
1284     } else {
1285         utRedoTransaction = utTransactions + lastTransaction;
1286     }
1287     utAssert(utChecksum == 0);
1288     return firstTransaction - lastTransaction;
1289 }
1290 
1291 /*--------------------------------------------------------------------------------------------------
1292   Load the persistent database into memory if it exists.  Return true if it does, otherwise false.
1293   If useTextDatabaseFormat is true, we'll keep the database in text instead of binary.  If
1294   keepBackup is true, we'll rename the old database file before writing the new one.
1295 --------------------------------------------------------------------------------------------------*/
utStartPersistence(char * directory,bool useTextDatabaseFormat,bool keepBackup)1296 bool utStartPersistence(
1297     char *directory,
1298     bool useTextDatabaseFormat,
1299     bool keepBackup)
1300 {
1301     char *fileName;
1302 
1303     if(!utAccess(directory, "w")) {
1304         utExit("Cannot write to directory %s\n", directory);
1305     }
1306     utDatabaseDirectory = utAllocString(directory);
1307     utUseTextDatabaseFormat = useTextDatabaseFormat;
1308     utKeepBackup = keepBackup;
1309     resetCommandBuffer();
1310     utTransactionInProgress = false;
1311     fileName = utSprintf("%s%cdatabase", directory, UTDIRSEP);
1312     if(utFileExists(fileName)) {
1313         utDatabaseSize = utFindFileSize(fileName);
1314         if(useTextDatabaseFormat) {
1315             utLoadTextDatabase(NULL);
1316         } else {
1317             utLoadBinaryDatabase(NULL);
1318         }
1319         resetCommandBuffer(); /* Loading a text database can issue record commands */
1320         openRecentChanges("rb");
1321         applyRecentChanges();
1322         utPersistenceInitialized = true;
1323         return true;
1324     }
1325     if(useTextDatabaseFormat) {
1326         utSaveTextDatabase(NULL);
1327     } else {
1328         utSaveBinaryDatabase(NULL);
1329     }
1330     utDatabaseSize = utFindFileSize(utSprintf("%s%cdatabase", directory, UTDIRSEP));
1331     openRecentChanges("wb");
1332     fclose(utRecentChangesFile);
1333     openRecentChanges("ab");
1334     utPersistenceInitialized = true;
1335     return false;
1336 }
1337 
1338 /*--------------------------------------------------------------------------------------------------
1339   Destroy the field.
1340 --------------------------------------------------------------------------------------------------*/
utFieldDestroy(utField field)1341 void utFieldDestroy(
1342     utField field)
1343 {
1344     free(utFieldGetName(field));
1345     if(utFieldGetDestName(field) != NULL) {
1346         free(utFieldGetDestName(field));
1347     }
1348 }
1349 
1350 /*--------------------------------------------------------------------------------------------------
1351   Destroy the module.
1352 --------------------------------------------------------------------------------------------------*/
utClassDestroy(utClass theClass)1353 void utClassDestroy(
1354     utClass theClass)
1355 {
1356     free(utClassGetName(theClass));
1357 }
1358 
1359 /*--------------------------------------------------------------------------------------------------
1360   Destroy the entry.
1361 --------------------------------------------------------------------------------------------------*/
utEntryDestroy(utEntry entry)1362 void utEntryDestroy(
1363     utEntry entry)
1364 {
1365     free(utEntryGetName(entry));
1366 }
1367 
1368 /*--------------------------------------------------------------------------------------------------
1369   Destroy the enum.
1370 --------------------------------------------------------------------------------------------------*/
utEnumDestroy(utEnum theEnum)1371 void utEnumDestroy(
1372     utEnum theEnum)
1373 {
1374     utEntry entry;
1375     uint16 xEntry;
1376 
1377     for(xEntry = 0; xEntry < utEnumGetNumEntries(theEnum); xEntry++) {
1378         entry = utEntries + utEnumGetFirstEntryIndex(theEnum) + xEntry;
1379         utEntryDestroy(entry);
1380     }
1381     free(utEnumGetName(theEnum));
1382 }
1383 
1384 /*--------------------------------------------------------------------------------------------------
1385   Destroy the module.
1386 --------------------------------------------------------------------------------------------------*/
utModuleDestroy(utModule module)1387 void utModuleDestroy(
1388     utModule module)
1389 {
1390     utClass theClass, nClass;
1391     utField field, nField;
1392     utEnum theEnum, nEnum;
1393 
1394     for(field = utModuleGetFirstField(module); field != utFieldNull; field = nField) {
1395         nField = utModuleGetNextModuleField(module, field);
1396         utFieldDestroy(field);
1397     }
1398     for(theClass = utModuleGetFirstClass(module); theClass != utClassNull; theClass = nClass) {
1399         nClass = utModuleGetNextModuleClass(module, theClass);
1400         utClassDestroy(theClass);
1401     }
1402     for(theEnum = utModuleGetFirstEnum(module); theEnum != utEnumNull; theEnum = nEnum) {
1403         nEnum = utModuleGetNextModuleEnum(module, theEnum);
1404         utEnumDestroy(theEnum);
1405     }
1406     free(utModuleGetPrefix(module));
1407 }
1408 
1409 /*--------------------------------------------------------------------------------------------------
1410   Stop the persistent database.  Flush changes to disk, and close files.
1411 --------------------------------------------------------------------------------------------------*/
utStopPersistence(void)1412 void utStopPersistence(void)
1413 {
1414     flushRecentChanges();
1415     fclose(utRecentChangesFile);
1416     if(utRedoTransaction != NULL) {
1417         utTruncateFile(utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP),
1418             utTransactionGetPosition(utRedoTransaction));
1419     }
1420     utFree(utDatabaseDirectory);
1421     utPersistenceInitialized = false;
1422 }
1423 
1424 /*--------------------------------------------------------------------------------------------------
1425   Load the persistent database into memory if it exists.  Return true if it does, otherwise false.
1426 --------------------------------------------------------------------------------------------------*/
utAllocPersistenceObjects(void)1427 void utAllocPersistenceObjects(void)
1428 {
1429     utAllocatedModules = 2;
1430     utUsedModules = 0;
1431     utModules = utNewA(struct utModuleStruct, utAllocatedModules);
1432     utAllocatedClasses = 2;
1433     utUsedClasses = 0;
1434     utClasses = utNewA(struct utClassStruct, utAllocatedClasses);
1435     utAllocatedFields = 2;
1436     utUsedFields = 0;
1437     utFields = utNewA(struct utFieldStruct, utAllocatedFields);
1438     utBufferPosition = 0;
1439     utBufferSize = UT_CHANGE_BUFFER_LENGTH;
1440     utCommandBuffer = utNewA(uint8, utBufferSize);
1441     utAllocatedTransactions = 2;
1442     utUsedTransactions = 0;
1443     utTransactions = utNewA(struct utTransactionStruct, utAllocatedTransactions);
1444     utTransactionAllocatedCommands = 2;
1445     utTransactionUsedCommands = 0;
1446     utTransactionCommands = utNewA(uint32, utTransactionAllocatedCommands);
1447     utAllocatedCommands = 2;
1448     utUsedCommands = 0;
1449     utCommands = utNewA(uint8, utAllocatedCommands);
1450     utAllocatedEnums = 2;
1451     utUsedEnums = 0;
1452     utEnums = utNewA(struct utEnumStruct, utAllocatedEnums);
1453     utAllocatedEntries = 2;
1454     utUsedEntries = 0;
1455     utEntries = utNewA(struct utEntryStruct, utAllocatedEntries);
1456     utAllocatedUnions = 2;
1457     utUsedUnions = 0;
1458     utUnions = utNewA(struct utUnionStruct, utAllocatedUnions);
1459     utAllocatedUnioncases = 2;
1460     utUsedUnioncases = 0;
1461     utUnioncases = utNewA(struct utUnioncaseStruct, utAllocatedUnioncases);
1462     utRedoTransaction = NULL;
1463 }
1464 
1465 /*--------------------------------------------------------------------------------------------------
1466   Stop the persistent database.  Flush changes to disk, and close files.
1467 --------------------------------------------------------------------------------------------------*/
utFreePersistenceObjects(void)1468 void utFreePersistenceObjects(void)
1469 {
1470     utModule module;
1471     uint8 xModule;
1472 
1473     for(xModule = 0; xModule < utUsedModules; xModule++) {
1474         module = utModules + xModule;
1475         utModuleDestroy(module);
1476     }
1477     utUsedModules = 0;
1478     utUsedClasses = 0;
1479     utUsedFields = 0;
1480     utFree(utModules);
1481     utFree(utClasses);
1482     utFree(utFields);
1483     utFree(utCommandBuffer);
1484     utFree(utTransactions);
1485     utFree(utTransactionCommands);
1486     utFree(utCommands);
1487     utFree(utEnums);
1488     utFree(utEntries);
1489     utFree(utUnions);
1490     utFree(utUnioncases);
1491 }
1492 
1493 /*--------------------------------------------------------------------------------------------------
1494   Reset the database, so that we can read in a new one.
1495 --------------------------------------------------------------------------------------------------*/
utResetDatabase(void)1496 void utResetDatabase(void)
1497 {
1498     utModule module;
1499     uint8 xModule;
1500 
1501     for(xModule = 0; xModule < utUsedModules; xModule++) {
1502         module = utModules + xModule;
1503         if(utModuleInitialized(module)) {
1504             utModuleGetStop(module)();
1505             utModuleGetStart(module)();
1506         }
1507     }
1508     utInitSymTable();
1509 }
1510 
1511 /*--------------------------------------------------------------------------------------------------
1512   Dump a transaction complete command.
1513   TRANSACTION_COMPLETE <32-bit checksum>
1514 --------------------------------------------------------------------------------------------------*/
dumpTransactionComplete(uint8 * command)1515 static void dumpTransactionComplete(
1516     uint8 *command)
1517 {
1518     uint32 checksum = readUint32(command);
1519 
1520     utDebug("transaction complete with checksum 0x%x\n", checksum);
1521 }
1522 
1523 /*--------------------------------------------------------------------------------------------------
1524   Dump a field command.
1525   WRITE_FIELD <16-bit field #> <32-bit index> <value>
1526 --------------------------------------------------------------------------------------------------*/
dumpField(uint8 * command)1527 static void dumpField(
1528     uint8 *command)
1529 {
1530     uint16 xField = readUint16(command);
1531     utField field = utFields + xField;
1532     utClass theClass = utClasses + utFieldGetClassIndex(field);
1533     utModule module = utModules + utClassGetModuleIndex(theClass);
1534     uint64 objectNumber;
1535 
1536     command += 2;
1537     objectNumber = utFindIntValue(command, utClassGetReferenceSize(theClass));
1538     command += utClassGetReferenceSize(theClass);
1539     utDebug("%s%ss.%s[%llu] = 0x%s\n", utModuleGetPrefix(module), utClassGetName(theClass), utFieldGetName(field), objectNumber,
1540         utFindHexString(command, utFieldGetSize(field)));
1541 }
1542 
1543 /*--------------------------------------------------------------------------------------------------
1544   Dump an array command.
1545   WRITE_ARRAY <16-bit field #> <32-bit index> <32-bit numValues> <values>
1546 --------------------------------------------------------------------------------------------------*/
dumpArray(uint8 * command)1547 static void dumpArray(
1548     uint8 *command)
1549 {
1550     uint16 xField = readUint16(command);
1551     utField field = utFields + xField;
1552     utClass theClass = utClasses + utFieldGetClassIndex(field);
1553     utModule module = utModules + utClassGetModuleIndex(theClass);
1554     uint32 length, bytes;
1555     uint32 dataIndex;
1556     uint32 xValue;
1557     bool firstTime = true;
1558 
1559     command += 2;
1560     dataIndex = readUint32(command);
1561     command += 4;
1562     length = readUint32(command);
1563     command += 4;
1564     bytes = utFieldGetSize(field)*length;
1565     utDebug("%s%ss.%s[%u] = (", utModuleGetPrefix(module), utClassGetName(theClass), utFieldGetName(field), dataIndex);
1566     for(xValue = 0; xValue < length; xValue++) {
1567         if(!firstTime) {
1568             utDebug(", ");
1569         }
1570         firstTime = false;
1571         utDebug("0x%s", utFindHexString(command, utFieldGetSize(field)));
1572     }
1573     utDebug(")\n");
1574 }
1575 
1576 /*--------------------------------------------------------------------------------------------------
1577   Dump a global command.
1578   WRITE_GLOBAL <8-bit moduleID> <16-bit offset> <8-bit numBytes> <values>
1579 --------------------------------------------------------------------------------------------------*/
dumpGlobal(uint8 * command)1580 static void dumpGlobal(
1581     uint8 *command)
1582 {
1583     uint8 moduleID = readUint8(command);
1584     utModule module = utModules + moduleID;
1585     uint16 offset;
1586     uint8 numBytes;
1587 
1588     command++;
1589     offset = readUint16(command);
1590     command += 2;
1591     numBytes = readUint8(command);
1592     command++;
1593     utDebug("%sRootData[%u] = 0x%s\n", utModuleGetPrefix(module), offset, utFindHexString(command, numBytes));
1594 }
1595 
1596 /*--------------------------------------------------------------------------------------------------
1597   Dump a resize command.
1598   RESIZE_FIELD <16-bit field #> <64-bit size>
1599 --------------------------------------------------------------------------------------------------*/
dumpResize(uint8 * command)1600 static void dumpResize(
1601     uint8 *command)
1602 {
1603     uint16 xField = readUint16(command);
1604     utField field = utFields + xField;
1605     utClass theClass = utClasses + utFieldGetClassIndex(field);
1606     utModule module = utModules + utClassGetModuleIndex(theClass);
1607     uint64 size;
1608 
1609     command += 2;
1610     size = readUint64(command);
1611     command += 8;
1612     utDebug("%s%ss.%s resized to %llu\n", utModuleGetPrefix(module), utClassGetName(theClass), utFieldGetName(field), size);
1613 }
1614 
1615 /*--------------------------------------------------------------------------------------------------
1616   Dump the command
1617 --------------------------------------------------------------------------------------------------*/
utDumpCommand(uint8 * command)1618 void utDumpCommand(
1619     uint8 *command)
1620 {
1621     if((*command) & 0x80) {
1622         utDebug("undo ");
1623     }
1624     switch(*command++ & 0x7f) {
1625     case UT_TRANSACTION_COMPLETE:
1626         dumpTransactionComplete(command);
1627         break;
1628     case UT_WRITE_FIELD:
1629         dumpField(command);
1630         break;
1631     case UT_WRITE_ARRAY:
1632         dumpArray(command);
1633         break;
1634     case UT_WRITE_GLOBAL:
1635         dumpGlobal(command);
1636         break;
1637     case UT_RESIZE_FIELD:
1638         dumpResize(command);
1639         break;
1640     default:
1641         utExit("Unknown command in recentChanges file");
1642     }
1643 }
1644 
1645 /*--------------------------------------------------------------------------------------------------
1646   Dump the commands in the buffer to the log file.
1647 --------------------------------------------------------------------------------------------------*/
dumpCommands(uint8 * commands,uint32 length)1648 static void dumpCommands(
1649     uint8 *commands,
1650     uint32 length)
1651 {
1652     uint32 xCommand = 0;
1653     uint32 size;
1654     uint32 checksum = 0;
1655 
1656     while(xCommand < length) {
1657         utDebug("%8x %8x ", checksum, xCommand);
1658         size = findCommandSize(commands + xCommand);
1659         utDumpCommand(commands + xCommand);
1660         if(commands[xCommand] == UT_TRANSACTION_COMPLETE) {
1661             checksum = 0;
1662         }
1663         while(size-- != 0) {
1664             checksum = (checksum ^ commands[xCommand++])*1103515245 + 12345;
1665         }
1666     }
1667 }
1668 
1669 /*--------------------------------------------------------------------------------------------------
1670   Dump the recent changes buffer to the log file in text.
1671 --------------------------------------------------------------------------------------------------*/
utDumpRecentChanges(void)1672 void utDumpRecentChanges(void)
1673 {
1674     char *fileName = utSprintf("%s%crecentChanges", utDatabaseDirectory, UTDIRSEP);
1675     uint32 fileSize = utFindFileSize(fileName);
1676     uint8 *commands = utNewA(uint8, fileSize);
1677 
1678     fclose(utRecentChangesFile);
1679     openRecentChanges("rb");
1680     fread(commands, sizeof(uint8), fileSize, utRecentChangesFile);
1681     fclose(utRecentChangesFile);
1682     dumpCommands(commands, fileSize);
1683     openRecentChanges("ab");
1684     utFree(commands);
1685 }
1686 
1687