1 /***********************************************************************************************************************************
2 Test Type Performance
3 
4 Test the performance of various types and data structures.  Generally speaking, the starting values should be high enough to "blow
5 up" in terms of execution time if there are performance problems without taking very long if everything is running smoothly.
6 
7 These starting values can then be scaled up for profiling and stress testing as needed.  In general we hope to scale to 1000 without
8 running out of memory on the test systems or taking an undue amount of time.  It should be noted that in this context scaling to
9 1000 is nowhere near turning it up to 11.
10 ***********************************************************************************************************************************/
11 #include "common/ini.h"
12 #include "common/io/bufferRead.h"
13 #include "common/io/bufferWrite.h"
14 #include "common/stat.h"
15 #include "common/time.h"
16 #include "common/type/list.h"
17 #include "common/type/object.h"
18 #include "info/manifest.h"
19 #include "postgres/version.h"
20 
21 #include "common/harnessInfo.h"
22 #include "common/harnessStorage.h"
23 
24 /***********************************************************************************************************************************
25 Test sort comparator
26 ***********************************************************************************************************************************/
27 static int
testComparator(const void * item1,const void * item2)28 testComparator(const void *item1, const void *item2)
29 {
30     int int1 = *(int *)item1;
31     int int2 = *(int *)item2;
32 
33     if (int1 < int2)
34         return -1;
35 
36     if (int1 > int2)
37         return 1;
38 
39     return 0;
40 }
41 
42 /***********************************************************************************************************************************
43 Test callback to count ini load results
44 ***********************************************************************************************************************************/
45 static void
testIniLoadCountCallback(void * data,const String * section,const String * key,const String * value,const Variant * valueVar)46 testIniLoadCountCallback(void *data, const String *section, const String *key, const String *value, const Variant *valueVar)
47 {
48     (*(unsigned int *)data)++;
49     (void)section;
50     (void)key;
51     (void)value;
52     (void)valueVar;
53 }
54 
55 /***********************************************************************************************************************************
56 Driver to test manifestNewBuild(). Generates files for a valid-looking PostgreSQL cluster that can be scaled to any size.
57 ***********************************************************************************************************************************/
58 typedef struct
59 {
60     STORAGE_COMMON_MEMBER;
61     uint64_t fileTotal;
62 } StorageTestManifestNewBuild;
63 
64 static StorageInfo
storageTestManifestNewBuildInfo(THIS_VOID,const String * file,StorageInfoLevel level,StorageInterfaceInfoParam param)65 storageTestManifestNewBuildInfo(THIS_VOID, const String *file, StorageInfoLevel level, StorageInterfaceInfoParam param)
66 {
67     (void)thisVoid; (void)level; (void)param;
68 
69     StorageInfo result =
70     {
71         .level = storageInfoLevelDetail,
72         .exists = true,
73         .type = storageTypePath,
74         .mode = 0600,
75         .userId = 100,
76         .groupId = 100,
77         .user = STRDEF("test"),
78         .group = STRDEF("test"),
79     };
80 
81     if (strEq(file, STRDEF("/pg")))
82     {
83         result.type = storageTypePath;
84     }
85     else
86         THROW_FMT(AssertError, "unhandled file info '%s'", strZ(file));
87 
88     return result;
89 }
90 
91 static bool
storageTestManifestNewBuildInfoList(THIS_VOID,const String * path,StorageInfoLevel level,StorageInfoListCallback callback,void * callbackData,StorageInterfaceInfoListParam param)92 storageTestManifestNewBuildInfoList(
93     THIS_VOID, const String *path, StorageInfoLevel level, StorageInfoListCallback callback, void *callbackData,
94     StorageInterfaceInfoListParam param)
95 {
96     THIS(StorageTestManifestNewBuild);
97     (void)path; (void)level; (void)param;
98 
99     MEM_CONTEXT_TEMP_RESET_BEGIN()
100     {
101         StorageInfo result =
102         {
103             .level = storageInfoLevelDetail,
104             .exists = true,
105             .type = storageTypePath,
106             .mode = 0700,
107             .userId = 100,
108             .groupId = 100,
109             .user = STRDEF("test"),
110             .group = STRDEF("test"),
111         };
112 
113         if (strEq(path, STRDEF("/pg")))
114         {
115             result.name = STRDEF("base");
116             callback(callbackData, &result);
117         }
118         else if (strEq(path, STRDEF("/pg/base")))
119         {
120             result.name = STRDEF("1000000000");
121             callback(callbackData, &result);
122         }
123         else if (strEq(path, STRDEF("/pg/base/1000000000")))
124         {
125             result.type = storageTypeFile;
126             result.size = 8192;
127             result.mode = 0600;
128             result.timeModified = 1595627966;
129 
130             for (unsigned int fileIdx = 0; fileIdx < this->fileTotal; fileIdx++)
131             {
132                 result.name = strNewFmt("%u", 1000000000 + fileIdx);
133                 callback(callbackData, &result);
134                 MEM_CONTEXT_TEMP_RESET(10000);
135             }
136         }
137         else
138             THROW_FMT(AssertError, "unhandled file list info '%s'", strZ(path));
139     }
140     MEM_CONTEXT_TEMP_END();
141 
142     return true;
143 }
144 
145 /***********************************************************************************************************************************
146 Test Run
147 ***********************************************************************************************************************************/
148 void
testRun(void)149 testRun(void)
150 {
151     FUNCTION_HARNESS_VOID();
152 
153     // *****************************************************************************************************************************
154     if (testBegin("lstFind()"))
155     {
156         CHECK(TEST_SCALE <= 10000);
157         int testMax = 100000 * (int)TEST_SCALE;
158 
159         // Generate a large list of values (use int instead of string so there fewer allocations)
160         List *list = lstNewP(sizeof(int), .comparator = testComparator);
161 
162         for (int listIdx = 0; listIdx < testMax; listIdx++)
163             lstAdd(list, &listIdx);
164 
165         CHECK(lstSize(list) == (unsigned int)testMax);
166 
167         TEST_LOG_FMT("generated %d item list", testMax);
168 
169         // Search for all values with an ascending sort
170         lstSort(list, sortOrderAsc);
171 
172         TimeMSec timeBegin = timeMSec();
173 
174         for (int listIdx = 0; listIdx < testMax; listIdx++)
175             CHECK(*(int *)lstFind(list, &listIdx) == listIdx);
176 
177         TEST_LOG_FMT("asc search completed in %ums", (unsigned int)(timeMSec() - timeBegin));
178 
179         // Search for all values with an descending sort
180         lstSort(list, sortOrderDesc);
181 
182         timeBegin = timeMSec();
183 
184         for (int listIdx = 0; listIdx < testMax; listIdx++)
185             CHECK(*(int *)lstFind(list, &listIdx) == listIdx);
186 
187         TEST_LOG_FMT("desc search completed in %ums", (unsigned int)(timeMSec() - timeBegin));
188     }
189 
190     // *****************************************************************************************************************************
191     if (testBegin("lstRemoveIdx()"))
192     {
193         CHECK(TEST_SCALE <= 10000);
194         int testMax = 1000000 * (int)TEST_SCALE;
195 
196         // Generate a large list of values (use int instead of string so there fewer allocations)
197         List *list = lstNewP(sizeof(int));
198 
199         for (int listIdx = 0; listIdx < testMax; listIdx++)
200             lstAdd(list, &listIdx);
201 
202         CHECK(lstSize(list) == (unsigned int)testMax);
203 
204         TEST_LOG_FMT("generated %d item list", testMax);
205 
206         // Remove all values from index 0
207         TimeMSec timeBegin = timeMSec();
208 
209         for (int listIdx = 0; listIdx < testMax; listIdx++)
210             lstRemoveIdx(list, 0);
211 
212         TEST_LOG_FMT("remove completed in %ums", (unsigned int)(timeMSec() - timeBegin));
213 
214         CHECK(lstEmpty(list));
215     }
216 
217     // *****************************************************************************************************************************
218     if (testBegin("iniLoad()"))
219     {
220         CHECK(TEST_SCALE <= 10000);
221 
222         String *iniStr = strNewZ("[section1]\n");
223         unsigned int iniMax = 100000 * (unsigned int)TEST_SCALE;
224 
225         for (unsigned int keyIdx = 0; keyIdx < iniMax; keyIdx++)
226             strCatFmt(iniStr, "key%u=\"value%u\"\n", keyIdx, keyIdx);
227 
228         TEST_LOG_FMT("ini size = %s, keys = %u", strZ(strSizeFormat(strSize(iniStr))), iniMax);
229 
230         TimeMSec timeBegin = timeMSec();
231         unsigned int iniTotal = 0;
232 
233         TEST_RESULT_VOID(iniLoad(ioBufferReadNew(BUFSTR(iniStr)), testIniLoadCountCallback, &iniTotal), "parse ini");
234         TEST_LOG_FMT("parse completed in %ums", (unsigned int)(timeMSec() - timeBegin));
235         TEST_RESULT_INT(iniTotal, iniMax, "    check ini total");
236     }
237 
238     // Build/load/save a larger manifest to test performance and memory usage. The default sizing is for a "typical" large cluster
239     // but this can be scaled to test larger cluster sizes.
240     // *****************************************************************************************************************************
241     if (testBegin("manifestNewBuild()/manifestNewLoad()/manifestSave()"))
242     {
243         CHECK(TEST_SCALE <= 1000000);
244 
245         // Create a storage driver to test manifest build with an arbitrary number of files
246         StorageTestManifestNewBuild driver =
247         {
248             .interface = storageInterfaceTestDummy,
249             .fileTotal = 100000 * (unsigned int)TEST_SCALE,
250         };
251 
252         driver.interface.info = storageTestManifestNewBuildInfo;
253         driver.interface.infoList = storageTestManifestNewBuildInfoList;
254 
255         const Storage *const storagePg = storageNew(
256             strIdFromZ(stringIdBit6, "test"), STRDEF("/pg"), 0, 0, false, NULL, &driver, driver.interface);
257 
258         // -------------------------------------------------------------------------------------------------------------------------
259         TEST_TITLE("build manifest");
260 
261         MemContext *testContext = memContextNew("test");
262         memContextKeep();
263         Manifest *manifest = NULL;
264         TimeMSec timeBegin = timeMSec();
265 
266         MEM_CONTEXT_BEGIN(testContext)
267         {
268             TEST_ASSIGN(manifest, manifestNewBuild(storagePg, PG_VERSION_91, 999999999, false, false, NULL, NULL), "build files");
269         }
270         MEM_CONTEXT_END();
271 
272         TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
273         TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
274 
275         TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, "   check file total");
276 
277         // -------------------------------------------------------------------------------------------------------------------------
278         TEST_TITLE("save manifest");
279 
280         Buffer *contentSave = bufNew(0);
281         timeBegin = timeMSec();
282 
283         manifestSave(manifest, ioBufferWriteNew(contentSave));
284 
285         TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
286 
287         memContextFree(testContext);
288 
289         // -------------------------------------------------------------------------------------------------------------------------
290         TEST_TITLE("load manifest");
291 
292         testContext = memContextNew("test");
293         memContextKeep();
294         timeBegin = timeMSec();
295 
296         MEM_CONTEXT_BEGIN(testContext)
297         {
298             manifest = manifestNewLoad(ioBufferReadNew(contentSave));
299         }
300         MEM_CONTEXT_END();
301 
302         TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
303         TEST_LOG_FMT("memory used %zu", memContextSize(testContext));
304 
305         TEST_RESULT_UINT(manifestFileTotal(manifest), driver.fileTotal, "   check file total");
306 
307         // -------------------------------------------------------------------------------------------------------------------------
308         TEST_TITLE("find all files");
309 
310         timeBegin = timeMSec();
311 
312         for (unsigned int fileIdx = 0; fileIdx < manifestFileTotal(manifest); fileIdx++)
313         {
314             const ManifestFile *file = manifestFile(manifest, fileIdx);
315             CHECK(file == manifestFileFind(manifest, file->name));
316         }
317 
318         TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
319     }
320 
321     // Make sure statistics collector performs well
322     // *****************************************************************************************************************************
323     if (testBegin("statistics collector"))
324     {
325         CHECK(TEST_SCALE <= 1000000);
326 
327         // Setup a list of stats to use for testing
328         #define TEST_STAT_TOTAL 100
329         String *statList[TEST_STAT_TOTAL];
330 
331         for (unsigned int statIdx = 0; statIdx < TEST_STAT_TOTAL; statIdx++)
332             statList[statIdx] = strNewFmt("STAT%u", statIdx);
333 
334         uint64_t runTotal = (uint64_t)TEST_SCALE * (uint64_t)100000;
335 
336         // -------------------------------------------------------------------------------------------------------------------------
337         TEST_TITLE_FMT("update %d stats %" PRIu64 " times", TEST_STAT_TOTAL, runTotal);
338 
339         TimeMSec timeBegin = timeMSec();
340 
341         for (uint64_t runIdx = 0; runIdx < runTotal; runIdx++)
342         {
343             for (unsigned int statIdx = 0; statIdx < TEST_STAT_TOTAL; statIdx++)
344                 statInc(statList[statIdx]);
345         }
346 
347         TEST_LOG_FMT("completed in %ums", (unsigned int)(timeMSec() - timeBegin));
348 
349         // -------------------------------------------------------------------------------------------------------------------------
350         TEST_TITLE("check stats have correct values");
351 
352         KeyValue *statKv = statToKv();
353 
354         for (unsigned int statIdx = 0; statIdx < TEST_STAT_TOTAL; statIdx++)
355         {
356             TEST_RESULT_UINT(
357                 varUInt64(kvGet(varKv(kvGet(statKv, VARSTR(statList[statIdx]))), STAT_VALUE_TOTAL_VAR)), runTotal,
358                 strZ(strNewFmt("check stat %u", statIdx)));
359         }
360     }
361 
362     FUNCTION_HARNESS_RETURN_VOID();
363 }
364