1 #include <stdio.h>
2 #include <stdlib.h>
3 #include <string.h>
4 #include <time.h>
5 #include <sys/time.h>
6 #include <errno.h>
7 #include <unistd.h>
8 #include <sys/stat.h>
9 #include <fcntl.h>
10 #include "fastcommon/shared_func.h"
11 #include "fastcommon/logger.h"
12 #include "test_types.h"
13 #include "common_func.h"
14 #include "dfs_func.h"
15
16 #define PROCESS_COUNT 10
17
18 typedef struct {
19 int file_type; //index
20 char *file_id;
21 } FileEntry;
22
23 typedef struct {
24 int bytes; //file size
25 char *filename;
26 int count; //total file count
27 int delete_count;
28 int success_count; //success upload count
29 int64_t time_used; //unit: ms
30 } TestFileInfo;
31
32 #ifdef DEBUG //for debug
33 static TestFileInfo files[FILE_TYPE_COUNT] = {
34 {5 * 1024, "5K", 1000 / PROCESS_COUNT, 0, 0, 0},
35 {50 * 1024, "50K", 2000 / PROCESS_COUNT, 0, 0, 0},
36 {200 * 1024, "200K", 1000 / PROCESS_COUNT, 0, 0, 0},
37 {1 * 1024 * 1024, "1M", 200 / PROCESS_COUNT, 0, 0, 0},
38 {10 * 1024 * 1024, "10M", 20 / PROCESS_COUNT, 0, 0, 0},
39 {100 * 1024 * 1024, "100M", 10 / PROCESS_COUNT, 0, 0, 0}
40 };
41
42 #else
43
44 static TestFileInfo files[FILE_TYPE_COUNT] = {
45 {5 * 1024, "5K", 1000000 / PROCESS_COUNT, 0, 0, 0},
46 {50 * 1024, "50K", 2000000 / PROCESS_COUNT, 0, 0, 0},
47 {200 * 1024, "200K", 1000000 / PROCESS_COUNT, 0, 0, 0},
48 {1 * 1024 * 1024, "1M", 200000 / PROCESS_COUNT, 0, 0, 0},
49 {10 * 1024 * 1024, "10M", 20000 / PROCESS_COUNT, 0, 0, 0},
50 {100 * 1024 * 1024, "100M", 1000 / PROCESS_COUNT, 0, 0, 0}
51 };
52
53 #endif
54
55 static StorageStat storages[MAX_STORAGE_COUNT];
56 static int storage_count = 0;
57 static time_t start_time;
58 static int total_count = 0;
59 static int success_count = 0;
60 static FILE *fpFail = NULL;
61
62 static int proccess_index = 0;
63 static int file_count = 0;
64 static FileEntry *file_entries = NULL;
65
66 static int load_file_ids();
67 static int test_init();
68 static int save_stats_by_overall();
69 static int save_stats_by_file_type();
70 static int save_stats_by_storage_ip();
71 static int add_to_storage_stat(const char *storage_ip, const int result, const int time_used);
72
main(int argc,char ** argv)73 int main(int argc, char **argv)
74 {
75 int result;
76 int i;
77 int file_type;
78 char storage_ip[IP_ADDRESS_SIZE];
79 char *conf_filename;
80 struct timeval tv_start;
81 struct timeval tv_end;
82 int time_used;
83
84 if (argc < 2)
85 {
86 printf("Usage: %s <process_index> [config_filename]\n", argv[0]);
87 return EINVAL;
88 }
89
90 log_init();
91 proccess_index = atoi(argv[1]);
92 if (proccess_index < 0 || proccess_index >= PROCESS_COUNT)
93 {
94 printf("Invalid proccess index: %d\n", proccess_index);
95 return EINVAL;
96 }
97
98 if (argc >= 3)
99 {
100 conf_filename = argv[2];
101 }
102 else
103 {
104 conf_filename = "/etc/fdfs/client.conf";
105 }
106
107 if ((result = load_file_ids()) != 0)
108 {
109 return result;
110 }
111
112 if ((result=test_init()) != 0)
113 {
114 return result;
115 }
116
117 if ((result=dfs_init(proccess_index, conf_filename)) != 0)
118 {
119 return result;
120 }
121
122 if ((result=my_daemon_init()) != 0)
123 {
124 return result;
125 }
126
127 /*
128 printf("file_count = %d\n", file_count);
129 printf("file_entries[0]=%s\n", file_entries[0].file_id);
130 printf("file_entries[%d]=%s\n", file_count-1, file_entries[file_count-1].file_id);
131 */
132
133 memset(&storages, 0, sizeof(storages));
134 memset(storage_ip, 0, sizeof(storage_ip));
135
136 start_time = time(NULL);
137 result = 0;
138 total_count = 0;
139 success_count = 0;
140 for (i=0; i<file_count; i++)
141 {
142 file_type = file_entries[i].file_type;
143 files[file_type].delete_count++;
144 total_count++;
145
146 gettimeofday(&tv_start, NULL);
147 *storage_ip = '\0';
148 result = delete_file(file_entries[i].file_id, storage_ip);
149 gettimeofday(&tv_end, NULL);
150 time_used = TIME_SUB_MS(tv_end, tv_start);
151 files[file_type].time_used += time_used;
152
153 add_to_storage_stat(storage_ip, result, time_used);
154 if (result == 0) //success
155 {
156 success_count++;
157 files[file_type].success_count++;
158 }
159 else //fail
160 {
161 fprintf(fpFail, "%d %d %s %s %d %d\n", (int)tv_end.tv_sec,
162 files[file_type].bytes, file_entries[i].file_id,
163 storage_ip, result, time_used);
164 fflush(fpFail);
165 }
166
167 if (total_count % 10000 == 0)
168 {
169 if ((result=save_stats_by_overall()) != 0)
170 {
171 break;
172 }
173 if ((result=save_stats_by_file_type()) != 0)
174 {
175 break;
176 }
177
178 if ((result=save_stats_by_storage_ip()) != 0)
179 {
180 break;
181 }
182 }
183 }
184
185 save_stats_by_overall();
186 save_stats_by_file_type();
187 save_stats_by_storage_ip();
188
189 fclose(fpFail);
190
191 dfs_destroy();
192
193 printf("proccess %d, time used: %ds\n", proccess_index, (int)(time(NULL) - start_time));
194 return result;
195 }
196
save_stats_by_file_type()197 static int save_stats_by_file_type()
198 {
199 int k;
200 char filename[64];
201 FILE *fp;
202
203 sprintf(filename, "%s.%d", STAT_FILENAME_BY_FILE_TYPE, proccess_index);
204 if ((fp=fopen(filename, "wb")) == NULL)
205 {
206 printf("open file %s fail, errno: %d, error info: %s\n",
207 filename, errno, STRERROR(errno));
208 return errno != 0 ? errno : EPERM;
209 }
210
211 fprintf(fp, "#file_type total_count success_count time_used(ms)\n");
212 for (k=0; k<FILE_TYPE_COUNT; k++)
213 {
214 fprintf(fp, "%s %d %d %"PRId64"\n", \
215 files[k].filename, files[k].delete_count, \
216 files[k].success_count, files[k].time_used);
217 }
218
219 fclose(fp);
220 return 0;
221 }
222
save_stats_by_storage_ip()223 static int save_stats_by_storage_ip()
224 {
225 int k;
226 char filename[64];
227 FILE *fp;
228
229 sprintf(filename, "%s.%d", STAT_FILENAME_BY_STORAGE_IP, proccess_index);
230 if ((fp=fopen(filename, "wb")) == NULL)
231 {
232 printf("open file %s fail, errno: %d, error info: %s\n",
233 filename, errno, STRERROR(errno));
234 return errno != 0 ? errno : EPERM;
235 }
236
237 fprintf(fp, "#ip_addr total_count success_count time_used(ms)\n");
238 for (k=0; k<storage_count; k++)
239 {
240 fprintf(fp, "%s %d %d %"PRId64"\n", \
241 storages[k].ip_addr, storages[k].total_count, \
242 storages[k].success_count, storages[k].time_used);
243 }
244
245 fclose(fp);
246 return 0;
247 }
248
save_stats_by_overall()249 static int save_stats_by_overall()
250 {
251 char filename[64];
252 FILE *fp;
253
254 sprintf(filename, "%s.%d", STAT_FILENAME_BY_OVERALL, proccess_index);
255 if ((fp=fopen(filename, "wb")) == NULL)
256 {
257 printf("open file %s fail, errno: %d, error info: %s\n",
258 filename, errno, STRERROR(errno));
259 return errno != 0 ? errno : EPERM;
260 }
261
262 fprintf(fp, "#total_count success_count time_used(s)\n");
263 fprintf(fp, "%d %d %d\n", total_count, success_count, (int)(time(NULL) - start_time));
264
265 fclose(fp);
266 return 0;
267 }
268
add_to_storage_stat(const char * storage_ip,const int result,const int time_used)269 static int add_to_storage_stat(const char *storage_ip, const int result, const int time_used)
270 {
271 StorageStat *pStorage;
272 StorageStat *pEnd;
273
274 pEnd = storages + storage_count;
275 for (pStorage=storages; pStorage<pEnd; pStorage++)
276 {
277 if (strcmp(storage_ip, pStorage->ip_addr) == 0)
278 {
279 break;
280 }
281 }
282
283 if (pStorage == pEnd) //not found
284 {
285 if (storage_count >= MAX_STORAGE_COUNT)
286 {
287 printf("storage_count %d >= %d\n", storage_count, MAX_STORAGE_COUNT);
288 return ENOSPC;
289 }
290
291 strcpy(pStorage->ip_addr, storage_ip);
292 storage_count++;
293 }
294
295 pStorage->time_used += time_used;
296 pStorage->total_count++;
297 if (result == 0)
298 {
299 pStorage->success_count++;
300 }
301
302 return 0;
303 }
304
get_file_type_index(const int file_bytes)305 static int get_file_type_index(const int file_bytes)
306 {
307 TestFileInfo *pFile;
308 TestFileInfo *pEnd;
309
310 pEnd = files + FILE_TYPE_COUNT;
311 for (pFile=files; pFile<pEnd; pFile++)
312 {
313 if (file_bytes == pFile->bytes)
314 {
315 return pFile - files;
316 }
317 }
318
319 return -1;
320 }
321
load_file_ids()322 static int load_file_ids()
323 {
324 int i;
325 int result;
326 int64_t file_size;
327 int bytes;
328 char filename[64];
329 char *file_buff;
330 char *p;
331 int nLineCount;
332 char *pStart;
333 char *pEnd;
334 char *pFind;
335
336 sprintf(filename, "upload/%s.%d", FILENAME_FILE_ID, proccess_index);
337 if ((result=getFileContent(filename, &file_buff, &file_size)) != 0)
338 {
339 printf("file: "__FILE__", line: %d, "
340 "getFileContent %s fail, errno: %d, error info: %s\n", __LINE__,
341 filename, errno, STRERROR(errno));
342
343 return result;
344 }
345
346 nLineCount = 0;
347 p = file_buff;
348 while (*p != '\0')
349 {
350 if (*p == '\n')
351 {
352 nLineCount++;
353 }
354
355 p++;
356 }
357
358 file_count = nLineCount;
359 if (file_count == 0)
360 {
361 printf("file: "__FILE__", line: %d, "
362 "file count == 0 in file %s\n", __LINE__, filename);
363 free(file_buff);
364 return EINVAL;
365 }
366
367 file_entries = (FileEntry *)malloc(sizeof(FileEntry) * file_count);
368 if (file_entries == NULL)
369 {
370 printf("file: "__FILE__", line: %d, "
371 "malloc %d bytes fail\n", __LINE__, \
372 (int)sizeof(FileEntry) * file_count);
373 free(file_buff);
374 return ENOMEM;
375 }
376 memset(file_entries, 0, sizeof(FileEntry) * file_count);
377
378 i = 0;
379 p = file_buff;
380 pStart = file_buff;
381 while (i < file_count)
382 {
383 if (*p == '\n')
384 {
385 *p = '\0';
386 pFind = strchr(pStart, ' ');
387 if (pFind == NULL)
388 {
389 printf("file: "__FILE__", line: %d, "
390 "can't find ' ' in file %s\n", __LINE__, filename);
391 result = EINVAL;
392 break;
393 }
394
395 pFind++;
396 pEnd = strchr(pFind, ' ');
397 if (pEnd == NULL)
398 {
399 printf("file: "__FILE__", line: %d, "
400 "can't find ' ' in file %s\n", __LINE__, filename);
401 result = EINVAL;
402 break;
403 }
404 *pEnd = '\0';
405 bytes = atoi(pFind);
406
407 pFind = pEnd + 1; //skip space
408 pEnd = strchr(pFind, ' ');
409 if (pEnd == NULL)
410 {
411 printf("file: "__FILE__", line: %d, "
412 "can't find ' ' in file %s\n", __LINE__, filename);
413 result = EINVAL;
414 break;
415 }
416 *pEnd = '\0';
417
418 file_entries[i].file_type = get_file_type_index(bytes);
419 if (file_entries[i].file_type < 0)
420 {
421 printf("file: "__FILE__", line: %d, "
422 "invalid file bytes: %d in file %s\n", __LINE__, bytes, filename);
423 result = EINVAL;
424 break;
425 }
426
427 file_entries[i].file_id = strdup(pFind);
428 if (file_entries[i].file_id == NULL)
429 {
430 printf("file: "__FILE__", line: %d, "
431 "malloc %d bytes fail\n", __LINE__, \
432 (int)strlen(pFind) + 1);
433 result = ENOMEM;
434 break;
435 }
436
437 i++;
438 pStart = ++p;
439 }
440 else
441 {
442 p++;
443 }
444 }
445
446 free(file_buff);
447
448 return result;
449 }
450
test_init()451 static int test_init()
452 {
453 char filename[64];
454
455 if (access("delete", 0) != 0 && mkdir("delete", 0755) != 0)
456 {
457 }
458
459 if (chdir("delete") != 0)
460 {
461 printf("chdir fail, errno: %d, error info: %s\n", errno, STRERROR(errno));
462 return errno != 0 ? errno : EPERM;
463 }
464
465 sprintf(filename, "%s.%d", FILENAME_FAIL, proccess_index);
466 if ((fpFail=fopen(filename, "wb")) == NULL)
467 {
468 printf("open file %s fail, errno: %d, error info: %s\n",
469 filename, errno, STRERROR(errno));
470 return errno != 0 ? errno : EPERM;
471 }
472
473 return 0;
474 }
475
476