1 #include <config.h>
2
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <ctype.h>
6
7 #ifdef DMALLOC
8 #include <dmalloc.h>
9 #endif
10
11 #include "suck_config.h"
12 #include "suck.h"
13 #include "both.h"
14 #include "dedupe.h"
15 #include "suckutils.h"
16 #include "phrases.h"
17 #include "timer.h"
18 #include "ssort.h"
19
20 /* this is almost a duplicate of chkhistory() routine. */
21
dedupe_list(PMaster master)22 void dedupe_list(PMaster master) {
23 /* throw the items into an array, qsort em, and then just compare items in a row to dedupe */
24
25 PList *array, ptr, curr, prev;
26 int i=0, nrfound=0;
27
28 print_phrases(master->msgs, dedupe_phrases[0], NULL);
29 fflush(master->msgs); /* so msg gets printed */
30
31 TimerFunc(TIMER_START, 0L, NULL);
32
33 /* 1. throw em into the array */
34 if((array = calloc(master->nritems, sizeof(PList))) == NULL) {
35 error_log(ERRLOG_REPORT, dedupe_phrases[1], NULL);
36 }
37 else {
38 ptr = master->head;
39 i = 0;
40 while(ptr != NULL) {
41 array[i] = ptr;
42 i++;
43 ptr = ptr->next;
44 }
45
46 TimerFunc(TIMER_START, 0L, NULL) ;
47
48 /* step 2, sort em */
49 ssort(array, master->nritems, 1); /* start with depth of 1 cause all start with < */
50
51 /* TimerFunc(TIMER_TIMEONLY, 0L, master->msgs); */
52
53
54 /* step 3, mark dupes */
55 for(i=0;i<(master->nritems-1);i++) {
56 if(cmp_msgid(array[i]->msgnr,array[i+1]->msgnr) == TRUE) {
57 /* if this is one we've already downloaded, or one we've already marked as dupe */
58 /* delete the other one */
59 if(array[i]->delete == TRUE || array[i]->downloaded == TRUE) {
60 array[i+1]->delete = TRUE;
61 }
62 else {
63 array[i]->delete = TRUE;
64 }
65
66 nrfound++;
67 }
68 }
69 /* step 4, delete em */
70 curr = master->head;
71 prev = NULL;
72 while(curr != NULL) {
73 if( curr->delete == TRUE) {
74 /* nuke it */
75 master->nritems--;
76 if(prev == NULL) {
77 /* remove master node */
78 master->head = curr->next;
79 free_one_node(curr);
80 curr = master->head;
81 }
82 else {
83 prev->next = curr->next;
84 free_one_node(curr);
85 curr = prev->next;
86 }
87 }
88 else {
89 prev = curr;
90 curr = curr->next;
91 }
92 }
93
94 /* all done free up mem */
95 free(array);
96 }
97 TimerFunc(TIMER_TIMEONLY, 0l, master->msgs);
98
99 print_phrases(master->msgs, dedupe_phrases[2], str_int(master->nritems), str_int(nrfound), NULL);
100 }
101
102