1 /* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
2
3 #include "lib.h"
4 #include "array.h"
5 #include "mail-index-private.h"
6
mail_index_map_parse_extensions(struct mail_index_map * map)7 int mail_index_map_parse_extensions(struct mail_index_map *map)
8 {
9 struct mail_index *index = map->index;
10 const struct mail_index_ext_header *ext_hdr;
11 unsigned int i, old_count, offset;
12 const char *name, *error;
13 uint32_t ext_id, ext_map_idx, ext_offset;
14
15 /* extension headers always start from 64bit offsets, so if base header
16 doesn't happen to be 64bit aligned we'll skip some bytes */
17 offset = MAIL_INDEX_HEADER_SIZE_ALIGN(map->hdr.base_header_size);
18 if (offset >= map->hdr.header_size && map->extension_pool == NULL) {
19 /* nothing to do, skip allocations and all */
20 return 0;
21 }
22
23 old_count = array_count(&index->extensions);
24 mail_index_map_init_extbufs(map, old_count + 5);
25
26 ext_id = (uint32_t)-1;
27 for (i = 0; i < old_count; i++)
28 array_push_back(&map->ext_id_map, &ext_id);
29
30 for (i = 0; offset < map->hdr.header_size; i++) {
31 ext_offset = offset;
32
33 if (mail_index_map_ext_get_next(map, &offset,
34 &ext_hdr, &name) < 0) {
35 mail_index_set_error(index, "Corrupted index file %s: "
36 "Header extension #%d (%s) goes outside header",
37 index->filepath, i, name);
38 return -1;
39 }
40
41 if (mail_index_map_ext_hdr_check(&map->hdr, ext_hdr,
42 name, &error) < 0) {
43 mail_index_set_error(index, "Corrupted index file %s: "
44 "Broken extension #%d (%s): %s",
45 index->filepath, i, name, error);
46 return -1;
47 }
48 if (mail_index_map_lookup_ext(map, name, &ext_map_idx)) {
49 mail_index_set_error(index, "Corrupted index file %s: "
50 "Duplicate header extension %s",
51 index->filepath, name);
52 return -1;
53 }
54
55 (void)mail_index_map_register_ext(map, name, ext_offset, ext_hdr);
56 }
57 return 0;
58 }
59
mail_index_map_parse_keywords(struct mail_index_map * map)60 int mail_index_map_parse_keywords(struct mail_index_map *map)
61 {
62 struct mail_index *index = map->index;
63 const struct mail_index_ext *ext;
64 const struct mail_index_keyword_header *kw_hdr;
65 const struct mail_index_keyword_header_rec *kw_rec;
66 const char *name;
67 unsigned int i, name_area_end_offset, old_count;
68 uint32_t idx;
69
70 if (!mail_index_map_lookup_ext(map, MAIL_INDEX_EXT_KEYWORDS, &idx)) {
71 if (array_is_created(&map->keyword_idx_map))
72 array_clear(&map->keyword_idx_map);
73 return 0;
74 }
75 ext = array_idx(&map->extensions, idx);
76
77 /* Extension header contains:
78 - struct mail_index_keyword_header
79 - struct mail_index_keyword_header_rec * keywords_count
80 - const char names[] * keywords_count
81
82 The mail_index_keyword_header_rec are rather unnecessary nowadays.
83 They were originally an optimization when dovecot.index header kept
84 changing constantly, but nowadays the changes are usually read from
85 the .log changes, so re-reading dovecot.index header isn't common.
86 In a later version we could even remove it.
87 */
88 i_assert(ext->hdr_offset < map->hdr.header_size);
89 kw_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset);
90 kw_rec = (const void *)(kw_hdr + 1);
91 name = (const char *)(kw_rec + kw_hdr->keywords_count);
92
93 old_count = !array_is_created(&map->keyword_idx_map) ? 0 :
94 array_count(&map->keyword_idx_map);
95
96 /* make sure the header is valid */
97 if (kw_hdr->keywords_count < old_count) {
98 mail_index_set_error(index, "Corrupted index file %s: "
99 "Keywords removed unexpectedly",
100 index->filepath);
101 return -1;
102 }
103
104 if ((size_t)(name - (const char *)kw_hdr) > ext->hdr_size) {
105 mail_index_set_error(index, "Corrupted index file %s: "
106 "keywords_count larger than header size",
107 index->filepath);
108 return -1;
109 }
110
111 name_area_end_offset = (const char *)kw_hdr + ext->hdr_size - name;
112 for (i = 0; i < kw_hdr->keywords_count; i++) {
113 if (kw_rec[i].name_offset > name_area_end_offset) {
114 mail_index_set_error(index, "Corrupted index file %s: "
115 "name_offset points outside allocated header",
116 index->filepath);
117 return -1;
118 }
119 }
120 if (name[name_area_end_offset-1] != '\0') {
121 mail_index_set_error(index, "Corrupted index file %s: "
122 "Keyword header doesn't end with NUL",
123 index->filepath);
124 return -1;
125 }
126
127 /* create file -> index mapping */
128 if (!array_is_created(&map->keyword_idx_map))
129 i_array_init(&map->keyword_idx_map, kw_hdr->keywords_count);
130
131 size_t name_offset = 0;
132 /* Check that existing headers are still the same. */
133 for (i = 0; i < array_count(&map->keyword_idx_map); i++) {
134 const char *keyword = name + kw_rec[i].name_offset;
135 const unsigned int *old_idx;
136 unsigned int kw_idx;
137
138 if (kw_rec[i].name_offset != name_offset) {
139 /* this shouldn't happen, but the old code didn't check
140 for this so for safety keep this as a warning. */
141 e_warning(index->event,
142 "Corrupted index file %s: "
143 "Mismatching keyword name_offset",
144 index->filepath);
145 }
146 name_offset += strlen(keyword) + 1;
147
148 old_idx = array_idx(&map->keyword_idx_map, i);
149 if (!mail_index_keyword_lookup(index, keyword, &kw_idx) ||
150 kw_idx != *old_idx) {
151 mail_index_set_error(index, "Corrupted index file %s: "
152 "Keywords changed unexpectedly",
153 index->filepath);
154 return -1;
155 }
156 }
157
158 /* Register the newly seen keywords */
159 i = array_count(&map->keyword_idx_map);
160 for (; i < kw_hdr->keywords_count; i++) {
161 const char *keyword = name + kw_rec[i].name_offset;
162 unsigned int kw_idx;
163
164 if (kw_rec[i].name_offset != name_offset) {
165 /* this shouldn't happen, but the old code didn't check
166 for this so for safety keep this as a warning. */
167 e_warning(index->event,
168 "Corrupted index file %s: "
169 "Mismatching keyword name_offset",
170 index->filepath);
171 }
172 name_offset += strlen(keyword) + 1;
173
174 if (*keyword == '\0') {
175 mail_index_set_error(index, "Corrupted index file %s: "
176 "Empty keyword name in header",
177 index->filepath);
178 return -1;
179 }
180 mail_index_keyword_lookup_or_create(index, keyword, &kw_idx);
181 array_push_back(&map->keyword_idx_map, &kw_idx);
182 }
183 return 0;
184 }
185
mail_index_check_header_compat(struct mail_index * index,const struct mail_index_header * hdr,uoff_t file_size,const char ** error_r)186 bool mail_index_check_header_compat(struct mail_index *index,
187 const struct mail_index_header *hdr,
188 uoff_t file_size, const char **error_r)
189 {
190 enum mail_index_header_compat_flags compat_flags = 0;
191
192 #ifndef WORDS_BIGENDIAN
193 compat_flags |= MAIL_INDEX_COMPAT_LITTLE_ENDIAN;
194 #endif
195
196 if (hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
197 /* major version change */
198 *error_r = t_strdup_printf("Major version changed (%u != %u)",
199 hdr->major_version, MAIL_INDEX_MAJOR_VERSION);
200 return FALSE;
201 }
202 if ((hdr->flags & MAIL_INDEX_HDR_FLAG_CORRUPTED) != 0) {
203 /* we've already complained about it */
204 *error_r = "Header's corrupted flag is set";
205 return FALSE;
206 }
207
208 if (hdr->compat_flags != compat_flags) {
209 /* architecture change */
210 *error_r = "CPU architecture changed";
211 return FALSE;
212 }
213
214 if (hdr->base_header_size < MAIL_INDEX_HEADER_MIN_SIZE ||
215 hdr->header_size < hdr->base_header_size) {
216 *error_r = t_strdup_printf(
217 "Corrupted header sizes (base %u, full %u)",
218 hdr->base_header_size, hdr->header_size);
219 return FALSE;
220 }
221 if (hdr->header_size > file_size) {
222 *error_r = t_strdup_printf(
223 "Header size is larger than file (%u > %"PRIuUOFF_T")",
224 hdr->header_size, file_size);
225 return FALSE;
226 }
227
228 if (hdr->indexid != index->indexid) {
229 if (index->indexid != 0) {
230 mail_index_set_error(index, "Index file %s: "
231 "indexid changed: %u -> %u",
232 index->filepath, index->indexid,
233 hdr->indexid);
234 }
235 index->indexid = hdr->indexid;
236 mail_transaction_log_indexid_changed(index->log);
237 }
238 return TRUE;
239 }
240
mail_index_map_clear_recent_flags(struct mail_index_map * map)241 static void mail_index_map_clear_recent_flags(struct mail_index_map *map)
242 {
243 struct mail_index_record *rec;
244 uint32_t seq;
245
246 for (seq = 1; seq <= map->hdr.messages_count; seq++) {
247 rec = MAIL_INDEX_REC_AT_SEQ(map, seq);
248 rec->flags &= ENUM_NEGATE(MAIL_RECENT);
249 }
250 }
251
mail_index_map_check_header(struct mail_index_map * map,const char ** error_r)252 int mail_index_map_check_header(struct mail_index_map *map,
253 const char **error_r)
254 {
255 struct mail_index *index = map->index;
256 const struct mail_index_header *hdr = &map->hdr;
257
258 if (!mail_index_check_header_compat(index, hdr, UOFF_T_MAX, error_r))
259 return 0;
260
261 /* following some extra checks that only take a bit of CPU */
262 if (hdr->record_size < sizeof(struct mail_index_record)) {
263 *error_r = t_strdup_printf(
264 "record_size too small (%u < %zu)",
265 hdr->record_size, sizeof(struct mail_index_record));
266 return -1;
267 }
268
269 if (hdr->uid_validity == 0 && hdr->next_uid != 1) {
270 *error_r = t_strdup_printf(
271 "uidvalidity=0, but next_uid=%u", hdr->next_uid);
272 return 0;
273 }
274 if (hdr->next_uid == 0) {
275 *error_r = "next_uid=0";
276 return 0;
277 }
278 if (hdr->messages_count > map->rec_map->records_count) {
279 *error_r = t_strdup_printf(
280 "messages_count is higher in header than record map (%u > %u)",
281 hdr->messages_count, map->rec_map->records_count);
282 return 0;
283 }
284
285 if (hdr->seen_messages_count > hdr->messages_count) {
286 *error_r = t_strdup_printf(
287 "seen_messages_count %u > messages_count %u",
288 hdr->seen_messages_count, hdr->messages_count);
289 return 0;
290 }
291 if (hdr->deleted_messages_count > hdr->messages_count) {
292 *error_r = t_strdup_printf(
293 "deleted_messages_count %u > messages_count %u",
294 hdr->deleted_messages_count, hdr->messages_count);
295 return 0;
296 }
297 switch (hdr->minor_version) {
298 case 0:
299 /* upgrade silently from v1.0 */
300 map->hdr.unused_old_recent_messages_count = 0;
301 if (hdr->first_recent_uid == 0)
302 map->hdr.first_recent_uid = 1;
303 if (index->need_recreate == NULL)
304 index->need_recreate = i_strdup("Upgrading from index version 1.0");
305 /* fall through */
306 case 1:
307 /* pre-v1.1.rc6: make sure the \Recent flags are gone */
308 mail_index_map_clear_recent_flags(map);
309 map->hdr.minor_version = MAIL_INDEX_MINOR_VERSION;
310 /* fall through */
311 case 2:
312 /* pre-v2.2 (although should have been done in v2.1 already):
313 make sure the old unused fields are cleared */
314 map->hdr.unused_old_sync_size_part1 = 0;
315 map->hdr.log2_rotate_time = 0;
316 map->hdr.last_temp_file_scan = 0;
317 }
318 if (hdr->first_recent_uid == 0) {
319 *error_r = "first_recent_uid=0";
320 return 0;
321 }
322 if (hdr->first_recent_uid > hdr->next_uid) {
323 *error_r = t_strdup_printf(
324 "first_recent_uid %u > next_uid %u",
325 hdr->first_recent_uid, hdr->next_uid);
326 return 0;
327 }
328 if (hdr->first_unseen_uid_lowwater > hdr->next_uid) {
329 *error_r = t_strdup_printf(
330 "first_unseen_uid_lowwater %u > next_uid %u",
331 hdr->first_unseen_uid_lowwater, hdr->next_uid);
332 return 0;
333 }
334 if (hdr->first_deleted_uid_lowwater > hdr->next_uid) {
335 *error_r = t_strdup_printf(
336 "first_deleted_uid_lowwater %u > next_uid %u",
337 hdr->first_deleted_uid_lowwater, hdr->next_uid);
338 return 0;
339 }
340
341 if (hdr->messages_count > 0) {
342 /* last message's UID must be smaller than next_uid.
343 also make sure it's not zero. */
344 const struct mail_index_record *rec;
345
346 rec = MAIL_INDEX_REC_AT_SEQ(map, hdr->messages_count);
347 if (rec->uid == 0) {
348 *error_r = "last message has uid=0";
349 return -1;
350 }
351 if (rec->uid >= hdr->next_uid) {
352 *error_r = t_strdup_printf(
353 "last message uid %u >= next_uid %u",
354 rec->uid, hdr->next_uid);
355 return 0;
356 }
357 }
358 return 1;
359 }
360