1 /*
2 * Copyright (c) 2007-2011, Novell Inc.
3 *
4 * This program is licensed under the BSD license, read LICENSE.BSD
5 * for further information
6 */
7
8 /*
9 * repo_write.c
10 *
11 * Write Repo data out to a file in solv format
12 *
13 * See doc/README.format for a description
14 * of the binary file format
15 *
16 */
17
18 #include <sys/types.h>
19 #include <limits.h>
20 #include <fcntl.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <assert.h>
25 #include <errno.h>
26
27 #include "pool.h"
28 #include "util.h"
29 #include "repo_write.h"
30 #include "repopage.h"
31
32 /*------------------------------------------------------------------*/
33 /* Id map optimizations */
34
35 typedef struct needid {
36 Id need;
37 Id map;
38 } NeedId;
39
40
41 #define NEEDIDOFF(id) (ISRELDEP(id) ? (needid[0].map + GETRELID(id)) : id)
42
43 /*
44 * increment need Id
45 * idarray: array of Ids, ID_NULL terminated
46 * needid: array of Id->NeedId
47 *
48 * return size of array (including trailing zero)
49 *
50 */
51
52 static inline void
incneedid(Id id,NeedId * needid)53 incneedid(Id id, NeedId *needid)
54 {
55 needid[NEEDIDOFF(id)].need++;
56 }
57
58 static int
incneedidarray(Id * idarray,NeedId * needid)59 incneedidarray(Id *idarray, NeedId *needid)
60 {
61 Id id;
62 int n = 0;
63
64 while ((id = *idarray++) != 0)
65 {
66 n++;
67 needid[NEEDIDOFF(id)].need++;
68 }
69 return n + 1;
70 }
71
72
73 /*
74 *
75 */
76
77 static int
needid_cmp_need(const void * ap,const void * bp,void * dp)78 needid_cmp_need(const void *ap, const void *bp, void *dp)
79 {
80 const NeedId *a = ap;
81 const NeedId *b = bp;
82 int r;
83 r = b->need - a->need;
84 if (r)
85 return r;
86 return a->map - b->map;
87 }
88
89 static int
needid_cmp_need_s(const void * ap,const void * bp,void * dp)90 needid_cmp_need_s(const void *ap, const void *bp, void *dp)
91 {
92 const NeedId *a = ap;
93 const NeedId *b = bp;
94 Stringpool *spool = dp;
95 const char *as;
96 const char *bs;
97
98 int r;
99 r = b->need - a->need;
100 if (r)
101 return r;
102 as = spool->stringspace + spool->strings[a->map];
103 bs = spool->stringspace + spool->strings[b->map];
104 return strcmp(as, bs);
105 }
106
107
108 /*------------------------------------------------------------------*/
109 /* output helper routines, used for writing the header */
110 /* (the data itself is accumulated in memory and written with
111 * write_blob) */
112
113 /*
114 * unsigned 32-bit
115 */
116
117 static void
write_u32(Repodata * data,unsigned int x)118 write_u32(Repodata *data, unsigned int x)
119 {
120 FILE *fp = data->fp;
121 if (data->error)
122 return;
123 if (putc(x >> 24, fp) == EOF ||
124 putc(x >> 16, fp) == EOF ||
125 putc(x >> 8, fp) == EOF ||
126 putc(x, fp) == EOF)
127 {
128 data->error = pool_error(data->repo->pool, -1, "write error u32: %s", strerror(errno));
129 }
130 }
131
132
133 /*
134 * unsigned 8-bit
135 */
136
137 static void
write_u8(Repodata * data,unsigned int x)138 write_u8(Repodata *data, unsigned int x)
139 {
140 if (data->error)
141 return;
142 if (putc(x, data->fp) == EOF)
143 {
144 data->error = pool_error(data->repo->pool, -1, "write error u8: %s", strerror(errno));
145 }
146 }
147
148 /*
149 * data blob
150 */
151
152 static void
write_blob(Repodata * data,void * blob,int len)153 write_blob(Repodata *data, void *blob, int len)
154 {
155 if (data->error)
156 return;
157 if (len && fwrite(blob, len, 1, data->fp) != 1)
158 {
159 data->error = pool_error(data->repo->pool, -1, "write error blob: %s", strerror(errno));
160 }
161 }
162
163 /*
164 * Id
165 */
166
167 static void
write_id(Repodata * data,Id x)168 write_id(Repodata *data, Id x)
169 {
170 FILE *fp = data->fp;
171 if (data->error)
172 return;
173 if (x >= (1 << 14))
174 {
175 if (x >= (1 << 28))
176 putc((x >> 28) | 128, fp);
177 if (x >= (1 << 21))
178 putc((x >> 21) | 128, fp);
179 putc((x >> 14) | 128, fp);
180 }
181 if (x >= (1 << 7))
182 putc((x >> 7) | 128, fp);
183 if (putc(x & 127, fp) == EOF)
184 {
185 data->error = pool_error(data->repo->pool, -1, "write error id: %s", strerror(errno));
186 }
187 }
188
189 static inline void
write_str(Repodata * data,const char * str)190 write_str(Repodata *data, const char *str)
191 {
192 if (data->error)
193 return;
194 if (fputs(str, data->fp) == EOF || putc(0, data->fp) == EOF)
195 {
196 data->error = pool_error(data->repo->pool, -1, "write error str: %s", strerror(errno));
197 }
198 }
199
200 /*
201 * Array of Ids
202 */
203
204 static void
write_idarray(Repodata * data,Pool * pool,NeedId * needid,Id * ids)205 write_idarray(Repodata *data, Pool *pool, NeedId *needid, Id *ids)
206 {
207 Id id;
208 if (!ids)
209 return;
210 if (!*ids)
211 {
212 write_u8(data, 0);
213 return;
214 }
215 for (;;)
216 {
217 id = *ids++;
218 if (needid)
219 id = needid[NEEDIDOFF(id)].need;
220 if (id >= 64)
221 id = (id & 63) | ((id & ~63) << 1);
222 if (!*ids)
223 {
224 write_id(data, id);
225 return;
226 }
227 write_id(data, id | 64);
228 }
229 }
230
231 struct extdata {
232 unsigned char *buf;
233 int len;
234 };
235
236 struct cbdata {
237 Pool *pool;
238 Repo *repo;
239 Repodata *target;
240
241 Stringpool *ownspool;
242 Dirpool *owndirpool;
243 int clonepool; /* are the pool ids cloned into ownspool? */
244
245 Id *keymap; /* keymap for this repodata */
246
247 NeedId *needid;
248
249 Id *schema; /* schema construction space */
250 Id *sp; /* pointer in above */
251
252 Id *subschemata;
253 int nsubschemata;
254 int current_sub;
255
256 struct extdata *extdata;
257
258 Id *dirused;
259
260 Id vstart; /* offset of key in vertical data */
261
262 Id maxdata;
263 Id lastlen;
264
265 int doingsolvables; /* working on solvables data */
266 int filelistmode;
267
268 Id lastdirid; /* last dir id seen in this repodata */
269 Id lastdirid_own; /* last dir id put in own pool */
270 };
271
272 #define NEEDID_BLOCK 1023
273 #define SCHEMATA_BLOCK 31
274 #define EXTDATA_BLOCK 4095
275
276 static inline void
data_addid(struct extdata * xd,Id sx)277 data_addid(struct extdata *xd, Id sx)
278 {
279 unsigned int x = (unsigned int)sx;
280 unsigned char *dp;
281
282 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
283 dp = xd->buf + xd->len;
284
285 if (x >= (1 << 14))
286 {
287 if (x >= (1 << 28))
288 *dp++ = (x >> 28) | 128;
289 if (x >= (1 << 21))
290 *dp++ = (x >> 21) | 128;
291 *dp++ = (x >> 14) | 128;
292 }
293 if (x >= (1 << 7))
294 *dp++ = (x >> 7) | 128;
295 *dp++ = x & 127;
296 xd->len = dp - xd->buf;
297 }
298
299 static inline void
data_addideof(struct extdata * xd,Id sx,int eof)300 data_addideof(struct extdata *xd, Id sx, int eof)
301 {
302 unsigned int x = (unsigned int)sx;
303 unsigned char *dp;
304
305 xd->buf = solv_extend(xd->buf, xd->len, 5, 1, EXTDATA_BLOCK);
306 dp = xd->buf + xd->len;
307
308 if (x >= (1 << 13))
309 {
310 if (x >= (1 << 27))
311 *dp++ = (x >> 27) | 128;
312 if (x >= (1 << 20))
313 *dp++ = (x >> 20) | 128;
314 *dp++ = (x >> 13) | 128;
315 }
316 if (x >= (1 << 6))
317 *dp++ = (x >> 6) | 128;
318 *dp++ = eof ? (x & 63) : (x & 63) | 64;
319 xd->len = dp - xd->buf;
320 }
321
322 static inline int
data_addideof_len(Id sx)323 data_addideof_len(Id sx)
324 {
325 unsigned int x = (unsigned int)sx;
326 if (x >= (1 << 13))
327 {
328 if (x >= (1 << 27))
329 return 5;
330 return x >= (1 << 20) ? 4 : 3;
331 }
332 return x >= (1 << 6) ? 2 : 1;
333 }
334
335 static void
data_addid64(struct extdata * xd,unsigned int x,unsigned int hx)336 data_addid64(struct extdata *xd, unsigned int x, unsigned int hx)
337 {
338 if (hx)
339 {
340 if (hx > 7)
341 {
342 data_addid(xd, (Id)(hx >> 3));
343 xd->buf[xd->len - 1] |= 128;
344 hx &= 7;
345 }
346 data_addid(xd, (Id)(x | 0x80000000));
347 xd->buf[xd->len - 5] = (x >> 28) | (hx << 4) | 128;
348 }
349 else
350 data_addid(xd, (Id)x);
351 }
352
353 #define USE_REL_IDARRAY
354 #ifdef USE_REL_IDARRAY
355
356 static int
cmp_ids(const void * pa,const void * pb,void * dp)357 cmp_ids(const void *pa, const void *pb, void *dp)
358 {
359 Id a = *(Id *)pa;
360 Id b = *(Id *)pb;
361 return a - b;
362 }
363
364 static void
data_adddepids(struct extdata * xd,Pool * pool,NeedId * needid,Id * ids,Id marker)365 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
366 {
367 int len, i;
368 Id lids[64], *sids;
369 Id id, old;
370
371 if (!ids)
372 return;
373 if (!*ids)
374 {
375 data_addid(xd, 0);
376 return;
377 }
378 for (len = 0; len < 64 && ids[len]; len++)
379 {
380 Id id = ids[len];
381 if (needid)
382 id = needid[NEEDIDOFF(id)].need;
383 lids[len] = id;
384 }
385 if (ids[len])
386 {
387 for (i = len + 1; ids[i]; i++)
388 ;
389 sids = solv_malloc2(i, sizeof(Id));
390 memcpy(sids, lids, 64 * sizeof(Id));
391 for (; ids[len]; len++)
392 {
393 Id id = ids[len];
394 if (needid)
395 id = needid[NEEDIDOFF(id)].need;
396 sids[len] = id;
397 }
398 }
399 else
400 sids = lids;
401
402 /* That bloody solvable:prereqmarker needs to stay in position :-( */
403 if (needid)
404 marker = needid[marker].need;
405 for (i = 0; i < len; i++)
406 if (sids[i] == marker)
407 break;
408 if (i > 1)
409 solv_sort(sids, i, sizeof(Id), cmp_ids, 0);
410 if ((len - i) > 2)
411 solv_sort(sids + i + 1, len - i - 1, sizeof(Id), cmp_ids, 0);
412
413 old = 0;
414
415 /* The differencing above produces many runs of ones and twos. I tried
416 fairly elaborate schemes to RLE those, but they give only very mediocre
417 improvements in compression, as coding the escapes costs quite some
418 space. Even if they are coded only as bits in IDs. The best improvement
419 was about 2.7% for the whole .solv file. It's probably better to
420 invest some complexity into sharing idarrays, than RLEing. */
421 for (i = 0; i < len - 1; i++)
422 {
423 id = sids[i];
424 /* Ugly PREREQ handling. A "difference" of 0 is the prereq marker,
425 hence all real differences are offsetted by 1. Otherwise we would
426 have to handle negative differences, which would cost code space for
427 the encoding of the sign. We loose the exact mapping of prereq here,
428 but we know the result, so we can recover from that in the reader. */
429 if (id == marker)
430 id = old = 0;
431 else
432 {
433 id = id - old + 1;
434 old = sids[i];
435 }
436 /* XXX If difference is zero we have multiple equal elements,
437 we might want to skip writing them out. */
438 data_addideof(xd, id, 0);
439 }
440 id = sids[i];
441 if (id == marker)
442 id = 0;
443 else
444 id = id - old + 1;
445 data_addideof(xd, id, 1);
446 if (sids != lids)
447 solv_free(sids);
448 }
449
450 #else
451
452 static void
data_adddepids(struct extdata * xd,Pool * pool,NeedId * needid,Id * ids,Id marker)453 data_adddepids(struct extdata *xd, Pool *pool, NeedId *needid, Id *ids, Id marker)
454 {
455 Id id;
456 if (!ids || !*ids)
457 {
458 data_addid(xd, 0);
459 return;
460 }
461 while ((id = *ids++) != 0)
462 {
463 if (needid)
464 id = needid[NEEDIDOFF(id)].need;
465 data_addideof(xd, id, *ids ? 0 : 1);
466 }
467 }
468
469 #endif
470
471 static inline void
data_addblob(struct extdata * xd,unsigned char * blob,int len)472 data_addblob(struct extdata *xd, unsigned char *blob, int len)
473 {
474 xd->buf = solv_extend(xd->buf, xd->len, len, 1, EXTDATA_BLOCK);
475 memcpy(xd->buf + xd->len, blob, len);
476 xd->len += len;
477 }
478
479 /* grow needid array so that it contains the specified id */
480 static void
grow_needid(struct cbdata * cbdata,Id id)481 grow_needid(struct cbdata *cbdata, Id id)
482 {
483 int oldoff = cbdata->needid[0].map;
484 int newoff = (id + 1 + NEEDID_BLOCK) & ~NEEDID_BLOCK;
485 int nrels = cbdata->pool->nrels;
486 cbdata->needid = solv_realloc2(cbdata->needid, newoff + nrels, sizeof(NeedId));
487 if (nrels)
488 memmove(cbdata->needid + newoff, cbdata->needid + oldoff, nrels * sizeof(NeedId));
489 memset(cbdata->needid + oldoff, 0, (newoff - oldoff) * sizeof(NeedId));
490 cbdata->needid[0].map = newoff;
491 }
492
493 static Id
putinownpool(struct cbdata * cbdata,Repodata * data,Id id)494 putinownpool(struct cbdata *cbdata, Repodata *data, Id id)
495 {
496 Stringpool *ss = data->localpool ? &data->spool : &cbdata->pool->ss;
497 const char *str = stringpool_id2str(ss, id);
498 id = stringpool_str2id(cbdata->ownspool, str, 1);
499 if (id >= cbdata->needid[0].map)
500 grow_needid(cbdata, id);
501 return id;
502 }
503
504 static Id
putinowndirpool_slow(struct cbdata * cbdata,Repodata * data,Dirpool * dp,Id dir)505 putinowndirpool_slow(struct cbdata *cbdata, Repodata *data, Dirpool *dp, Id dir)
506 {
507 Id compid, parent;
508
509 parent = dirpool_parent(dp, dir);
510 if (parent)
511 parent = putinowndirpool_slow(cbdata, data, dp, parent);
512 compid = dirpool_compid(dp, dir);
513 if (cbdata->ownspool && compid > 1 && (!cbdata->clonepool || data->localpool))
514 compid = putinownpool(cbdata, data, compid);
515 return dirpool_add_dir(cbdata->owndirpool, parent, compid, 1);
516 }
517
518 static inline Id
putinowndirpool(struct cbdata * cbdata,Repodata * data,Id dir)519 putinowndirpool(struct cbdata *cbdata, Repodata *data, Id dir)
520 {
521 if (dir && dir == cbdata->lastdirid)
522 return cbdata->lastdirid_own;
523 cbdata->lastdirid = dir;
524 cbdata->lastdirid_own = putinowndirpool_slow(cbdata, data, &data->dirpool, dir);
525 return cbdata->lastdirid_own;
526 }
527
528 /*
529 * pass 1 callback:
530 * collect key/id/dirid usage information, create needed schemas
531 */
532 static int
collect_needed_cb(void * vcbdata,Solvable * s,Repodata * data,Repokey * key,KeyValue * kv)533 collect_needed_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
534 {
535 struct cbdata *cbdata = vcbdata;
536 Id id;
537 int rm;
538
539 #if 0
540 fprintf(stderr, "solvable %d (%s): key (%d)%s %d\n", s ? (int)(s - cbdata->pool->solvables) : 0, s ? pool_id2str(cbdata->pool, s->name) : "", key->name, pool_id2str(cbdata->pool, key->name), key->type);
541 #endif
542 if (key->name == REPOSITORY_SOLVABLES)
543 return SEARCH_NEXT_KEY; /* we do not want this one */
544
545 rm = cbdata->keymap[key - data->keys];
546 if (!rm)
547 return SEARCH_NEXT_KEY; /* we do not want this one */
548
549 /* record key in schema */
550 if (cbdata->sp[-1] != rm)
551 *cbdata->sp++ = rm;
552
553 switch(key->type)
554 {
555 case REPOKEY_TYPE_ID:
556 case REPOKEY_TYPE_IDARRAY:
557 id = kv->id;
558 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
559 id = putinownpool(cbdata, data, id);
560 incneedid(id, cbdata->needid);
561 break;
562 case REPOKEY_TYPE_DIR:
563 case REPOKEY_TYPE_DIRNUMNUMARRAY:
564 case REPOKEY_TYPE_DIRSTRARRAY:
565 id = kv->id;
566 if (cbdata->owndirpool)
567 putinowndirpool(cbdata, data, id);
568 else
569 cbdata->dirused[id] = 1;
570 break;
571 case REPOKEY_TYPE_FIXARRAY:
572 case REPOKEY_TYPE_FLEXARRAY:
573 if (kv->entry)
574 {
575 /* finish schema, rewind to start */
576 Id *sp = cbdata->sp - 1;
577 *sp = 0;
578 while (sp[-1])
579 sp--;
580 if (sp[-2] >= 0)
581 cbdata->subschemata[sp[-2]] = repodata_schema2id(cbdata->target, sp, 1);
582 cbdata->sp = sp - 2;
583 }
584 if (kv->eof != 2)
585 {
586 /* start new schema */
587 if (kv->entry == 0 || key->type == REPOKEY_TYPE_FLEXARRAY)
588 {
589 cbdata->subschemata = solv_extend(cbdata->subschemata, cbdata->nsubschemata, 1, sizeof(Id), SCHEMATA_BLOCK);
590 *cbdata->sp++ = cbdata->nsubschemata++;
591 }
592 else
593 *cbdata->sp++ = -1;
594 *cbdata->sp++ = 0;
595 }
596 break;
597 default:
598 break;
599 }
600 return 0;
601 }
602
603 static void
collect_needed_solvable(struct cbdata * cbdata,Solvable * s,Id * keymap)604 collect_needed_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
605 {
606 /* set schema info, keep in sync with collect_data_solvable */
607 Repo *repo = s->repo;
608 Id *sp = cbdata->sp;
609 NeedId *needid = cbdata->needid;
610 Repodata *target = cbdata->target;
611 Id *idarraydata = repo->idarraydata;
612
613 if (keymap[SOLVABLE_NAME])
614 {
615 *sp++ = keymap[SOLVABLE_NAME];
616 needid[s->name].need++;
617 }
618 if (keymap[SOLVABLE_ARCH])
619 {
620 *sp++ = keymap[SOLVABLE_ARCH];
621 needid[s->arch].need++;
622 }
623 if (keymap[SOLVABLE_EVR])
624 {
625 *sp++ = keymap[SOLVABLE_EVR];
626 needid[s->evr].need++;
627 }
628 if (s->vendor && keymap[SOLVABLE_VENDOR])
629 {
630 *sp++ = keymap[SOLVABLE_VENDOR];
631 needid[s->vendor].need++;
632 }
633 if (s->provides && keymap[SOLVABLE_PROVIDES])
634 {
635 *sp++ = keymap[SOLVABLE_PROVIDES];
636 target->keys[keymap[SOLVABLE_PROVIDES]].size += incneedidarray(idarraydata + s->provides, needid);
637 }
638 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
639 {
640 *sp++ = keymap[SOLVABLE_OBSOLETES];
641 target->keys[keymap[SOLVABLE_OBSOLETES]].size += incneedidarray(idarraydata + s->obsoletes, needid);
642 }
643 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
644 {
645 *sp++ = keymap[SOLVABLE_CONFLICTS];
646 target->keys[keymap[SOLVABLE_CONFLICTS]].size += incneedidarray(idarraydata + s->conflicts, needid);
647 }
648 if (s->requires && keymap[SOLVABLE_REQUIRES])
649 {
650 *sp++ = keymap[SOLVABLE_REQUIRES];
651 target->keys[keymap[SOLVABLE_REQUIRES]].size += incneedidarray(idarraydata + s->requires, needid);
652 }
653 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
654 {
655 *sp++ = keymap[SOLVABLE_RECOMMENDS];
656 target->keys[keymap[SOLVABLE_RECOMMENDS]].size += incneedidarray(idarraydata + s->recommends, needid);
657 }
658 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
659 {
660 *sp++ = keymap[SOLVABLE_SUGGESTS];
661 target->keys[keymap[SOLVABLE_SUGGESTS]].size += incneedidarray(idarraydata + s->suggests, needid);
662 }
663 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
664 {
665 *sp++ = keymap[SOLVABLE_SUPPLEMENTS];
666 target->keys[keymap[SOLVABLE_SUPPLEMENTS]].size += incneedidarray(idarraydata + s->supplements, needid);
667 }
668 if (s->enhances && keymap[SOLVABLE_ENHANCES])
669 {
670 *sp++ = keymap[SOLVABLE_ENHANCES];
671 target->keys[keymap[SOLVABLE_ENHANCES]].size += incneedidarray(idarraydata + s->enhances, needid);
672 }
673 if (repo->rpmdbid && keymap[RPM_RPMDBID])
674 {
675 *sp++ = keymap[RPM_RPMDBID];
676 target->keys[keymap[RPM_RPMDBID]].size++;
677 }
678 cbdata->sp = sp;
679 }
680
681
682 /*
683 * pass 2 callback:
684 * encode all of the data into the correct buffers
685 */
686 static int
collect_data_cb(void * vcbdata,Solvable * s,Repodata * data,Repokey * key,KeyValue * kv)687 collect_data_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
688 {
689 struct cbdata *cbdata = vcbdata;
690 int rm;
691 Id id, storage;
692 struct extdata *xd;
693 NeedId *needid;
694
695 if (key->name == REPOSITORY_SOLVABLES)
696 return SEARCH_NEXT_KEY;
697
698 rm = cbdata->keymap[key - data->keys];
699 if (!rm)
700 return SEARCH_NEXT_KEY; /* we do not want this one */
701 storage = cbdata->target->keys[rm].storage;
702
703 xd = cbdata->extdata + 0; /* incore buffer */
704 if (storage == KEY_STORAGE_VERTICAL_OFFSET)
705 {
706 xd += rm; /* vertical buffer */
707 if (cbdata->vstart == -1)
708 cbdata->vstart = xd->len;
709 }
710 switch(key->type)
711 {
712 case REPOKEY_TYPE_DELETED:
713 case REPOKEY_TYPE_VOID:
714 case REPOKEY_TYPE_CONSTANT:
715 case REPOKEY_TYPE_CONSTANTID:
716 break;
717 case REPOKEY_TYPE_ID:
718 id = kv->id;
719 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
720 id = putinownpool(cbdata, data, id);
721 needid = cbdata->needid;
722 id = needid[NEEDIDOFF(id)].need;
723 data_addid(xd, id);
724 break;
725 case REPOKEY_TYPE_IDARRAY:
726 id = kv->id;
727 if (!ISRELDEP(id) && cbdata->ownspool && id > 1 && (!cbdata->clonepool || data->localpool))
728 id = putinownpool(cbdata, data, id);
729 needid = cbdata->needid;
730 id = needid[NEEDIDOFF(id)].need;
731 data_addideof(xd, id, kv->eof);
732 break;
733 case REPOKEY_TYPE_STR:
734 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
735 break;
736 case REPOKEY_TYPE_MD5:
737 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_MD5);
738 break;
739 case REPOKEY_TYPE_SHA1:
740 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA1);
741 break;
742 case REPOKEY_TYPE_SHA224:
743 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA224);
744 break;
745 case REPOKEY_TYPE_SHA256:
746 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA256);
747 break;
748 case REPOKEY_TYPE_SHA384:
749 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA384);
750 break;
751 case REPOKEY_TYPE_SHA512:
752 data_addblob(xd, (unsigned char *)kv->str, SIZEOF_SHA512);
753 break;
754 break;
755 case REPOKEY_TYPE_NUM:
756 data_addid64(xd, kv->num, kv->num2);
757 break;
758 case REPOKEY_TYPE_DIR:
759 id = kv->id;
760 if (cbdata->owndirpool)
761 id = putinowndirpool(cbdata, data, id);
762 id = cbdata->dirused[id];
763 data_addid(xd, id);
764 break;
765 case REPOKEY_TYPE_BINARY:
766 data_addid(xd, kv->num);
767 if (kv->num)
768 data_addblob(xd, (unsigned char *)kv->str, kv->num);
769 break;
770 case REPOKEY_TYPE_DIRNUMNUMARRAY:
771 id = kv->id;
772 if (cbdata->owndirpool)
773 id = putinowndirpool(cbdata, data, id);
774 id = cbdata->dirused[id];
775 data_addid(xd, id);
776 data_addid(xd, kv->num);
777 data_addideof(xd, kv->num2, kv->eof);
778 break;
779 case REPOKEY_TYPE_DIRSTRARRAY:
780 id = kv->id;
781 if (cbdata->owndirpool)
782 id = putinowndirpool(cbdata, data, id);
783 id = cbdata->dirused[id];
784 if (rm == cbdata->filelistmode)
785 {
786 /* postpone adding to xd, just update len to get the correct offsets into the incore data*/
787 xd->len += data_addideof_len(id) + strlen(kv->str) + 1;
788 break;
789 }
790 data_addideof(xd, id, kv->eof);
791 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
792 break;
793 case REPOKEY_TYPE_FIXARRAY:
794 case REPOKEY_TYPE_FLEXARRAY:
795 if (!kv->entry)
796 data_addid(xd, kv->num);
797 if (kv->eof != 2 && (!kv->entry || key->type == REPOKEY_TYPE_FLEXARRAY))
798 data_addid(xd, cbdata->subschemata[cbdata->current_sub++]);
799 if (xd == cbdata->extdata + 0 && !kv->parent && !cbdata->doingsolvables)
800 {
801 if (xd->len - cbdata->lastlen > cbdata->maxdata)
802 cbdata->maxdata = xd->len - cbdata->lastlen;
803 cbdata->lastlen = xd->len;
804 }
805 break;
806 default:
807 cbdata->target->error = pool_error(cbdata->pool, -1, "unknown type for %d: %d\n", key->name, key->type);
808 break;
809 }
810 if (storage == KEY_STORAGE_VERTICAL_OFFSET && kv->eof)
811 {
812 /* we can re-use old data in the blob here! */
813 data_addid(cbdata->extdata + 0, cbdata->vstart); /* add offset into incore data */
814 data_addid(cbdata->extdata + 0, xd->len - cbdata->vstart); /* add length into incore data */
815 cbdata->vstart = -1;
816 }
817 return 0;
818 }
819
820 /* special version of collect_data_cb that collects just one single REPOKEY_TYPE_DIRSTRARRAY vertical data */
821 static int
collect_filelist_cb(void * vcbdata,Solvable * s,Repodata * data,Repokey * key,KeyValue * kv)822 collect_filelist_cb(void *vcbdata, Solvable *s, Repodata *data, Repokey *key, KeyValue *kv)
823 {
824 struct cbdata *cbdata = vcbdata;
825 int rm;
826 Id id;
827 struct extdata *xd;
828
829 rm = cbdata->keymap[key - data->keys];
830 if (rm != cbdata->filelistmode)
831 return SEARCH_NEXT_KEY; /* we do not want this one */
832 id = kv->id;
833 if (cbdata->owndirpool)
834 id = putinowndirpool(cbdata, data, id);
835 id = cbdata->dirused[id];
836 xd = cbdata->extdata + rm; /* vertical buffer */
837 data_addideof(xd, id, kv->eof);
838 data_addblob(xd, (unsigned char *)kv->str, strlen(kv->str) + 1);
839 return 0;
840 }
841
842 static void
collect_data_solvable(struct cbdata * cbdata,Solvable * s,Id * keymap)843 collect_data_solvable(struct cbdata *cbdata, Solvable *s, Id *keymap)
844 {
845 Repo *repo = s->repo;
846 Pool *pool = repo->pool;
847 struct extdata *xd = cbdata->extdata;
848 NeedId *needid = cbdata->needid;
849 Id *idarraydata = repo->idarraydata;
850
851 if (keymap[SOLVABLE_NAME])
852 data_addid(xd, needid[s->name].need);
853 if (keymap[SOLVABLE_ARCH])
854 data_addid(xd, needid[s->arch].need);
855 if (keymap[SOLVABLE_EVR])
856 data_addid(xd, needid[s->evr].need);
857 if (s->vendor && keymap[SOLVABLE_VENDOR])
858 data_addid(xd, needid[s->vendor].need);
859 if (s->provides && keymap[SOLVABLE_PROVIDES])
860 data_adddepids(xd, pool, needid, idarraydata + s->provides, SOLVABLE_FILEMARKER);
861 if (s->obsoletes && keymap[SOLVABLE_OBSOLETES])
862 data_adddepids(xd, pool, needid, idarraydata + s->obsoletes, 0);
863 if (s->conflicts && keymap[SOLVABLE_CONFLICTS])
864 data_adddepids(xd, pool, needid, idarraydata + s->conflicts, 0);
865 if (s->requires && keymap[SOLVABLE_REQUIRES])
866 data_adddepids(xd, pool, needid, idarraydata + s->requires, SOLVABLE_PREREQMARKER);
867 if (s->recommends && keymap[SOLVABLE_RECOMMENDS])
868 data_adddepids(xd, pool, needid, idarraydata + s->recommends, 0);
869 if (s->suggests && keymap[SOLVABLE_SUGGESTS])
870 data_adddepids(xd, pool, needid, idarraydata + s->suggests, 0);
871 if (s->supplements && keymap[SOLVABLE_SUPPLEMENTS])
872 data_adddepids(xd, pool, needid, idarraydata + s->supplements, 0);
873 if (s->enhances && keymap[SOLVABLE_ENHANCES])
874 data_adddepids(xd, pool, needid, idarraydata + s->enhances, 0);
875 if (repo->rpmdbid && keymap[RPM_RPMDBID])
876 data_addid(xd, repo->rpmdbid[(s - pool->solvables) - repo->start]);
877 }
878
879 /* traverse through directory with first child "dir" */
880 static int
traverse_dirs(Dirpool * dp,Id * dirmap,Id n,Id dir,Id * used)881 traverse_dirs(Dirpool *dp, Id *dirmap, Id n, Id dir, Id *used)
882 {
883 Id sib, child;
884 Id parent, lastn;
885
886 parent = n;
887 /* special case for '/', which has to come first */
888 if (parent == 1)
889 dirmap[n++] = 1;
890 for (sib = dir; sib; sib = dirpool_sibling(dp, sib))
891 {
892 if (used && !used[sib])
893 continue;
894 if (sib == 1 && parent == 1)
895 continue; /* already did that one above */
896 dirmap[n++] = sib;
897 }
898
899 /* check if our block has some content */
900 if (parent == n)
901 return n - 1; /* nope, drop parent id again */
902
903 /* now go through all the siblings we just added and
904 * do recursive calls on them */
905 lastn = n;
906 for (; parent < lastn; parent++)
907 {
908 sib = dirmap[parent];
909 if (used && used[sib] != 2) /* 2: used as parent */
910 continue;
911 child = dirpool_child(dp, sib);
912 if (child)
913 {
914 dirmap[n++] = -parent; /* start new block */
915 n = traverse_dirs(dp, dirmap, n, child, used);
916 }
917 }
918 return n;
919 }
920
921 static void
write_compressed_page(Repodata * data,unsigned char * page,int len)922 write_compressed_page(Repodata *data, unsigned char *page, int len)
923 {
924 int clen;
925 unsigned char cpage[REPOPAGE_BLOBSIZE];
926
927 clen = repopagestore_compress_page(page, len, cpage, len - 1);
928 if (!clen)
929 {
930 write_u32(data, len * 2);
931 write_blob(data, page, len);
932 }
933 else
934 {
935 write_u32(data, clen * 2 + 1);
936 write_blob(data, cpage, clen);
937 }
938 }
939
940 static Id verticals[] = {
941 SOLVABLE_AUTHORS,
942 SOLVABLE_DESCRIPTION,
943 SOLVABLE_MESSAGEDEL,
944 SOLVABLE_MESSAGEINS,
945 SOLVABLE_EULA,
946 SOLVABLE_DISKUSAGE,
947 SOLVABLE_FILELIST,
948 SOLVABLE_CHECKSUM,
949 DELTA_CHECKSUM,
950 DELTA_SEQ_NUM,
951 SOLVABLE_PKGID,
952 SOLVABLE_HDRID,
953 SOLVABLE_LEADSIGID,
954 SOLVABLE_CHANGELOG_AUTHOR,
955 SOLVABLE_CHANGELOG_TEXT,
956 0
957 };
958
959 static char *languagetags[] = {
960 "solvable:summary:",
961 "solvable:description:",
962 "solvable:messageins:",
963 "solvable:messagedel:",
964 "solvable:eula:",
965 0
966 };
967
968 int
repo_write_stdkeyfilter(Repo * repo,Repokey * key,void * kfdata)969 repo_write_stdkeyfilter(Repo *repo, Repokey *key, void *kfdata)
970 {
971 const char *keyname;
972 int i;
973
974 for (i = 0; verticals[i]; i++)
975 if (key->name == verticals[i])
976 return KEY_STORAGE_VERTICAL_OFFSET;
977 keyname = pool_id2str(repo->pool, key->name);
978 for (i = 0; languagetags[i] != 0; i++)
979 if (!strncmp(keyname, languagetags[i], strlen(languagetags[i])))
980 return KEY_STORAGE_VERTICAL_OFFSET;
981 return KEY_STORAGE_INCORE;
982 }
983
984 static int
write_compressed_extdata(Repodata * target,struct extdata * xd,unsigned char * vpage,int lpage)985 write_compressed_extdata(Repodata *target, struct extdata *xd, unsigned char *vpage, int lpage)
986 {
987 unsigned char *dp = xd->buf;
988 int l = xd->len;
989 while (l)
990 {
991 int ll = REPOPAGE_BLOBSIZE - lpage;
992 if (l < ll)
993 ll = l;
994 memcpy(vpage + lpage, dp, ll);
995 dp += ll;
996 lpage += ll;
997 l -= ll;
998 if (lpage == REPOPAGE_BLOBSIZE)
999 {
1000 write_compressed_page(target, vpage, lpage);
1001 lpage = 0;
1002 }
1003 }
1004 return lpage;
1005 }
1006
1007
1008 static Id *
create_keyskip(Repo * repo,Id entry,unsigned char * repodataused,Id ** oldkeyskip)1009 create_keyskip(Repo *repo, Id entry, unsigned char *repodataused, Id **oldkeyskip)
1010 {
1011 Repodata *data, *last = 0;
1012 Id *keyskip;
1013 int rdid, cnt = 0;
1014
1015 if (repo->nrepodata <= 2)
1016 return 0;
1017 keyskip = *oldkeyskip;
1018 if (keyskip)
1019 {
1020 if (keyskip[1] >= 0x10000000)
1021 keyskip = solv_free(keyskip);
1022 else
1023 keyskip[1] = keyskip[2];
1024 }
1025 FOR_REPODATAS(repo, rdid, data)
1026 {
1027 if (!repodataused[rdid])
1028 continue;
1029 if (entry != SOLVID_META)
1030 {
1031 if (entry < data->start || entry >= data->end)
1032 continue;
1033 /* if repodataused is set we know that the state is AVAILABLE */
1034 if (!data->incoreoffset[entry - data->start])
1035 continue;
1036 }
1037 if (last)
1038 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1039 last = data;
1040 cnt++;
1041 }
1042 if (cnt <= 1) /* just one repodata means we don't need a keyskip */
1043 {
1044 *oldkeyskip = keyskip;
1045 return 0;
1046 }
1047 keyskip = repodata_fill_keyskip(last, entry, keyskip);
1048 if (keyskip)
1049 keyskip[2] = keyskip[1] + repo->nrepodata;
1050 *oldkeyskip = keyskip;
1051 return keyskip;
1052 }
1053
1054 /*
1055 * Repo
1056 */
1057
1058 Repowriter *
repowriter_create(Repo * repo)1059 repowriter_create(Repo *repo)
1060 {
1061 Repowriter *writer = solv_calloc(1, sizeof(*writer));
1062 writer->repo = repo;
1063 writer->keyfilter = repo_write_stdkeyfilter;
1064 writer->repodatastart = 1;
1065 writer->repodataend = repo->nrepodata;
1066 writer->solvablestart = repo->start;
1067 writer->solvableend = repo->end;
1068 return writer;
1069 }
1070
1071 Repowriter *
repowriter_free(Repowriter * writer)1072 repowriter_free(Repowriter *writer)
1073 {
1074 return solv_free(writer);
1075 }
1076
1077 void
repowriter_set_flags(Repowriter * writer,int flags)1078 repowriter_set_flags(Repowriter *writer, int flags)
1079 {
1080 writer->flags = flags;
1081 }
1082
1083 void
repowriter_set_keyfilter(Repowriter * writer,int (* keyfilter)(Repo * repo,Repokey * key,void * kfdata),void * kfdata)1084 repowriter_set_keyfilter(Repowriter *writer, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata)
1085 {
1086 writer->keyfilter = keyfilter;
1087 writer->kfdata = kfdata;
1088 }
1089
1090 void
repowriter_set_keyqueue(Repowriter * writer,Queue * keyq)1091 repowriter_set_keyqueue(Repowriter *writer, Queue *keyq)
1092 {
1093 writer->keyq = keyq;
1094 }
1095
1096 void
repowriter_set_repodatarange(Repowriter * writer,int repodatastart,int repodataend)1097 repowriter_set_repodatarange(Repowriter *writer, int repodatastart, int repodataend)
1098 {
1099 writer->repodatastart = repodatastart;
1100 writer->repodataend = repodataend;
1101 }
1102
1103 void
repowriter_set_solvablerange(Repowriter * writer,int solvablestart,int solvableend)1104 repowriter_set_solvablerange(Repowriter *writer, int solvablestart, int solvableend)
1105 {
1106 writer->solvablestart = solvablestart;
1107 writer->solvableend = solvableend;
1108 }
1109
1110 /*
1111 * the code works the following way:
1112 *
1113 * 1) find which keys should be written
1114 * 2) collect usage information for keys/ids/dirids, create schema
1115 * data
1116 * 3) use usage information to create mapping tables, so that often
1117 * used ids get a lower number
1118 * 4) encode data into buffers using the mapping tables
1119 * 5) write everything to disk
1120 */
1121 int
repowriter_write(Repowriter * writer,FILE * fp)1122 repowriter_write(Repowriter *writer, FILE *fp)
1123 {
1124 Repo *repo = writer->repo;
1125 Pool *pool = repo->pool;
1126 int i, j, n;
1127 Solvable *s;
1128 NeedId *needid, *needidp;
1129 int nstrings, nrels;
1130 unsigned int sizeid;
1131 unsigned int solv_flags;
1132 Id *oldkeyskip = 0;
1133 Id *keyskip = 0;
1134 int searchflags = 0;
1135
1136 Id id, *sp;
1137
1138 Id *keymap; /* maps repo key to my key, 0 -> not used */
1139 int nkeymap;
1140 int *keymapstart; /* maps repo number to keymap offset */
1141
1142 Id *dirmap;
1143 int ndirmap;
1144 Id *keyused;
1145
1146 unsigned char *repodataused;
1147 int anyrepodataused = 0;
1148
1149 int solvablestart, solvableend;
1150 Id *solvschemata;
1151 int anysolvableused = 0;
1152 int nsolvables;
1153
1154 struct cbdata cbdata;
1155
1156 int clonepool;
1157 Repokey *key;
1158 int poolusage, dirpoolusage;
1159 int reloff;
1160
1161 Repodata *data, *dirpooldata;
1162
1163 Repodata target;
1164
1165 Stringpool *spool;
1166 Dirpool *dirpool;
1167
1168 Id mainschema, *mainschemakeys;
1169
1170 struct extdata *xd;
1171
1172 Id type_constantid = 0;
1173
1174
1175 memset(&cbdata, 0, sizeof(cbdata));
1176 cbdata.pool = pool;
1177 cbdata.repo = repo;
1178 cbdata.target = ⌖
1179
1180 repodata_initdata(&target, repo, 1);
1181
1182 /* go through all repodata and find the keys we need */
1183 /* also unify keys */
1184
1185 /* start with all KEY_STORAGE_SOLVABLE ids */
1186
1187 n = ID_NUM_INTERNAL;
1188 FOR_REPODATAS(repo, i, data)
1189 n += data->nkeys;
1190 nkeymap = n;
1191 keymap = solv_calloc(nkeymap, sizeof(Id));
1192 keymapstart = solv_calloc(repo->nrepodata, sizeof(Id));
1193 repodataused = solv_calloc(repo->nrepodata, 1);
1194
1195 clonepool = 0;
1196 poolusage = 0;
1197
1198 if (!(writer->flags & REPOWRITER_NO_STORAGE_SOLVABLE))
1199 {
1200 /* add keys for STORAGE_SOLVABLE */
1201 for (i = SOLVABLE_NAME; i <= RPM_RPMDBID; i++)
1202 {
1203 Repokey keyd;
1204 keyd.name = i;
1205 if (i < SOLVABLE_PROVIDES)
1206 keyd.type = REPOKEY_TYPE_ID;
1207 else if (i < RPM_RPMDBID)
1208 #ifdef USE_REL_IDARRAY
1209 keyd.type = REPOKEY_TYPE_REL_IDARRAY;
1210 #else
1211 keyd.type = REPOKEY_TYPE_IDARRAY;
1212 #endif
1213 else
1214 keyd.type = REPOKEY_TYPE_NUM;
1215 keyd.size = 0;
1216 keyd.storage = KEY_STORAGE_SOLVABLE;
1217 if (writer->keyfilter)
1218 {
1219 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1220 if (keyd.storage == KEY_STORAGE_DROPPED)
1221 continue;
1222 keyd.storage = KEY_STORAGE_SOLVABLE;
1223 }
1224 poolusage = 1;
1225 clonepool = 1;
1226 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1227 }
1228 }
1229
1230 if (repo->nsolvables)
1231 {
1232 Repokey keyd;
1233 keyd.name = REPOSITORY_SOLVABLES;
1234 keyd.type = REPOKEY_TYPE_FLEXARRAY;
1235 keyd.size = 0;
1236 keyd.storage = KEY_STORAGE_INCORE;
1237 keymap[keyd.name] = repodata_key2id(&target, &keyd, 1);
1238 }
1239
1240 dirpoolusage = 0;
1241
1242 spool = 0;
1243 dirpool = 0;
1244 dirpooldata = 0;
1245 n = ID_NUM_INTERNAL;
1246 FOR_REPODATAS(repo, i, data)
1247 {
1248 int idused, dirused;
1249 if (i < writer->repodatastart || i >= writer->repodataend)
1250 continue;
1251 if (writer->keyfilter && (writer->flags & REPOWRITER_LEGACY) != 0)
1252 {
1253 /* ask keyfilter if we want this repodata */
1254 Repokey keyd;
1255 /* check if we want this repodata */
1256 memset(&keyd, 0, sizeof(keyd));
1257 keyd.name = 1;
1258 keyd.type = 1;
1259 keyd.size = i;
1260 if (writer->keyfilter(repo, &keyd, writer->kfdata) == -1)
1261 continue;
1262 }
1263 keymapstart[i] = n;
1264 keymap[n++] = 0; /* key 0 */
1265 idused = dirused = 0;
1266 for (j = 1; j < data->nkeys; j++, n++)
1267 {
1268 key = data->keys + j;
1269 if (key->name == REPOSITORY_SOLVABLES && key->type == REPOKEY_TYPE_FLEXARRAY)
1270 {
1271 keymap[n] = keymap[key->name];
1272 continue;
1273 }
1274 if (key->type == REPOKEY_TYPE_DELETED && (writer->flags & REPOWRITER_KEEP_TYPE_DELETED) == 0)
1275 {
1276 keymap[n] = 0;
1277 continue;
1278 }
1279 if (key->type == REPOKEY_TYPE_CONSTANTID && data->localpool)
1280 {
1281 Repokey keyd = *key;
1282 keyd.size = repodata_globalize_id(data, key->size, 1);
1283 id = repodata_key2id(&target, &keyd, 0);
1284 }
1285 else
1286 id = repodata_key2id(&target, key, 0);
1287 if (!id)
1288 {
1289 /* a new key. ask keyfilter if we want it before creating it */
1290 Repokey keyd = *key;
1291 keyd.storage = KEY_STORAGE_INCORE;
1292 if (keyd.type == REPOKEY_TYPE_CONSTANTID)
1293 keyd.size = repodata_globalize_id(data, key->size, 1);
1294 else if (keyd.type != REPOKEY_TYPE_CONSTANT)
1295 keyd.size = 0;
1296 if (writer->keyfilter)
1297 {
1298 keyd.storage = writer->keyfilter(repo, &keyd, writer->kfdata);
1299 if (keyd.storage == KEY_STORAGE_DROPPED)
1300 {
1301 keymap[n] = 0;
1302 continue;
1303 }
1304 }
1305 if (data->state != REPODATA_STUB)
1306 id = repodata_key2id(&target, &keyd, 1);
1307 }
1308 keymap[n] = id;
1309 /* load repodata if not already loaded */
1310 if (data->state == REPODATA_STUB)
1311 {
1312 int oldnkeys = data->nkeys;
1313 repodata_load(data);
1314 if (oldnkeys != data->nkeys)
1315 {
1316 nkeymap += data->nkeys - oldnkeys; /* grow/shrink keymap */
1317 keymap = solv_realloc2(keymap, nkeymap, sizeof(Id));
1318 }
1319 if (data->state == REPODATA_AVAILABLE)
1320 {
1321 /* redo this repodata! */
1322 j = 0;
1323 n = keymapstart[i];
1324 continue;
1325 }
1326 }
1327 if (data->state != REPODATA_AVAILABLE && data->state != REPODATA_LOADING)
1328 {
1329 /* too bad! */
1330 keymap[n] = 0;
1331 continue;
1332 }
1333
1334 repodataused[i] = 1;
1335 anyrepodataused = 1;
1336 if (key->type == REPOKEY_TYPE_CONSTANTID || key->type == REPOKEY_TYPE_ID ||
1337 key->type == REPOKEY_TYPE_IDARRAY || key->type == REPOKEY_TYPE_REL_IDARRAY)
1338 idused = 1;
1339 else if (key->type == REPOKEY_TYPE_DIR || key->type == REPOKEY_TYPE_DIRNUMNUMARRAY || key->type == REPOKEY_TYPE_DIRSTRARRAY)
1340 {
1341 idused = 1; /* dirs also use ids */
1342 dirused = 1;
1343 }
1344 }
1345 if (idused)
1346 {
1347 if (data->localpool)
1348 {
1349 if (poolusage)
1350 poolusage = 3; /* need own pool */
1351 else
1352 {
1353 poolusage = 2;
1354 spool = &data->spool;
1355 }
1356 }
1357 else
1358 {
1359 if (poolusage == 0)
1360 poolusage = 1;
1361 else if (poolusage != 1)
1362 poolusage = 3; /* need own pool */
1363 }
1364 }
1365 if (dirused)
1366 {
1367 if (dirpoolusage)
1368 dirpoolusage = 3; /* need own dirpool */
1369 else
1370 {
1371 dirpoolusage = 2;
1372 dirpool = &data->dirpool;
1373 dirpooldata = data;
1374 }
1375 }
1376 }
1377 nkeymap = n; /* update */
1378
1379 /* 0: no pool needed at all */
1380 /* 1: use global pool */
1381 /* 2: use repodata local pool */
1382 /* 3: need own pool */
1383 if (poolusage != 3)
1384 clonepool = 0;
1385 if (poolusage == 3)
1386 {
1387 spool = &target.spool;
1388 target.localpool = 1; /* so we can use repodata_translate */
1389 /* hack: reuse global pool data so we don't have to map pool ids */
1390 if (clonepool)
1391 {
1392 stringpool_free(spool);
1393 stringpool_clone(spool, &pool->ss);
1394 cbdata.clonepool = 1;
1395 }
1396 cbdata.ownspool = spool;
1397 }
1398 else if (poolusage == 0 || poolusage == 1)
1399 {
1400 poolusage = 1;
1401 spool = &pool->ss;
1402 }
1403
1404 if (dirpoolusage == 3)
1405 {
1406 /* dirpoolusage == 3 means that at least two repodata
1407 * areas have dir keys. This means that two areas have
1408 * idused set to 1, which results in poolusage being
1409 * either 1 (global pool) or 3 (own pool) */
1410 dirpool = &target.dirpool;
1411 dirpooldata = 0;
1412 cbdata.owndirpool = dirpool;
1413 }
1414 else if (dirpool)
1415 cbdata.dirused = solv_calloc(dirpool->ndirs, sizeof(Id));
1416
1417
1418 /********************************************************************/
1419 #if 0
1420 fprintf(stderr, "poolusage: %d\n", poolusage);
1421 fprintf(stderr, "dirpoolusage: %d\n", dirpoolusage);
1422 fprintf(stderr, "clonepool: %d\n", clonepool);
1423 fprintf(stderr, "nkeys: %d\n", target.nkeys);
1424 for (i = 1; i < target.nkeys; i++)
1425 fprintf(stderr, " %2d: %s[%d] %d %d %d\n", i, pool_id2str(pool, target.keys[i].name), target.keys[i].name, target.keys[i].type, target.keys[i].size, target.keys[i].storage);
1426 #endif
1427
1428 /********************************************************************/
1429
1430 searchflags = SEARCH_SUB|SEARCH_ARRAYSENTINEL;
1431 if ((writer->flags & REPOWRITER_KEEP_TYPE_DELETED) != 0)
1432 searchflags |= SEARCH_KEEP_TYPE_DELETED;
1433
1434 /* set needed count of all strings and rels,
1435 * find which keys are used in the solvables
1436 * put all strings in own spool
1437 */
1438
1439 reloff = spool->nstrings;
1440 if (cbdata.ownspool)
1441 reloff = (reloff + NEEDID_BLOCK) & ~NEEDID_BLOCK;
1442 else if (poolusage == 2)
1443 {
1444 /* we'll need to put the key data into the spool,
1445 * so leave some room. 3 * nkeys is an upper bound */
1446 reloff += 3 * target.nkeys;
1447 }
1448
1449 needid = solv_calloc(reloff + pool->nrels, sizeof(*needid));
1450 needid[0].map = reloff; /* remember size in case we need to grow */
1451
1452 cbdata.needid = needid;
1453 cbdata.schema = solv_calloc(target.nkeys + 2, sizeof(Id));
1454
1455 /* create main schema */
1456 cbdata.sp = cbdata.schema + 1;
1457
1458 /* collect meta data from all repodatas */
1459 /* XXX: merge arrays of equal keys? */
1460 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1461 FOR_REPODATAS(repo, j, data)
1462 {
1463 if (!repodataused[j])
1464 continue;
1465 cbdata.keymap = keymap + keymapstart[j];
1466 cbdata.lastdirid = 0; /* clear dir mapping cache */
1467 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1468 }
1469 needid = cbdata.needid; /* maybe relocated */
1470 sp = cbdata.sp;
1471 /* add solvables if needed (may revert later) */
1472 if (repo->nsolvables)
1473 {
1474 *sp++ = keymap[REPOSITORY_SOLVABLES];
1475 target.keys[keymap[REPOSITORY_SOLVABLES]].size++;
1476 }
1477 *sp = 0;
1478 /* stash away main schema (including terminating zero) */
1479 mainschemakeys = solv_memdup2(cbdata.schema + 1, sp - cbdata.schema, sizeof(Id));
1480
1481 /* collect data for all solvables */
1482 solvschemata = solv_calloc(repo->nsolvables, sizeof(Id)); /* allocate upper bound */
1483 solvablestart = writer->solvablestart < repo->start ? repo->start : writer->solvablestart;
1484 solvableend = writer->solvableend > repo->end ? repo->end : writer->solvableend;
1485 anysolvableused = 0;
1486 nsolvables = 0; /* solvables we are going to write, will be <= repo->nsolvables */
1487 cbdata.doingsolvables = 1;
1488 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
1489 {
1490 if (s->repo != repo)
1491 continue;
1492
1493 cbdata.sp = cbdata.schema + 1;
1494 collect_needed_solvable(&cbdata, s, keymap);
1495
1496 if (anyrepodataused)
1497 {
1498 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1499 FOR_REPODATAS(repo, j, data)
1500 {
1501 if (!repodataused[j] || i < data->start || i >= data->end)
1502 continue;
1503 cbdata.keymap = keymap + keymapstart[j];
1504 cbdata.lastdirid = 0;
1505 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_needed_cb, &cbdata);
1506 }
1507 needid = cbdata.needid; /* maybe relocated */
1508 }
1509 *cbdata.sp = 0;
1510 solvschemata[nsolvables] = repodata_schema2id(cbdata.target, cbdata.schema + 1, 1);
1511 if (solvschemata[nsolvables])
1512 anysolvableused = 1;
1513 nsolvables++;
1514 }
1515 cbdata.doingsolvables = 0;
1516
1517 if (repo->nsolvables && !anysolvableused)
1518 {
1519 /* strip off REPOSITORY_SOLVABLES from the main schema */
1520 for (sp = mainschemakeys; *sp; sp++)
1521 ;
1522 sp[-1] = 0; /* strip last entry */
1523 }
1524 mainschema = repodata_schema2id(cbdata.target, mainschemakeys, 1);
1525 mainschemakeys = solv_free(mainschemakeys);
1526
1527 /********************************************************************/
1528
1529 /* remove unused keys */
1530 keyused = solv_calloc(target.nkeys, sizeof(Id));
1531 for (i = 1; i < (int)target.schemadatalen; i++)
1532 keyused[target.schemadata[i]] = 1;
1533 keyused[0] = 0;
1534 for (n = i = 1; i < target.nkeys; i++)
1535 {
1536 if (!keyused[i])
1537 continue;
1538 if (i != n)
1539 target.keys[n] = target.keys[i];
1540 keyused[i] = n++;
1541 }
1542 target.nkeys = n;
1543
1544 /* update schema data to the new key ids */
1545 for (i = 1; i < (int)target.schemadatalen; i++)
1546 target.schemadata[i] = keyused[target.schemadata[i]];
1547 /* update keymap to the new key ids */
1548 for (i = 0; i < nkeymap; i++)
1549 keymap[i] = keyused[keymap[i]];
1550 keyused = solv_free(keyused);
1551
1552 /* copy keys if requested */
1553 if (writer->keyq)
1554 {
1555 queue_empty(writer->keyq);
1556 for (i = 1; i < target.nkeys; i++)
1557 queue_push2(writer->keyq, target.keys[i].name, target.keys[i].type);
1558 }
1559
1560 /********************************************************************/
1561
1562 /* check if we can do the special filelist memory optimization
1563 * we do the check before the keys are mapped.
1564 * The optimization is done if there is just one vertical key and
1565 * it is of type REPOKEY_TYPE_DIRSTRARRAY */
1566 if (anysolvableused && anyrepodataused)
1567 {
1568 for (i = 1; i < target.nkeys; i++)
1569 {
1570 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1571 continue;
1572 if (target.keys[i].type != REPOKEY_TYPE_DIRSTRARRAY || cbdata.filelistmode != 0)
1573 {
1574 cbdata.filelistmode = 0;
1575 break;
1576 }
1577 cbdata.filelistmode = i;
1578 }
1579 }
1580
1581 /********************************************************************/
1582
1583 if (poolusage > 1)
1584 {
1585 /* put all the keys in our string pool */
1586 /* put mapped ids right into target.keys */
1587 for (i = 1, key = target.keys + i; i < target.nkeys; i++, key++)
1588 {
1589 key->name = stringpool_str2id(spool, pool_id2str(pool, key->name), 1);
1590 id = stringpool_str2id(spool, pool_id2str(pool, key->type), 1);
1591 if (key->type == REPOKEY_TYPE_CONSTANTID)
1592 {
1593 type_constantid = id;
1594 key->size = stringpool_str2id(spool, pool_id2str(pool, key->size), 1);
1595 }
1596 key->type = id;
1597 }
1598 if (poolusage == 2)
1599 stringpool_freehash(spool); /* free some mem */
1600 if (cbdata.ownspool && spool->nstrings > needid[0].map)
1601 {
1602 grow_needid(&cbdata, spool->nstrings - 1);
1603 needid = cbdata.needid; /* we relocated */
1604 }
1605 }
1606 else
1607 type_constantid = REPOKEY_TYPE_CONSTANTID;
1608
1609 /* increment needid of the keys */
1610 for (i = 1; i < target.nkeys; i++)
1611 {
1612 if (target.keys[i].type == type_constantid)
1613 needid[target.keys[i].size].need++;
1614 needid[target.keys[i].name].need++;
1615 needid[target.keys[i].type].need++;
1616 }
1617
1618 /********************************************************************/
1619
1620 /* increment need id of all relations
1621 * if we refer to another relation, make sure that the
1622 * need value is it is bigger than our value so that
1623 * ordering works.
1624 */
1625 reloff = needid[0].map;
1626 for (i = pool->nrels - 1, needidp = needid + (reloff + i); i > 0; i--, needidp--)
1627 if (needidp->need)
1628 break;
1629 if (i)
1630 {
1631 /* we have some relations with a non-zero need */
1632 Reldep *rd;
1633
1634 for (rd = pool->rels + i; i > 0; i--, rd--)
1635 {
1636 int need = needid[reloff + i].need;
1637 if (!need)
1638 continue;
1639 id = rd->name;
1640 if (ISRELDEP(id))
1641 {
1642 id = GETRELID(id);
1643 if (needid[reloff + id].need < need + 1)
1644 needid[reloff + id].need = need + 1;
1645 }
1646 else
1647 {
1648 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1649 {
1650 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1651 if (id >= cbdata.needid[0].map)
1652 {
1653 grow_needid(&cbdata, id);
1654 needid = cbdata.needid; /* we relocated */
1655 reloff = needid[0].map; /* we have a new offset */
1656 }
1657 }
1658 needid[id].need++;
1659 }
1660
1661 id = rd->evr;
1662 if (ISRELDEP(id))
1663 {
1664 id = GETRELID(id);
1665 if (needid[reloff + id].need < need + 1)
1666 needid[reloff + id].need = need + 1;
1667 }
1668 else
1669 {
1670 if (cbdata.ownspool && id > 1 && !cbdata.clonepool)
1671 {
1672 id = stringpool_str2id(cbdata.ownspool, pool_id2str(pool, id), 1);
1673 if (id >= cbdata.needid[0].map)
1674 {
1675 grow_needid(&cbdata, id);
1676 needid = cbdata.needid; /* we relocated */
1677 reloff = needid[0].map; /* we have a new offset */
1678 }
1679 }
1680 needid[id].need++;
1681 }
1682 }
1683 }
1684
1685 /********************************************************************/
1686
1687 /* increment need id for used dir components */
1688 if (cbdata.owndirpool)
1689 {
1690 /* if we have own dirpool, all entries in it are used.
1691 also, all comp ids are already mapped by putinowndirpool(),
1692 so we can simply increment needid.
1693 (owndirpool != 0, dirused == 0, dirpooldata == 0) */
1694 for (i = 1; i < dirpool->ndirs; i++)
1695 {
1696 id = dirpool->dirs[i];
1697 if (id <= 0)
1698 continue;
1699 needid[id].need++;
1700 }
1701 }
1702 else if (dirpool)
1703 {
1704 Id parent;
1705 /* else we re-use a dirpool of repodata "dirpooldata".
1706 dirused tells us which of the ids are used.
1707 we need to map comp ids if we generate a new pool.
1708 (owndirpool == 0, dirused != 0, dirpooldata != 0) */
1709 for (i = dirpool->ndirs - 1; i > 0; i--)
1710 {
1711 if (!cbdata.dirused[i])
1712 continue;
1713 parent = dirpool_parent(dirpool, i); /* always < i */
1714 cbdata.dirused[parent] = 2; /* 2: used as parent */
1715 id = dirpool->dirs[i];
1716 if (id <= 0)
1717 continue;
1718 if (cbdata.ownspool && id > 1 && (!cbdata.clonepool || dirpooldata->localpool))
1719 {
1720 id = putinownpool(&cbdata, dirpooldata, id);
1721 needid = cbdata.needid;
1722 }
1723 needid[id].need++;
1724 }
1725 if (!cbdata.dirused[0])
1726 {
1727 cbdata.dirused = solv_free(cbdata.dirused);
1728 dirpool = 0;
1729 }
1730 }
1731
1732
1733 /********************************************************************/
1734
1735 /*
1736 * create mapping table, new keys are sorted by needid[].need
1737 *
1738 * needid[key].need : old key -> new key
1739 * needid[key].map : new key -> old key
1740 */
1741
1742 /* zero out id 0 and rel 0 just in case */
1743 reloff = needid[0].map;
1744 needid[0].need = 0;
1745 needid[reloff].need = 0;
1746
1747 for (i = 1; i < reloff + pool->nrels; i++)
1748 needid[i].map = i;
1749
1750 /* make first entry '' */
1751 needid[1].need = 1;
1752 solv_sort(needid + 2, spool->nstrings - 2, sizeof(*needid), needid_cmp_need_s, spool);
1753 solv_sort(needid + reloff, pool->nrels, sizeof(*needid), needid_cmp_need, 0);
1754 /* now needid is in new order, needid[newid].map -> oldid */
1755
1756 /* calculate string space size, also zero out needid[].need */
1757 sizeid = 0;
1758 for (i = 1; i < reloff; i++)
1759 {
1760 if (!needid[i].need)
1761 break; /* as we have sorted, every entry after this also has need == 0 */
1762 needid[i].need = 0;
1763 sizeid += strlen(spool->stringspace + spool->strings[needid[i].map]) + 1;
1764 }
1765 nstrings = i; /* our new string id end */
1766
1767 /* make needid[oldid].need point to newid */
1768 for (i = 1; i < nstrings; i++)
1769 needid[needid[i].map].need = i;
1770
1771 /* same as above for relations */
1772 for (i = 0; i < pool->nrels; i++)
1773 {
1774 if (!needid[reloff + i].need)
1775 break;
1776 needid[reloff + i].need = 0;
1777 }
1778 nrels = i; /* our new rel id end */
1779
1780 for (i = 0; i < nrels; i++)
1781 needid[needid[reloff + i].map].need = nstrings + i;
1782
1783 /* now we have: needid[oldid].need -> newid
1784 needid[newid].map -> oldid
1785 both for strings and relations */
1786
1787
1788 /********************************************************************/
1789
1790 ndirmap = 0;
1791 dirmap = 0;
1792 if (dirpool && dirpool->ndirs)
1793 {
1794 /* create our new target directory structure by traversing through all
1795 * used dirs. This will concatenate blocks with the same parent
1796 * directory into single blocks.
1797 * Instead of components, traverse_dirs stores the old dirids,
1798 * we will change this in the second step below */
1799 /* (dirpooldata and dirused are 0 if we have our own dirpool) */
1800 if (cbdata.dirused && !cbdata.dirused[1])
1801 {
1802 cbdata.dirused[1] = 1; /* always want / entry */
1803 cbdata.dirused[0] = 2; /* always want / entry */
1804 }
1805 dirmap = solv_calloc(dirpool->ndirs, sizeof(Id));
1806 dirmap[0] = 0;
1807 ndirmap = traverse_dirs(dirpool, dirmap, 1, dirpool_child(dirpool, 0), cbdata.dirused);
1808
1809 /* (re)create dirused, so that it maps from "old dirid" to "new dirid" */
1810 /* change dirmap so that it maps from "new dirid" to "new compid" */
1811 if (!cbdata.dirused)
1812 cbdata.dirused = solv_malloc2(dirpool->ndirs, sizeof(Id));
1813 memset(cbdata.dirused, 0, dirpool->ndirs * sizeof(Id));
1814 for (i = 1; i < ndirmap; i++)
1815 {
1816 if (dirmap[i] <= 0)
1817 continue;
1818 cbdata.dirused[dirmap[i]] = i;
1819 id = dirpool->dirs[dirmap[i]];
1820 if (dirpooldata && cbdata.ownspool && id > 1)
1821 id = putinownpool(&cbdata, dirpooldata, id);
1822 dirmap[i] = needid[id].need;
1823 }
1824 /* now the new target directory structure is complete (dirmap), and we have
1825 * dirused[olddirid] -> newdirid */
1826 }
1827
1828 /********************************************************************/
1829
1830 /* collect all data
1831 * we use extdata[0] for incore data and extdata[keyid] for vertical data
1832 *
1833 * this must match the code above that creates the schema data!
1834 */
1835
1836 cbdata.extdata = solv_calloc(target.nkeys, sizeof(struct extdata));
1837
1838 xd = cbdata.extdata;
1839 cbdata.current_sub = 0;
1840 /* add main schema */
1841 cbdata.lastlen = 0;
1842 data_addid(xd, mainschema);
1843
1844 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
1845 FOR_REPODATAS(repo, j, data)
1846 {
1847 if (!repodataused[j])
1848 continue;
1849 cbdata.keymap = keymap + keymapstart[j];
1850 cbdata.lastdirid = 0;
1851 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1852 }
1853 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1854 cbdata.maxdata = xd->len - cbdata.lastlen;
1855 cbdata.lastlen = xd->len;
1856
1857 if (anysolvableused)
1858 {
1859 data_addid(xd, nsolvables); /* FLEXARRAY nentries */
1860 cbdata.doingsolvables = 1;
1861
1862 for (i = solvablestart, s = pool->solvables + i, n = 0; i < solvableend; i++, s++)
1863 {
1864 if (s->repo != repo)
1865 continue;
1866 data_addid(xd, solvschemata[n]);
1867 collect_data_solvable(&cbdata, s, keymap);
1868 if (anyrepodataused)
1869 {
1870 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
1871 cbdata.vstart = -1;
1872 FOR_REPODATAS(repo, j, data)
1873 {
1874 if (!repodataused[j] || i < data->start || i >= data->end)
1875 continue;
1876 cbdata.keymap = keymap + keymapstart[j];
1877 cbdata.lastdirid = 0;
1878 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_data_cb, &cbdata);
1879 }
1880 }
1881 if (xd->len - cbdata.lastlen > cbdata.maxdata)
1882 cbdata.maxdata = xd->len - cbdata.lastlen;
1883 cbdata.lastlen = xd->len;
1884 n++;
1885 }
1886 cbdata.doingsolvables = 0;
1887 }
1888
1889 assert(cbdata.current_sub == cbdata.nsubschemata);
1890 cbdata.subschemata = solv_free(cbdata.subschemata);
1891 cbdata.nsubschemata = 0;
1892
1893 /********************************************************************/
1894
1895 target.fp = fp;
1896
1897 /* write header */
1898
1899 /* write file header */
1900 write_u32(&target, 'S' << 24 | 'O' << 16 | 'L' << 8 | 'V');
1901 write_u32(&target, SOLV_VERSION_8);
1902
1903
1904 /* write counts */
1905 write_u32(&target, nstrings);
1906 write_u32(&target, nrels);
1907 write_u32(&target, ndirmap);
1908 write_u32(&target, anysolvableused ? nsolvables : 0);
1909 write_u32(&target, target.nkeys);
1910 write_u32(&target, target.nschemata);
1911 solv_flags = 0;
1912 solv_flags |= SOLV_FLAG_PREFIX_POOL;
1913 solv_flags |= SOLV_FLAG_SIZE_BYTES;
1914 write_u32(&target, solv_flags);
1915
1916 if (nstrings)
1917 {
1918 /*
1919 * calculate prefix encoding of the strings
1920 */
1921 unsigned char *prefixcomp = solv_malloc(nstrings);
1922 unsigned int compsum = 0;
1923 char *old_str = "";
1924
1925 prefixcomp[0] = 0;
1926 for (i = 1; i < nstrings; i++)
1927 {
1928 char *str = spool->stringspace + spool->strings[needid[i].map];
1929 int same;
1930 for (same = 0; same < 255; same++)
1931 if (!old_str[same] || old_str[same] != str[same])
1932 break;
1933 prefixcomp[i] = same;
1934 compsum += same;
1935 old_str = str;
1936 }
1937
1938 /*
1939 * write strings
1940 */
1941 write_u32(&target, sizeid);
1942 /* we save compsum bytes but need 1 extra byte for every string */
1943 write_u32(&target, sizeid + nstrings - 1 - compsum);
1944 for (i = 1; i < nstrings; i++)
1945 {
1946 char *str = spool->stringspace + spool->strings[needid[i].map];
1947 write_u8(&target, prefixcomp[i]);
1948 write_str(&target, str + prefixcomp[i]);
1949 }
1950 solv_free(prefixcomp);
1951 }
1952 else
1953 {
1954 write_u32(&target, 0);
1955 write_u32(&target, 0);
1956 }
1957
1958 /*
1959 * write RelDeps
1960 */
1961 for (i = 0; i < nrels; i++)
1962 {
1963 Reldep *ran = pool->rels + (needid[reloff + i].map - reloff);
1964 write_id(&target, needid[NEEDIDOFF(ran->name)].need);
1965 write_id(&target, needid[NEEDIDOFF(ran->evr)].need);
1966 write_u8(&target, ran->flags);
1967 }
1968
1969 /*
1970 * write dirs (skip both root and / entry)
1971 */
1972 for (i = 2; i < ndirmap; i++)
1973 {
1974 if (dirmap[i] > 0)
1975 write_id(&target, dirmap[i]);
1976 else
1977 write_id(&target, nstrings - dirmap[i]);
1978 }
1979 solv_free(dirmap);
1980
1981 /*
1982 * write keys
1983 */
1984 for (i = 1; i < target.nkeys; i++)
1985 {
1986 write_id(&target, needid[target.keys[i].name].need);
1987 write_id(&target, needid[target.keys[i].type].need);
1988 if (target.keys[i].storage != KEY_STORAGE_VERTICAL_OFFSET)
1989 {
1990 if (target.keys[i].type == type_constantid)
1991 write_id(&target, needid[target.keys[i].size].need);
1992 else
1993 write_id(&target, target.keys[i].size);
1994 }
1995 else
1996 write_id(&target, cbdata.extdata[i].len);
1997 write_id(&target, target.keys[i].storage);
1998 }
1999
2000 /*
2001 * write schemata
2002 */
2003 write_id(&target, target.schemadatalen); /* XXX -1? */
2004 for (i = 1; i < target.nschemata; i++)
2005 write_idarray(&target, pool, 0, repodata_id2schema(&target, i));
2006
2007 /*
2008 * write incore data
2009 */
2010 write_id(&target, cbdata.maxdata);
2011 write_id(&target, cbdata.extdata[0].len);
2012 if (cbdata.extdata[0].len)
2013 write_blob(&target, cbdata.extdata[0].buf, cbdata.extdata[0].len);
2014 solv_free(cbdata.extdata[0].buf);
2015
2016 /*
2017 * write vertical data if we have any
2018 */
2019 for (i = 1; i < target.nkeys; i++)
2020 if (cbdata.extdata[i].len)
2021 break;
2022 if (i < target.nkeys)
2023 {
2024 /* have vertical data, write it in pages */
2025 unsigned char vpage[REPOPAGE_BLOBSIZE];
2026 int lpage = 0;
2027
2028 write_u32(&target, REPOPAGE_BLOBSIZE);
2029 if (!cbdata.filelistmode)
2030 {
2031 for (i = 1; i < target.nkeys; i++)
2032 if (cbdata.extdata[i].len)
2033 lpage = write_compressed_extdata(&target, cbdata.extdata + i, vpage, lpage);
2034 }
2035 else
2036 {
2037 /* ok, just one single extdata which is of type REPOKEY_TYPE_DIRSTRARRAY */
2038 xd = cbdata.extdata + i;
2039 xd->len = 0;
2040 keyskip = create_keyskip(repo, SOLVID_META, repodataused, &oldkeyskip);
2041 FOR_REPODATAS(repo, j, data)
2042 {
2043 if (!repodataused[j])
2044 continue;
2045 cbdata.keymap = keymap + keymapstart[j];
2046 cbdata.lastdirid = 0;
2047 repodata_search_keyskip(data, SOLVID_META, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2048 }
2049 for (i = solvablestart, s = pool->solvables + i; i < solvableend; i++, s++)
2050 {
2051 if (s->repo != repo)
2052 continue;
2053 keyskip = create_keyskip(repo, i, repodataused, &oldkeyskip);
2054 FOR_REPODATAS(repo, j, data)
2055 {
2056 if (!repodataused[j] || i < data->start || i >= data->end)
2057 continue;
2058 cbdata.keymap = keymap + keymapstart[j];
2059 cbdata.lastdirid = 0;
2060 repodata_search_keyskip(data, i, 0, searchflags, keyskip, collect_filelist_cb, &cbdata);
2061 }
2062 if (xd->len > 1024 * 1024)
2063 {
2064 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2065 xd->len = 0;
2066 }
2067 }
2068 if (xd->len)
2069 lpage = write_compressed_extdata(&target, xd, vpage, lpage);
2070 }
2071 if (lpage)
2072 write_compressed_page(&target, vpage, lpage);
2073 }
2074
2075 for (i = 1; i < target.nkeys; i++)
2076 solv_free(cbdata.extdata[i].buf);
2077 solv_free(cbdata.extdata);
2078
2079 target.fp = 0;
2080 repodata_freedata(&target);
2081
2082 solv_free(needid);
2083 solv_free(solvschemata);
2084 solv_free(cbdata.schema);
2085
2086 solv_free(keymap);
2087 solv_free(keymapstart);
2088 solv_free(cbdata.dirused);
2089 solv_free(repodataused);
2090 solv_free(oldkeyskip);
2091 return target.error;
2092 }
2093
2094 int
repo_write(Repo * repo,FILE * fp)2095 repo_write(Repo *repo, FILE *fp)
2096 {
2097 int res;
2098 Repowriter *writer = repowriter_create(repo);
2099 res = repowriter_write(writer, fp);
2100 repowriter_free(writer);
2101 return res;
2102 }
2103
2104 int
repodata_write(Repodata * data,FILE * fp)2105 repodata_write(Repodata *data, FILE *fp)
2106 {
2107 int res;
2108 Repowriter *writer = repowriter_create(data->repo);
2109 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2110 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE);
2111 res = repowriter_write(writer, fp);
2112 repowriter_free(writer);
2113 return res;
2114 }
2115
2116 /* deprecated functions, do not use in new code! */
2117 int
repo_write_filtered(Repo * repo,FILE * fp,int (* keyfilter)(Repo * repo,Repokey * key,void * kfdata),void * kfdata,Queue * keyq)2118 repo_write_filtered(Repo *repo, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2119 {
2120 int res;
2121 Repowriter *writer = repowriter_create(repo);
2122 repowriter_set_flags(writer, REPOWRITER_LEGACY);
2123 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2124 repowriter_set_keyqueue(writer, keyq);
2125 res = repowriter_write(writer, fp);
2126 repowriter_free(writer);
2127 return res;
2128 }
2129
2130 int
repodata_write_filtered(Repodata * data,FILE * fp,int (* keyfilter)(Repo * repo,Repokey * key,void * kfdata),void * kfdata,Queue * keyq)2131 repodata_write_filtered(Repodata *data, FILE *fp, int (*keyfilter)(Repo *repo, Repokey *key, void *kfdata), void *kfdata, Queue *keyq)
2132 {
2133 int res;
2134 Repowriter *writer = repowriter_create(data->repo);
2135 repowriter_set_repodatarange(writer, data->repodataid, data->repodataid + 1);
2136 repowriter_set_flags(writer, REPOWRITER_NO_STORAGE_SOLVABLE | REPOWRITER_LEGACY);
2137 repowriter_set_keyfilter(writer, keyfilter, kfdata);
2138 repowriter_set_keyqueue(writer, keyq);
2139 res = repowriter_write(writer, fp);
2140 repowriter_free(writer);
2141 return res;
2142 }
2143
2144