1 /*
2 * Copyright (c) 2019 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * This code uses concepts and configuration based on 'synth', by
8 * John R. Marino <draco@marino.st>, which was written in ada.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 *
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in
18 * the documentation and/or other materials provided with the
19 * distribution.
20 * 3. Neither the name of The DragonFly Project nor the names of its
21 * contributors may be used to endorse or promote products derived
22 * from this software without specific, prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
27 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
28 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
29 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
30 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
31 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
32 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
33 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
34 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 */
37
38 #include "dsynth.h"
39
40 #define PKG_HSIZE 32768
41 #define PKG_HMASK 32767
42
43 static int parsepkglist_file(const char *path, int debugstop);
44 static void childGetPackageInfo(bulk_t *bulk);
45 static void childGetBinaryDistInfo(bulk_t *bulk);
46 static void childOptimizeEnv(bulk_t *bulk);
47 static pkg_t *resolveDeps(pkg_t *dep_list, pkg_t ***list_tailp, int gentopo);
48 static void resolveFlavors(pkg_t *pkg, char *flavors, int gentopo);
49 static void resolveDepString(pkg_t *pkg, char *depstr,
50 int gentopo, int dep_type);
51 static pkg_t *processPackageListBulk(int total);
52 static int scan_and_queue_dir(const char *path, const char *level1, int level);
53 static int scan_binary_repo(const char *path);
54 #if 0
55 static void pkgfree(pkg_t *pkg);
56 #endif
57
58 static int PrepareSystemFlag;
59
60 static pkg_t *PkgHash1[PKG_HSIZE]; /* by portdir */
61 static pkg_t *PkgHash2[PKG_HSIZE]; /* by pkgfile */
62
63 /*
64 * Allocate a new pkg structure plus basic initialization.
65 */
66 static __inline pkg_t *
allocpkg(void)67 allocpkg(void)
68 {
69 pkg_t *pkg;
70
71 pkg = calloc(1, sizeof(*pkg));
72 pkg->idepon_list.next = &pkg->idepon_list;
73 pkg->idepon_list.prev = &pkg->idepon_list;
74 pkg->deponi_list.next = &pkg->deponi_list;
75 pkg->deponi_list.prev = &pkg->deponi_list;
76
77 return pkg;
78 }
79
80 /*
81 * Simple hash for lookups
82 */
83 static __inline int
pkghash(const char * str)84 pkghash(const char *str)
85 {
86 int hv = 0xABC32923;
87 while (*str) {
88 hv = (hv << 5) ^ *str;
89 ++str;
90 }
91 hv = hv ^ (hv / PKG_HSIZE) ^ (hv / PKG_HSIZE / PKG_HSIZE);
92 return (hv & PKG_HMASK);
93 }
94
95 static __inline const char *
deptype2str(int dep_type)96 deptype2str(int dep_type)
97 {
98 switch (dep_type) {
99 case DEP_TYPE_FETCH:
100 return("FETCH_DEPENDS");
101 case DEP_TYPE_EXT:
102 return("EXTRACT_DEPENDS");
103 case DEP_TYPE_PATCH:
104 return("PATCH_DEPENDS");
105 case DEP_TYPE_BUILD:
106 return("BUILD_DEPENDS");
107 case DEP_TYPE_LIB:
108 return("LIB_DEPENDS");
109 case DEP_TYPE_RUN:
110 return("RUN_DEPENDS");
111 default:
112 return("UNKNOWN");
113 }
114 }
115
116 static void
pkg_enter(pkg_t * pkg)117 pkg_enter(pkg_t *pkg)
118 {
119 pkg_t **pkgp;
120 pkg_t *scan;
121
122 if (pkg->portdir) {
123 pkgp = &PkgHash1[pkghash(pkg->portdir)];
124 while ((scan = *pkgp) != NULL) {
125 if (strcmp(pkg->portdir, scan->portdir) == 0)
126 break;
127 pkgp = &scan->hnext1;
128 }
129 ddassert(scan == NULL || (scan->flags & PKGF_PLACEHOLD));
130 if (scan && (scan->flags & PKGF_PLACEHOLD)) {
131 ddassert(scan->idepon_list.next == &scan->idepon_list);
132 ddassert(scan->deponi_list.next == &scan->deponi_list);
133 *pkgp = pkg;
134 pkg->hnext1 = scan->hnext1;
135 free(scan->portdir);
136 free(scan);
137 scan = NULL;
138 }
139 if (scan == NULL)
140 *pkgp = pkg;
141 }
142
143 if (pkg->pkgfile) {
144 pkgp = &PkgHash2[pkghash(pkg->pkgfile)];
145 while ((scan = *pkgp) != NULL) {
146 if (strcmp(pkg->pkgfile, scan->pkgfile) == 0)
147 break;
148 pkgp = &scan->hnext2;
149 }
150 if (scan == NULL)
151 *pkgp = pkg;
152 }
153 }
154
155 static pkg_t *
pkg_find(const char * match)156 pkg_find(const char *match)
157 {
158 pkg_t **pkgp;
159 pkg_t *pkg;
160
161 pkgp = &PkgHash1[pkghash(match)];
162 for (pkg = *pkgp; pkg; pkg = pkg->hnext1) {
163 if (strcmp(pkg->portdir, match) == 0)
164 return pkg;
165 }
166 pkgp = &PkgHash2[pkghash(match)];
167 for (pkg = *pkgp; pkg; pkg = pkg->hnext2) {
168 if (strcmp(pkg->pkgfile, match) == 0)
169 return pkg;
170 }
171 return NULL;
172 }
173
174 /*
175 * Parse a specific list of ports via origin name (portdir/subdir)
176 */
177 pkg_t *
ParsePackageList(int n,char ** ary,int debugstop)178 ParsePackageList(int n, char **ary, int debugstop)
179 {
180 pkg_t *list;
181 int total;
182 int fail;
183 int i;
184
185 total = 0;
186 fail = 0;
187 initbulk(childGetPackageInfo, MaxBulk);
188
189 /*
190 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual
191 * selection, "d" meaning DEBUGSTOP mode, or NULL.
192 */
193 queuebulk("ports-mgmt", "pkg", NULL, "x");
194
195 for (i = 0; i < n; ++i) {
196 char *l1;
197 char *l2;
198 char *l3;
199 struct stat st;
200
201 l1 = strdup(ary[i]);
202 if (stat(l1, &st) == 0 && S_ISREG(st.st_mode)) {
203 total += parsepkglist_file(l1, debugstop);
204 continue;
205 }
206
207 l2 = strchr(l1, '/');
208 if (l2 == NULL) {
209 printf("Bad portdir specification: %s\n", l1);
210 free(l1);
211 fail = 1;
212 continue;
213 }
214 *l2++ = 0;
215 l3 = strchr(l2, '@');
216 if (l3)
217 *l3++ = 0;
218
219 /*
220 * Silently ignore any manually specified ports-mgmt/pkg,
221 * which we already auto-added.
222 */
223 if (strcmp(l1, "ports-mgmt") != 0 ||
224 strcmp(l2, "pkg") != 0)
225 {
226 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
227 }
228 ++total;
229 free(l1);
230 }
231 printf("Processing %d ports\n", total);
232
233 list = processPackageListBulk(total);
234 if (fail) {
235 dfatal("Bad specifications, exiting");
236 exit(1);
237 }
238
239 return list;
240 }
241
242 static
243 int
parsepkglist_file(const char * path,int debugstop)244 parsepkglist_file(const char *path, int debugstop)
245 {
246 FILE *fp;
247 char *base;
248 char *l1;
249 char *l2;
250 char *l3;
251 size_t len;
252 int total;
253
254 if ((fp = fopen(path, "r")) == NULL) {
255 dpanic_errno("Cannot read %s\n", path);
256 /* NOT REACHED */
257 return 0;
258 }
259
260 total = 0;
261
262 while ((base = fgetln(fp, &len)) != NULL) {
263 if (len == 0 || base[len-1] != '\n')
264 continue;
265 base[--len] = 0;
266 l1 = strtok(base, " \t\r\n");
267 if (l1 == NULL) {
268 printf("Badly formatted pkg info line: %s\n", base);
269 continue;
270 }
271 l2 = strchr(l1, '/');
272 if (l2 == NULL) {
273 printf("Badly formatted specification: %s\n", l1);
274 continue;
275 }
276 *l2++ = 0;
277 l3 = strchr(l2, '@');
278 if (l3)
279 *l3++ = 0;
280 queuebulk(l1, l2, l3, (debugstop ? "d" : NULL));
281 ++total;
282 }
283 fclose(fp);
284
285 return total;
286 }
287
288 /*
289 * Parse packages from the list installed on the system.
290 */
291 pkg_t *
GetLocalPackageList(void)292 GetLocalPackageList(void)
293 {
294 pkg_t *list;
295 FILE *fp;
296 char *base;
297 char *data;
298 char *l1;
299 char *l2;
300 char *l3;
301 int total;
302 int state;
303 size_t len;
304
305 PrepareSystemFlag = 1;
306 initbulk(childGetPackageInfo, MaxBulk);
307 total = 0;
308 state = 0;
309 l1 = NULL;
310 l2 = NULL;
311 l3 = NULL;
312
313 fp = popen("pkg info -a -o -A", "r");
314
315 /*
316 * Always include ports-mgmt/pkg. s4 is "x" meaning not a manual
317 * selection, "d" meaning DEBUGSTOP mode, or NULL.
318 */
319 queuebulk("ports-mgmt", "pkg", NULL, "x");
320
321 while ((base = fgetln(fp, &len)) != NULL) {
322 if (len == 0 || base[len-1] != '\n')
323 continue;
324 base[--len] = 0;
325
326 data = strchr(base, ':');
327 if (data == NULL)
328 continue;
329 *data++ = 0;
330
331 base = strtok(base, " \t\r");
332 data = strtok(data, " \t\r");
333
334 if (base == NULL || data == NULL)
335 continue;
336
337 if (strcmp(base, "Origin") == 0) {
338 if (state == 1) {
339 queuebulk(l1, l2, NULL, NULL);
340 state = 0;
341 ++total;
342 }
343
344 if (strchr(data, '/') == NULL) {
345 printf("Badly formatted origin: %s\n", l1);
346 }
347 if (l1)
348 free(l1);
349 if (l3)
350 free(l3);
351 l1 = strdup(data);
352 l2 = strchr(l1, '/');
353 *l2++ = 0;
354 l3 = strchr(l2, '@'); /* typically NULL */
355 if (l3) {
356 *l3++ = 0;
357 l3 = strdup(l3);
358 }
359
360 /*
361 * Don't queue ports-mgmt/pkg twice, we already
362 * queued it manually.
363 */
364 if (strcmp(l1, "ports-mgmt") != 0 ||
365 strcmp(l2, "pkg") != 0) {
366 state = 1;
367 }
368 continue;
369 }
370 if (state == 1 && strcmp(base, "flavor") == 0) {
371 queuebulk(l1, l2, data, NULL);
372 state = 0;
373 ++total;
374 }
375 }
376 if (state == 1) {
377 queuebulk(l1, l2, NULL, NULL);
378 /*state = 0; not needed */
379 }
380 if (l1)
381 free(l1);
382 if (l3)
383 free(l3);
384
385 pclose(fp);
386
387 printf("Processing %d ports\n", total);
388
389 list = processPackageListBulk(total);
390
391 return list;
392 }
393
394 pkg_t *
GetFullPackageList(void)395 GetFullPackageList(void)
396 {
397 int total;
398
399 initbulk(childGetPackageInfo, MaxBulk);
400 total = scan_and_queue_dir(DPortsPath, NULL, 1);
401 printf("Scanning %d ports\n", total);
402
403 return processPackageListBulk(total);
404 }
405
406 /*
407 * Caller has queued the process list for bulk operation. We retrieve
408 * the results and clean up the bulk operation (we may have to do a second
409 * bulk operation so we have to be the ones to clean it up).
410 */
411 static pkg_t *
processPackageListBulk(int total)412 processPackageListBulk(int total)
413 {
414 bulk_t *bulk;
415 pkg_t *scan;
416 pkg_t *list;
417 pkg_t *dep_list;
418 pkg_t **list_tail;
419 int count;
420 int stop_fail;
421 int stop_base_list;
422 int remove_corrupt;
423
424 list = NULL;
425 list_tail = &list;
426 count = 0;
427 remove_corrupt = 0;
428
429 while ((bulk = getbulk()) != NULL) {
430 ++count;
431 if ((count & 255) == 0) {
432 printf("%6.2f%%\r",
433 (double)count * 100.0 / (double)total + 0.001);
434 fflush(stdout);
435 }
436 if (bulk->list) {
437 *list_tail = bulk->list;
438 bulk->list = NULL;
439 while ((scan = *list_tail) != NULL) {
440 if (bulk->s4 == NULL || bulk->s4[0] != 'x')
441 scan->flags |= PKGF_MANUALSEL;
442 pkg_enter(scan);
443 list_tail = &scan->bnext;
444 }
445 }
446 freebulk(bulk);
447 }
448 printf("100.00%%\n");
449 printf("\nTotal %d\n", count);
450 fflush(stdout);
451
452 /*
453 * Resolve all dependencies for the related packages, potentially
454 * adding anything that could not be found to the list. This will
455 * continue to issue bulk operations and process the result until
456 * no dependencies are left.
457 */
458 printf("Resolving dependencies...");
459 fflush(stdout);
460 dep_list = list;
461 while (dep_list) {
462 dep_list = resolveDeps(dep_list, &list_tail, 0);
463 }
464 printf("done\n");
465
466 donebulk();
467
468 /*
469 * Generate the topology
470 */
471 resolveDeps(list, NULL, 1);
472
473 /*
474 * Do a final count, ignore place holders.
475 *
476 * Also set stop_fail if appropriate. Check for direct specifications
477 * which fail to probe and any direct dependencies of those
478 * specifications, but don't recurse (for now)... don't check indirect
479 * dependencies (i.e. A -> B -> C where A is directly specified, B
480 * is adirect dependency, and C fails to probe).
481 */
482 count = 0;
483 stop_fail = 0;
484 stop_base_list = 0;
485 for (scan = list; scan; scan = scan->bnext) {
486 if ((scan->flags & PKGF_ERROR) == 0) {
487 ++count;
488 }
489 if ((scan->flags & PKGF_MANUALSEL) && MaskProbeAbort == 0) {
490 pkglink_t *link;
491
492 /*
493 * Directly specified package failed to probe
494 */
495 if (scan->flags & PKGF_CORRUPT) {
496 ++stop_fail;
497 ++stop_base_list;
498 }
499
500 /*
501 * Directly specified package had a direct dependency
502 * that failed to probe (don't go further).
503 */
504 PKGLIST_FOREACH(link, &scan->idepon_list) {
505 if (link->pkg &&
506 (link->pkg->flags & PKGF_CORRUPT)) {
507 ++stop_fail;
508 }
509 }
510 }
511 }
512 printf("Total Returned %d\n", count);
513
514 /*
515 * Check to see if any PKGF_MANUALSEL packages
516 */
517 if (stop_fail) {
518 printf("%d packages failed to probe\n", stop_fail);
519 if (PrepareSystemFlag) {
520 if (stop_fail == stop_base_list) {
521 printf(
522 "prepare-system: Some of your installed packages no longer exist in\n"
523 "dports, do you wish to continue rebuilding what does exist?\n");
524 if (askyn("Continue anyway? "))
525 remove_corrupt = 1;
526 } else {
527 printf(
528 "prepare-system: Some of your installed packages have dependencies\n"
529 "which could not be found in dports, cannot continue, aborting\n");
530 }
531 } else {
532 if (ForceOpt) {
533 remove_corrupt = 1;
534 printf("continuing despite pkglist "
535 "errors (-f)\n");
536 } else {
537 printf("unable to continue, aborting\n");
538 }
539 }
540 if (remove_corrupt == 0)
541 exit(1);
542 }
543
544 /*
545 * Remove corrupt packages before continuing
546 */
547 if (remove_corrupt) {
548 list_tail = &list;
549 while ((scan = *list_tail) != NULL) {
550 if (scan->flags & PKGF_CORRUPT)
551 *list_tail = scan->bnext;
552 else
553 list_tail = &scan->bnext;
554 }
555 }
556
557 /*
558 * Scan our binary distributions and related dependencies looking
559 * for any packages that have already been built.
560 */
561 initbulk(childGetBinaryDistInfo, MaxBulk);
562 total = scan_binary_repo(RepositoryPath);
563 count = 0;
564 printf("Scanning %d packages\n", total);
565
566 while ((bulk = getbulk()) != NULL) {
567 ++count;
568 if ((count & 255) == 0) {
569 printf("%6.2f%%\r",
570 (double)count * 100.0 / (double)total + 0.001);
571 fflush(stdout);
572 }
573 freebulk(bulk);
574 }
575 printf("100.00%%\n");
576 printf("\nTotal %d\n", count);
577 fflush(stdout);
578 donebulk();
579
580 printf("all done\n");
581
582 return list;
583 }
584
585 pkg_t *
GetPkgPkg(pkg_t ** listp)586 GetPkgPkg(pkg_t **listp)
587 {
588 bulk_t *bulk;
589 pkg_t *scan;
590 pkg_t *s2;
591
592 for (scan = *listp; scan; scan = scan->bnext) {
593 if (strcmp(scan->portdir, "ports-mgmt/pkg") == 0)
594 return scan;
595 }
596
597 /*
598 * This will force pkg to be built, but generally this code
599 * is not reached because the package list processing code
600 * adds ports-mgmt/pkg unconditionally.
601 */
602 initbulk(childGetPackageInfo, MaxBulk);
603 queuebulk("ports-mgmt", "pkg", NULL, "x");
604 bulk = getbulk();
605 dassert(bulk, "Cannot find ports-mgmt/pkg");
606 scan = bulk->list;
607 bulk->list = NULL;
608 freebulk(bulk);
609 donebulk();
610
611
612 /*
613 * Include added packages to the total and add the initial bulk
614 * built packages to the list so they get counted.
615 */
616 for (s2 = scan; s2->bnext; s2 = s2->bnext)
617 ++BuildTotal;
618 for (s2 = scan; s2->bnext; s2 = s2->bnext)
619 ;
620 s2->bnext = *listp;
621 *listp = scan;
622 ++BuildTotal;
623
624 return scan;
625 }
626
627 /*
628 * Try to optimize the environment by supplying information that
629 * the ports system would generally have to run stuff to get on
630 * every package.
631 *
632 * See childOptimizeEnv() for the actual handling. We execute
633 * a single make -V... -V... for ports-mgmt/pkg from within the
634 * bulk system (which handles the environment and disables
635 * /etc/make.conf), and we then call addbuildenv() as appropriate.
636 *
637 * _PERL5_FROM_BIN
638 * add others...
639 */
640 void
OptimizeEnv(void)641 OptimizeEnv(void)
642 {
643 bulk_t *bulk;
644
645 initbulk(childOptimizeEnv, MaxBulk);
646 queuebulk("ports-mgmt", "pkg", NULL, NULL);
647 bulk = getbulk();
648 freebulk(bulk);
649 donebulk();
650 }
651
652 /*
653 * Run through the list resolving dependencies and constructing the topology
654 * linkages. This may append packages to the list. Dependencies to dummy
655 * nodes which do not specify a flavor do not need special handling, the
656 * search code in build.c will properly follow the first flavor.
657 */
658 static pkg_t *
resolveDeps(pkg_t * list,pkg_t *** list_tailp,int gentopo)659 resolveDeps(pkg_t *list, pkg_t ***list_tailp, int gentopo)
660 {
661 pkg_t *ret_list = NULL;
662 pkg_t *scan;
663 pkg_t *use;
664 bulk_t *bulk;
665
666 for (scan = list; scan; scan = scan->bnext) {
667 use = pkg_find(scan->portdir);
668 resolveFlavors(use, scan->flavors, gentopo);
669 resolveDepString(use, scan->fetch_deps,
670 gentopo, DEP_TYPE_FETCH);
671 resolveDepString(use, scan->ext_deps,
672 gentopo, DEP_TYPE_EXT);
673 resolveDepString(use, scan->patch_deps,
674 gentopo, DEP_TYPE_PATCH);
675 resolveDepString(use, scan->build_deps,
676 gentopo, DEP_TYPE_BUILD);
677 resolveDepString(use, scan->lib_deps,
678 gentopo, DEP_TYPE_LIB);
679 resolveDepString(use, scan->run_deps,
680 gentopo, DEP_TYPE_RUN);
681 }
682
683 /*
684 * No bulk ops are queued when doing the final topology
685 * generation.
686 *
687 * Avoid entering duplicate results from the bulk ops. Duplicate
688 * results are mostly filtered out, but not always. A dummy node
689 * representing multiple flavors will parse-out the flavors
690 */
691 if (gentopo)
692 return NULL;
693 while ((bulk = getbulk()) != NULL) {
694 if (bulk->list) {
695 if (ret_list == NULL)
696 ret_list = bulk->list;
697 **list_tailp = bulk->list;
698 bulk->list = NULL;
699 while ((scan = **list_tailp) != NULL) {
700 pkg_enter(scan);
701 *list_tailp = &scan->bnext;
702 }
703 }
704 freebulk(bulk);
705 }
706 return (ret_list);
707 }
708
709 /*
710 * Resolve a generic node that has flavors, queue to retrieve info for
711 * each flavor and setup linkages as appropriate.
712 */
713 static void
resolveFlavors(pkg_t * pkg,char * flavors,int gentopo)714 resolveFlavors(pkg_t *pkg, char *flavors, int gentopo)
715 {
716 char *flavor_base;
717 char *flavor_scan;
718 char *flavor;
719 char *portdir;
720 char *s1;
721 char *s2;
722 pkg_t *dpkg;
723 pkglink_t *link;
724
725 if ((pkg->flags & PKGF_DUMMY) == 0)
726 return;
727 if (pkg->flavors == NULL || pkg->flavors[0] == 0)
728 return;
729 flavor_base = strdup(flavors);
730 flavor_scan = flavor_base;
731
732 for (;;) {
733 do {
734 flavor = strsep(&flavor_scan, " \t");
735 } while (flavor && *flavor == 0);
736 if (flavor == NULL)
737 break;
738
739 /*
740 * Iterate each flavor generating "s1/s2@flavor".
741 *
742 * queuebulk() info for each flavor, and set-up the
743 * linkages in the topology generation pass.
744 */
745 asprintf(&portdir, "%s@%s", pkg->portdir, flavor);
746 s1 = strdup(pkg->portdir);
747 s2 = strchr(s1, '/');
748 *s2++ = 0;
749
750 dpkg = pkg_find(portdir);
751 if (dpkg && gentopo) {
752 /*
753 * Setup linkages
754 */
755 free(portdir);
756
757 link = calloc(1, sizeof(*link));
758 link->pkg = dpkg;
759 link->next = &pkg->idepon_list;
760 link->prev = pkg->idepon_list.prev;
761 link->next->prev = link;
762 link->prev->next = link;
763 link->dep_type = DEP_TYPE_BUILD;
764
765 link = calloc(1, sizeof(*link));
766 link->pkg = pkg;
767 link->next = &dpkg->deponi_list;
768 link->prev = dpkg->deponi_list.prev;
769 link->next->prev = link;
770 link->prev->next = link;
771 link->dep_type = DEP_TYPE_BUILD;
772 ++dpkg->depi_count;
773 } else if (gentopo == 0 && dpkg == NULL) {
774 /*
775 * Use a place-holder to prevent duplicate
776 * dependencies from being processed. The placeholder
777 * will be replaced by the actual dependency.
778 */
779 dpkg = allocpkg();
780 dpkg->portdir = portdir;
781 dpkg->flags = PKGF_PLACEHOLD;
782 pkg_enter(dpkg);
783 queuebulk(s1, s2, flavor, NULL);
784 }
785 free(s1);
786 }
787 free(flavor_base);
788 }
789
790 static void
resolveDepString(pkg_t * pkg,char * depstr,int gentopo,int dep_type)791 resolveDepString(pkg_t *pkg, char *depstr, int gentopo, int dep_type)
792 {
793 char *copy_base;
794 char *copy;
795 char *dep;
796 char *log_component;
797 char *sep;
798 char *tag;
799 char *flavor;
800 pkg_t *dpkg;
801
802 if (depstr == NULL || depstr[0] == 0)
803 return;
804
805 copy_base = strdup(depstr);
806 copy = copy_base;
807 log_component = copy;
808
809 for (;;) {
810 do {
811 dep = strsep(©, " \t");
812 } while (dep && *dep == 0);
813 if (dep == NULL)
814 break;
815
816 /*
817 * Ignore dependencies prefixed with ${NONEXISTENT}
818 */
819 if (strncmp(dep, "/nonexistent:", 13) == 0)
820 continue;
821
822 log_component = dep;
823 dep = strchr(dep, ':');
824 if (dep == NULL || *dep != ':') {
825 printf("Error parsing %s dependency for "
826 "%s: '%s' at index %zd '%s' "
827 "(looking for ':')\n",
828 deptype2str(dep_type),
829 pkg->portdir, depstr,
830 log_component - copy_base,
831 log_component);
832 continue;
833 }
834 ++dep;
835
836 /*
837 * Strip-off any DPortsPath prefix. EXTRACT_DEPENDS
838 * often (always?) generates this prefix.
839 */
840 if (strncmp(dep, DPortsPath, strlen(DPortsPath)) == 0) {
841 dep += strlen(DPortsPath);
842 if (*dep == '/')
843 ++dep;
844 }
845
846 /*
847 * Strip-off any tag (such as :patch). We don't try to
848 * organize dependencies at this fine a grain (for now).
849 */
850 tag = strchr(dep, ':');
851 if (tag)
852 *tag++ = 0;
853 log_component = dep;
854
855 /*
856 * Locate the dependency
857 */
858 if ((dpkg = pkg_find(dep)) != NULL) {
859 if (gentopo) {
860 pkglink_t *link;
861
862 /*
863 * NOTE: idep_count is calculated recursively
864 * at build-time
865 */
866 ddprintf(0, "Add Dependency %s -> %s\n",
867 pkg->portdir, dpkg->portdir);
868 link = calloc(1, sizeof(*link));
869 link->pkg = dpkg;
870 link->next = &pkg->idepon_list;
871 link->prev = pkg->idepon_list.prev;
872 link->next->prev = link;
873 link->prev->next = link;
874 link->dep_type = dep_type;
875
876 link = calloc(1, sizeof(*link));
877 link->pkg = pkg;
878 link->next = &dpkg->deponi_list;
879 link->prev = dpkg->deponi_list.prev;
880 link->next->prev = link;
881 link->prev->next = link;
882 link->dep_type = dep_type;
883 ++dpkg->depi_count;
884 }
885 continue;
886 }
887
888 /*
889 * This shouldn't happen because we already took a first
890 * pass and should have generated the pkgs.
891 */
892 if (gentopo) {
893 printf("Topology Generate failed for %s: %s\n",
894 pkg->portdir, copy_base);
895 continue;
896 }
897
898 /*
899 * Separate out the two dports directory components and
900 * extract the optional '@flavor' specification.
901 */
902 sep = strchr(dep, '/');
903 if (sep == NULL) {
904 printf("Error parsing %s dependency for "
905 "%s: '%s' at index %zd '%s' "
906 "(looking for '/')\n",
907 deptype2str(dep_type),
908 pkg->portdir, depstr,
909 log_component - copy_base,
910 log_component);
911 continue;
912 }
913 *sep++ = 0;
914
915 /*
916 * The flavor hangs off the separator, not the tag
917 */
918 flavor = strrchr(sep, '@');
919 #if 0
920 if (tag)
921 flavor = strrchr(tag, '@');
922 else
923 flavor = strrchr(sep, '@');
924 #endif
925 if (flavor)
926 *flavor++ = 0;
927
928 if (flavor)
929 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s@%s\n",
930 pkg->portdir, dep, sep, flavor);
931 else
932 ddprintf(0, "QUEUE DEPENDENCY FROM PKG %s: %s/%s\n",
933 pkg->portdir, dep, sep);
934
935 /*
936 * Use a place-holder to prevent duplicate dependencies from
937 * being processed. The placeholder will be replaced by
938 * the actual dependency.
939 */
940 dpkg = allocpkg();
941 if (flavor)
942 asprintf(&dpkg->portdir, "%s/%s@%s", dep, sep, flavor);
943 else
944 asprintf(&dpkg->portdir, "%s/%s", dep, sep);
945 dpkg->flags = PKGF_PLACEHOLD;
946 pkg_enter(dpkg);
947
948 queuebulk(dep, sep, flavor, NULL);
949 }
950 free(copy_base);
951 }
952
953 void
FreePackageList(pkg_t * pkgs __unused)954 FreePackageList(pkg_t *pkgs __unused)
955 {
956 dfatal("not implemented");
957 }
958
959 /*
960 * Scan some or all dports to allocate the related pkg structure. Dependencies
961 * are stored but not processed.
962 *
963 * Threaded function
964 */
965 static void
childGetPackageInfo(bulk_t * bulk)966 childGetPackageInfo(bulk_t *bulk)
967 {
968 pkg_t *pkg;
969 char *flavor;
970 char *ptr;
971 FILE *fp;
972 int line;
973 size_t len;
974 char *portpath;
975 char *flavarg;
976 char *localbase;
977 const char *cav[MAXCAC];
978 pid_t pid;
979 int cac;
980
981 /*
982 * If the package has flavors we will loop on each one. If a flavor
983 * is not passed in s3 we will loop on all flavors, otherwise we will
984 * only process the passed-in flavor.
985 */
986 flavor = bulk->s3; /* usually NULL */
987
988 bulk->list = NULL;
989
990 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
991 if (flavor)
992 asprintf(&flavarg, "FLAVOR=%s", flavor);
993 else
994 flavarg = NULL;
995
996 cac = 0;
997 cav[cac++] = MAKE_BINARY;
998 cav[cac++] = "-C";
999 cav[cac++] = portpath;
1000 if (flavarg)
1001 cav[cac++] = flavarg;
1002
1003 /*
1004 * Prevent postgresql, mysql, and other package Makefile tests
1005 * from accessing the host system's /usr/local by setting LOCALBASE
1006 * to an empty directory.
1007 */
1008 asprintf(&localbase, "LOCALBASE=%s/empty", BuildBase);
1009 cav[cac++] = localbase;
1010
1011 /*
1012 * Variables we need to retrieve (order is specific to the switch
1013 * below)
1014 */
1015 cav[cac++] = "-VPKGVERSION";
1016 cav[cac++] = "-VPKGFILE:T";
1017 cav[cac++] = "-VALLFILES";
1018 cav[cac++] = "-VDIST_SUBDIR";
1019 cav[cac++] = "-VMAKE_JOBS_NUMBER";
1020 cav[cac++] = "-VIGNORE";
1021 cav[cac++] = "-VFETCH_DEPENDS";
1022 cav[cac++] = "-VEXTRACT_DEPENDS";
1023 cav[cac++] = "-VPATCH_DEPENDS";
1024 cav[cac++] = "-VBUILD_DEPENDS";
1025 cav[cac++] = "-VLIB_DEPENDS";
1026 cav[cac++] = "-VRUN_DEPENDS";
1027 cav[cac++] = "-VSELECTED_OPTIONS";
1028 cav[cac++] = "-VDESELECTED_OPTIONS";
1029 cav[cac++] = "-VUSE_LINUX";
1030 cav[cac++] = "-VFLAVORS";
1031 cav[cac++] = "-VUSES";
1032
1033 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac,
1034 &pid, NULL, 1, 1);
1035 freestrp(&flavarg);
1036 freestrp(&localbase);
1037
1038 pkg = allocpkg();
1039 if (flavor)
1040 asprintf(&pkg->portdir, "%s/%s@%s", bulk->s1, bulk->s2, flavor);
1041 else
1042 asprintf(&pkg->portdir, "%s/%s", bulk->s1, bulk->s2);
1043
1044 line = 1;
1045 while ((ptr = fgetln(fp, &len)) != NULL) {
1046 if (len == 0 || ptr[len-1] != '\n') {
1047 dfatal("Bad package info for %s/%s response line %d",
1048 bulk->s1, bulk->s2, line);
1049 }
1050 ptr[--len] = 0;
1051
1052 switch(line) {
1053 case 1: /* PKGVERSION */
1054 asprintf(&pkg->version, "%s", ptr);
1055 break;
1056 case 2: /* PKGFILE */
1057 asprintf(&pkg->pkgfile, "%s", ptr);
1058 break;
1059 case 3: /* ALLFILES (aka DISTFILES + patch files) */
1060 asprintf(&pkg->distfiles, "%s", ptr);
1061 break;
1062 case 4: /* DIST_SUBDIR */
1063 pkg->distsubdir = strdup_or_null(ptr);
1064 break;
1065 case 5: /* MAKE_JOBS_NUMBER */
1066 pkg->make_jobs_number = strtol(ptr, NULL, 0);
1067 break;
1068 case 6: /* IGNORE */
1069 pkg->ignore = strdup_or_null(ptr);
1070 break;
1071 case 7: /* FETCH_DEPENDS */
1072 pkg->fetch_deps = strdup_or_null(ptr);
1073 break;
1074 case 8: /* EXTRACT_DEPENDS */
1075 pkg->ext_deps = strdup_or_null(ptr);
1076 break;
1077 case 9: /* PATCH_DEPENDS */
1078 pkg->patch_deps = strdup_or_null(ptr);
1079 break;
1080 case 10: /* BUILD_DEPENDS */
1081 pkg->build_deps = strdup_or_null(ptr);
1082 break;
1083 case 11: /* LIB_DEPENDS */
1084 pkg->lib_deps = strdup_or_null(ptr);
1085 break;
1086 case 12: /* RUN_DEPENDS */
1087 pkg->run_deps = strdup_or_null(ptr);
1088 break;
1089 case 13: /* SELECTED_OPTIONS */
1090 pkg->pos_options = strdup_or_null(ptr);
1091 break;
1092 case 14: /* DESELECTED_OPTIONS */
1093 pkg->neg_options = strdup_or_null(ptr);
1094 break;
1095 case 15: /* USE_LINUX */
1096 if (ptr[0])
1097 pkg->use_linux = 1;
1098 break;
1099 case 16: /* FLAVORS */
1100 asprintf(&pkg->flavors, "%s", ptr);
1101 break;
1102 case 17: /* USES */
1103 asprintf(&pkg->uses, "%s", ptr);
1104 if (strstr(pkg->uses, "metaport"))
1105 pkg->flags |= PKGF_META;
1106 break;
1107 default:
1108 printf("EXTRA LINE: %s\n", ptr);
1109 break;
1110 }
1111 ++line;
1112 }
1113 if (line == 1) {
1114 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1115 pkg->flags |= PKGF_NOTFOUND;
1116 } else if (line != 17 + 1) {
1117 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1118 pkg->flags |= PKGF_CORRUPT;
1119 }
1120 if (dexec_close(fp, pid)) {
1121 printf("make -V* command for %s/%s failed\n",
1122 bulk->s1, bulk->s2);
1123 pkg->flags |= PKGF_CORRUPT;
1124 }
1125 ddassert(bulk->s1);
1126
1127 /*
1128 * DEBUGSTOP mode
1129 */
1130 if (bulk->s4 && bulk->s4[0] == 'd')
1131 pkg->flags |= PKGF_DEBUGSTOP;
1132
1133 /*
1134 * Mark as a dummy node, the front-end will iterate the flavors
1135 * and create sub-nodes for us.
1136 *
1137 * Get rid of elements returned that are for the first flavor.
1138 * We are creating a dummy node here, not the node for the first
1139 * flavor.
1140 */
1141 if (flavor == NULL && pkg->flavors && pkg->flavors[0]) {
1142 pkg->flags |= PKGF_DUMMY;
1143 freestrp(&pkg->fetch_deps);
1144 freestrp(&pkg->ext_deps);
1145 freestrp(&pkg->patch_deps);
1146 freestrp(&pkg->build_deps);
1147 freestrp(&pkg->lib_deps);
1148 freestrp(&pkg->run_deps);
1149 freestrp(&pkg->pkgfile);
1150 }
1151
1152 /*
1153 * Checksum the port directory tree. This just rollsup crcs of the
1154 * path names and a few stat fields (mtime, size) in order to detect
1155 * if any modification has been made to the port.
1156 */
1157 pkg->crc32 = crcDirTree(portpath);
1158
1159 /*
1160 * Only one pkg is put on the return list now. This code no
1161 * longer creates pseudo-nodes for flavors (the frontend requests
1162 * each flavor instead).
1163 */
1164 bulk->list = pkg;
1165 free(portpath);
1166 }
1167
1168 /*
1169 * Query the package (at least to make sure it hasn't been truncated)
1170 * and mark it as PACKAGED if found.
1171 *
1172 * This is a pre-req prior to doing builds, so we cannot assume that
1173 * the template has its pkg-static binary yet.
1174 *
1175 * Threaded function
1176 */
1177 static void
childGetBinaryDistInfo(bulk_t * bulk)1178 childGetBinaryDistInfo(bulk_t *bulk)
1179 {
1180 char *ptr;
1181 FILE *fp;
1182 size_t len;
1183 pkg_t *pkg;
1184 const char *cav[MAXCAC];
1185 char *repopath;
1186 char buf[1024];
1187 pid_t pid;
1188 int cac;
1189 int deleteme;
1190
1191 asprintf(&repopath, "%s/%s", RepositoryPath, bulk->s1);
1192
1193 cac = 0;
1194 cav[cac++] = PKG_BINARY;
1195 cav[cac++] = "query";
1196 cav[cac++] = "-F";
1197 cav[cac++] = repopath;
1198 cav[cac++] = "%n-%v";
1199
1200 fp = dexec_open(NULL, cav, cac, &pid, NULL, 1, 0);
1201 deleteme = DeleteObsoletePkgs;
1202
1203 while ((ptr = fgetln(fp, &len)) != NULL) {
1204 if (len == 0 || ptr[len-1] != '\n')
1205 continue;
1206 ptr[len-1] = 0;
1207
1208 /*
1209 * As of pkg 1.17 the binary dist files use a .pkg suffix
1210 * regardless of the compression format, so always check
1211 * those.
1212 */
1213 snprintf(buf, sizeof(buf), "%s%s", ptr, ".pkg");
1214 pkg = pkg_find(buf);
1215 if (pkg == NULL) {
1216 snprintf(buf, sizeof(buf), "%s%s", ptr, UsePkgSufx);
1217 pkg = pkg_find(buf);
1218 }
1219 if (pkg) {
1220 if (FetchOnlyOpt) {
1221 ddprintf(0, "Note: Package is already packaged, ignore for -f: %s\n", ptr);
1222 deleteme = 0;
1223 } else {
1224 pkg->flags |= PKGF_PACKAGED;
1225 deleteme = 0;
1226 }
1227 } else {
1228 ddprintf(0, "Note: package scan, not in list, "
1229 "skipping %s{.%s,.pkg}\n", ptr, UsePkgSufx);
1230 }
1231 }
1232 if (dexec_close(fp, pid)) {
1233 printf("pkg query command failed for %s\n", repopath);
1234 }
1235 if (deleteme) {
1236 dlog(DLOG_ALL | DLOG_STDOUT,
1237 "Deleting obsolete package %s\n", repopath);
1238 remove(repopath);
1239 }
1240 free(repopath);
1241 }
1242
1243 static void
childOptimizeEnv(bulk_t * bulk)1244 childOptimizeEnv(bulk_t *bulk)
1245 {
1246 char *portpath;
1247 char *localbase;
1248 char *ptr;
1249 FILE *fp;
1250 int line;
1251 size_t len;
1252 const char *cav[MAXCAC];
1253 pid_t pid;
1254 int cac;
1255
1256 asprintf(&portpath, "%s/%s/%s", DPortsPath, bulk->s1, bulk->s2);
1257
1258 /*
1259 * Prevent postgresql, mysql, and other package Makefile tests
1260 * from accessing the host system's /usr/local by setting LOCALBASE
1261 * to an empty directory.
1262 */
1263 asprintf(&localbase, "LOCALBASE=%s/empty", BuildBase);
1264
1265 cac = 0;
1266 cav[cac++] = MAKE_BINARY;
1267 cav[cac++] = "-C";
1268 cav[cac++] = portpath;
1269 cav[cac++] = localbase;
1270 cav[cac++] = "-V_PERL5_FROM_BIN";
1271
1272 fp = dexec_open(portpath + strlen(DPortsPath) + 1, cav, cac,
1273 &pid, NULL, 1, 1);
1274 free(localbase);
1275 free(portpath);
1276
1277 line = 1;
1278 while ((ptr = fgetln(fp, &len)) != NULL) {
1279 if (len == 0 || ptr[len-1] != '\n') {
1280 dfatal("Bad package info for %s/%s response line %d",
1281 bulk->s1, bulk->s2, line);
1282 }
1283 ptr[--len] = 0;
1284
1285 switch(line) {
1286 case 1: /* _PERL5_FROM_BIN */
1287 addbuildenv("_PERL5_FROM_BIN", ptr, BENV_ENVIRONMENT);
1288 break;
1289 default:
1290 printf("childOptimizeEnv: EXTRA LINE: %s\n", ptr);
1291 break;
1292 }
1293 ++line;
1294 }
1295 if (line == 1) {
1296 printf("DPort not found: %s/%s\n", bulk->s1, bulk->s2);
1297 } else if (line != 1 + 1) {
1298 printf("DPort corrupt: %s/%s\n", bulk->s1, bulk->s2);
1299 }
1300 if (dexec_close(fp, pid)) {
1301 printf("childOptimizeEnv() failed\n");
1302 }
1303 }
1304
1305 static int
scan_and_queue_dir(const char * path,const char * level1,int level)1306 scan_and_queue_dir(const char *path, const char *level1, int level)
1307 {
1308 DIR *dir;
1309 char *s1;
1310 char *s2;
1311 struct dirent *den;
1312 struct stat st;
1313 int count = 0;
1314
1315 dir = opendir(path);
1316 dassert(dir, "Cannot open dports path \"%s\"", path);
1317
1318 while ((den = readdir(dir)) != NULL) {
1319 if (den->d_namlen == 1 && den->d_name[0] == '.')
1320 continue;
1321 if (den->d_namlen == 2 &&
1322 den->d_name[0] == '.' && den->d_name[1] == '.')
1323 continue;
1324 asprintf(&s1, "%s/%s", path, den->d_name);
1325 if (lstat(s1, &st) < 0 || !S_ISDIR(st.st_mode)) {
1326 free(s1);
1327 continue;
1328 }
1329 if (level == 1) {
1330 count += scan_and_queue_dir(s1, den->d_name, 2);
1331 free(s1);
1332 continue;
1333 }
1334 asprintf(&s2, "%s/Makefile", s1);
1335 if (lstat(s2, &st) == 0) {
1336 queuebulk(level1, den->d_name, NULL, NULL);
1337 ++count;
1338 }
1339 free(s1);
1340 free(s2);
1341 }
1342 closedir(dir);
1343
1344 return count;
1345 }
1346
1347 static int
scan_binary_repo(const char * path)1348 scan_binary_repo(const char *path)
1349 {
1350 DIR *dir;
1351 struct dirent *den;
1352 int count;
1353
1354 count = 0;
1355 dir = opendir(path);
1356 dassert(dir, "Cannot open repository path \"%s\"", path);
1357
1358 /*
1359 * NOTE: Test includes the '.' in the suffix.
1360 */
1361 while ((den = readdir(dir)) != NULL) {
1362 const char *suffix;
1363
1364 suffix = strrchr(den->d_name, '.');
1365 if (suffix && suffix != den->d_name &&
1366 strcmp(suffix, UsePkgSufx) == 0)
1367 {
1368 queuebulk(den->d_name, NULL, NULL, NULL);
1369 ++count;
1370 }
1371
1372 /*
1373 * As of 1.17, pkg generates .pkg files.
1374 */
1375 if (suffix && suffix != den->d_name &&
1376 strcmp(suffix, ".pkg") == 0)
1377 {
1378 queuebulk(den->d_name, NULL, NULL, NULL);
1379 ++count;
1380 }
1381 }
1382 closedir(dir);
1383
1384 return count;
1385 }
1386
1387 #if 0
1388 static void
1389 pkgfree(pkg_t *pkg)
1390 {
1391 freestrp(&pkg->portdir);
1392 freestrp(&pkg->version);
1393 freestrp(&pkg->pkgfile);
1394 freestrp(&pkg->ignore);
1395 freestrp(&pkg->fetch_deps);
1396 freestrp(&pkg->ext_deps);
1397 freestrp(&pkg->patch_deps);
1398 freestrp(&pkg->build_deps);
1399 freestrp(&pkg->lib_deps);
1400 freestrp(&pkg->run_deps);
1401 freestrp(&pkg->pos_options);
1402 freestrp(&pkg->neg_options);
1403 freestrp(&pkg->flavors);
1404 free(pkg);
1405 }
1406 #endif
1407