1# Copyright (c) Facebook, Inc. and its affiliates.
2#
3# This source code is licensed under the MIT license found in the
4# LICENSE file in the root directory of this source tree.
5
6from __future__ import absolute_import, division, print_function, unicode_literals
7
8import io
9import os
10
11from .builder import (
12    AutoconfBuilder,
13    Boost,
14    CargoBuilder,
15    CMakeBuilder,
16    BistroBuilder,
17    Iproute2Builder,
18    MakeBuilder,
19    NinjaBootstrap,
20    NopBuilder,
21    OpenNSABuilder,
22    OpenSSLBuilder,
23    SqliteBuilder,
24    CMakeBootStrapBuilder,
25)
26from .expr import parse_expr
27from .fetcher import (
28    ArchiveFetcher,
29    GitFetcher,
30    PreinstalledNopFetcher,
31    ShipitTransformerFetcher,
32    SimpleShipitTransformerFetcher,
33    SystemPackageFetcher,
34)
35from .py_wheel_builder import PythonWheelBuilder
36
37
38try:
39    import configparser
40except ImportError:
41    import ConfigParser as configparser
42
43REQUIRED = "REQUIRED"
44OPTIONAL = "OPTIONAL"
45
46SCHEMA = {
47    "manifest": {
48        "optional_section": False,
49        "fields": {
50            "name": REQUIRED,
51            "fbsource_path": OPTIONAL,
52            "shipit_project": OPTIONAL,
53            "shipit_fbcode_builder": OPTIONAL,
54        },
55    },
56    "dependencies": {"optional_section": True, "allow_values": False},
57    "depends.environment": {"optional_section": True},
58    "git": {
59        "optional_section": True,
60        "fields": {"repo_url": REQUIRED, "rev": OPTIONAL, "depth": OPTIONAL},
61    },
62    "download": {
63        "optional_section": True,
64        "fields": {"url": REQUIRED, "sha256": REQUIRED},
65    },
66    "build": {
67        "optional_section": True,
68        "fields": {
69            "builder": REQUIRED,
70            "subdir": OPTIONAL,
71            "build_in_src_dir": OPTIONAL,
72        },
73    },
74    "msbuild": {"optional_section": True, "fields": {"project": REQUIRED}},
75    "cargo": {
76        "optional_section": True,
77        "fields": {
78            "build_doc": OPTIONAL,
79            "workspace_dir": OPTIONAL,
80            "manifests_to_build": OPTIONAL,
81        },
82    },
83    "cmake.defines": {"optional_section": True},
84    "autoconf.args": {"optional_section": True},
85    "autoconf.envcmd.LDFLAGS": {"optional_section": True},
86    "rpms": {"optional_section": True},
87    "debs": {"optional_section": True},
88    "preinstalled.env": {"optional_section": True},
89    "b2.args": {"optional_section": True},
90    "make.build_args": {"optional_section": True},
91    "make.install_args": {"optional_section": True},
92    "make.test_args": {"optional_section": True},
93    "header-only": {"optional_section": True, "fields": {"includedir": REQUIRED}},
94    "shipit.pathmap": {"optional_section": True},
95    "shipit.strip": {"optional_section": True},
96    "install.files": {"optional_section": True},
97}
98
99# These sections are allowed to vary for different platforms
100# using the expression syntax to enable/disable sections
101ALLOWED_EXPR_SECTIONS = [
102    "autoconf.args",
103    "autoconf.envcmd.LDFLAGS",
104    "build",
105    "cmake.defines",
106    "dependencies",
107    "make.build_args",
108    "make.install_args",
109    "b2.args",
110    "download",
111    "git",
112    "install.files",
113    "rpms",
114    "debs",
115]
116
117
118def parse_conditional_section_name(name, section_def):
119    expr = name[len(section_def) + 1 :]
120    return parse_expr(expr, ManifestContext.ALLOWED_VARIABLES)
121
122
123def validate_allowed_fields(file_name, section, config, allowed_fields):
124    for field in config.options(section):
125        if not allowed_fields.get(field):
126            raise Exception(
127                ("manifest file %s section '%s' contains " "unknown field '%s'")
128                % (file_name, section, field)
129            )
130
131    for field in allowed_fields:
132        if allowed_fields[field] == REQUIRED and not config.has_option(section, field):
133            raise Exception(
134                ("manifest file %s section '%s' is missing " "required field '%s'")
135                % (file_name, section, field)
136            )
137
138
139def validate_allow_values(file_name, section, config):
140    for field in config.options(section):
141        value = config.get(section, field)
142        if value is not None:
143            raise Exception(
144                (
145                    "manifest file %s section '%s' has '%s = %s' but "
146                    "this section doesn't allow specifying values "
147                    "for its entries"
148                )
149                % (file_name, section, field, value)
150            )
151
152
153def validate_section(file_name, section, config):
154    section_def = SCHEMA.get(section)
155    if not section_def:
156        for name in ALLOWED_EXPR_SECTIONS:
157            if section.startswith(name + "."):
158                # Verify that the conditional parses, but discard it
159                try:
160                    parse_conditional_section_name(section, name)
161                except Exception as exc:
162                    raise Exception(
163                        ("manifest file %s section '%s' has invalid " "conditional: %s")
164                        % (file_name, section, str(exc))
165                    )
166                section_def = SCHEMA.get(name)
167                canonical_section_name = name
168                break
169        if not section_def:
170            raise Exception(
171                "manifest file %s contains unknown section '%s'" % (file_name, section)
172            )
173    else:
174        canonical_section_name = section
175
176    allowed_fields = section_def.get("fields")
177    if allowed_fields:
178        validate_allowed_fields(file_name, section, config, allowed_fields)
179    elif not section_def.get("allow_values", True):
180        validate_allow_values(file_name, section, config)
181    return canonical_section_name
182
183
184class ManifestParser(object):
185    def __init__(self, file_name, fp=None):
186        # allow_no_value enables listing parameters in the
187        # autoconf.args section one per line
188        config = configparser.RawConfigParser(allow_no_value=True)
189        config.optionxform = str  # make it case sensitive
190
191        if fp is None:
192            with open(file_name, "r") as fp:
193                config.read_file(fp)
194        elif isinstance(fp, type("")):
195            # For testing purposes, parse from a string (str
196            # or unicode)
197            config.read_file(io.StringIO(fp))
198        else:
199            config.read_file(fp)
200
201        # validate against the schema
202        seen_sections = set()
203
204        for section in config.sections():
205            seen_sections.add(validate_section(file_name, section, config))
206
207        for section in SCHEMA.keys():
208            section_def = SCHEMA[section]
209            if (
210                not section_def.get("optional_section", False)
211                and section not in seen_sections
212            ):
213                raise Exception(
214                    "manifest file %s is missing required section %s"
215                    % (file_name, section)
216                )
217
218        self._config = config
219        self.name = config.get("manifest", "name")
220        self.fbsource_path = self.get("manifest", "fbsource_path")
221        self.shipit_project = self.get("manifest", "shipit_project")
222        self.shipit_fbcode_builder = self.get("manifest", "shipit_fbcode_builder")
223
224        if self.name != os.path.basename(file_name):
225            raise Exception(
226                "filename of the manifest '%s' does not match the manifest name '%s'"
227                % (file_name, self.name)
228            )
229
230    def get(self, section, key, defval=None, ctx=None):
231        ctx = ctx or {}
232
233        for s in self._config.sections():
234            if s == section:
235                if self._config.has_option(s, key):
236                    return self._config.get(s, key)
237                return defval
238
239            if s.startswith(section + "."):
240                expr = parse_conditional_section_name(s, section)
241                if not expr.eval(ctx):
242                    continue
243
244                if self._config.has_option(s, key):
245                    return self._config.get(s, key)
246
247        return defval
248
249    def get_section_as_args(self, section, ctx=None):
250        """Intended for use with the make.[build_args/install_args] and
251        autoconf.args sections, this method collects the entries and returns an
252        array of strings.
253        If the manifest contains conditional sections, ctx is used to
254        evaluate the condition and merge in the values.
255        """
256        args = []
257        ctx = ctx or {}
258
259        for s in self._config.sections():
260            if s != section:
261                if not s.startswith(section + "."):
262                    continue
263                expr = parse_conditional_section_name(s, section)
264                if not expr.eval(ctx):
265                    continue
266            for field in self._config.options(s):
267                value = self._config.get(s, field)
268                if value is None:
269                    args.append(field)
270                else:
271                    args.append("%s=%s" % (field, value))
272        return args
273
274    def get_section_as_ordered_pairs(self, section, ctx=None):
275        """Used for eg: shipit.pathmap which has strong
276        ordering requirements"""
277        res = []
278        ctx = ctx or {}
279
280        for s in self._config.sections():
281            if s != section:
282                if not s.startswith(section + "."):
283                    continue
284                expr = parse_conditional_section_name(s, section)
285                if not expr.eval(ctx):
286                    continue
287
288            for key in self._config.options(s):
289                value = self._config.get(s, key)
290                res.append((key, value))
291        return res
292
293    def get_section_as_dict(self, section, ctx=None):
294        d = {}
295        ctx = ctx or {}
296
297        for s in self._config.sections():
298            if s != section:
299                if not s.startswith(section + "."):
300                    continue
301                expr = parse_conditional_section_name(s, section)
302                if not expr.eval(ctx):
303                    continue
304            for field in self._config.options(s):
305                value = self._config.get(s, field)
306                d[field] = value
307        return d
308
309    def update_hash(self, hasher, ctx):
310        """Compute a hash over the configuration for the given
311        context.  The goal is for the hash to change if the config
312        for that context changes, but not if a change is made to
313        the config only for a different platform than that expressed
314        by ctx.  The hash is intended to be used to help invalidate
315        a future cache for the third party build products.
316        The hasher argument is a hash object returned from hashlib."""
317        for section in sorted(SCHEMA.keys()):
318            hasher.update(section.encode("utf-8"))
319
320            # Note: at the time of writing, nothing in the implementation
321            # relies on keys in any config section being ordered.
322            # In theory we could have conflicting flags in different
323            # config sections and later flags override earlier flags.
324            # For the purposes of computing a hash we're not super
325            # concerned about this: manifest changes should be rare
326            # enough and we'd rather that this trigger an invalidation
327            # than strive for a cache hit at this time.
328            pairs = self.get_section_as_ordered_pairs(section, ctx)
329            pairs.sort(key=lambda pair: pair[0])
330            for key, value in pairs:
331                hasher.update(key.encode("utf-8"))
332                if value is not None:
333                    hasher.update(value.encode("utf-8"))
334
335    def is_first_party_project(self):
336        """returns true if this is an FB first-party project"""
337        return self.shipit_project is not None
338
339    def get_required_system_packages(self, ctx):
340        """Returns dictionary of packager system -> list of packages"""
341        return {
342            "rpm": self.get_section_as_args("rpms", ctx),
343            "deb": self.get_section_as_args("debs", ctx),
344        }
345
346    def _is_satisfied_by_preinstalled_environment(self, ctx):
347        envs = self.get_section_as_args("preinstalled.env", ctx)
348        if not envs:
349            return False
350        for key in envs:
351            val = os.environ.get(key, None)
352            print(f"Testing ENV[{key}]: {repr(val)}")
353            if val is None:
354                return False
355            if len(val) == 0:
356                return False
357
358        return True
359
360    def create_fetcher(self, build_options, ctx):
361        use_real_shipit = (
362            ShipitTransformerFetcher.available() and build_options.use_shipit
363        )
364        if (
365            not use_real_shipit
366            and self.fbsource_path
367            and build_options.fbsource_dir
368            and self.shipit_project
369        ):
370            return SimpleShipitTransformerFetcher(build_options, self)
371
372        if (
373            self.fbsource_path
374            and build_options.fbsource_dir
375            and self.shipit_project
376            and ShipitTransformerFetcher.available()
377        ):
378            # We can use the code from fbsource
379            return ShipitTransformerFetcher(build_options, self.shipit_project)
380
381        # Can we satisfy this dep with system packages?
382        if build_options.allow_system_packages:
383            if self._is_satisfied_by_preinstalled_environment(ctx):
384                return PreinstalledNopFetcher()
385
386            packages = self.get_required_system_packages(ctx)
387            package_fetcher = SystemPackageFetcher(build_options, packages)
388            if package_fetcher.packages_are_installed():
389                return package_fetcher
390
391        repo_url = self.get("git", "repo_url", ctx=ctx)
392        if repo_url:
393            rev = self.get("git", "rev")
394            depth = self.get("git", "depth")
395            return GitFetcher(build_options, self, repo_url, rev, depth)
396
397        url = self.get("download", "url", ctx=ctx)
398        if url:
399            # We need to defer this import until now to avoid triggering
400            # a cycle when the facebook/__init__.py is loaded.
401            try:
402                from getdeps.facebook.lfs import LFSCachingArchiveFetcher
403
404                return LFSCachingArchiveFetcher(
405                    build_options, self, url, self.get("download", "sha256", ctx=ctx)
406                )
407            except ImportError:
408                # This FB internal module isn't shippped to github,
409                # so just use its base class
410                return ArchiveFetcher(
411                    build_options, self, url, self.get("download", "sha256", ctx=ctx)
412                )
413
414        raise KeyError(
415            "project %s has no fetcher configuration matching %s" % (self.name, ctx)
416        )
417
418    def create_builder(  # noqa:C901
419        self,
420        build_options,
421        src_dir,
422        build_dir,
423        inst_dir,
424        ctx,
425        loader,
426        final_install_prefix=None,
427        extra_cmake_defines=None,
428    ):
429        builder = self.get("build", "builder", ctx=ctx)
430        if not builder:
431            raise Exception("project %s has no builder for %r" % (self.name, ctx))
432        build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx)
433        if build_in_src_dir == "true":
434            # Some scripts don't work when they are configured and build in
435            # a different directory than source (or when the build directory
436            # is not a subdir of source).
437            build_dir = src_dir
438            subdir = self.get("build", "subdir", None, ctx=ctx)
439            if subdir is not None:
440                build_dir = os.path.join(build_dir, subdir)
441            print("build_dir is %s" % build_dir)  # just to quiet lint
442
443        if builder == "make" or builder == "cmakebootstrap":
444            build_args = self.get_section_as_args("make.build_args", ctx)
445            install_args = self.get_section_as_args("make.install_args", ctx)
446            test_args = self.get_section_as_args("make.test_args", ctx)
447            if builder == "cmakebootstrap":
448                return CMakeBootStrapBuilder(
449                    build_options,
450                    ctx,
451                    self,
452                    src_dir,
453                    None,
454                    inst_dir,
455                    build_args,
456                    install_args,
457                    test_args,
458                )
459            else:
460                return MakeBuilder(
461                    build_options,
462                    ctx,
463                    self,
464                    src_dir,
465                    None,
466                    inst_dir,
467                    build_args,
468                    install_args,
469                    test_args,
470                )
471
472        if builder == "autoconf":
473            args = self.get_section_as_args("autoconf.args", ctx)
474            conf_env_args = {}
475            ldflags_cmd = self.get_section_as_args("autoconf.envcmd.LDFLAGS", ctx)
476            if ldflags_cmd:
477                conf_env_args["LDFLAGS"] = ldflags_cmd
478            return AutoconfBuilder(
479                build_options,
480                ctx,
481                self,
482                src_dir,
483                build_dir,
484                inst_dir,
485                args,
486                conf_env_args,
487            )
488
489        if builder == "boost":
490            args = self.get_section_as_args("b2.args", ctx)
491            return Boost(build_options, ctx, self, src_dir, build_dir, inst_dir, args)
492
493        if builder == "bistro":
494            return BistroBuilder(
495                build_options,
496                ctx,
497                self,
498                src_dir,
499                build_dir,
500                inst_dir,
501            )
502
503        if builder == "cmake":
504            defines = self.get_section_as_dict("cmake.defines", ctx)
505            return CMakeBuilder(
506                build_options,
507                ctx,
508                self,
509                src_dir,
510                build_dir,
511                inst_dir,
512                defines,
513                loader,
514                final_install_prefix,
515                extra_cmake_defines,
516            )
517
518        if builder == "python-wheel":
519            return PythonWheelBuilder(
520                build_options, ctx, self, src_dir, build_dir, inst_dir
521            )
522
523        if builder == "sqlite":
524            return SqliteBuilder(build_options, ctx, self, src_dir, build_dir, inst_dir)
525
526        if builder == "ninja_bootstrap":
527            return NinjaBootstrap(
528                build_options, ctx, self, build_dir, src_dir, inst_dir
529            )
530
531        if builder == "nop":
532            return NopBuilder(build_options, ctx, self, src_dir, inst_dir)
533
534        if builder == "openssl":
535            return OpenSSLBuilder(
536                build_options, ctx, self, build_dir, src_dir, inst_dir
537            )
538
539        if builder == "iproute2":
540            return Iproute2Builder(
541                build_options, ctx, self, src_dir, build_dir, inst_dir
542            )
543
544        if builder == "cargo":
545            build_doc = self.get("cargo", "build_doc", False, ctx)
546            workspace_dir = self.get("cargo", "workspace_dir", None, ctx)
547            manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx)
548            return CargoBuilder(
549                build_options,
550                ctx,
551                self,
552                src_dir,
553                build_dir,
554                inst_dir,
555                build_doc,
556                workspace_dir,
557                manifests_to_build,
558                loader,
559            )
560
561        if builder == "OpenNSA":
562            return OpenNSABuilder(build_options, ctx, self, src_dir, inst_dir)
563
564        raise KeyError("project %s has no known builder" % (self.name))
565
566
567class ManifestContext(object):
568    """ProjectContext contains a dictionary of values to use when evaluating boolean
569    expressions in a project manifest.
570
571    This object should be passed as the `ctx` parameter in ManifestParser.get() calls.
572    """
573
574    ALLOWED_VARIABLES = {"os", "distro", "distro_vers", "fb", "test"}
575
576    def __init__(self, ctx_dict):
577        assert set(ctx_dict.keys()) == self.ALLOWED_VARIABLES
578        self.ctx_dict = ctx_dict
579
580    def get(self, key):
581        return self.ctx_dict[key]
582
583    def set(self, key, value):
584        assert key in self.ALLOWED_VARIABLES
585        self.ctx_dict[key] = value
586
587    def copy(self):
588        return ManifestContext(dict(self.ctx_dict))
589
590    def __str__(self):
591        s = ", ".join(
592            "%s=%s" % (key, value) for key, value in sorted(self.ctx_dict.items())
593        )
594        return "{" + s + "}"
595
596
597class ContextGenerator(object):
598    """ContextGenerator allows creating ManifestContext objects on a per-project basis.
599    This allows us to evaluate different projects with slightly different contexts.
600
601    For instance, this can be used to only enable tests for some projects."""
602
603    def __init__(self, default_ctx):
604        self.default_ctx = ManifestContext(default_ctx)
605        self.ctx_by_project = {}
606
607    def set_value_for_project(self, project_name, key, value):
608        project_ctx = self.ctx_by_project.get(project_name)
609        if project_ctx is None:
610            project_ctx = self.default_ctx.copy()
611            self.ctx_by_project[project_name] = project_ctx
612        project_ctx.set(key, value)
613
614    def set_value_for_all_projects(self, key, value):
615        self.default_ctx.set(key, value)
616        for ctx in self.ctx_by_project.values():
617            ctx.set(key, value)
618
619    def get_context(self, project_name):
620        return self.ctx_by_project.get(project_name, self.default_ctx)
621