1# -*- Mode:Python; indent-tabs-mode:nil; tab-width:4 -*-
2#
3# Modified by Anthony Fok on 2018-10-01 to add support for ppc64el and s390x
4#
5# Copyright (C) 2015-2017 Canonical Ltd
6#
7# This program is free software: you can redistribute it and/or modify
8# it under the terms of the GNU General Public License version 3 as
9# published by the Free Software Foundation.
10#
11# This program is distributed in the hope that it will be useful,
12# but WITHOUT ANY WARRANTY; without even the implied warranty of
13# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14# GNU General Public License for more details.
15#
16# You should have received a copy of the GNU General Public License
17# along with this program.  If not, see <http://www.gnu.org/licenses/>.
18
19"""The nodejs plugin is useful for node/npm based parts.
20
21The plugin uses node to install dependencies from `package.json`. It
22also sets up binaries defined in `package.json` into the `PATH`.
23
24This plugin uses the common plugin keywords as well as those for "sources".
25For more information check the 'plugins' topic for the former and the
26'sources' topic for the latter.
27
28Additionally, this plugin uses the following plugin-specific keywords:
29
30    - node-packages:
31      (list)
32      A list of dependencies to fetch using npm.
33    - node-engine:
34      (string)
35      The version of nodejs you want the snap to run on.
36    - npm-run:
37      (list)
38      A list of targets to `npm run`.
39      These targets will be run in order, after `npm install`
40    - npm-flags:
41      (list)
42      A list of flags for npm.
43    - node-package-manager
44      (string; default: npm)
45      The language package manager to use to drive installation
46      of node packages. Can be either `npm` (default) or `yarn`.
47"""
48
49import collections
50import contextlib
51import json
52import logging
53import os
54import shutil
55import subprocess
56import sys
57
58import snapcraft
59from snapcraft import sources
60from snapcraft.file_utils import link_or_copy_tree
61from snapcraft.internal import errors
62
63logger = logging.getLogger(__name__)
64
65_NODEJS_BASE = "node-v{version}-linux-{arch}"
66_NODEJS_VERSION = "12.18.4"
67_NODEJS_TMPL = "https://nodejs.org/dist/v{version}/{base}.tar.gz"
68_NODEJS_ARCHES = {"i386": "x86", "amd64": "x64", "armhf": "armv7l", "arm64": "arm64", "ppc64el": "ppc64le", "s390x": "s390x"}
69_YARN_URL = "https://yarnpkg.com/latest.tar.gz"
70
71
72class NodePlugin(snapcraft.BasePlugin):
73    @classmethod
74    def schema(cls):
75        schema = super().schema()
76
77        schema["properties"]["node-packages"] = {
78            "type": "array",
79            "minitems": 1,
80            "uniqueItems": True,
81            "items": {"type": "string"},
82            "default": [],
83        }
84        schema["properties"]["node-engine"] = {
85            "type": "string",
86            "default": _NODEJS_VERSION,
87        }
88        schema["properties"]["node-package-manager"] = {
89            "type": "string",
90            "default": "npm",
91            "enum": ["npm", "yarn"],
92        }
93        schema["properties"]["npm-run"] = {
94            "type": "array",
95            "minitems": 1,
96            "uniqueItems": False,
97            "items": {"type": "string"},
98            "default": [],
99        }
100        schema["properties"]["npm-flags"] = {
101            "type": "array",
102            "minitems": 1,
103            "uniqueItems": False,
104            "items": {"type": "string"},
105            "default": [],
106        }
107
108        if "required" in schema:
109            del schema["required"]
110
111        return schema
112
113    @classmethod
114    def get_build_properties(cls):
115        # Inform Snapcraft of the properties associated with building. If these
116        # change in the YAML Snapcraft will consider the build step dirty.
117        return ["node-packages", "npm-run", "npm-flags"]
118
119    @classmethod
120    def get_pull_properties(cls):
121        # Inform Snapcraft of the properties associated with pulling. If these
122        # change in the YAML Snapcraft will consider the build step dirty.
123        return ["node-engine", "node-package-manager"]
124
125    @property
126    def _nodejs_tar(self):
127        if self._nodejs_tar_handle is None:
128            self._nodejs_tar_handle = sources.Tar(
129                self._nodejs_release_uri, self._npm_dir
130            )
131        return self._nodejs_tar_handle
132
133    @property
134    def _yarn_tar(self):
135        if self._yarn_tar_handle is None:
136            self._yarn_tar_handle = sources.Tar(_YARN_URL, self._npm_dir)
137        return self._yarn_tar_handle
138
139    def __init__(self, name, options, project):
140        super().__init__(name, options, project)
141        self._source_package_json = os.path.join(
142            os.path.abspath(self.options.source), "package.json"
143        )
144        self._npm_dir = os.path.join(self.partdir, "npm")
145        self._manifest = collections.OrderedDict()
146        self._nodejs_release_uri = get_nodejs_release(
147            self.options.node_engine, self.project.deb_arch
148        )
149        self._nodejs_tar_handle = None
150        self._yarn_tar_handle = None
151
152    def pull(self):
153        super().pull()
154        os.makedirs(self._npm_dir, exist_ok=True)
155        self._nodejs_tar.download()
156        if self.options.node_package_manager == "yarn":
157            self._yarn_tar.download()
158        # do the install in the pull phase to download all dependencies.
159        if self.options.node_package_manager == "npm":
160            self._npm_install(rootdir=self.sourcedir)
161        else:
162            self._yarn_install(rootdir=self.sourcedir)
163
164    def clean_pull(self):
165        super().clean_pull()
166
167        # Remove the npm directory (if any)
168        if os.path.exists(self._npm_dir):
169            shutil.rmtree(self._npm_dir)
170
171    def build(self):
172        super().build()
173        if self.options.node_package_manager == "npm":
174            installed_node_packages = self._npm_install(rootdir=self.builddir)
175            # Copy the content of the symlink to the build directory
176            # LP: #1702661
177            modules_dir = os.path.join(self.installdir, "lib", "node_modules")
178            _copy_symlinked_content(modules_dir)
179        else:
180            installed_node_packages = self._yarn_install(rootdir=self.builddir)
181            lock_file_path = os.path.join(self.sourcedir, "yarn.lock")
182            if os.path.isfile(lock_file_path):
183                with open(lock_file_path) as lock_file:
184                    self._manifest["yarn-lock-contents"] = lock_file.read()
185
186        self._manifest["node-packages"] = [
187            "{}={}".format(name, installed_node_packages[name])
188            for name in installed_node_packages
189        ]
190
191    def _npm_install(self, rootdir):
192        self._nodejs_tar.provision(
193            self.installdir, clean_target=False, keep_tarball=True
194        )
195        npm_cmd = ["npm"] + self.options.npm_flags
196        npm_install = npm_cmd + ["--cache-min=Infinity", "install"]
197        for pkg in self.options.node_packages:
198            self.run(npm_install + ["--global"] + [pkg], cwd=rootdir)
199        if os.path.exists(os.path.join(rootdir, "package.json")):
200            self.run(npm_install, cwd=rootdir)
201            self.run(npm_install + ["--global"], cwd=rootdir)
202        for target in self.options.npm_run:
203            self.run(npm_cmd + ["run", target], cwd=rootdir)
204        return self._get_installed_node_packages("npm", self.installdir)
205
206    def _yarn_install(self, rootdir):
207        self._nodejs_tar.provision(
208            self.installdir, clean_target=False, keep_tarball=True
209        )
210        self._yarn_tar.provision(self._npm_dir, clean_target=False, keep_tarball=True)
211        yarn_cmd = [os.path.join(self._npm_dir, "bin", "yarn")]
212        yarn_cmd.extend(self.options.npm_flags)
213        if "http_proxy" in os.environ:
214            yarn_cmd.extend(["--proxy", os.environ["http_proxy"]])
215        if "https_proxy" in os.environ:
216            yarn_cmd.extend(["--https-proxy", os.environ["https_proxy"]])
217        flags = []
218        if rootdir == self.builddir:
219            yarn_add = yarn_cmd + ["global", "add"]
220            flags.extend(
221                [
222                    "--offline",
223                    "--prod",
224                    "--global-folder",
225                    self.installdir,
226                    "--prefix",
227                    self.installdir,
228                ]
229            )
230        else:
231            yarn_add = yarn_cmd + ["add"]
232        for pkg in self.options.node_packages:
233            self.run(yarn_add + [pkg] + flags, cwd=rootdir)
234
235        # local packages need to be added as if they were remote, we
236        # remove the local package.json so `yarn add` doesn't pollute it.
237        if os.path.exists(self._source_package_json):
238            with contextlib.suppress(FileNotFoundError):
239                os.unlink(os.path.join(rootdir, "package.json"))
240            shutil.copy(
241                self._source_package_json, os.path.join(rootdir, "package.json")
242            )
243            self.run(yarn_add + ["file:{}".format(rootdir)] + flags, cwd=rootdir)
244
245        # npm run would require to bring back package.json
246        if self.options.npm_run and os.path.exists(self._source_package_json):
247            # The current package.json is the yarn prefilled one.
248            with contextlib.suppress(FileNotFoundError):
249                os.unlink(os.path.join(rootdir, "package.json"))
250            os.link(self._source_package_json, os.path.join(rootdir, "package.json"))
251        for target in self.options.npm_run:
252            self.run(
253                yarn_cmd + ["run", target],
254                cwd=rootdir,
255                env=self._build_environment(rootdir),
256            )
257        return self._get_installed_node_packages("npm", self.installdir)
258
259    def _get_installed_node_packages(self, package_manager, cwd):
260        try:
261            output = self.run_output(
262                [package_manager, "ls", "--global", "--json"], cwd=cwd
263            )
264        except subprocess.CalledProcessError as error:
265            # XXX When dependencies have missing dependencies, an error like
266            # this is printed to stderr:
267            # npm ERR! peer dep missing: glob@*, required by glob-promise@3.1.0
268            # retcode is not 0, which raises an exception.
269            output = error.output.decode(sys.getfilesystemencoding()).strip()
270        packages = collections.OrderedDict()
271        dependencies = json.loads(output, object_pairs_hook=collections.OrderedDict)[
272            "dependencies"
273        ]
274        while dependencies:
275            key, value = dependencies.popitem(last=False)
276            # XXX Just as above, dependencies without version are the ones
277            # missing.
278            if "version" in value:
279                packages[key] = value["version"]
280            if "dependencies" in value:
281                dependencies.update(value["dependencies"])
282        return packages
283
284    def get_manifest(self):
285        return self._manifest
286
287    def _build_environment(self, rootdir):
288        env = os.environ.copy()
289        if rootdir.endswith("src"):
290            hidden_path = os.path.join(rootdir, "node_modules", ".bin")
291            if env.get("PATH"):
292                new_path = "{}:{}".format(hidden_path, env.get("PATH"))
293            else:
294                new_path = hidden_path
295            env["PATH"] = new_path
296        return env
297
298
299def _get_nodejs_base(node_engine, machine):
300    if machine not in _NODEJS_ARCHES:
301        raise errors.SnapcraftEnvironmentError(
302            "architecture not supported ({})".format(machine)
303        )
304    return _NODEJS_BASE.format(version=node_engine, arch=_NODEJS_ARCHES[machine])
305
306
307def get_nodejs_release(node_engine, arch):
308    return _NODEJS_TMPL.format(
309        version=node_engine, base=_get_nodejs_base(node_engine, arch)
310    )
311
312
313def _copy_symlinked_content(modules_dir):
314    """Copy symlinked content.
315
316    When running newer versions of npm, symlinks to the local tree are
317    created from the part's installdir to the root of the builddir of the
318    part (this only affects some build configurations in some projects)
319    which is valid when running from the context of the part but invalid
320    as soon as the artifacts migrate across the steps,
321    i.e.; stage and prime.
322
323    If modules_dir does not exist we simply return.
324    """
325    if not os.path.exists(modules_dir):
326        return
327    modules = [os.path.join(modules_dir, d) for d in os.listdir(modules_dir)]
328    symlinks = [l for l in modules if os.path.islink(l)]
329    for link_path in symlinks:
330        link_target = os.path.realpath(link_path)
331        os.unlink(link_path)
332        link_or_copy_tree(link_target, link_path)
333