1# This Source Code Form is subject to the terms of the Mozilla Public
2# License, v. 2.0. If a copy of the MPL was not distributed with this
3# file, # You can obtain one at http://mozilla.org/MPL/2.0/.
4
5# Utility package for working with moz.yaml files.
6#
7# Requires `pyyaml` and `voluptuous`
8# (both are in-tree under third_party/python)
9
10from __future__ import absolute_import, print_function, unicode_literals
11
12import errno
13import os
14import re
15import sys
16
17HERE = os.path.abspath(os.path.dirname(__file__))
18lib_path = os.path.join(HERE, '..', '..', '..', 'third_party', 'python')
19sys.path.append(os.path.join(lib_path, 'voluptuous'))
20sys.path.append(os.path.join(lib_path, 'pyyaml', 'lib'))
21
22import voluptuous
23import yaml
24from voluptuous import (All, FqdnUrl, Length, Match, Msg, Required, Schema,
25                        Unique, )
26from yaml.error import MarkedYAMLError
27
28# TODO ensure this matches the approved list of licenses
29VALID_LICENSES = [
30    # Standard Licenses (as per https://spdx.org/licenses/)
31    'Apache-2.0',
32    'BSD-2-Clause',
33    'BSD-3-Clause-Clear',
34    'GPL-3.0',
35    'ISC',
36    'ICU',
37    'LGPL-2.1',
38    'LGPL-3.0',
39    'MIT',
40    'MPL-1.1',
41    'MPL-2.0',
42    # Unique Licenses
43    'ACE',  # http://www.cs.wustl.edu/~schmidt/ACE-copying.html
44    'Anti-Grain-Geometry',  # http://www.antigrain.com/license/index.html
45    'JPNIC',  # https://www.nic.ad.jp/ja/idn/idnkit/download/index.html
46    'Khronos',  # https://www.khronos.org/openmaxdl
47    'Unicode',  # http://www.unicode.org/copyright.html
48]
49
50"""
51---
52# Third-Party Library Template
53# All fields are mandatory unless otherwise noted
54
55# Version of this schema
56schema: 1
57
58bugzilla:
59  # Bugzilla product and component for this directory and subdirectories
60  product: product name
61  component: component name
62
63# Document the source of externally hosted code
64origin:
65
66  # Short name of the package/library
67  name: name of the package
68
69  description: short (one line) description
70
71  # Full URL for the package's homepage/etc
72  # Usually different from repository url
73  url: package's homepage url
74
75  # Human-readable identifier for this version/release
76  # Generally "version NNN", "tag SSS", "bookmark SSS"
77  release: identifier
78
79  # The package's license, where possible using the mnemonic from
80  # https://spdx.org/licenses/
81  # Multiple licenses can be specified (as a YAML list)
82  # A "LICENSE" file must exist containing the full license text
83  license: MPL-2.0
84
85# Configuration for the automated vendoring system.
86# Files are always vendored into a directory structure that matches the source
87# repository, into the same directory as the moz.yaml file
88# optional
89vendoring:
90
91  # Repository URL to vendor from
92  # eg. https://github.com/kinetiknz/nestegg.git
93  # Any repository host can be specified here, however initially we'll only
94  # support automated vendoring from selected sources initiall.
95  url: source url (generally repository clone url)
96
97  # Revision to pull in
98  # Must be a long or short commit SHA (long preferred)
99  revision: sha
100
101  # List of patch files to apply after vendoring. Applied in the order
102  # specified, and alphabetically if globbing is used. Patches must apply
103  # cleanly before changes are pushed
104  # All patch files are implicitly added to the keep file list.
105  # optional
106  patches:
107    - file
108    - path/to/file
109    - path/*.patch
110
111  # List of files that are not deleted while vendoring
112  # Implicitly contains "moz.yaml", any files referenced as patches
113  # optional
114  keep:
115    - file
116    - path/to/file
117    - another/path
118    - *.mozilla
119
120  # Files/paths that will not be vendored from source repository
121  # Implicitly contains ".git", and ".gitignore"
122  # optional
123  exclude:
124    - file
125    - path/to/file
126    - another/path
127    - docs
128    - src/*.test
129
130  # Files/paths that will always be vendored, even if they would
131  # otherwise be excluded by "exclude".
132  # optional
133  include:
134    - file
135    - path/to/file
136    - another/path
137    - docs/LICENSE.*
138
139  # If neither "exclude" or "include" are set, all files will be vendored
140  # Files/paths in "include" will always be vendored, even if excluded
141  # eg. excluding "docs/" then including "docs/LICENSE" will vendor just the
142  #     LICENSE file from the docs directory
143
144  # All three file/path parameters ("keep", "exclude", and "include") support
145  # filenames, directory names, and globs/wildcards.
146
147  # In-tree scripts to be executed after vendoring but before pushing.
148  # optional
149  run_after:
150    - script
151    - another script
152"""
153
154RE_SECTION = re.compile(r'^(\S[^:]*):').search
155RE_FIELD = re.compile(r'^\s\s([^:]+):\s+(\S+)$').search
156
157
158class VerifyError(Exception):
159    def __init__(self, filename, error):
160        self.filename = filename
161        self.error = error
162
163    def __str__(self):
164        return '%s: %s' % (self.filename, self.error)
165
166
167def load_moz_yaml(filename, verify=True, require_license_file=True):
168    """Loads and verifies the specified manifest."""
169
170    # Load and parse YAML.
171    try:
172        with open(filename, 'r') as f:
173            manifest = yaml.safe_load(f)
174    except IOError as e:
175        if e.errno == errno.ENOENT:
176            raise VerifyError(filename,
177                              'Failed to find manifest: %s' % filename)
178        raise
179    except MarkedYAMLError as e:
180        raise VerifyError(filename, e)
181
182    if not verify:
183        return manifest
184
185    # Verify schema.
186    if 'schema' not in manifest:
187        raise VerifyError(filename, 'Missing manifest "schema"')
188    if manifest['schema'] == 1:
189        schema = _schema_1()
190        schema_additional = _schema_1_additional
191    else:
192        raise VerifyError(filename, 'Unsupported manifest schema')
193
194    try:
195        schema(manifest)
196        schema_additional(filename, manifest,
197                          require_license_file=require_license_file)
198    except (voluptuous.Error, ValueError) as e:
199        raise VerifyError(filename, e)
200
201    return manifest
202
203
204def update_moz_yaml(filename, release, revision, verify=True, write=True):
205    """Update origin:release and vendoring:revision without stripping
206    comments or reordering fields."""
207
208    if verify:
209        load_moz_yaml(filename)
210
211    lines = []
212    with open(filename) as f:
213        found_release = False
214        found_revision = False
215        section = None
216        for line in f.readlines():
217            m = RE_SECTION(line)
218            if m:
219                section = m.group(1)
220            else:
221                m = RE_FIELD(line)
222                if m:
223                    (name, value) = m.groups()
224                    if section == 'origin' and name == 'release':
225                        line = '  release: %s\n' % release
226                        found_release = True
227                    elif section == 'vendoring' and name == 'revision':
228                        line = '  revision: %s\n' % revision
229                        found_revision = True
230            lines.append(line)
231
232        if not found_release and found_revision:
233            raise ValueError('Failed to find origin:release and '
234                             'vendoring:revision')
235
236    if write:
237        with open(filename, 'w') as f:
238            f.writelines(lines)
239
240
241def _schema_1():
242    """Returns Voluptuous Schema object."""
243    return Schema({
244        Required('schema'): 1,
245        Required('bugzilla'): {
246            Required('product'): All(str, Length(min=1)),
247            Required('component'): All(str, Length(min=1)),
248        },
249        'origin': {
250            Required('name'): All(str, Length(min=1)),
251            Required('description'): All(str, Length(min=1)),
252            Required('url'): FqdnUrl(),
253            Required('license'): Msg(License(), msg='Unsupported License'),
254            Required('release'): All(str, Length(min=1)),
255        },
256        'vendoring': {
257            Required('url'): FqdnUrl(),
258            Required('revision'): Match(r'^[a-fA-F0-9]{12,40}$'),
259            'patches': Unique([str]),
260            'keep': Unique([str]),
261            'exclude': Unique([str]),
262            'include': Unique([str]),
263            'run_after': Unique([str]),
264        },
265    })
266
267
268def _schema_1_additional(filename, manifest, require_license_file=True):
269    """Additional schema/validity checks"""
270
271    # LICENSE file must exist.
272    if require_license_file and 'origin' in manifest:
273        files = [f.lower() for f in os.listdir(os.path.dirname(filename))
274                 if f.lower().startswith('license')]
275        if not ('license' in files
276                or 'license.txt' in files
277                or 'license.rst' in files
278                or 'license.html' in files
279                or 'license.md' in files):
280            license = manifest['origin']['license']
281            if isinstance(license, list):
282                license = '/'.join(license)
283            raise ValueError('Failed to find %s LICENSE file' % license)
284
285    # Cannot vendor without an origin.
286    if 'vendoring' in manifest and 'origin' not in manifest:
287        raise ValueError('"vendoring" requires an "origin"')
288
289    # Check for a simple YAML file
290    with open(filename, 'r') as f:
291        has_schema = False
292        for line in f.readlines():
293            m = RE_SECTION(line)
294            if m:
295                if m.group(1) == 'schema':
296                    has_schema = True
297                    break
298        if not has_schema:
299            raise ValueError('Not simple YAML')
300
301    # Verify YAML can be updated.
302    if 'vendor' in manifest:
303        update_moz_yaml(filename, '', '', verify=False, write=True)
304
305
306class License(object):
307    """Voluptuous validator which verifies the license(s) are valid as per our
308    whitelist."""
309
310    def __call__(self, values):
311        if isinstance(values, str):
312            values = [values]
313        elif not isinstance(values, list):
314            raise ValueError('Must be string or list')
315        for v in values:
316            if v not in VALID_LICENSES:
317                raise ValueError('Bad License')
318        return values
319
320    def __repr__(self):
321        return 'License'
322