1#!/usr/bin/env python
2# This Source Code Form is subject to the terms of the Mozilla Public
3# License, v. 2.0. If a copy of the MPL was not distributed with this
4# file, You can obtain one at http://mozilla.org/MPL/2.0/.
5
6import mock
7import mozunit
8import os
9import shutil
10import struct
11import subprocess
12import sys
13import tempfile
14import unittest
15import buildconfig
16
17from mock import patch
18from mozpack.manifests import InstallManifest
19import mozpack.path as mozpath
20
21import symbolstore
22from symbolstore import realpath
23
24# Some simple functions to mock out files that the platform-specific dumpers will accept.
25# dump_syms itself will not be run (we mock that call out), but we can't override
26# the ShouldProcessFile method since we actually want to test that.
27
28
29def write_elf(filename):
30    open(filename, "wb").write(
31        struct.pack("<7B45x", 0x7F, ord("E"), ord("L"), ord("F"), 1, 1, 1)
32    )
33
34
35def write_macho(filename):
36    open(filename, "wb").write(struct.pack("<I28x", 0xFEEDFACE))
37
38
39def write_dll(filename):
40    open(filename, "w").write("aaa")
41    # write out a fake PDB too
42    open(os.path.splitext(filename)[0] + ".pdb", "w").write("aaa")
43
44
45def target_platform():
46    return buildconfig.substs["OS_TARGET"]
47
48
49def host_platform():
50    return buildconfig.substs["HOST_OS_ARCH"]
51
52
53writer = {
54    "WINNT": write_dll,
55    "Linux": write_elf,
56    "Sunos5": write_elf,
57    "Darwin": write_macho,
58}[target_platform()]
59extension = {"WINNT": ".dll", "Linux": ".so", "Sunos5": ".so", "Darwin": ".dylib"}[
60    target_platform()
61]
62file_output = [
63    {"WINNT": "bogus data", "Linux": "ELF executable", "Darwin": "Mach-O executable"}[
64        target_platform()
65    ]
66]
67
68
69def add_extension(files):
70    return [f + extension for f in files]
71
72
73class HelperMixin(object):
74    """
75    Test that passing filenames to exclude from processing works.
76    """
77
78    def setUp(self):
79        self.test_dir = tempfile.mkdtemp()
80        if not self.test_dir.endswith(os.sep):
81            self.test_dir += os.sep
82        symbolstore.srcdirRepoInfo = {}
83        symbolstore.vcsFileInfoCache = {}
84
85        # Remove environment variables that can influence tests.
86        for e in ("MOZ_SOURCE_CHANGESET", "MOZ_SOURCE_REPO"):
87            try:
88                del os.environ[e]
89            except KeyError:
90                pass
91
92    def tearDown(self):
93        shutil.rmtree(self.test_dir)
94        symbolstore.srcdirRepoInfo = {}
95        symbolstore.vcsFileInfoCache = {}
96
97    def make_dirs(self, f):
98        d = os.path.dirname(f)
99        if d and not os.path.exists(d):
100            os.makedirs(d)
101
102    def make_file(self, path):
103        self.make_dirs(path)
104        with open(path, "wb"):
105            pass
106
107    def add_test_files(self, files):
108        for f in files:
109            f = os.path.join(self.test_dir, f)
110            self.make_dirs(f)
111            writer(f)
112
113
114def mock_dump_syms(module_id, filename, extra=[]):
115    return (
116        ["MODULE os x86 %s %s" % (module_id, filename)]
117        + extra
118        + ["FILE 0 foo.c", "PUBLIC xyz 123"]
119    )
120
121
122class TestCopyDebug(HelperMixin, unittest.TestCase):
123    def setUp(self):
124        HelperMixin.setUp(self)
125        self.symbol_dir = tempfile.mkdtemp()
126        self.mock_call = patch("subprocess.call").start()
127        self.stdouts = []
128        self.mock_popen = patch("subprocess.Popen").start()
129        stdout_iter = self.next_mock_stdout()
130
131        def next_popen(*args, **kwargs):
132            m = mock.MagicMock()
133            # Get the iterators over whatever output was provided.
134            stdout_ = next(stdout_iter)
135            # Eager evaluation for communicate(), below.
136            stdout_ = list(stdout_)
137            # stdout is really an iterator, so back to iterators we go.
138            m.stdout = iter(stdout_)
139            m.wait.return_value = 0
140            # communicate returns the full text of stdout and stderr.
141            m.communicate.return_value = ("\n".join(stdout_), "")
142            return m
143
144        self.mock_popen.side_effect = next_popen
145        shutil.rmtree = patch("shutil.rmtree").start()
146
147    def tearDown(self):
148        HelperMixin.tearDown(self)
149        patch.stopall()
150        shutil.rmtree(self.symbol_dir)
151
152    def next_mock_stdout(self):
153        if not self.stdouts:
154            yield iter([])
155        for s in self.stdouts:
156            yield iter(s)
157
158    def test_copy_debug_universal(self):
159        """
160        Test that dumping symbols for multiple architectures only copies debug symbols once
161        per file.
162        """
163        copied = []
164
165        def mock_copy_debug(filename, debug_file, guid, code_file, code_id):
166            copied.append(
167                filename[len(self.symbol_dir):]
168                if filename.startswith(self.symbol_dir)
169                else filename
170            )
171
172        self.add_test_files(add_extension(["foo"]))
173        # Windows doesn't call file(1) to figure out if the file should be processed.
174        if target_platform() != "WINNT":
175            self.stdouts.append(file_output)
176        self.stdouts.append(mock_dump_syms("X" * 33, add_extension(["foo"])[0]))
177        self.stdouts.append(mock_dump_syms("Y" * 33, add_extension(["foo"])[0]))
178
179        def mock_dsymutil(args, **kwargs):
180            filename = args[-1]
181            os.makedirs(filename + ".dSYM")
182            return 0
183
184        self.mock_call.side_effect = mock_dsymutil
185        d = symbolstore.GetPlatformSpecificDumper(
186            dump_syms="dump_syms",
187            symbol_path=self.symbol_dir,
188            copy_debug=True,
189            archs="abc xyz",
190        )
191        d.CopyDebug = mock_copy_debug
192        d.Process(os.path.join(self.test_dir, add_extension(["foo"])[0]))
193        self.assertEqual(1, len(copied))
194
195    @patch.dict("buildconfig.substs._dict", {"MAKECAB": "makecab"})
196    def test_copy_debug_copies_binaries(self):
197        """
198        Test that CopyDebug copies binaries as well on Windows.
199        """
200        test_file = os.path.join(self.test_dir, "foo.dll")
201        write_dll(test_file)
202        code_file = "foo.dll"
203        code_id = "abc123"
204        self.stdouts.append(
205            mock_dump_syms(
206                "X" * 33, "foo.pdb", ["INFO CODE_ID %s %s" % (code_id, code_file)]
207            )
208        )
209
210        def mock_compress(args, **kwargs):
211            filename = args[-1]
212            open(filename, "w").write("stuff")
213            return 0
214
215        self.mock_call.side_effect = mock_compress
216        d = symbolstore.Dumper_Win32(
217            dump_syms="dump_syms", symbol_path=self.symbol_dir, copy_debug=True
218        )
219        d.Process(test_file)
220        self.assertTrue(
221            os.path.isfile(
222                os.path.join(self.symbol_dir, code_file, code_id, code_file[:-1] + "_")
223            )
224        )
225
226
227class TestGetVCSFilename(HelperMixin, unittest.TestCase):
228    def setUp(self):
229        HelperMixin.setUp(self)
230
231    def tearDown(self):
232        HelperMixin.tearDown(self)
233
234    @patch("subprocess.Popen")
235    def testVCSFilenameHg(self, mock_Popen):
236        # mock calls to `hg parent` and `hg showconfig paths.default`
237        mock_communicate = mock_Popen.return_value.communicate
238        mock_communicate.side_effect = [
239            ("abcd1234", ""),
240            ("http://example.com/repo", ""),
241        ]
242        os.mkdir(os.path.join(self.test_dir, ".hg"))
243        filename = os.path.join(self.test_dir, "foo.c")
244        self.assertEqual(
245            "hg:example.com/repo:foo.c:abcd1234",
246            symbolstore.GetVCSFilename(filename, [self.test_dir])[0],
247        )
248
249    @patch("subprocess.Popen")
250    def testVCSFilenameHgMultiple(self, mock_Popen):
251        # mock calls to `hg parent` and `hg showconfig paths.default`
252        mock_communicate = mock_Popen.return_value.communicate
253        mock_communicate.side_effect = [
254            ("abcd1234", ""),
255            ("http://example.com/repo", ""),
256            ("0987ffff", ""),
257            ("http://example.com/other", ""),
258        ]
259        srcdir1 = os.path.join(self.test_dir, "one")
260        srcdir2 = os.path.join(self.test_dir, "two")
261        os.makedirs(os.path.join(srcdir1, ".hg"))
262        os.makedirs(os.path.join(srcdir2, ".hg"))
263        filename1 = os.path.join(srcdir1, "foo.c")
264        filename2 = os.path.join(srcdir2, "bar.c")
265        self.assertEqual(
266            "hg:example.com/repo:foo.c:abcd1234",
267            symbolstore.GetVCSFilename(filename1, [srcdir1, srcdir2])[0],
268        )
269        self.assertEqual(
270            "hg:example.com/other:bar.c:0987ffff",
271            symbolstore.GetVCSFilename(filename2, [srcdir1, srcdir2])[0],
272        )
273
274    def testVCSFilenameEnv(self):
275        # repo URL and changeset read from environment variables if defined.
276        os.environ["MOZ_SOURCE_REPO"] = "https://somewhere.com/repo"
277        os.environ["MOZ_SOURCE_CHANGESET"] = "abcdef0123456"
278        os.mkdir(os.path.join(self.test_dir, ".hg"))
279        filename = os.path.join(self.test_dir, "foo.c")
280        self.assertEqual(
281            "hg:somewhere.com/repo:foo.c:abcdef0123456",
282            symbolstore.GetVCSFilename(filename, [self.test_dir])[0],
283        )
284
285
286# SHA-512 of a zero-byte file
287EMPTY_SHA512 = (
288    "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff"
289)
290EMPTY_SHA512 += "8318d2877eec2f63b931bd47417a81a538327af927da3e"
291
292
293class TestGeneratedFilePath(HelperMixin, unittest.TestCase):
294    def setUp(self):
295        HelperMixin.setUp(self)
296
297    def tearDown(self):
298        HelperMixin.tearDown(self)
299
300    def test_generated_file_path(self):
301        # Make an empty generated file
302        g = os.path.join(self.test_dir, "generated")
303        rel_path = "a/b/generated"
304        with open(g, "wb"):
305            pass
306        expected = "s3:bucket:{}/{}:".format(EMPTY_SHA512, rel_path)
307        self.assertEqual(
308            expected, symbolstore.get_generated_file_s3_path(g, rel_path, "bucket")
309        )
310
311
312if host_platform() == "WINNT":
313
314    class TestRealpath(HelperMixin, unittest.TestCase):
315        def test_realpath(self):
316            # self.test_dir is going to be 8.3 paths...
317            junk = os.path.join(self.test_dir, 'x')
318            with open(junk, 'w') as o:
319                o.write('x')
320            fixed_dir = os.path.dirname(realpath(junk))
321            files = [
322                "one\\two.c",
323                "three\\Four.d",
324                "Five\\Six.e",
325                "seven\\Eight\\nine.F",
326            ]
327            for rel_path in files:
328                full_path = os.path.normpath(os.path.join(self.test_dir, rel_path))
329                self.make_dirs(full_path)
330                with open(full_path, 'w') as o:
331                    o.write('x')
332                fixed_path = realpath(full_path.lower())
333                fixed_path = os.path.relpath(fixed_path, fixed_dir)
334                self.assertEqual(rel_path, fixed_path)
335
336
337if target_platform() == "WINNT":
338
339    class TestSourceServer(HelperMixin, unittest.TestCase):
340        @patch("subprocess.call")
341        @patch("subprocess.Popen")
342        @patch.dict("buildconfig.substs._dict", {"PDBSTR": "pdbstr"})
343        def test_HGSERVER(self, mock_Popen, mock_call):
344            """
345            Test that HGSERVER gets set correctly in the source server index.
346            """
347            symbolpath = os.path.join(self.test_dir, "symbols")
348            os.makedirs(symbolpath)
349            srcdir = os.path.join(self.test_dir, "srcdir")
350            os.makedirs(os.path.join(srcdir, ".hg"))
351            sourcefile = os.path.join(srcdir, "foo.c")
352            test_files = add_extension(["foo"])
353            self.add_test_files(test_files)
354            # mock calls to `dump_syms`, `hg parent` and
355            # `hg showconfig paths.default`
356            mock_Popen.return_value.stdout = iter(
357                [
358                    "MODULE os x86 %s %s" % ("X" * 33, test_files[0]),
359                    "FILE 0 %s" % sourcefile,
360                    "PUBLIC xyz 123",
361                ]
362            )
363            mock_Popen.return_value.wait.return_value = 0
364            mock_communicate = mock_Popen.return_value.communicate
365            mock_communicate.side_effect = [
366                ("abcd1234", ""),
367                ("http://example.com/repo", ""),
368            ]
369            # And mock the call to pdbstr to capture the srcsrv stream data.
370            global srcsrv_stream
371            srcsrv_stream = None
372
373            def mock_pdbstr(args, cwd="", **kwargs):
374                for arg in args:
375                    if arg.startswith("-i:"):
376                        global srcsrv_stream
377                        srcsrv_stream = open(os.path.join(cwd, arg[3:]), "r").read()
378                return 0
379
380            mock_call.side_effect = mock_pdbstr
381            d = symbolstore.GetPlatformSpecificDumper(
382                dump_syms="dump_syms",
383                symbol_path=symbolpath,
384                srcdirs=[srcdir],
385                vcsinfo=True,
386                srcsrv=True,
387                copy_debug=True,
388            )
389            # stub out CopyDebug
390            d.CopyDebug = lambda *args: True
391            d.Process(os.path.join(self.test_dir, test_files[0]))
392            self.assertNotEqual(srcsrv_stream, None)
393            hgserver = [
394                x.rstrip()
395                for x in srcsrv_stream.splitlines()
396                if x.startswith("HGSERVER=")
397            ]
398            self.assertEqual(len(hgserver), 1)
399            self.assertEqual(hgserver[0].split("=")[1], "http://example.com/repo")
400
401
402class TestInstallManifest(HelperMixin, unittest.TestCase):
403    def setUp(self):
404        HelperMixin.setUp(self)
405        self.srcdir = os.path.join(self.test_dir, "src")
406        os.mkdir(self.srcdir)
407        self.objdir = os.path.join(self.test_dir, "obj")
408        os.mkdir(self.objdir)
409        self.manifest = InstallManifest()
410        self.canonical_mapping = {}
411        for s in ["src1", "src2"]:
412            srcfile = realpath(os.path.join(self.srcdir, s))
413            objfile = realpath(os.path.join(self.objdir, s))
414            self.canonical_mapping[objfile] = srcfile
415            self.manifest.add_copy(srcfile, s)
416        self.manifest_file = os.path.join(self.test_dir, "install-manifest")
417        self.manifest.write(self.manifest_file)
418
419    def testMakeFileMapping(self):
420        """
421        Test that valid arguments are validated.
422        """
423        arg = "%s,%s" % (self.manifest_file, self.objdir)
424        ret = symbolstore.validate_install_manifests([arg])
425        self.assertEqual(len(ret), 1)
426        manifest, dest = ret[0]
427        self.assertTrue(isinstance(manifest, InstallManifest))
428        self.assertEqual(dest, self.objdir)
429
430        file_mapping = symbolstore.make_file_mapping(ret)
431        for obj, src in self.canonical_mapping.items():
432            self.assertTrue(obj in file_mapping)
433            self.assertEqual(file_mapping[obj], src)
434
435    def testMissingFiles(self):
436        """
437        Test that missing manifest files or install directories give errors.
438        """
439        missing_manifest = os.path.join(self.test_dir, "missing-manifest")
440        arg = "%s,%s" % (missing_manifest, self.objdir)
441        with self.assertRaises(IOError) as e:
442            symbolstore.validate_install_manifests([arg])
443            self.assertEqual(e.filename, missing_manifest)
444
445        missing_install_dir = os.path.join(self.test_dir, "missing-dir")
446        arg = "%s,%s" % (self.manifest_file, missing_install_dir)
447        with self.assertRaises(IOError) as e:
448            symbolstore.validate_install_manifests([arg])
449            self.assertEqual(e.filename, missing_install_dir)
450
451    def testBadManifest(self):
452        """
453        Test that a bad manifest file give errors.
454        """
455        bad_manifest = os.path.join(self.test_dir, 'bad-manifest')
456        with open(bad_manifest, 'w') as f:
457            f.write('junk\n')
458        arg = '%s,%s' % (bad_manifest, self.objdir)
459        with self.assertRaises(IOError) as e:
460            symbolstore.validate_install_manifests([arg])
461            self.assertEqual(e.filename, bad_manifest)
462
463    def testBadArgument(self):
464        """
465        Test that a bad manifest argument gives an error.
466        """
467        with self.assertRaises(ValueError):
468            symbolstore.validate_install_manifests(["foo"])
469
470
471class TestFileMapping(HelperMixin, unittest.TestCase):
472    def setUp(self):
473        HelperMixin.setUp(self)
474        self.srcdir = os.path.join(self.test_dir, "src")
475        os.mkdir(self.srcdir)
476        self.objdir = os.path.join(self.test_dir, "obj")
477        os.mkdir(self.objdir)
478        self.symboldir = os.path.join(self.test_dir, "symbols")
479        os.mkdir(self.symboldir)
480
481    @patch("subprocess.Popen")
482    def testFileMapping(self, mock_Popen):
483        files = [("a/b", "mozilla/b"), ("c/d", "foo/d")]
484        if os.sep != "/":
485            files = [[f.replace("/", os.sep) for f in x] for x in files]
486        file_mapping = {}
487        dumped_files = []
488        expected_files = []
489        self.make_dirs(os.path.join(self.objdir, "x", "y"))
490        for s, o in files:
491            srcfile = os.path.join(self.srcdir, s)
492            self.make_file(srcfile)
493            expected_files.append(realpath(srcfile))
494            objfile = os.path.join(self.objdir, o)
495            self.make_file(objfile)
496            file_mapping[realpath(objfile)] = realpath(srcfile)
497            dumped_files.append(os.path.join(self.objdir, "x", "y", "..", "..", o))
498        # mock the dump_syms output
499        file_id = ("X" * 33, "somefile")
500
501        def mk_output(files):
502            return iter(
503                ["MODULE os x86 %s %s\n" % file_id]
504                + ["FILE %d %s\n" % (i, s) for i, s in enumerate(files)]
505                + ["PUBLIC xyz 123\n"]
506            )
507
508        mock_Popen.return_value.stdout = mk_output(dumped_files)
509        mock_Popen.return_value.wait.return_value = 0
510
511        d = symbolstore.Dumper('dump_syms', self.symboldir,
512                               file_mapping=file_mapping)
513        f = os.path.join(self.objdir, 'somefile')
514        open(f, 'w').write('blah')
515        d.Process(f)
516        expected_output = "".join(mk_output(expected_files))
517        symbol_file = os.path.join(
518            self.symboldir, file_id[1], file_id[0], file_id[1] + ".sym"
519        )
520        self.assertEqual(open(symbol_file, "r").read(), expected_output)
521
522
523class TestFunctional(HelperMixin, unittest.TestCase):
524    """Functional tests of symbolstore.py, calling it with a real
525    dump_syms binary and passing in a real binary to dump symbols from.
526
527    Since the rest of the tests in this file mock almost everything and
528    don't use the actual process pool like buildsymbols does, this tests
529    that the way symbolstore.py gets called in buildsymbols works.
530    """
531
532    def setUp(self):
533        HelperMixin.setUp(self)
534        self.skip_test = False
535        if buildconfig.substs["MOZ_BUILD_APP"] != "browser":
536            self.skip_test = True
537        if buildconfig.substs.get("ENABLE_STRIP"):
538            self.skip_test = True
539        # Bug 1608146.
540        if buildconfig.substs.get("MOZ_CODE_COVERAGE"):
541            self.skip_test = True
542        self.topsrcdir = buildconfig.topsrcdir
543        self.script_path = os.path.join(
544            self.topsrcdir, "toolkit", "crashreporter", "tools", "symbolstore.py"
545        )
546        if target_platform() == "WINNT":
547            self.dump_syms = buildconfig.substs["DUMP_SYMS"]
548        else:
549            self.dump_syms = os.path.join(
550                buildconfig.topobjdir, "dist", "host", "bin", "dump_syms"
551            )
552
553        if target_platform() == "WINNT":
554            self.target_bin = os.path.join(
555                buildconfig.topobjdir, "dist", "bin", "firefox.exe"
556            )
557        else:
558            self.target_bin = os.path.join(
559                buildconfig.topobjdir, "dist", "bin", "firefox-bin"
560            )
561
562    def tearDown(self):
563        HelperMixin.tearDown(self)
564
565    def testSymbolstore(self):
566        if self.skip_test:
567            raise unittest.SkipTest('Skipping test in non-Firefox product')
568        dist_include_manifest = os.path.join(buildconfig.topobjdir,
569                                             '_build_manifests/install/dist_include')
570        dist_include = os.path.join(buildconfig.topobjdir, 'dist/include')
571        browser_app = os.path.join(buildconfig.topobjdir, 'browser/app')
572        output = subprocess.check_output([sys.executable,
573                                          self.script_path,
574                                          '--vcs-info',
575                                          '-s', self.topsrcdir,
576                                          '--install-manifest=%s,%s' % (dist_include_manifest,
577                                                                        dist_include),
578                                          self.dump_syms,
579                                          self.test_dir,
580                                          self.target_bin],
581                                         universal_newlines=True,
582                                         stderr=None,
583                                         cwd=browser_app)
584        lines = [l for l in output.splitlines() if l.strip()]
585        self.assertEqual(1, len(lines),
586                         'should have one filename in the output; got %s' % repr(output))
587        symbol_file = os.path.join(self.test_dir, lines[0])
588        self.assertTrue(os.path.isfile(symbol_file))
589        symlines = open(symbol_file, "r").readlines()
590        file_lines = [l for l in symlines if l.startswith("FILE")]
591
592        def check_hg_path(lines, match):
593            match_lines = [l for l in file_lines if match in l]
594            self.assertTrue(
595                len(match_lines) >= 1, "should have a FILE line for " + match
596            )
597            # Skip this check for local git repositories.
598            if not os.path.isdir(mozpath.join(self.topsrcdir, ".hg")):
599                return
600            for line in match_lines:
601                filename = line.split(None, 2)[2]
602                self.assertEqual("hg:", filename[:3])
603
604        # Check that nsBrowserApp.cpp is listed as a FILE line, and that
605        # it was properly mapped to the source repo.
606        check_hg_path(file_lines, "nsBrowserApp.cpp")
607        # Also check Sprintf.h to verify that files from dist/include
608        # are properly mapped.
609        check_hg_path(file_lines, "mfbt/Sprintf.h")
610
611
612if __name__ == "__main__":
613    mozunit.main()
614