1# Copyright 2020 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4"""Test apps for running tests using xcodebuild."""
5
6import os
7import plistlib
8import subprocess
9import time
10
11import shard_util
12import test_runner
13
14
15OUTPUT_DISABLED_TESTS_TEST_ARG = '--write-compiled-tests-json-to-writable-path'
16
17
18#TODO(crbug.com/1046911): Remove usage of KIF filters.
19def get_kif_test_filter(tests, invert=False):
20  """Returns the KIF test filter to filter the given test cases.
21
22  Args:
23    tests: List of test cases to filter.
24    invert: Whether to invert the filter or not. Inverted, the filter will match
25      everything except the given test cases.
26
27  Returns:
28    A string which can be supplied to GKIF_SCENARIO_FILTER.
29  """
30  # A pipe-separated list of test cases with the "KIF." prefix omitted.
31  # e.g. NAME:a|b|c matches KIF.a, KIF.b, KIF.c.
32  # e.g. -NAME:a|b|c matches everything except KIF.a, KIF.b, KIF.c.
33  test_filter = '|'.join(test.split('KIF.', 1)[-1] for test in tests)
34  if invert:
35    return '-NAME:%s' % test_filter
36  return 'NAME:%s' % test_filter
37
38
39def get_gtest_filter(tests, invert=False):
40  """Returns the GTest filter to filter the given test cases.
41
42  Args:
43    tests: List of test cases to filter.
44    invert: Whether to invert the filter or not. Inverted, the filter will match
45      everything except the given test cases.
46
47  Returns:
48    A string which can be supplied to --gtest_filter.
49  """
50  # A colon-separated list of tests cases.
51  # e.g. a:b:c matches a, b, c.
52  # e.g. -a:b:c matches everything except a, b, c.
53  test_filter = ':'.join(test for test in tests)
54  if invert:
55    return '-%s' % test_filter
56  return test_filter
57
58
59def get_bundle_id(app_path):
60  """Get bundle identifier for app.
61
62  Args:
63    app_path: (str) A path to app.
64  """
65  return subprocess.check_output([
66      '/usr/libexec/PlistBuddy',
67      '-c',
68      'Print:CFBundleIdentifier',
69      os.path.join(app_path, 'Info.plist'),
70  ]).rstrip()
71
72
73class GTestsApp(object):
74  """Gtests app to run.
75
76  Stores data about egtests:
77    test_app: full path to an app.
78  """
79
80  def __init__(self,
81               test_app,
82               included_tests=None,
83               excluded_tests=None,
84               test_args=None,
85               env_vars=None,
86               release=False,
87               host_app_path=None):
88    """Initialize Egtests.
89
90    Args:
91      test_app: (str) full path to egtests app.
92      included_tests: (list) Specific tests to run
93         E.g.
94          [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
95      excluded_tests: (list) Specific tests not to run
96         E.g.
97          [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
98      test_args: List of strings to pass as arguments to the test when
99        launching.
100      env_vars: List of environment variables to pass to the test itself.
101      release: (bool) Whether the app is release build.
102
103    Raises:
104      AppNotFoundError: If the given app does not exist
105    """
106    if not os.path.exists(test_app):
107      raise test_runner.AppNotFoundError(test_app)
108    self.test_app_path = test_app
109    self.project_path = os.path.dirname(self.test_app_path)
110    self.test_args = test_args or []
111    self.env_vars = {}
112    for env_var in env_vars or []:
113      env_var = env_var.split('=', 1)
114      self.env_vars[env_var[0]] = None if len(env_var) == 1 else env_var[1]
115    self.included_tests = included_tests or []
116    self.excluded_tests = excluded_tests or []
117    self.disabled_tests = []
118    self.module_name = os.path.splitext(os.path.basename(test_app))[0]
119    self.release = release
120    self.host_app_path = host_app_path
121
122  def fill_xctest_run(self, out_dir):
123    """Fills xctestrun file by egtests.
124
125    Args:
126      out_dir: (str) A path where xctestrun will store.
127
128    Returns:
129      A path to xctestrun file.
130    """
131    folder = os.path.abspath(os.path.join(out_dir, os.pardir))
132    if not os.path.exists(folder):
133      os.makedirs(folder)
134    xctestrun = os.path.join(folder, 'run_%d.xctestrun' % int(time.time()))
135    if not os.path.exists(xctestrun):
136      with open(xctestrun, 'w'):
137        pass
138    # Creates a dict with data about egtests to run - fill all required fields:
139    # egtests_module, egtest_app_path, egtests_xctest_path and
140    # filtered tests if filter is specified.
141    # Write data in temp xctest run file.
142    plistlib.writePlist(self.fill_xctestrun_node(), xctestrun)
143    return xctestrun
144
145  def fill_xctestrun_node(self):
146    """Fills only required nodes for egtests in xctestrun file.
147
148    Returns:
149      A node with filled required fields about egtests.
150    """
151    module = self.module_name + '_module'
152
153    # If --run-with-custom-webkit is passed as a test arg, set up
154    # DYLD_FRAMEWORK_PATH and DYLD_LIBRARY_PATH to load the custom webkit
155    # modules.
156    dyld_path = self.project_path
157    if '--run-with-custom-webkit' in self.test_args:
158      if self.host_app_path:
159        webkit_path = os.path.join(self.host_app_path, 'WebKitFrameworks')
160      else:
161        webkit_path = os.path.join(self.test_app_path, 'WebKitFrameworks')
162      dyld_path = dyld_path + ':' + webkit_path
163
164    module_data = {
165        'TestBundlePath': self.test_app_path,
166        'TestHostPath': self.test_app_path,
167        'TestHostBundleIdentifier': get_bundle_id(self.test_app_path),
168        'TestingEnvironmentVariables': {
169            'DYLD_LIBRARY_PATH':
170                '%s:__PLATFORMS__/iPhoneSimulator.platform/Developer/Library' %
171                dyld_path,
172            'DYLD_FRAMEWORK_PATH':
173                '%s:__PLATFORMS__/iPhoneSimulator.platform/'
174                'Developer/Library/Frameworks' % dyld_path,
175        }
176    }
177
178    xctestrun_data = {module: module_data}
179    kif_filter = []
180    gtest_filter = []
181
182    if self.included_tests:
183      kif_filter = get_kif_test_filter(self.included_tests, invert=False)
184      gtest_filter = get_gtest_filter(self.included_tests, invert=False)
185    elif self.excluded_tests:
186      kif_filter = get_kif_test_filter(self.excluded_tests, invert=True)
187      gtest_filter = get_gtest_filter(self.excluded_tests, invert=True)
188
189    if kif_filter:
190      self.env_vars['GKIF_SCENARIO_FILTER'] = gtest_filter
191    if gtest_filter:
192      # Removed previous gtest-filter if exists.
193      self.test_args = [el for el in self.test_args
194                        if not el.startswith('--gtest_filter=')]
195      self.test_args.append('--gtest_filter=%s' % gtest_filter)
196
197    if self.env_vars:
198      xctestrun_data[module].update({'EnvironmentVariables': self.env_vars})
199    if self.test_args:
200      xctestrun_data[module].update({'CommandLineArguments': self.test_args})
201
202    if self.excluded_tests:
203      xctestrun_data[module].update({
204          'SkipTestIdentifiers': self.excluded_tests
205      })
206    if self.included_tests:
207      xctestrun_data[module].update({
208          'OnlyTestIdentifiers': self.included_tests
209      })
210    return xctestrun_data
211
212  def command(self, out_dir, destination, shards):
213    """Returns the command that launches tests using xcodebuild.
214
215    Format of command:
216    xcodebuild test-without-building -xctestrun file.xctestrun \
217      -parallel-testing-enabled YES -parallel-testing-worker-count %d% \
218      [-destination "destination"]  -resultBundlePath %output_path%
219
220    Args:
221      out_dir: (str) An output directory.
222      destination: (str) A destination of running simulator.
223      shards: (int) A number of shards.
224
225    Returns:
226      A list of strings forming the command to launch the test.
227    """
228    cmd = [
229        'xcodebuild', 'test-without-building',
230        '-xctestrun', self.fill_xctest_run(out_dir),
231        '-destination', destination,
232        '-resultBundlePath', out_dir
233    ]
234    if shards > 1:
235      cmd += ['-parallel-testing-enabled', 'YES',
236              '-parallel-testing-worker-count', str(shards)]
237    return cmd
238
239  def get_all_tests(self):
240    """Gets all tests to run in this object."""
241    # Method names that starts with test* and also are in *TestCase classes
242    # but they are not test-methods.
243    # TODO(crbug.com/982435): Rename not test methods with test-suffix.
244    none_tests = ['ChromeTestCase/testServer', 'FindInPageTestCase/testURL']
245    # TODO(crbug.com/1123681): Move all_tests to class var. Set all_tests,
246    # disabled_tests values in initialization to avoid multiple calls to otool.
247    all_tests = []
248    # Only store the tests when there is the test arg.
249    store_disabled_tests = OUTPUT_DISABLED_TESTS_TEST_ARG in self.test_args
250    self.disabled_tests = []
251    for test_class, test_method in shard_util.fetch_test_names(
252        self.test_app_path,
253        self.host_app_path,
254        self.release,
255        enabled_tests_only=False):
256      test_name = '%s/%s' % (test_class, test_method)
257      if (test_name not in none_tests and
258          # inlcuded_tests contains the tests to execute, which may be a subset
259          # of all tests b/c of the iOS test sharding logic in run.py. Filter by
260          # self.included_tests if specified
261          (test_class in self.included_tests if self.included_tests else True)):
262        if test_method.startswith('test'):
263          all_tests.append(test_name)
264        elif store_disabled_tests:
265          self.disabled_tests.append(test_name)
266    return all_tests
267
268
269class EgtestsApp(GTestsApp):
270  """Egtests to run.
271
272  Stores data about egtests:
273    egtests_app: full path to egtests app.
274    project_path: root project folder.
275    module_name: egtests module name.
276    included_tests: List of tests to run.
277    excluded_tests: List of tests not to run.
278  """
279
280  def __init__(self,
281               egtests_app,
282               included_tests=None,
283               excluded_tests=None,
284               test_args=None,
285               env_vars=None,
286               release=False,
287               host_app_path=None):
288    """Initialize Egtests.
289
290    Args:
291      egtests_app: (str) full path to egtests app.
292      included_tests: (list) Specific tests to run
293         E.g.
294          [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
295      excluded_tests: (list) Specific tests not to run
296         E.g.
297          [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
298      test_args: List of strings to pass as arguments to the test when
299        launching.
300      env_vars: List of environment variables to pass to the test itself.
301      host_app_path: (str) full path to host app.
302
303    Raises:
304      AppNotFoundError: If the given app does not exist
305    """
306    super(EgtestsApp,
307          self).__init__(egtests_app, included_tests, excluded_tests, test_args,
308                         env_vars, release, host_app_path)
309
310  def _xctest_path(self):
311    """Gets xctest-file from egtests/PlugIns folder.
312
313    Returns:
314      A path for xctest in the format of /PlugIns/file.xctest
315
316    Raises:
317      PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
318      XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
319    """
320    plugins_dir = os.path.join(self.test_app_path, 'PlugIns')
321    if not os.path.exists(plugins_dir):
322      raise test_runner.PlugInsNotFoundError(plugins_dir)
323    plugin_xctest = None
324    if os.path.exists(plugins_dir):
325      for plugin in os.listdir(plugins_dir):
326        if plugin.endswith('.xctest'):
327          plugin_xctest = os.path.join(plugins_dir, plugin)
328    if not plugin_xctest:
329      raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
330    return plugin_xctest.replace(self.test_app_path, '')
331
332  def fill_xctestrun_node(self):
333    """Fills only required nodes for egtests in xctestrun file.
334
335    Returns:
336      A node with filled required fields about egtests.
337    """
338    xctestrun_data = super(EgtestsApp, self).fill_xctestrun_node()
339    module_data = xctestrun_data[self.module_name + '_module']
340
341    module_data['TestingEnvironmentVariables']['DYLD_INSERT_LIBRARIES'] = (
342        '__PLATFORMS__/iPhoneSimulator.platform/Developer/'
343        'usr/lib/libXCTestBundleInject.dylib')
344    module_data['TestBundlePath'] = '__TESTHOST__/%s' % self._xctest_path()
345    module_data['TestingEnvironmentVariables'][
346        'XCInjectBundleInto'] = '__TESTHOST__/%s' % self.module_name
347
348    if self.host_app_path:
349      # Module data specific to EG2 tests
350      module_data['IsUITestBundle'] = True
351      module_data['IsXCTRunnerHostedTestBundle'] = True
352      module_data['UITargetAppPath'] = '%s' % self.host_app_path
353      # Special handling for Xcode10.2
354      dependent_products = [
355          module_data['UITargetAppPath'],
356          module_data['TestBundlePath'],
357          module_data['TestHostPath']
358      ]
359      module_data['DependentProductPaths'] = dependent_products
360    # Module data specific to EG1 tests
361    else:
362      module_data['IsAppHostedTestBundle'] = True
363
364    return xctestrun_data
365
366
367class DeviceXCTestUnitTestsApp(GTestsApp):
368  """XCTest hosted unit tests to run on devices.
369
370  This is for the XCTest framework hosted unit tests running on devices.
371
372  Stores data about tests:
373    tests_app: full path to tests app.
374    project_path: root project folder.
375    module_name: egtests module name.
376    included_tests: List of tests to run.
377    excluded_tests: List of tests not to run.
378  """
379
380  def __init__(self,
381               tests_app,
382               included_tests=None,
383               excluded_tests=None,
384               test_args=None,
385               env_vars=None,
386               release=False):
387    """Initialize the class.
388
389    Args:
390      tests_app: (str) full path to tests app.
391      included_tests: (list) Specific tests to run
392         E.g.
393          [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
394      excluded_tests: (list) Specific tests not to run
395         E.g.
396          [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
397      test_args: List of strings to pass as arguments to the test when
398        launching. Test arg to run as XCTest based unit test will be appended.
399      env_vars: List of environment variables to pass to the test itself.
400
401    Raises:
402      AppNotFoundError: If the given app does not exist
403    """
404    test_args = list(test_args or [])
405    test_args.append('--enable-run-ios-unittests-with-xctest')
406    super(DeviceXCTestUnitTestsApp,
407          self).__init__(tests_app, included_tests, excluded_tests, test_args,
408                         env_vars, release, None)
409
410  # TODO(crbug.com/1077277): Refactor class structure and remove duplicate code.
411  def _xctest_path(self):
412    """Gets xctest-file from egtests/PlugIns folder.
413
414    Returns:
415      A path for xctest in the format of /PlugIns/file.xctest
416
417    Raises:
418      PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
419      XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
420    """
421    plugins_dir = os.path.join(self.test_app_path, 'PlugIns')
422    if not os.path.exists(plugins_dir):
423      raise test_runner.PlugInsNotFoundError(plugins_dir)
424    plugin_xctest = None
425    if os.path.exists(plugins_dir):
426      for plugin in os.listdir(plugins_dir):
427        if plugin.endswith('.xctest'):
428          plugin_xctest = os.path.join(plugins_dir, plugin)
429    if not plugin_xctest:
430      raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
431    return plugin_xctest.replace(self.test_app_path, '')
432
433  def fill_xctestrun_node(self):
434    """Fills only required nodes for XCTest hosted unit tests in xctestrun file.
435
436    Returns:
437      A node with filled required fields about tests.
438    """
439    xctestrun_data = {
440        'TestTargetName': {
441            'IsAppHostedTestBundle': True,
442            'TestBundlePath': '__TESTHOST__/%s' % self._xctest_path(),
443            'TestHostBundleIdentifier': get_bundle_id(self.test_app_path),
444            'TestHostPath': '%s' % self.test_app_path,
445            'TestingEnvironmentVariables': {
446                'DYLD_INSERT_LIBRARIES':
447                    '__TESTHOST__/Frameworks/libXCTestBundleInject.dylib',
448                'DYLD_LIBRARY_PATH':
449                    '__PLATFORMS__/iPhoneOS.platform/Developer/Library',
450                'DYLD_FRAMEWORK_PATH':
451                    '__PLATFORMS__/iPhoneOS.platform/Developer/'
452                    'Library/Frameworks',
453                'XCInjectBundleInto':
454                    '__TESTHOST__/%s' % self.module_name
455            }
456        }
457    }
458
459    if self.env_vars:
460      self.xctestrun_data['TestTargetName'].update(
461          {'EnvironmentVariables': self.env_vars})
462
463    gtest_filter = []
464    if self.included_tests:
465      gtest_filter = get_gtest_filter(self.included_tests, invert=False)
466    elif self.excluded_tests:
467      gtest_filter = get_gtest_filter(self.excluded_tests, invert=True)
468    if gtest_filter:
469      # Removed previous gtest-filter if exists.
470      self.test_args = [
471          el for el in self.test_args if not el.startswith('--gtest_filter=')
472      ]
473      self.test_args.append('--gtest_filter=%s' % gtest_filter)
474
475    xctestrun_data['TestTargetName'].update(
476        {'CommandLineArguments': self.test_args})
477
478    return xctestrun_data
479
480
481class SimulatorXCTestUnitTestsApp(GTestsApp):
482  """XCTest hosted unit tests to run on simulators.
483
484  This is for the XCTest framework hosted unit tests running on simulators.
485
486  Stores data about tests:
487    tests_app: full path to tests app.
488    project_path: root project folder.
489    module_name: egtests module name.
490    included_tests: List of tests to run.
491    excluded_tests: List of tests not to run.
492  """
493
494  def __init__(self,
495               tests_app,
496               included_tests=None,
497               excluded_tests=None,
498               test_args=None,
499               env_vars=None,
500               release=False):
501    """Initialize the class.
502
503    Args:
504      tests_app: (str) full path to tests app.
505      included_tests: (list) Specific tests to run
506         E.g.
507          [ 'TestCaseClass1/testMethod1', 'TestCaseClass2/testMethod2']
508      excluded_tests: (list) Specific tests not to run
509         E.g.
510          [ 'TestCaseClass1', 'TestCaseClass2/testMethod2']
511      test_args: List of strings to pass as arguments to the test when
512        launching. Test arg to run as XCTest based unit test will be appended.
513      env_vars: List of environment variables to pass to the test itself.
514
515    Raises:
516      AppNotFoundError: If the given app does not exist
517    """
518    test_args = list(test_args or [])
519    test_args.append('--enable-run-ios-unittests-with-xctest')
520    super(SimulatorXCTestUnitTestsApp,
521          self).__init__(tests_app, included_tests, excluded_tests, test_args,
522                         env_vars, release, None)
523
524  # TODO(crbug.com/1077277): Refactor class structure and remove duplicate code.
525  def _xctest_path(self):
526    """Gets xctest-file from egtests/PlugIns folder.
527
528    Returns:
529      A path for xctest in the format of /PlugIns/file.xctest
530
531    Raises:
532      PlugInsNotFoundError: If no PlugIns folder found in egtests.app.
533      XCTestPlugInNotFoundError: If no xctest-file found in PlugIns.
534    """
535    plugins_dir = os.path.join(self.test_app_path, 'PlugIns')
536    if not os.path.exists(plugins_dir):
537      raise test_runner.PlugInsNotFoundError(plugins_dir)
538    plugin_xctest = None
539    if os.path.exists(plugins_dir):
540      for plugin in os.listdir(plugins_dir):
541        if plugin.endswith('.xctest'):
542          plugin_xctest = os.path.join(plugins_dir, plugin)
543    if not plugin_xctest:
544      raise test_runner.XCTestPlugInNotFoundError(plugin_xctest)
545    return plugin_xctest.replace(self.test_app_path, '')
546
547  def fill_xctestrun_node(self):
548    """Fills only required nodes for XCTest hosted unit tests in xctestrun file.
549
550    Returns:
551      A node with filled required fields about tests.
552    """
553    xctestrun_data = {
554        'TestTargetName': {
555            'IsAppHostedTestBundle': True,
556            'TestBundlePath': '__TESTHOST__/%s' % self._xctest_path(),
557            'TestHostBundleIdentifier': get_bundle_id(self.test_app_path),
558            'TestHostPath': '%s' % self.test_app_path,
559            'TestingEnvironmentVariables': {
560                'DYLD_INSERT_LIBRARIES':
561                    '__PLATFORMS__/iPhoneSimulator.platform/Developer/usr/lib/'
562                    'libXCTestBundleInject.dylib',
563                'DYLD_LIBRARY_PATH':
564                    '__PLATFORMS__/iPhoneSimulator.platform/Developer/Library',
565                'DYLD_FRAMEWORK_PATH':
566                    '__PLATFORMS__/iPhoneSimulator.platform/Developer/'
567                    'Library/Frameworks',
568                'XCInjectBundleInto':
569                    '__TESTHOST__/%s' % self.module_name
570            }
571        }
572    }
573
574    if self.env_vars:
575      self.xctestrun_data['TestTargetName'].update(
576          {'EnvironmentVariables': self.env_vars})
577
578    gtest_filter = []
579    if self.included_tests:
580      gtest_filter = get_gtest_filter(self.included_tests, invert=False)
581    elif self.excluded_tests:
582      gtest_filter = get_gtest_filter(self.excluded_tests, invert=True)
583    if gtest_filter:
584      # Removed previous gtest-filter if exists.
585      self.test_args = [
586          el for el in self.test_args if not el.startswith('--gtest_filter=')
587      ]
588      self.test_args.append('--gtest_filter=%s' % gtest_filter)
589
590    xctestrun_data['TestTargetName'].update(
591        {'CommandLineArguments': self.test_args})
592
593    return xctestrun_data
594