1#
2# Copyright (c) 1995, 2018, Oracle and/or its affiliates. All rights reserved.
3# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4#
5# This code is free software; you can redistribute it and/or modify it
6# under the terms of the GNU General Public License version 2 only, as
7# published by the Free Software Foundation.  Oracle designates this
8# particular file as subject to the "Classpath" exception as provided
9# by Oracle in the LICENSE file that accompanied this code.
10#
11# This code is distributed in the hope that it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14# version 2 for more details (a copy is included in the LICENSE file that
15# accompanied this code).
16#
17# You should have received a copy of the GNU General Public License version
18# 2 along with this work; if not, write to the Free Software Foundation,
19# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22# or visit www.oracle.com if you need additional information or have any
23# questions.
24#
25
26#
27# Common logic to run various tests for a component, to be included by the
28# component specific test makefiles.
29#
30
31# Default values for some properties that can be overridden by components.
32USE_JTREG_VERSION ?= 4.2
33JTREG_VM_TYPE ?= -agentvm
34USE_JTREG_ASSERT ?= true
35LIMIT_JTREG_VM_MEMORY ?= true
36
37X:=
38SPACE:=$(X) $(X)
39
40.DEFAULT : all
41
42# Empty these to get rid of some default rules
43.SUFFIXES:
44.SUFFIXES: .java
45CO=
46GET=
47
48# Utilities used
49AWK       = awk
50CAT       = cat
51CD        = cd
52CHMOD     = chmod
53CP        = cp
54CUT       = cut
55DIRNAME   = dirname
56ECHO      = echo
57EGREP     = egrep
58EXPAND    = expand
59FIND      = find
60MKDIR     = mkdir
61PWD       = pwd
62RM        = rm -f
63SED       = sed
64SORT      = sort
65TEE       = tee
66UNAME     = uname
67UNIQ      = uniq
68WC        = wc
69ZIPEXE    = zip
70
71# Get OS name from uname (Cygwin inexplicably adds _NT-5.1)
72UNAME_S := $(shell $(UNAME) -s | $(CUT) -f1 -d_)
73
74# Commands to run on paths to make mixed paths for java on windows
75ifeq ($(findstring CYGWIN,$(UNAME_S)), CYGWIN)
76  # Location of developer shared files
77  SLASH_JAVA = J:
78  GETMIXEDPATH = cygpath -m
79  PLATFORM = windows
80else
81  # Location of developer shared files
82  SLASH_JAVA = /java
83  GETMIXEDPATH = $(ECHO)
84  PLATFORM = unix # we only care about windows or bsd.
85  ifeq ($(UNAME_S), Darwin)
86    PLATFORM = bsd
87  endif
88  ifeq ($(findstring BSD,$(UNAME_S)), BSD)
89    PLATFORM = bsd
90  endif
91endif
92
93# convert list of directories to dos paths
94define MixedDirs
95$(foreach i,$1,$(shell $(GETMIXEDPATH) "${i}"))
96endef
97
98ifdef ALT_SLASH_JAVA
99  SLASH_JAVA = $(ALT_SLASH_JAVA)
100endif
101
102# Root of this test area (important to use full paths in some places)
103TEST_ROOT := $(shell $(PWD))
104
105# Root of all test results
106ifdef TEST_OUTPUT_DIR
107  $(shell $(MKDIR) -p $(TEST_OUTPUT_DIR)/jtreg)
108  ABS_TEST_OUTPUT_DIR := \
109    $(shell $(CD) $(TEST_OUTPUT_DIR)/jtreg && $(PWD))
110else
111  ifdef ALT_OUTPUTDIR
112    ABS_OUTPUTDIR = $(shell $(CD) $(ALT_OUTPUTDIR) && $(PWD))
113  else
114    ABS_OUTPUTDIR = $(shell $(CD) $(TEST_ROOT)/.. && $(PWD))
115  endif
116
117  ABS_PLATFORM_BUILD_ROOT = $(ABS_OUTPUTDIR)
118  ABS_TEST_OUTPUT_DIR := $(ABS_PLATFORM_BUILD_ROOT)/testoutput/$(UNIQUE_DIR)
119endif
120
121# If unset, set up the PRODUCT_HOME variable to the jdk to test
122ifndef PRODUCT_HOME
123  # Try to use images/jdk if it exists
124  ABS_JDK_IMAGE = $(ABS_PLATFORM_BUILD_ROOT)/images/jdk
125  PRODUCT_HOME :=                               \
126    $(shell                                     \
127      if [ -d $(ABS_JDK_IMAGE) ] ; then         \
128         $(ECHO) "$(ABS_JDK_IMAGE)";            \
129       else                                     \
130         $(ECHO) "$(ABS_PLATFORM_BUILD_ROOT)";  \
131       fi)
132  PRODUCT_HOME := $(PRODUCT_HOME)
133endif
134
135# On Windows, setup the _NT_SYMBOL_PATH if possible.
136ifeq ($(PLATFORM), windows)
137  ifndef _NT_SYMBOL_PATH
138    ifdef PRODUCT_SYMBOLS_HOME
139      _NT_SYMBOL_PATH := \
140          $(subst $(SPACE),;,$(strip $(call MixedDirs, $(sort $(dir $(wildcard \
141          $(addprefix $(PRODUCT_SYMBOLS_HOME)/bin/, *.pdb */*.pdb)))))))
142      export _NT_SYMBOL_PATH
143    endif
144  endif
145  JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH='$(_NT_SYMBOL_PATH)'
146endif
147
148ifneq ($(NATIVE_TEST_PATH), )
149  # jtreg -nativepath <dir>
150  #
151  # Local make tests will be TEST_IMAGE_DIR
152  ifdef TEST_IMAGE_DIR
153    TESTNATIVE_DIR = $(TEST_IMAGE_DIR)
154  endif
155  ifdef TESTNATIVE_DIR
156    JTREG_NATIVE_PATH = -nativepath:$(shell $(GETMIXEDPATH) "$(TESTNATIVE_DIR)/$(NATIVE_TEST_PATH)")
157  endif
158endif
159
160ifeq ($(USE_FAILURE_HANDLER), true)
161  # jtreg failure handler config
162  ifeq ($(FAILURE_HANDLER_DIR), )
163    ifneq ($(TESTNATIVE_DIR), )
164      FAILURE_HANDLER_DIR := $(TESTNATIVE_DIR)/failure_handler
165    endif
166  endif
167  ifneq ($(FAILURE_HANDLER_DIR), )
168    FAILURE_HANDLER_DIR_MIXED := $(shell $(GETMIXEDPATH) "$(FAILURE_HANDLER_DIR)")
169    JTREG_FAILURE_HANDLER_OPTIONS := \
170        -timeoutHandlerDir:$(FAILURE_HANDLER_DIR_MIXED)/jtregFailureHandler.jar \
171        -observerDir:$(FAILURE_HANDLER_DIR_MIXED)/jtregFailureHandler.jar \
172        -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \
173        -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \
174        -timeoutHandlerTimeout:0
175  ifeq ($(PLATFORM), windows)
176      JTREG_FAILURE_HANDLER_OPTIONS += -J-Djava.library.path="$(FAILURE_HANDLER_DIR_MIXED)"
177    endif
178  endif
179endif
180
181# How to create the test bundle (pass or fail, we want to create this)
182#   Follow command with ";$(BUNDLE_UP_AND_EXIT)", so it always gets executed.
183ifneq ($(ARCHIVE_BUNDLE), )
184  ZIP_UP_RESULTS = ( $(MKDIR) -p `$(DIRNAME) $(ARCHIVE_BUNDLE)`     \
185	           && $(CD) $(ABS_TEST_OUTPUT_DIR)             \
186	           && $(CHMOD) -R a+r . \
187	           && $(ZIPEXE) -q -r $(ARCHIVE_BUNDLE) . ) ;
188  CLEAN_ARCHIVE_BUNDLE = @$(RM) $(ARCHIVE_BUNDLE)
189endif
190
191# AddressSanitizer
192ifeq ($(ASAN_ENABLED), yes)
193  export ASAN_OPTIONS="handle_segv=0 detect_leaks=0"
194  JTREG_BASIC_OPTIONS += -e:ASAN_OPTIONS=$(ASAN_OPTIONS)
195  ifneq ($(DEVKIT_LIB_DIR),)
196    export LD_LIBRARY_PATH:=$(LD_LIBRARY_PATH):$(DEVKIT_LIB_DIR)
197    JTREG_BASIC_OPTIONS += -e:LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)
198  endif
199endif
200
201# important results files
202SUMMARY_TXT = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/JTreport/text/summary.txt")
203STATS_TXT_NAME = Stats.txt
204STATS_TXT = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/$(STATS_TXT_NAME)")
205RUNLIST   = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/runlist.txt")
206PASSLIST  = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/passlist.txt")
207FAILLIST  = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/faillist.txt")
208EXITCODE  = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/exitcode.txt")
209
210TESTEXIT = \
211  if [ ! -s $(EXITCODE) ] ; then \
212    $(ECHO) "ERROR: EXITCODE file not filled in."; \
213    $(ECHO) "1" > $(EXITCODE); \
214  fi ; \
215  testExitCode=`$(CAT) $(EXITCODE)`; \
216  $(ECHO) "EXIT CODE: $${testExitCode}"; \
217  exit $${testExitCode}
218
219ifeq ($(TREAT_EXIT_CODE_1_AS_0), true)
220  ADJUST_EXIT_CODE := \
221  if [ $${jtregExitCode} = 1 ] ; then \
222    jtregExitCode=0; \
223  fi
224else
225  # colon is the shell no-op
226  ADJUST_EXIT_CODE := :
227endif
228
229BUNDLE_UP_AND_EXIT = \
230( \
231  jtregExitCode=$$? && \
232  _summary="$(SUMMARY_TXT)"; \
233  $(ADJUST_EXIT_CODE) ; \
234  $(RM) -f $(STATS_TXT) $(RUNLIST) $(PASSLIST) $(FAILLIST) $(EXITCODE); \
235  $(ECHO) "$${jtregExitCode}" > $(EXITCODE); \
236  if [ -r "$${_summary}" ] ; then \
237    $(ECHO) "Summary: $(UNIQUE_DIR)" > $(STATS_TXT); \
238    $(EXPAND) $${_summary} | $(EGREP) -v ' Not run\.' > $(RUNLIST); \
239    $(EGREP) ' Passed\.' $(RUNLIST) \
240      | $(EGREP) -v ' Error\.' \
241      | $(EGREP) -v ' Failed\.' > $(PASSLIST); \
242    ( $(EGREP) ' Failed\.' $(RUNLIST); \
243      $(EGREP) ' Error\.' $(RUNLIST); \
244      $(EGREP) -v ' Passed\.' $(RUNLIST) ) \
245      | $(SORT) | $(UNIQ) > $(FAILLIST); \
246    if [ $${jtregExitCode} != 0 -o -s $(FAILLIST) ] ; then \
247      $(EXPAND) $(FAILLIST) \
248        | $(CUT) -d' ' -f1 \
249        | $(SED) -e 's@^@FAILED: @' >> $(STATS_TXT); \
250      if [ $${jtregExitCode} = 0 ] ; then \
251        jtregExitCode=1; \
252      fi; \
253    fi; \
254    runc="`$(CAT) $(RUNLIST)      | $(WC) -l | $(AWK) '{print $$1;}'`"; \
255    passc="`$(CAT) $(PASSLIST)    | $(WC) -l | $(AWK) '{print $$1;}'`"; \
256    failc="`$(CAT) $(FAILLIST)    | $(WC) -l | $(AWK) '{print $$1;}'`"; \
257    exclc="FIXME CODETOOLS-7900176"; \
258    $(ECHO) "TEST STATS: name=$(UNIQUE_DIR)  run=$${runc}  pass=$${passc}  fail=$${failc}" \
259      >> $(STATS_TXT); \
260  else \
261    $(ECHO) "Missing file: $${_summary}" >> $(STATS_TXT); \
262  fi; \
263  if [ -f $(STATS_TXT) ] ; then \
264    $(CAT) $(STATS_TXT); \
265  fi; \
266  $(ZIP_UP_RESULTS) \
267  $(TESTEXIT) \
268)
269
270################################################################
271
272# Prep for output
273# Change execute permissions on shared library files.
274# Files in repositories should never have execute permissions, but
275# there are some tests that have pre-built shared libraries, and these
276# windows dll files must have execute permission. Adding execute
277# permission may happen automatically on windows when using certain
278# versions of mercurial but it cannot be guaranteed. And blindly
279# adding execute permission might be seen as a mercurial 'change', so
280# we avoid adding execute permission to repository files. But testing
281# from a plain source tree needs the chmod a+rx. Applying the chmod to
282# all shared libraries not just dll files. And with CYGWIN and sshd
283# service, you may need CYGWIN=ntsec for this to work.
284prep:
285	@$(MKDIR) -p $(ABS_TEST_OUTPUT_DIR)
286	@if [ ! -d $(TEST_ROOT)/../../.hg ] && [ ! -d $(TEST_ROOT)/../../../.hg ]; then  \
287	  $(FIND) $(TEST_ROOT) \( -name \*.dll -o -name \*.DLL -o -name \*.so \)  \
288	        -exec $(CHMOD) a+rx {} \; ;                                       \
289	fi
290
291ifeq ($(CLEAN_BEFORE_PREP), true)
292prep: clean
293
294endif
295
296# Cleanup
297clean:
298	@$(RM) -r $(ABS_TEST_OUTPUT_DIR)
299	$(CLEAN_ARCHIVE_BUNDLE)
300
301################################################################
302
303# jtreg tests
304
305# Expect JT_HOME to be set for jtreg tests. (home for jtreg)
306ifndef JT_HOME
307  JT_HOME = $(SLASH_JAVA)/re/jtreg/$(USE_JTREG_VERSION)/promoted/latest/binaries/jtreg
308endif
309
310# Problematic tests to be excluded
311EXTRA_PROBLEM_LISTS :=
312PROBLEM_LISTS := ProblemList.txt $(EXTRA_PROBLEM_LISTS)
313
314# Create exclude list for this platform and arch
315ifdef NO_EXCLUDES
316  JTREG_EXCLUSIONS =
317else
318  JTREG_EXCLUSIONS = $(addprefix -exclude:, $(wildcard $(PROBLEM_LISTS)))
319endif
320
321# ------------------------------------------------------------------
322
323# The TESTDIRS variable can be used to select the jtreg tests to run
324ifdef TESTDIRS
325  TEST_SELECTION = $(TESTDIRS)
326endif
327
328ifeq ($(UNAME_S), SunOS)
329  NUM_CORES := $(shell LC_MESSAGES=C /usr/sbin/psrinfo -v | grep -c on-line)
330endif
331ifeq ($(UNAME_S), Linux)
332  NUM_CORES := $(shell cat /proc/cpuinfo  | grep -c processor)
333endif
334ifeq ($(findstring BSD,$(UNAME_S)), BSD)
335  NUM_CORES := $(shell /sbin/sysctl -n hw.ncpu)
336endif
337ifeq ($(UNAME_S), Darwin)
338  NUM_CORES := $(shell /usr/sbin/sysctl -n hw.ncpu)
339endif
340ifeq ($(findstring CYGWIN,$(UNAME_S)), CYGWIN)
341  ifneq ($(NUMBER_OF_PROCESSORS), )
342    NUM_CORES := $(NUMBER_OF_PROCESSORS)
343  else
344    ifneq ($(HOTSPOT_BUILD_JOBS), )
345      NUM_CORES := $(HOTSPOT_BUILD_JOBS)
346    else
347      NUM_CORES := 1 # fallback
348    endif
349  endif
350endif
351
352ifndef CONCURRENCY_FACTOR
353  CONCURRENCY_FACTOR = 1
354endif
355
356# Concurrency based on min(cores / 2, 12) * CONCURRENCY_FACTOR
357CONCURRENCY := $(shell $(AWK) \
358  'BEGIN { \
359    c = $(NUM_CORES) / 2; \
360    if (c > 12) c = 12; \
361    c = c * $(CONCURRENCY_FACTOR); \
362    if (c < 1) c = 1; \
363    printf "%.0f", c; \
364  }')
365JTREG_BASIC_OPTIONS += -concurrency:$(CONCURRENCY)
366
367# Make sure MaxRAMPercentage is low enough to not cause OOM or swapping since
368# we may end up with a lot of JVM's
369MAX_RAM_PERCENTAGE := $(shell expr 25 / $(CONCURRENCY))
370JTREG_BASIC_OPTIONS += -vmoption:-XX:MaxRAMPercentage=$(MAX_RAM_PERCENTAGE)
371
372ifdef EXTRA_JTREG_OPTIONS
373  JTREG_BASIC_OPTIONS += $(EXTRA_JTREG_OPTIONS)
374endif
375
376# Default JTREG to run
377JTREG = $(JT_HOME)/bin/jtreg
378# run in agentvm/othervm mode
379JTREG_BASIC_OPTIONS += $(JTREG_VM_TYPE)
380# Only run automatic tests
381JTREG_BASIC_OPTIONS += -a
382# Always turn on assertions
383ifeq ($(USE_JTREG_ASSERT), true)
384  JTREG_ASSERT_OPTION = -ea -esa
385endif
386JTREG_BASIC_OPTIONS += $(JTREG_ASSERT_OPTION)
387# jtreg verbosity setting
388# Default is to report details on all failed or error tests, times too
389JTREG_VERBOSE ?= fail,error,time
390JTREG_BASIC_OPTIONS += $(if $(JTREG_VERBOSE),-v:$(JTREG_VERBOSE))
391# Retain all files for failing tests
392JTREG_BASIC_OPTIONS += -retain:fail,error
393# Ignore tests are not run and completely silent about it
394JTREG_IGNORE_OPTION = -ignore:quiet
395JTREG_BASIC_OPTIONS += $(JTREG_IGNORE_OPTION)
396# Multiply by 4 the timeout factor
397JTREG_TIMEOUT_OPTION =  -timeoutFactor:4
398JTREG_BASIC_OPTIONS += $(JTREG_TIMEOUT_OPTION)
399ifeq ($(LIMIT_JTREG_VM_MEMORY), true)
400  # Set the max memory for jtreg control vm
401  JTREG_MEMORY_OPTION = -J-Xmx512m
402  JTREG_BASIC_OPTIONS += $(JTREG_MEMORY_OPTION)
403  # Set the max memory for jtreg target test vms
404  JTREG_TESTVM_MEMORY_OPTION = -vmoption:-Xmx512m
405  JTREG_TEST_OPTIONS += $(JTREG_TESTVM_MEMORY_OPTION)
406endif
407# Make it possible to specify the JIB_DATA_DIR for tests using the
408# JIB Artifact resolver
409JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
410# Give tests access to JT_JAVA, see JDK-8141609
411JTREG_BASIC_OPTIONS += -e:JDK8_HOME=${JT_JAVA}
412# Give aot tests access to Visual Studio installation
413ifneq ($(VS120COMNTOOLS), )
414  JTREG_BASIC_OPTIONS += -e:VS120COMNTOOLS="$(shell $(GETMIXEDPATH) "$(patsubst %\,%,$(VS120COMNTOOLS))")"
415endif
416
417JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=${TEST_IMAGE_DIR}/hotspot/jtreg/graal
418
419# Set other vm and test options
420JTREG_TEST_OPTIONS += $(JAVA_ARGS:%=-javaoptions:%) $(JAVA_VM_ARGS:%=-vmoption:%)
421ifneq ($(JIB_HOME), )
422  JTREG_BASIC_OPTIONS += -e:JIB_HOME=$(shell $(GETMIXEDPATH) "$(JIB_HOME)")
423endif
424ifeq ($(IGNORE_MARKED_TESTS), true)
425  # Option to tell jtreg to not run tests marked with "ignore"
426  ifeq ($(PLATFORM), windows)
427    JTREG_KEY_OPTION = -k:!ignore
428  else
429    JTREG_KEY_OPTION = -k:\!ignore
430  endif
431  JTREG_BASIC_OPTIONS += $(JTREG_KEY_OPTION)
432endif
433
434# Make sure jtreg exists
435ifeq ($(USE_WINDOWS_EXISTENCE_CHECK), true)
436  jtreg_exists:
437	test -d $(shell $(GETMIXEDPATH) "$(JT_HOME)")
438	test -f $(shell $(GETMIXEDPATH) "$(JTREG)")
439
440else
441  jtreg_exists: $(JT_HOME)
442endif
443PHONY_LIST += jtreg_exists
444
445# Run jtreg
446jtreg_tests: prep jtreg_exists $(PRODUCT_HOME) $(TEST_PREREQS)
447	(                                                                    \
448	  ( JT_HOME=$(shell $(GETMIXEDPATH) "$(JT_HOME)");                   \
449	    export JT_HOME;                                                  \
450	    $(shell $(GETMIXEDPATH) "$(JTREG)")                              \
451	      $(JTREG_BASIC_OPTIONS)                                         \
452	      -r:$(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/JTreport")  \
453	      -w:$(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/JTwork")    \
454	      -jdk:$(shell $(GETMIXEDPATH) "$(PRODUCT_HOME)")                \
455	      $(JTREG_NATIVE_PATH)                                           \
456	      $(JTREG_FAILURE_HANDLER_OPTIONS)                               \
457	      $(JTREG_EXCLUSIONS)                                            \
458	      $(JTREG_TEST_OPTIONS)                                          \
459	      $(TEST_SELECTION)                                              \
460	  ) ;                                                                \
461	  $(BUNDLE_UP_AND_EXIT)                                              \
462	) 2>&1 | $(TEE) $(ABS_TEST_OUTPUT_DIR)/output.txt ; $(TESTEXIT)
463
464PHONY_LIST += jtreg_tests
465
466# Make it possible to call this with <component>_jtreg_tests
467%_jtreg_tests: jtreg_tests
468	# Must have a fake recipe here to prevent make from matching any other rule
469
470################################################################
471
472# Phony targets (e.g. these are not filenames)
473.PHONY: all clean prep $(PHONY_LIST)
474
475################################################################
476