1#
2# Copyright (c) 1995, 2018, Oracle and/or its affiliates. All rights reserved.
3# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4#
5# This code is free software; you can redistribute it and/or modify it
6# under the terms of the GNU General Public License version 2 only, as
7# published by the Free Software Foundation.  Oracle designates this
8# particular file as subject to the "Classpath" exception as provided
9# by Oracle in the LICENSE file that accompanied this code.
10#
11# This code is distributed in the hope that it will be useful, but WITHOUT
12# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14# version 2 for more details (a copy is included in the LICENSE file that
15# accompanied this code).
16#
17# You should have received a copy of the GNU General Public License version
18# 2 along with this work; if not, write to the Free Software Foundation,
19# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20#
21# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22# or visit www.oracle.com if you need additional information or have any
23# questions.
24#
25
26#
27# Common logic to run various tests for a component, to be included by the
28# component specific test makefiles.
29#
30
31# Default values for some properties that can be overridden by components.
32USE_JTREG_VERSION ?= 4.2
33JTREG_VM_TYPE ?= -agentvm
34USE_JTREG_ASSERT ?= true
35LIMIT_JTREG_VM_MEMORY ?= true
36
37X:=
38SPACE:=$(X) $(X)
39
40.DEFAULT : all
41
42# Empty these to get rid of some default rules
43.SUFFIXES:
44.SUFFIXES: .java
45CO=
46GET=
47
48# Utilities used
49AWK       = awk
50CAT       = cat
51CD        = cd
52CHMOD     = chmod
53CP        = cp
54CUT       = cut
55DIRNAME   = dirname
56ECHO      = echo
57EGREP     = egrep
58EXPAND    = expand
59FIND      = find
60MKDIR     = mkdir
61PWD       = pwd
62RM        = rm -f
63SED       = sed
64SORT      = sort
65TEE       = tee
66UNAME     = uname
67UNIQ      = uniq
68WC        = wc
69ZIPEXE    = zip
70
71# Get OS name from uname (Cygwin inexplicably adds _NT-5.1)
72UNAME_S := $(shell $(UNAME) -s | $(CUT) -f1 -d_)
73
74# Commands to run on paths to make mixed paths for java on windows
75ifeq ($(findstring CYGWIN,$(UNAME_S)), CYGWIN)
76  # Location of developer shared files
77  SLASH_JAVA = J:
78  GETMIXEDPATH = cygpath -m
79  PLATFORM = windows
80else
81  # Location of developer shared files
82  SLASH_JAVA = /java
83  GETMIXEDPATH = $(ECHO)
84  PLATFORM = unix # we only care about windows or bsd.
85  ifeq ($(UNAME_S), Darwin)
86    PLATFORM = bsd
87  endif
88  ifeq ($(findstring BSD,$(UNAME_S)), BSD)
89    PLATFORM = bsd
90  endif
91endif
92
93# convert list of directories to dos paths
94define MixedDirs
95$(foreach i,$1,$(shell $(GETMIXEDPATH) "${i}"))
96endef
97
98ifdef ALT_SLASH_JAVA
99  SLASH_JAVA = $(ALT_SLASH_JAVA)
100endif
101
102# Root of this test area (important to use full paths in some places)
103TEST_ROOT := $(shell $(PWD))
104
105# Root of all test results
106ifdef TEST_OUTPUT_DIR
107  $(shell $(MKDIR) -p $(TEST_OUTPUT_DIR)/jtreg)
108  ABS_TEST_OUTPUT_DIR := \
109    $(shell $(CD) $(TEST_OUTPUT_DIR)/jtreg && $(PWD))
110else
111  ifdef ALT_OUTPUTDIR
112    ABS_OUTPUTDIR = $(shell $(CD) $(ALT_OUTPUTDIR) && $(PWD))
113  else
114    ABS_OUTPUTDIR = $(shell $(CD) $(TEST_ROOT)/.. && $(PWD))
115  endif
116
117  ABS_PLATFORM_BUILD_ROOT = $(ABS_OUTPUTDIR)
118  ABS_TEST_OUTPUT_DIR := $(ABS_PLATFORM_BUILD_ROOT)/testoutput/$(UNIQUE_DIR)
119endif
120
121# If unset, set up the PRODUCT_HOME variable to the jdk to test
122ifndef PRODUCT_HOME
123  # Try to use images/jdk if it exists
124  ABS_JDK_IMAGE = $(ABS_PLATFORM_BUILD_ROOT)/images/jdk
125  PRODUCT_HOME :=                               \
126    $(shell                                     \
127      if [ -d $(ABS_JDK_IMAGE) ] ; then         \
128         $(ECHO) "$(ABS_JDK_IMAGE)";            \
129       else                                     \
130         $(ECHO) "$(ABS_PLATFORM_BUILD_ROOT)";  \
131       fi)
132  PRODUCT_HOME := $(PRODUCT_HOME)
133endif
134
135# On Windows, setup the _NT_SYMBOL_PATH if possible.
136ifeq ($(PLATFORM), windows)
137  ifndef _NT_SYMBOL_PATH
138    ifdef PRODUCT_SYMBOLS_HOME
139      _NT_SYMBOL_PATH := \
140          $(subst $(SPACE),;,$(strip $(call MixedDirs, $(sort $(dir $(wildcard \
141          $(addprefix $(PRODUCT_SYMBOLS_HOME)/bin/, *.pdb */*.pdb)))))))
142      export _NT_SYMBOL_PATH
143    endif
144  endif
145  JTREG_BASIC_OPTIONS += -e:_NT_SYMBOL_PATH='$(_NT_SYMBOL_PATH)'
146endif
147
148ifneq ($(NATIVE_TEST_PATH), )
149  # jtreg -nativepath <dir>
150  #
151  # Local make tests will be TEST_IMAGE_DIR
152  ifdef TEST_IMAGE_DIR
153    TESTNATIVE_DIR = $(TEST_IMAGE_DIR)
154  endif
155  ifdef TESTNATIVE_DIR
156    JTREG_NATIVE_PATH = -nativepath:$(shell $(GETMIXEDPATH) "$(TESTNATIVE_DIR)/$(NATIVE_TEST_PATH)")
157  endif
158endif
159
160ifeq ($(USE_FAILURE_HANDLER), true)
161  # jtreg failure handler config
162  ifeq ($(FAILURE_HANDLER_DIR), )
163    ifneq ($(TESTNATIVE_DIR), )
164      FAILURE_HANDLER_DIR := $(TESTNATIVE_DIR)/failure_handler
165    endif
166  endif
167  ifneq ($(FAILURE_HANDLER_DIR), )
168    FAILURE_HANDLER_DIR_MIXED := $(shell $(GETMIXEDPATH) "$(FAILURE_HANDLER_DIR)")
169    JTREG_FAILURE_HANDLER_OPTIONS := \
170        -timeoutHandlerDir:$(FAILURE_HANDLER_DIR_MIXED)/jtregFailureHandler.jar \
171        -observerDir:$(FAILURE_HANDLER_DIR_MIXED)/jtregFailureHandler.jar \
172        -timeoutHandler:jdk.test.failurehandler.jtreg.GatherProcessInfoTimeoutHandler \
173        -observer:jdk.test.failurehandler.jtreg.GatherDiagnosticInfoObserver \
174        -timeoutHandlerTimeout:0
175  ifeq ($(PLATFORM), windows)
176      JTREG_FAILURE_HANDLER_OPTIONS += -J-Djava.library.path="$(FAILURE_HANDLER_DIR_MIXED)"
177    endif
178  endif
179endif
180
181
182# Optionally create a CDS archive before running tests
183ifeq ($(GENERATE_CDS_ARCHIVE), true)
184  CDS_ARCHIVE_FILE := $(ABS_TEST_OUTPUT_DIR)/cds_archive.jsa
185
186  $(CDS_ARCHIVE_FILE): $(PRODUCT_HOME)
187	$(PRODUCT_HOME)/bin/java -XX:+UnlockDiagnosticVMOptions \
188	    -XX:SharedArchiveFile=$(shell $(GETMIXEDPATH) "$(CDS_ARCHIVE_FILE)") -Xshare:dump
189
190  CDS_VM_ARGS := -XX:+UnlockDiagnosticVMOptions -XX:SharedArchiveFile=$(shell $(GETMIXEDPATH) "$(CDS_ARCHIVE_FILE)")
191  JTREG_TEST_OPTIONS += $(addprefix -vmoption:, $(CDS_VM_ARGS))
192  TEST_PREREQS += $(CDS_ARCHIVE_FILE)
193endif
194
195# How to create the test bundle (pass or fail, we want to create this)
196#   Follow command with ";$(BUNDLE_UP_AND_EXIT)", so it always gets executed.
197ifneq ($(ARCHIVE_BUNDLE), )
198  ZIP_UP_RESULTS = ( $(MKDIR) -p `$(DIRNAME) $(ARCHIVE_BUNDLE)`     \
199	           && $(CD) $(ABS_TEST_OUTPUT_DIR)             \
200	           && $(CHMOD) -R a+r . \
201	           && $(ZIPEXE) -q -r $(ARCHIVE_BUNDLE) . ) ;
202  CLEAN_ARCHIVE_BUNDLE = @$(RM) $(ARCHIVE_BUNDLE)
203endif
204
205# AddressSanitizer
206ifeq ($(ASAN_ENABLED), yes)
207  export ASAN_OPTIONS="handle_segv=0 detect_leaks=0"
208  JTREG_BASIC_OPTIONS += -e:ASAN_OPTIONS=$(ASAN_OPTIONS)
209  ifneq ($(DEVKIT_LIB_DIR),)
210    export LD_LIBRARY_PATH:=$(LD_LIBRARY_PATH):$(DEVKIT_LIB_DIR)
211    JTREG_BASIC_OPTIONS += -e:LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)
212  endif
213endif
214
215# important results files
216SUMMARY_TXT = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/JTreport/text/summary.txt")
217STATS_TXT_NAME = Stats.txt
218STATS_TXT = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/$(STATS_TXT_NAME)")
219RUNLIST   = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/runlist.txt")
220PASSLIST  = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/passlist.txt")
221FAILLIST  = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/faillist.txt")
222EXITCODE  = $(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/exitcode.txt")
223
224TESTEXIT = \
225  if [ ! -s $(EXITCODE) ] ; then \
226    $(ECHO) "ERROR: EXITCODE file not filled in."; \
227    $(ECHO) "1" > $(EXITCODE); \
228  fi ; \
229  testExitCode=`$(CAT) $(EXITCODE)`; \
230  $(ECHO) "EXIT CODE: $${testExitCode}"; \
231  exit $${testExitCode}
232
233ifeq ($(TREAT_EXIT_CODE_1_AS_0), true)
234  ADJUST_EXIT_CODE := \
235  if [ $${jtregExitCode} = 1 ] ; then \
236    jtregExitCode=0; \
237  fi
238else
239  # colon is the shell no-op
240  ADJUST_EXIT_CODE := :
241endif
242
243BUNDLE_UP_AND_EXIT = \
244( \
245  jtregExitCode=$$? && \
246  _summary="$(SUMMARY_TXT)"; \
247  $(ADJUST_EXIT_CODE) ; \
248  $(RM) -f $(STATS_TXT) $(RUNLIST) $(PASSLIST) $(FAILLIST) $(EXITCODE); \
249  $(ECHO) "$${jtregExitCode}" > $(EXITCODE); \
250  if [ -r "$${_summary}" ] ; then \
251    $(ECHO) "Summary: $(UNIQUE_DIR)" > $(STATS_TXT); \
252    $(EXPAND) $${_summary} | $(EGREP) -v ' Not run\.' > $(RUNLIST); \
253    $(EGREP) ' Passed\.' $(RUNLIST) \
254      | $(EGREP) -v ' Error\.' \
255      | $(EGREP) -v ' Failed\.' > $(PASSLIST); \
256    ( $(EGREP) ' Failed\.' $(RUNLIST); \
257      $(EGREP) ' Error\.' $(RUNLIST); \
258      $(EGREP) -v ' Passed\.' $(RUNLIST) ) \
259      | $(SORT) | $(UNIQ) > $(FAILLIST); \
260    if [ $${jtregExitCode} != 0 -o -s $(FAILLIST) ] ; then \
261      $(EXPAND) $(FAILLIST) \
262        | $(CUT) -d' ' -f1 \
263        | $(SED) -e 's@^@FAILED: @' >> $(STATS_TXT); \
264      if [ $${jtregExitCode} = 0 ] ; then \
265        jtregExitCode=1; \
266      fi; \
267    fi; \
268    runc="`$(CAT) $(RUNLIST)      | $(WC) -l | $(AWK) '{print $$1;}'`"; \
269    passc="`$(CAT) $(PASSLIST)    | $(WC) -l | $(AWK) '{print $$1;}'`"; \
270    failc="`$(CAT) $(FAILLIST)    | $(WC) -l | $(AWK) '{print $$1;}'`"; \
271    exclc="FIXME CODETOOLS-7900176"; \
272    $(ECHO) "TEST STATS: name=$(UNIQUE_DIR)  run=$${runc}  pass=$${passc}  fail=$${failc}" \
273      >> $(STATS_TXT); \
274  else \
275    $(ECHO) "Missing file: $${_summary}" >> $(STATS_TXT); \
276  fi; \
277  if [ -f $(STATS_TXT) ] ; then \
278    $(CAT) $(STATS_TXT); \
279  fi; \
280  $(ZIP_UP_RESULTS) \
281  $(TESTEXIT) \
282)
283
284################################################################
285
286# Prep for output
287# Change execute permissions on shared library files.
288# Files in repositories should never have execute permissions, but
289# there are some tests that have pre-built shared libraries, and these
290# windows dll files must have execute permission. Adding execute
291# permission may happen automatically on windows when using certain
292# versions of mercurial but it cannot be guaranteed. And blindly
293# adding execute permission might be seen as a mercurial 'change', so
294# we avoid adding execute permission to repository files. But testing
295# from a plain source tree needs the chmod a+rx. Applying the chmod to
296# all shared libraries not just dll files. And with CYGWIN and sshd
297# service, you may need CYGWIN=ntsec for this to work.
298prep:
299	@$(MKDIR) -p $(ABS_TEST_OUTPUT_DIR)
300	@if [ ! -d $(TEST_ROOT)/../../.hg ] && [ ! -d $(TEST_ROOT)/../../../.hg ]; then  \
301	  $(FIND) $(TEST_ROOT) \( -name \*.dll -o -name \*.DLL -o -name \*.so \)  \
302	        -exec $(CHMOD) a+rx {} \; ;                                       \
303	fi
304
305ifeq ($(CLEAN_BEFORE_PREP), true)
306prep: clean
307
308endif
309
310# Cleanup
311clean:
312	@$(RM) -r $(ABS_TEST_OUTPUT_DIR)
313	$(CLEAN_ARCHIVE_BUNDLE)
314
315################################################################
316
317# jtreg tests
318
319# Expect JT_HOME to be set for jtreg tests. (home for jtreg)
320ifndef JT_HOME
321  JT_HOME = $(SLASH_JAVA)/re/jtreg/$(USE_JTREG_VERSION)/promoted/latest/binaries/jtreg
322endif
323
324# Problematic tests to be excluded
325EXTRA_PROBLEM_LISTS :=
326PROBLEM_LISTS := ProblemList.txt $(EXTRA_PROBLEM_LISTS)
327
328# Create exclude list for this platform and arch
329ifdef NO_EXCLUDES
330  JTREG_EXCLUSIONS =
331else
332  JTREG_EXCLUSIONS = $(addprefix -exclude:, $(wildcard $(PROBLEM_LISTS)))
333endif
334
335# ------------------------------------------------------------------
336
337# The TESTDIRS variable can be used to select the jtreg tests to run
338ifdef TESTDIRS
339  TEST_SELECTION = $(TESTDIRS)
340endif
341
342ifeq ($(UNAME_S), SunOS)
343  NUM_CORES := $(shell LC_MESSAGES=C /usr/sbin/psrinfo -v | grep -c on-line)
344endif
345ifeq ($(UNAME_S), Linux)
346  NUM_CORES := $(shell cat /proc/cpuinfo  | grep -c processor)
347endif
348ifeq ($(findstring BSD,$(UNAME_S)), BSD)
349  NUM_CORES := $(shell /sbin/sysctl -n hw.ncpu)
350endif
351ifeq ($(UNAME_S), Darwin)
352  NUM_CORES := $(shell /usr/sbin/sysctl -n hw.ncpu)
353endif
354ifeq ($(findstring CYGWIN,$(UNAME_S)), CYGWIN)
355  ifneq ($(NUMBER_OF_PROCESSORS), )
356    NUM_CORES := $(NUMBER_OF_PROCESSORS)
357  else
358    ifneq ($(HOTSPOT_BUILD_JOBS), )
359      NUM_CORES := $(HOTSPOT_BUILD_JOBS)
360    else
361      NUM_CORES := 1 # fallback
362    endif
363  endif
364endif
365
366ifndef CONCURRENCY_FACTOR
367  CONCURRENCY_FACTOR = 1
368endif
369
370# Concurrency based on min(cores / 2, 12) * CONCURRENCY_FACTOR
371CONCURRENCY := $(shell $(AWK) \
372  'BEGIN { \
373    c = $(NUM_CORES) / 2; \
374    if (c > 12) c = 12; \
375    c = c * $(CONCURRENCY_FACTOR); \
376    if (c < 1) c = 1; \
377    printf "%.0f", c; \
378  }')
379JTREG_BASIC_OPTIONS += -concurrency:$(CONCURRENCY)
380
381# Make sure MaxRAMPercentage is low enough to not cause OOM or swapping since
382# we may end up with a lot of JVM's
383MAX_RAM_PERCENTAGE := $(shell expr 25 / $(CONCURRENCY))
384JTREG_BASIC_OPTIONS += -vmoption:-XX:MaxRAMPercentage=$(MAX_RAM_PERCENTAGE)
385
386ifdef EXTRA_JTREG_OPTIONS
387  JTREG_BASIC_OPTIONS += $(EXTRA_JTREG_OPTIONS)
388endif
389
390# Default JTREG to run
391JTREG = $(JT_HOME)/bin/jtreg
392# run in agentvm/othervm mode
393JTREG_BASIC_OPTIONS += $(JTREG_VM_TYPE)
394# Only run automatic tests
395JTREG_BASIC_OPTIONS += -a
396# Always turn on assertions
397ifeq ($(USE_JTREG_ASSERT), true)
398  JTREG_ASSERT_OPTION = -ea -esa
399endif
400JTREG_BASIC_OPTIONS += $(JTREG_ASSERT_OPTION)
401# jtreg verbosity setting
402# Default is to report details on all failed or error tests, times too
403JTREG_VERBOSE ?= fail,error,time
404JTREG_BASIC_OPTIONS += $(if $(JTREG_VERBOSE),-v:$(JTREG_VERBOSE))
405# Retain all files for failing tests
406JTREG_BASIC_OPTIONS += -retain:fail,error
407# Ignore tests are not run and completely silent about it
408JTREG_IGNORE_OPTION = -ignore:quiet
409JTREG_BASIC_OPTIONS += $(JTREG_IGNORE_OPTION)
410# Multiply by 4 the timeout factor
411JTREG_TIMEOUT_OPTION =  -timeoutFactor:4
412JTREG_BASIC_OPTIONS += $(JTREG_TIMEOUT_OPTION)
413ifeq ($(LIMIT_JTREG_VM_MEMORY), true)
414  # Set the max memory for jtreg control vm
415  JTREG_MEMORY_OPTION = -J-Xmx512m
416  JTREG_BASIC_OPTIONS += $(JTREG_MEMORY_OPTION)
417  # Set the max memory for jtreg target test vms
418  JTREG_TESTVM_MEMORY_OPTION = -vmoption:-Xmx512m
419  JTREG_TEST_OPTIONS += $(JTREG_TESTVM_MEMORY_OPTION)
420endif
421# Make it possible to specify the JIB_DATA_DIR for tests using the
422# JIB Artifact resolver
423JTREG_BASIC_OPTIONS += -e:JIB_DATA_DIR
424# Give tests access to JT_JAVA, see JDK-8141609
425JTREG_BASIC_OPTIONS += -e:JDK8_HOME=${JT_JAVA}
426# Give aot tests access to Visual Studio installation
427ifneq ($(VS120COMNTOOLS), )
428  JTREG_BASIC_OPTIONS += -e:VS120COMNTOOLS="$(shell $(GETMIXEDPATH) "$(patsubst %\,%,$(VS120COMNTOOLS))")"
429endif
430
431JTREG_BASIC_OPTIONS += -e:TEST_IMAGE_GRAAL_DIR=${TEST_IMAGE_DIR}/hotspot/jtreg/graal
432
433# Set other vm and test options
434JTREG_TEST_OPTIONS += $(JAVA_ARGS:%=-javaoptions:%) $(JAVA_VM_ARGS:%=-vmoption:%)
435ifneq ($(JIB_HOME), )
436  JTREG_BASIC_OPTIONS += -e:JIB_HOME=$(shell $(GETMIXEDPATH) "$(JIB_HOME)")
437endif
438ifeq ($(IGNORE_MARKED_TESTS), true)
439  # Option to tell jtreg to not run tests marked with "ignore"
440  ifeq ($(PLATFORM), windows)
441    JTREG_KEY_OPTION = -k:!ignore
442  else
443    JTREG_KEY_OPTION = -k:\!ignore
444  endif
445  JTREG_BASIC_OPTIONS += $(JTREG_KEY_OPTION)
446endif
447
448# Make sure jtreg exists
449ifeq ($(USE_WINDOWS_EXISTENCE_CHECK), true)
450  jtreg_exists:
451	test -d $(shell $(GETMIXEDPATH) "$(JT_HOME)")
452	test -f $(shell $(GETMIXEDPATH) "$(JTREG)")
453
454else
455  jtreg_exists: $(JT_HOME)
456endif
457PHONY_LIST += jtreg_exists
458
459# Run jtreg
460jtreg_tests: prep jtreg_exists $(PRODUCT_HOME) $(TEST_PREREQS)
461	(                                                                    \
462	  ( JT_HOME=$(shell $(GETMIXEDPATH) "$(JT_HOME)");                   \
463	    export JT_HOME;                                                  \
464	    $(shell $(GETMIXEDPATH) "$(JTREG)")                              \
465	      $(JTREG_BASIC_OPTIONS)                                         \
466	      -r:$(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/JTreport")  \
467	      -w:$(shell $(GETMIXEDPATH) "$(ABS_TEST_OUTPUT_DIR)/JTwork")    \
468	      -jdk:$(shell $(GETMIXEDPATH) "$(PRODUCT_HOME)")                \
469	      $(JTREG_NATIVE_PATH)                                           \
470	      $(JTREG_FAILURE_HANDLER_OPTIONS)                               \
471	      $(JTREG_EXCLUSIONS)                                            \
472	      $(JTREG_TEST_OPTIONS)                                          \
473	      $(TEST_SELECTION)                                              \
474	  ) ;                                                                \
475	  $(BUNDLE_UP_AND_EXIT)                                              \
476	) 2>&1 | $(TEE) $(ABS_TEST_OUTPUT_DIR)/output.txt ; $(TESTEXIT)
477
478PHONY_LIST += jtreg_tests
479
480# Make it possible to call this with <component>_jtreg_tests
481%_jtreg_tests: jtreg_tests
482	# Must have a fake recipe here to prevent make from matching any other rule
483
484################################################################
485
486# Phony targets (e.g. these are not filenames)
487.PHONY: all clean prep $(PHONY_LIST)
488
489################################################################
490