1 /*******************************************************************************
2  * Copyright (c) 2000, 2018 IBM Corporation and others.
3  *
4  * This program and the accompanying materials
5  * are made available under the terms of the Eclipse Public License 2.0
6  * which accompanies this distribution, and is available at
7  * https://www.eclipse.org/legal/epl-2.0/
8  *
9  * SPDX-License-Identifier: EPL-2.0
10  *
11  * Contributors:
12  *     IBM Corporation - initial API and implementation
13  *******************************************************************************/
14 
15 package org.eclipse.test.performance;
16 
17 import java.lang.reflect.InvocationTargetException;
18 
19 import org.eclipse.core.runtime.Platform;
20 import org.eclipse.test.internal.performance.InternalDimensions;
21 import org.eclipse.test.internal.performance.InternalPerformanceMeter;
22 import org.eclipse.test.internal.performance.NullPerformanceMeter;
23 import org.eclipse.test.internal.performance.OSPerformanceMeterFactory;
24 import org.eclipse.test.internal.performance.PerformanceMeterFactory;
25 import org.eclipse.test.internal.performance.PerformanceTestPlugin;
26 import org.eclipse.test.internal.performance.data.Dim;
27 import org.eclipse.test.internal.performance.eval.AbsoluteBandChecker;
28 import org.eclipse.test.internal.performance.eval.AssertChecker;
29 import org.eclipse.test.internal.performance.eval.Evaluator;
30 import org.eclipse.test.internal.performance.eval.IEvaluator;
31 import org.eclipse.test.internal.performance.eval.RelativeBandChecker;
32 import org.osgi.framework.Bundle;
33 
34 import junit.framework.TestCase;
35 
36 /**
37  * Helper for performance measurements. Currently provides performance meter creation and checking of measurements.
38  *
39  * This class is not intended to be subclassed by clients.
40  *
41  * @since 3.1
42  */
43 public class Performance {
44 
45     /**
46      * A comment kind of a comment that explains a performance degradation.
47      */
48     public static final int         EXPLAINS_DEGRADATION_COMMENT       = 1;
49 
50     private static final String     PERFORMANCE_METER_FACTORY          = "/option/performanceMeterFactory"; //$NON-NLS-1$
51     private static final String     PERFORMANCE_METER_FACTORY_PROPERTY = "PerformanceMeterFactory";        //$NON-NLS-1$
52 
53     private static Performance      fgDefault;
54 
55     private PerformanceMeterFactory fPerformanceMeterFactory;
56     private IEvaluator              fDefaultEvaluator;
57 
58     /** Null performance meter singleton */
59     private NullPerformanceMeter    fNullPeformanceMeter;
60 
61     /**
62      * Private constructor to block instance creation.
63      */
Performance()64     private Performance() {
65         // empty
66     }
67 
68     /**
69      * Returns the singleton of <code>Performance</code>
70      *
71      * @return the singleton of <code>Performance</code>
72      */
getDefault()73     public static Performance getDefault() {
74         if (fgDefault == null)
75             fgDefault = new Performance();
76         return fgDefault;
77     }
78 
79     /**
80      * Asserts default properties of the measurements captured by the given performance meter.
81      *
82      * @param performanceMeter
83      *            the performance meter
84      * @throws RuntimeException
85      *             if the properties do not hold
86      */
assertPerformance(PerformanceMeter performanceMeter)87     public void assertPerformance(PerformanceMeter performanceMeter) {
88         if (fDefaultEvaluator == null) {
89             fDefaultEvaluator = new Evaluator();
90             fDefaultEvaluator.setAssertCheckers(new AssertChecker[] { new RelativeBandChecker(InternalDimensions.ELAPSED_PROCESS,
91                     0.0f, 1.10f),
92             // new RelativeBandChecker(InternalDimensions.CPU_TIME, 0.0f, 1.10f),
93             // new RelativeBandChecker(InternalDimensions.WORKING_SET, 0.0f, 3.00f),
94             // new RelativeBandChecker(InternalDimensions.USED_JAVA_HEAP, 0.0f, 2.00f),
95             // new RelativeBandChecker(InternalDimensions.SYSTEM_TIME, 0.0f, 1.10f)
96                     });
97         }
98         fDefaultEvaluator.evaluate(performanceMeter);
99     }
100 
101     /**
102      * Asserts that the measurement specified by the dimension captured in the given performance meter is within a certain range
103      * with respect to some reference value. If the performance meter doesn't provide the specified dimension, the call has no
104      * effect.
105      *
106      * @param performanceMeter
107      *            the performance meter
108      * @param dim
109      *            the Dimension to check
110      * @param lowerPercentage
111      *            a negative number indicating the percentage the measured value is allowed to be smaller than some reference value
112      * @param upperPercentage
113      *            a positive number indicating the percentage the measured value is allowed to be greater than some reference value
114      * @throws RuntimeException
115      *             if the properties do not hold
116      */
assertPerformanceInRelativeBand(PerformanceMeter performanceMeter, Dimension dim, int lowerPercentage, int upperPercentage)117     public void assertPerformanceInRelativeBand(PerformanceMeter performanceMeter, Dimension dim, int lowerPercentage,
118             int upperPercentage) {
119         Evaluator e = new Evaluator();
120         e.setAssertCheckers(new AssertChecker[] { new RelativeBandChecker((Dim) dim, 1.0 + (lowerPercentage / 100.0),
121                 1.0 + (upperPercentage / 100.0)), });
122         e.evaluate(performanceMeter);
123     }
124 
125     /**
126      * Asserts that the measurement specified by the dimension captured in the given performance meter is within a certain range
127      * with respect to some reference value. If the performance meter doesn't provide the specified dimension, the call has no
128      * effect.
129      *
130      * @param performanceMeter
131      *            the performance meter
132      * @param dim
133      *            the Dimension to check
134      * @param lowerBand
135      *            a negative number indicating the absolute amount the measured value is allowed to be smaller than some reference
136      *            value
137      * @param upperBand
138      *            a positive number indicating the absolute amount the measured value is allowed to be greater than some reference
139      *            value
140      * @throws RuntimeException
141      *             if the properties do not hold
142      */
assertPerformanceInAbsoluteBand(PerformanceMeter performanceMeter, Dimension dim, int lowerBand, int upperBand)143     public void assertPerformanceInAbsoluteBand(PerformanceMeter performanceMeter, Dimension dim, int lowerBand, int upperBand) {
144         Evaluator e = new Evaluator();
145         e.setAssertCheckers(new AssertChecker[] { new AbsoluteBandChecker((Dim) dim, lowerBand, upperBand), });
146         e.evaluate(performanceMeter);
147     }
148 
149     /**
150      * Creates a performance meter for the given scenario id.
151      *
152      * @param scenarioId
153      *            the scenario id
154      * @return a performance meter for the given scenario id
155      * @throws IllegalArgumentException
156      *             if a performance meter for the given scenario id has already been created
157      */
createPerformanceMeter(String scenarioId)158     public PerformanceMeter createPerformanceMeter(String scenarioId) {
159         return getPeformanceMeterFactory().createPerformanceMeter(scenarioId);
160     }
161 
162     /**
163      * Returns the null performance meter singleton.
164      *
165      * @return the null performance meter singleton
166      */
getNullPerformanceMeter()167     public PerformanceMeter getNullPerformanceMeter() {
168         if (fNullPeformanceMeter == null)
169             fNullPeformanceMeter = new NullPerformanceMeter();
170         return fNullPeformanceMeter;
171     }
172 
173     /**
174      * Returns a default scenario id for the given test. The test's name must have been set, such that <code>test.getName()</code>
175      * is not <code>null</code>.
176      *
177      * @param test
178      *            the test
179      * @return the default scenario id for the test
180      */
getDefaultScenarioId(TestCase test)181     public String getDefaultScenarioId(TestCase test) {
182         return test.getClass().getName() + '#' + test.getName() + "()"; //$NON-NLS-1$
183     }
184 
185     /**
186      * Returns a default scenario id for the given test class.
187      *
188      * @param test
189      *            the test class
190      * @return the default scenario id for the test
191      * @since 3.15
192      */
getDefaultScenarioId(Class<?> test)193     public String getDefaultScenarioId(Class<?> test) {
194         return test.getName() + '#' + test.getName() + "()"; //$NON-NLS-1$
195     }
196 
197     /**
198      * Returns a default scenario id for the given test and id. The test's name must have been set, such that
199      * <code>test.getName()</code> is not <code>null</code>. The id distinguishes multiple scenarios in the same test.
200      *
201      * @param test
202      *            the test
203      * @param id
204      *            the id
205      * @return the default scenario id for the test and the id
206      */
getDefaultScenarioId(TestCase test, String id)207     public String getDefaultScenarioId(TestCase test, String id) {
208         return getDefaultScenarioId(test) + '-' + id;
209     }
210 
getPeformanceMeterFactory()211     private PerformanceMeterFactory getPeformanceMeterFactory() {
212         if (fPerformanceMeterFactory == null)
213             fPerformanceMeterFactory = createPerformanceMeterFactory();
214         return fPerformanceMeterFactory;
215     }
216 
createPerformanceMeterFactory()217     private PerformanceMeterFactory createPerformanceMeterFactory() {
218         PerformanceMeterFactory factory;
219         factory = tryInstantiate(System.getProperty(PERFORMANCE_METER_FACTORY_PROPERTY));
220         if (factory != null)
221             return factory;
222 
223         factory = tryInstantiate(Platform.getDebugOption(PerformanceTestPlugin.PLUGIN_ID + PERFORMANCE_METER_FACTORY));
224         if (factory != null)
225             return factory;
226 
227         return createDefaultPerformanceMeterFactory();
228     }
229 
tryInstantiate(String className)230     private PerformanceMeterFactory tryInstantiate(String className) {
231         PerformanceMeterFactory instance = null;
232         if (className != null && className.length() > 0) {
233             try {
234                 Class<?> c = null;
235                 if (Platform.isRunning()) {
236                     int separator = className.indexOf(':');
237                     Bundle bundle = null;
238                     if (separator == -1) {
239                         bundle = PerformanceTestPlugin.getDefault().getBundle();
240                     } else {
241                         String bundleName = className.substring(0, separator);
242                         className = className.substring(separator + 1);
243                         bundle = Platform.getBundle(bundleName);
244                     }
245                     c = bundle.loadClass(className);
246                 } else {
247                     c = Class.forName(className);
248                 }
249                 instance = (PerformanceMeterFactory) c.getDeclaredConstructor().newInstance();
250             } catch (
251                     ClassNotFoundException |
252                     InstantiationException |
253                     IllegalAccessException |
254                     ClassCastException |
255                     IllegalArgumentException |
256                     InvocationTargetException |
257                     NoSuchMethodException |
258                     SecurityException e) {
259                 PerformanceTestPlugin.log(e);
260             }
261         }
262         return instance;
263     }
264 
createDefaultPerformanceMeterFactory()265     private PerformanceMeterFactory createDefaultPerformanceMeterFactory() {
266         return new OSPerformanceMeterFactory();
267     }
268 
269     /**
270      * Mark the scenario represented by the given PerformanceMeter to be included into the global and the component performance
271      * summary. The summary shows the given dimension of the scenario and labels the scenario with the short name.
272      *
273      * @param pm
274      *            the PerformanceMeter
275      * @param shortName
276      *            a short (shorter than 40 characters) descriptive name of the scenario
277      * @param dimension
278      *            the dimension to show in the summary
279      */
tagAsGlobalSummary(PerformanceMeter pm, String shortName, Dimension dimension)280     public void tagAsGlobalSummary(PerformanceMeter pm, String shortName, Dimension dimension) {
281         tagAsGlobalSummary(pm, shortName, new Dimension[] { dimension });
282     }
283 
284     /**
285      * Mark the scenario represented by the given PerformanceMeter to be included into the global and the component performance
286      * summary. The summary shows the given dimensions of the scenario and labels the scenario with the short name.
287      *
288      * @param pm
289      *            the PerformanceMeter
290      * @param shortName
291      *            a short (shorter than 40 characters) descriptive name of the scenario
292      * @param dimensions
293      *            an array of dimensions to show in the summary
294      */
tagAsGlobalSummary(PerformanceMeter pm, String shortName, Dimension[] dimensions)295     public void tagAsGlobalSummary(PerformanceMeter pm, String shortName, Dimension[] dimensions) {
296         if (pm instanceof InternalPerformanceMeter) {
297             InternalPerformanceMeter ipm = (InternalPerformanceMeter) pm;
298             ipm.tagAsSummary(true, shortName, dimensions);
299         }
300     }
301 
302     /**
303      * Mark the scenario represented by the given PerformanceMeter to be included into the component performance summary. The
304      * summary shows the given dimension of the scenario and labels the scenario with the short name.
305      *
306      * @param pm
307      *            the PerformanceMeter
308      * @param shortName
309      *            a short (shorter than 40 characters) descriptive name of the scenario
310      * @param dimension
311      *            the dimension to show in the summary
312      */
tagAsSummary(PerformanceMeter pm, String shortName, Dimension dimension)313     public void tagAsSummary(PerformanceMeter pm, String shortName, Dimension dimension) {
314         tagAsSummary(pm, shortName, new Dimension[] { dimension });
315     }
316 
317     /**
318      * Mark the scenario represented by the given PerformanceMeter to be included into the component performance summary. The
319      * summary shows the given dimensions of the scenario and labels the scenario with the short name.
320      *
321      * @param pm
322      *            the PerformanceMeter
323      * @param shortName
324      *            a short (shorter than 40 characters) descriptive name of the scenario
325      * @param dimensions
326      *            an array of dimensions to show in the summary
327      */
tagAsSummary(PerformanceMeter pm, String shortName, Dimension[] dimensions)328     public void tagAsSummary(PerformanceMeter pm, String shortName, Dimension[] dimensions) {
329         if (pm instanceof InternalPerformanceMeter) {
330             InternalPerformanceMeter ipm = (InternalPerformanceMeter) pm;
331             ipm.tagAsSummary(false, shortName, dimensions);
332         }
333     }
334 
335     /**
336      * Set a comment for the scenario represented by the given PerformanceMeter. Currently only comments with a commentKind of
337      * EXPLAINS_DEGRADATION_COMMENT are used. Their commentText is shown in a hover of the performance summaries graph if a
338      * performance degradation exists.
339      *
340      * @param pm
341      *            the PerformanceMeter
342      * @param commentKind
343      *            kind of comment. Must be EXPLAINS_DEGRADATION_COMMENT to have an effect.
344      * @param commentText
345      *            the comment (shorter than 400 characters)
346      */
setComment(PerformanceMeter pm, int commentKind, String commentText)347     public void setComment(PerformanceMeter pm, int commentKind, String commentText) {
348         if (commentKind == EXPLAINS_DEGRADATION_COMMENT) {
349             if (pm instanceof InternalPerformanceMeter) {
350                 InternalPerformanceMeter ipm = (InternalPerformanceMeter) pm;
351                 ipm.setComment(commentKind, commentText);
352             }
353         }
354     }
355 }
356