1 /**
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements.  See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership.  The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License.  You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18 package org.apache.hadoop.mapred.gridmix.emulators.resourceusage;
19 
20 import java.io.IOException;
21 import java.util.ArrayList;
22 import java.util.List;
23 
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.mapred.gridmix.Progressive;
26 import org.apache.hadoop.tools.rumen.ResourceUsageMetrics;
27 import org.apache.hadoop.util.ReflectionUtils;
28 import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
29 
30 /**
31  * <p>This is the driver class for managing all the resource usage emulators.
32  * {@link ResourceUsageMatcher} expects a comma separated list of
33  * {@link ResourceUsageEmulatorPlugin} implementations specified using
34  * {@link #RESOURCE_USAGE_EMULATION_PLUGINS} as the configuration parameter.</p>
35  *
36  * <p>Note that the order in which the emulators are invoked is same as the
37  * order in which they are configured.
38  */
39 public class ResourceUsageMatcher implements Progressive {
40   /**
41    * Configuration key to set resource usage emulators.
42    */
43   public static final String RESOURCE_USAGE_EMULATION_PLUGINS =
44     "gridmix.emulators.resource-usage.plugins";
45 
46   private List<ResourceUsageEmulatorPlugin> emulationPlugins =
47     new ArrayList<ResourceUsageEmulatorPlugin>();
48 
49   /**
50    * Configure the {@link ResourceUsageMatcher} to load the configured plugins
51    * and initialize them.
52    */
53   @SuppressWarnings("unchecked")
configure(Configuration conf, ResourceCalculatorPlugin monitor, ResourceUsageMetrics metrics, Progressive progress)54   public void configure(Configuration conf, ResourceCalculatorPlugin monitor,
55                         ResourceUsageMetrics metrics, Progressive progress) {
56     Class[] plugins = conf.getClasses(RESOURCE_USAGE_EMULATION_PLUGINS);
57     if (plugins == null) {
58       System.out.println("No resource usage emulator plugins configured.");
59     } else {
60       for (Class clazz : plugins) {
61         if (clazz != null) {
62           if (ResourceUsageEmulatorPlugin.class.isAssignableFrom(clazz)) {
63             ResourceUsageEmulatorPlugin plugin =
64               (ResourceUsageEmulatorPlugin) ReflectionUtils.newInstance(clazz,
65                                                                         conf);
66             emulationPlugins.add(plugin);
67           } else {
68             throw new RuntimeException("Misconfigured resource usage plugins. "
69                 + "Class " + clazz.getClass().getName() + " is not a resource "
70                 + "usage plugin as it does not extend "
71                 + ResourceUsageEmulatorPlugin.class.getName());
72           }
73         }
74       }
75     }
76 
77     // initialize the emulators once all the configured emulator plugins are
78     // loaded
79     for (ResourceUsageEmulatorPlugin emulator : emulationPlugins) {
80       emulator.initialize(conf, metrics, monitor, progress);
81     }
82   }
83 
matchResourceUsage()84   public void matchResourceUsage() throws IOException, InterruptedException {
85     for (ResourceUsageEmulatorPlugin emulator : emulationPlugins) {
86       // match the resource usage
87       emulator.emulate();
88     }
89   }
90 
91   /**
92    * Returns the average progress.
93    */
94   @Override
getProgress()95   public float getProgress() {
96     if (emulationPlugins.size() > 0) {
97       // return the average progress
98       float progress = 0f;
99       for (ResourceUsageEmulatorPlugin emulator : emulationPlugins) {
100         // consider weighted progress of each emulator
101         progress += emulator.getProgress();
102       }
103 
104       return progress / emulationPlugins.size();
105     }
106 
107     // if no emulators are configured then return 1
108     return 1f;
109 
110   }
111 }