1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase; 19 20 import java.io.IOException; 21 import java.lang.reflect.InvocationTargetException; 22 import java.lang.reflect.Method; 23 import java.util.Map; 24 25 import org.apache.commons.logging.Log; 26 import org.apache.commons.logging.LogFactory; 27 import org.apache.hadoop.conf.Configuration; 28 import org.apache.hadoop.hbase.classification.InterfaceAudience; 29 import org.apache.hadoop.hbase.classification.InterfaceStability; 30 import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil; 31 import org.apache.hadoop.hbase.util.VersionInfo; 32 import org.apache.hadoop.hbase.zookeeper.ZKConfig; 33 34 /** 35 * Adds HBase configuration files to a Configuration 36 */ 37 @InterfaceAudience.Public 38 @InterfaceStability.Stable 39 public class HBaseConfiguration extends Configuration { 40 private static final Log LOG = LogFactory.getLog(HBaseConfiguration.class); 41 42 /** 43 * Instantiating HBaseConfiguration() is deprecated. Please use 44 * HBaseConfiguration#create() to construct a plain Configuration 45 */ 46 @Deprecated HBaseConfiguration()47 public HBaseConfiguration() { 48 //TODO:replace with private constructor, HBaseConfiguration should not extend Configuration 49 super(); 50 addHbaseResources(this); 51 LOG.warn("instantiating HBaseConfiguration() is deprecated. Please use" 52 + " HBaseConfiguration#create() to construct a plain Configuration"); 53 } 54 55 /** 56 * Instantiating HBaseConfiguration() is deprecated. Please use 57 * HBaseConfiguration#create(conf) to construct a plain Configuration 58 */ 59 @Deprecated HBaseConfiguration(final Configuration c)60 public HBaseConfiguration(final Configuration c) { 61 //TODO:replace with private constructor 62 this(); 63 merge(this, c); 64 } 65 checkDefaultsVersion(Configuration conf)66 private static void checkDefaultsVersion(Configuration conf) { 67 if (conf.getBoolean("hbase.defaults.for.version.skip", Boolean.FALSE)) return; 68 String defaultsVersion = conf.get("hbase.defaults.for.version"); 69 String thisVersion = VersionInfo.getVersion(); 70 if (!thisVersion.equals(defaultsVersion)) { 71 throw new RuntimeException( 72 "hbase-default.xml file seems to be for an older version of HBase (" + 73 defaultsVersion + "), this version is " + thisVersion); 74 } 75 } 76 addHbaseResources(Configuration conf)77 public static Configuration addHbaseResources(Configuration conf) { 78 conf.addResource("hbase-default.xml"); 79 conf.addResource("hbase-site.xml"); 80 81 checkDefaultsVersion(conf); 82 HeapMemorySizeUtil.checkForClusterFreeMemoryLimit(conf); 83 return conf; 84 } 85 86 /** 87 * Creates a Configuration with HBase resources 88 * @return a Configuration with HBase resources 89 */ create()90 public static Configuration create() { 91 Configuration conf = new Configuration(); 92 // In case HBaseConfiguration is loaded from a different classloader than 93 // Configuration, conf needs to be set with appropriate class loader to resolve 94 // HBase resources. 95 conf.setClassLoader(HBaseConfiguration.class.getClassLoader()); 96 return addHbaseResources(conf); 97 } 98 99 /** 100 * @param that Configuration to clone. 101 * @return a Configuration created with the hbase-*.xml files plus 102 * the given configuration. 103 */ create(final Configuration that)104 public static Configuration create(final Configuration that) { 105 Configuration conf = create(); 106 merge(conf, that); 107 return conf; 108 } 109 110 /** 111 * Merge two configurations. 112 * @param destConf the configuration that will be overwritten with items 113 * from the srcConf 114 * @param srcConf the source configuration 115 **/ merge(Configuration destConf, Configuration srcConf)116 public static void merge(Configuration destConf, Configuration srcConf) { 117 for (Map.Entry<String, String> e : srcConf) { 118 destConf.set(e.getKey(), e.getValue()); 119 } 120 } 121 122 /** 123 * Returns a subset of the configuration properties, matching the given key prefix. 124 * The prefix is stripped from the return keys, ie. when calling with a prefix of "myprefix", 125 * the entry "myprefix.key1 = value1" would be returned as "key1 = value1". If an entry's 126 * key matches the prefix exactly ("myprefix = value2"), it will <strong>not</strong> be 127 * included in the results, since it would show up as an entry with an empty key. 128 */ subset(Configuration srcConf, String prefix)129 public static Configuration subset(Configuration srcConf, String prefix) { 130 Configuration newConf = new Configuration(false); 131 for (Map.Entry<String, String> entry : srcConf) { 132 if (entry.getKey().startsWith(prefix)) { 133 String newKey = entry.getKey().substring(prefix.length()); 134 // avoid entries that would produce an empty key 135 if (!newKey.isEmpty()) { 136 newConf.set(newKey, entry.getValue()); 137 } 138 } 139 } 140 return newConf; 141 } 142 143 /** 144 * Sets all the entries in the provided {@code Map<String, String>} as properties in the 145 * given {@code Configuration}. Each property will have the specified prefix prepended, 146 * so that the configuration entries are keyed by {@code prefix + entry.getKey()}. 147 */ setWithPrefix(Configuration conf, String prefix, Iterable<Map.Entry<String, String>> properties)148 public static void setWithPrefix(Configuration conf, String prefix, 149 Iterable<Map.Entry<String, String>> properties) { 150 for (Map.Entry<String, String> entry : properties) { 151 conf.set(prefix + entry.getKey(), entry.getValue()); 152 } 153 } 154 155 /** 156 * @return whether to show HBase Configuration in servlet 157 */ isShowConfInServlet()158 public static boolean isShowConfInServlet() { 159 boolean isShowConf = false; 160 try { 161 if (Class.forName("org.apache.hadoop.conf.ConfServlet") != null) { 162 isShowConf = true; 163 } 164 } catch (LinkageError e) { 165 // should we handle it more aggressively in addition to log the error? 166 LOG.warn("Error thrown: ", e); 167 } catch (ClassNotFoundException ce) { 168 LOG.debug("ClassNotFound: ConfServlet"); 169 // ignore 170 } 171 return isShowConf; 172 } 173 174 /** 175 * Get the value of the <code>name</code> property as an <code>int</code>, possibly 176 * referring to the deprecated name of the configuration property. 177 * If no such property exists, the provided default value is returned, 178 * or if the specified value is not a valid <code>int</code>, 179 * then an error is thrown. 180 * 181 * @param name property name. 182 * @param deprecatedName a deprecatedName for the property to use 183 * if non-deprecated name is not used 184 * @param defaultValue default value. 185 * @throws NumberFormatException when the value is invalid 186 * @return property value as an <code>int</code>, 187 * or <code>defaultValue</code>. 188 */ 189 // TODO: developer note: This duplicates the functionality of deprecated 190 // property support in Configuration in Hadoop 2. But since Hadoop-1 does not 191 // contain these changes, we will do our own as usual. Replace these when H2 is default. getInt(Configuration conf, String name, String deprecatedName, int defaultValue)192 public static int getInt(Configuration conf, String name, 193 String deprecatedName, int defaultValue) { 194 if (conf.get(deprecatedName) != null) { 195 LOG.warn(String.format("Config option \"%s\" is deprecated. Instead, use \"%s\"" 196 , deprecatedName, name)); 197 return conf.getInt(deprecatedName, defaultValue); 198 } else { 199 return conf.getInt(name, defaultValue); 200 } 201 } 202 203 /** 204 * Get the password from the Configuration instance using the 205 * getPassword method if it exists. If not, then fall back to the 206 * general get method for configuration elements. 207 * @param conf configuration instance for accessing the passwords 208 * @param alias the name of the password element 209 * @param defPass the default password 210 * @return String password or default password 211 * @throws IOException 212 */ getPassword(Configuration conf, String alias, String defPass)213 public static String getPassword(Configuration conf, String alias, 214 String defPass) throws IOException { 215 String passwd = null; 216 try { 217 Method m = Configuration.class.getMethod("getPassword", String.class); 218 char[] p = (char[]) m.invoke(conf, alias); 219 if (p != null) { 220 LOG.debug(String.format("Config option \"%s\" was found through" + 221 " the Configuration getPassword method.", alias)); 222 passwd = new String(p); 223 } 224 else { 225 LOG.debug(String.format( 226 "Config option \"%s\" was not found. Using provided default value", 227 alias)); 228 passwd = defPass; 229 } 230 } catch (NoSuchMethodException e) { 231 // this is a version of Hadoop where the credential 232 //provider API doesn't exist yet 233 LOG.debug(String.format( 234 "Credential.getPassword method is not available." + 235 " Falling back to configuration.")); 236 passwd = conf.get(alias, defPass); 237 } catch (SecurityException e) { 238 throw new IOException(e.getMessage(), e); 239 } catch (IllegalAccessException e) { 240 throw new IOException(e.getMessage(), e); 241 } catch (IllegalArgumentException e) { 242 throw new IOException(e.getMessage(), e); 243 } catch (InvocationTargetException e) { 244 throw new IOException(e.getMessage(), e); 245 } 246 return passwd; 247 } 248 249 /** 250 * Generates a {@link Configuration} instance by applying the ZooKeeper cluster key 251 * to the base Configuration. Note that additional configuration properties may be needed 252 * for a remote cluster, so it is preferable to use 253 * {@link #createClusterConf(Configuration, String, String)}. 254 * 255 * @param baseConf the base configuration to use, containing prefixed override properties 256 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 257 * 258 * @return the merged configuration with override properties and cluster key applied 259 * 260 * @see #createClusterConf(Configuration, String, String) 261 */ createClusterConf(Configuration baseConf, String clusterKey)262 public static Configuration createClusterConf(Configuration baseConf, String clusterKey) 263 throws IOException { 264 return createClusterConf(baseConf, clusterKey, null); 265 } 266 267 /** 268 * Generates a {@link Configuration} instance by applying property overrides prefixed by 269 * a cluster profile key to the base Configuration. Override properties are extracted by 270 * the {@link #subset(Configuration, String)} method, then the merged on top of the base 271 * Configuration and returned. 272 * 273 * @param baseConf the base configuration to use, containing prefixed override properties 274 * @param clusterKey the ZooKeeper quorum cluster key to apply, or {@code null} if none 275 * @param overridePrefix the property key prefix to match for override properties, 276 * or {@code null} if none 277 * @return the merged configuration with override properties and cluster key applied 278 */ createClusterConf(Configuration baseConf, String clusterKey, String overridePrefix)279 public static Configuration createClusterConf(Configuration baseConf, String clusterKey, 280 String overridePrefix) throws IOException { 281 Configuration clusterConf = HBaseConfiguration.create(baseConf); 282 if (clusterKey != null && !clusterKey.isEmpty()) { 283 applyClusterKeyToConf(clusterConf, clusterKey); 284 } 285 286 if (overridePrefix != null && !overridePrefix.isEmpty()) { 287 Configuration clusterSubset = HBaseConfiguration.subset(clusterConf, overridePrefix); 288 HBaseConfiguration.merge(clusterConf, clusterSubset); 289 } 290 return clusterConf; 291 } 292 293 /** 294 * Apply the settings in the given key to the given configuration, this is 295 * used to communicate with distant clusters 296 * @param conf configuration object to configure 297 * @param key string that contains the 3 required configuratins 298 * @throws IOException 299 */ applyClusterKeyToConf(Configuration conf, String key)300 private static void applyClusterKeyToConf(Configuration conf, String key) 301 throws IOException{ 302 ZKConfig.ZKClusterKey zkClusterKey = ZKConfig.transformClusterKey(key); 303 conf.set(HConstants.ZOOKEEPER_QUORUM, zkClusterKey.getQuorumString()); 304 conf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, zkClusterKey.getClientPort()); 305 conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, zkClusterKey.getZnodeParent()); 306 } 307 308 /** 309 * For debugging. Dump configurations to system output as xml format. 310 * Master and RS configurations can also be dumped using 311 * http services. e.g. "curl http://master:16010/dump" 312 */ main(String[] args)313 public static void main(String[] args) throws Exception { 314 HBaseConfiguration.create().writeXml(System.out); 315 } 316 } 317