1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hdfs; 19 20 import java.util.HashMap; 21 22 import org.apache.commons.logging.Log; 23 import org.apache.commons.logging.LogFactory; 24 import org.apache.hadoop.classification.InterfaceAudience; 25 import org.apache.hadoop.conf.Configuration; 26 import org.apache.hadoop.hdfs.DFSClient.Conf; 27 import org.apache.hadoop.hdfs.shortcircuit.DomainSocketFactory; 28 import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache; 29 import org.apache.hadoop.hdfs.util.ByteArrayManager; 30 31 import com.google.common.annotations.VisibleForTesting; 32 import com.google.common.cache.Cache; 33 34 /** 35 * ClientContext contains context information for a client. 36 * 37 * This allows us to share caches such as the socket cache across 38 * DFSClient instances. 39 */ 40 @InterfaceAudience.Private 41 public class ClientContext { 42 private static final Log LOG = LogFactory.getLog(ClientContext.class); 43 44 /** 45 * Global map of context names to caches contexts. 46 */ 47 private final static HashMap<String, ClientContext> CACHES = 48 new HashMap<String, ClientContext>(); 49 50 /** 51 * Name of context. 52 */ 53 private final String name; 54 55 /** 56 * String representation of the configuration. 57 */ 58 private final String confString; 59 60 /** 61 * Caches short-circuit file descriptors, mmap regions. 62 */ 63 private final ShortCircuitCache shortCircuitCache; 64 65 /** 66 * Caches TCP and UNIX domain sockets for reuse. 67 */ 68 private final PeerCache peerCache; 69 70 /** 71 * Stores information about socket paths. 72 */ 73 private final DomainSocketFactory domainSocketFactory; 74 75 /** 76 * Caches key Providers for the DFSClient 77 */ 78 private final KeyProviderCache keyProviderCache; 79 /** 80 * True if we should use the legacy BlockReaderLocal. 81 */ 82 private final boolean useLegacyBlockReaderLocal; 83 84 /** 85 * True if the legacy BlockReaderLocal is disabled. 86 * 87 * The legacy block reader local gets disabled completely whenever there is an 88 * error or miscommunication. The new block reader local code handles this 89 * case more gracefully inside DomainSocketFactory. 90 */ 91 private volatile boolean disableLegacyBlockReaderLocal = false; 92 93 /** Creating byte[] for {@link DFSOutputStream}. */ 94 private final ByteArrayManager byteArrayManager; 95 96 /** 97 * Whether or not we complained about a DFSClient fetching a CacheContext that 98 * didn't match its config values yet. 99 */ 100 private boolean printedConfWarning = false; 101 ClientContext(String name, Conf conf)102 private ClientContext(String name, Conf conf) { 103 this.name = name; 104 this.confString = confAsString(conf); 105 this.shortCircuitCache = new ShortCircuitCache( 106 conf.shortCircuitStreamsCacheSize, 107 conf.shortCircuitStreamsCacheExpiryMs, 108 conf.shortCircuitMmapCacheSize, 109 conf.shortCircuitMmapCacheExpiryMs, 110 conf.shortCircuitMmapCacheRetryTimeout, 111 conf.shortCircuitCacheStaleThresholdMs, 112 conf.shortCircuitSharedMemoryWatcherInterruptCheckMs); 113 this.peerCache = 114 new PeerCache(conf.socketCacheCapacity, conf.socketCacheExpiry); 115 this.keyProviderCache = new KeyProviderCache(conf.keyProviderCacheExpiryMs); 116 this.useLegacyBlockReaderLocal = conf.useLegacyBlockReaderLocal; 117 this.domainSocketFactory = new DomainSocketFactory(conf); 118 119 this.byteArrayManager = ByteArrayManager.newInstance(conf.writeByteArrayManagerConf); 120 } 121 confAsString(Conf conf)122 public static String confAsString(Conf conf) { 123 StringBuilder builder = new StringBuilder(); 124 builder.append("shortCircuitStreamsCacheSize = "). 125 append(conf.shortCircuitStreamsCacheSize). 126 append(", shortCircuitStreamsCacheExpiryMs = "). 127 append(conf.shortCircuitStreamsCacheExpiryMs). 128 append(", shortCircuitMmapCacheSize = "). 129 append(conf.shortCircuitMmapCacheSize). 130 append(", shortCircuitMmapCacheExpiryMs = "). 131 append(conf.shortCircuitMmapCacheExpiryMs). 132 append(", shortCircuitMmapCacheRetryTimeout = "). 133 append(conf.shortCircuitMmapCacheRetryTimeout). 134 append(", shortCircuitCacheStaleThresholdMs = "). 135 append(conf.shortCircuitCacheStaleThresholdMs). 136 append(", socketCacheCapacity = "). 137 append(conf.socketCacheCapacity). 138 append(", socketCacheExpiry = "). 139 append(conf.socketCacheExpiry). 140 append(", shortCircuitLocalReads = "). 141 append(conf.shortCircuitLocalReads). 142 append(", useLegacyBlockReaderLocal = "). 143 append(conf.useLegacyBlockReaderLocal). 144 append(", domainSocketDataTraffic = "). 145 append(conf.domainSocketDataTraffic). 146 append(", shortCircuitSharedMemoryWatcherInterruptCheckMs = "). 147 append(conf.shortCircuitSharedMemoryWatcherInterruptCheckMs). 148 append(", keyProviderCacheExpiryMs = "). 149 append(conf.keyProviderCacheExpiryMs); 150 151 return builder.toString(); 152 } 153 get(String name, Conf conf)154 public static ClientContext get(String name, Conf conf) { 155 ClientContext context; 156 synchronized(ClientContext.class) { 157 context = CACHES.get(name); 158 if (context == null) { 159 context = new ClientContext(name, conf); 160 CACHES.put(name, context); 161 } else { 162 context.printConfWarningIfNeeded(conf); 163 } 164 } 165 return context; 166 } 167 168 /** 169 * Get a client context, from a Configuration object. 170 * 171 * This method is less efficient than the version which takes a DFSClient#Conf 172 * object, and should be mostly used by tests. 173 */ 174 @VisibleForTesting getFromConf(Configuration conf)175 public static ClientContext getFromConf(Configuration conf) { 176 return get(conf.get(DFSConfigKeys.DFS_CLIENT_CONTEXT, 177 DFSConfigKeys.DFS_CLIENT_CONTEXT_DEFAULT), 178 new DFSClient.Conf(conf)); 179 } 180 printConfWarningIfNeeded(Conf conf)181 private void printConfWarningIfNeeded(Conf conf) { 182 String existing = this.getConfString(); 183 String requested = confAsString(conf); 184 if (!existing.equals(requested)) { 185 if (!printedConfWarning) { 186 printedConfWarning = true; 187 LOG.warn("Existing client context '" + name + "' does not match " + 188 "requested configuration. Existing: " + existing + 189 ", Requested: " + requested); 190 } 191 } 192 } 193 getConfString()194 public String getConfString() { 195 return confString; 196 } 197 getShortCircuitCache()198 public ShortCircuitCache getShortCircuitCache() { 199 return shortCircuitCache; 200 } 201 getPeerCache()202 public PeerCache getPeerCache() { 203 return peerCache; 204 } 205 getKeyProviderCache()206 public KeyProviderCache getKeyProviderCache() { 207 return keyProviderCache; 208 } 209 getUseLegacyBlockReaderLocal()210 public boolean getUseLegacyBlockReaderLocal() { 211 return useLegacyBlockReaderLocal; 212 } 213 getDisableLegacyBlockReaderLocal()214 public boolean getDisableLegacyBlockReaderLocal() { 215 return disableLegacyBlockReaderLocal; 216 } 217 setDisableLegacyBlockReaderLocal()218 public void setDisableLegacyBlockReaderLocal() { 219 disableLegacyBlockReaderLocal = true; 220 } 221 getDomainSocketFactory()222 public DomainSocketFactory getDomainSocketFactory() { 223 return domainSocketFactory; 224 } 225 getByteArrayManager()226 public ByteArrayManager getByteArrayManager() { 227 return byteArrayManager; 228 } 229 } 230