1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hdfs.server.blockmanagement; 19 20 import com.google.common.annotations.VisibleForTesting; 21 import com.google.common.base.Function; 22 import com.google.common.base.Joiner; 23 import com.google.common.base.Preconditions; 24 import com.google.common.collect.HashMultimap; 25 import com.google.common.collect.Iterators; 26 import com.google.common.collect.Multimap; 27 import com.google.common.collect.UnmodifiableIterator; 28 import org.apache.commons.logging.Log; 29 import org.apache.commons.logging.LogFactory; 30 import org.apache.hadoop.hdfs.protocol.DatanodeID; 31 import org.apache.hadoop.util.HostsFileReader; 32 33 import javax.annotation.Nullable; 34 import java.io.IOException; 35 import java.net.InetAddress; 36 import java.net.InetSocketAddress; 37 import java.net.URI; 38 import java.net.URISyntaxException; 39 import java.util.Collection; 40 import java.util.HashSet; 41 import java.util.Iterator; 42 import java.util.Map; 43 44 /** 45 * This class manages the include and exclude files for HDFS. 46 * <p/> 47 * These files control which DataNodes the NameNode expects to see in the 48 * cluster. Loosely speaking, the include file, if it exists and is not 49 * empty, is a list of everything we expect to see. The exclude file is 50 * a list of everything we want to ignore if we do see it. 51 * <p/> 52 * Entries may or may not specify a port. If they don't, we consider 53 * them to apply to every DataNode on that host. The code canonicalizes the 54 * entries into IP addresses. 55 * <p/> 56 * <p/> 57 * The code ignores all entries that the DNS fails to resolve their IP 58 * addresses. This is okay because by default the NN rejects the registrations 59 * of DNs when it fails to do a forward and reverse lookup. Note that DNS 60 * resolutions are only done during the loading time to minimize the latency. 61 */ 62 class HostFileManager { 63 private static final Log LOG = LogFactory.getLog(HostFileManager.class); 64 private HostSet includes = new HostSet(); 65 private HostSet excludes = new HostSet(); 66 readFile(String type, String filename)67 private static HostSet readFile(String type, String filename) 68 throws IOException { 69 HostSet res = new HostSet(); 70 if (!filename.isEmpty()) { 71 HashSet<String> entrySet = new HashSet<String>(); 72 HostsFileReader.readFileToSet(type, filename, entrySet); 73 for (String str : entrySet) { 74 InetSocketAddress addr = parseEntry(type, filename, str); 75 if (addr != null) { 76 res.add(addr); 77 } 78 } 79 } 80 return res; 81 } 82 83 @VisibleForTesting parseEntry(String type, String fn, String line)84 static InetSocketAddress parseEntry(String type, String fn, String line) { 85 try { 86 URI uri = new URI("dummy", line, null, null, null); 87 int port = uri.getPort() == -1 ? 0 : uri.getPort(); 88 InetSocketAddress addr = new InetSocketAddress(uri.getHost(), port); 89 if (addr.isUnresolved()) { 90 LOG.warn(String.format("Failed to resolve address `%s` in `%s`. " + 91 "Ignoring in the %s list.", line, fn, type)); 92 return null; 93 } 94 return addr; 95 } catch (URISyntaxException e) { 96 LOG.warn(String.format("Failed to parse `%s` in `%s`. " + "Ignoring in " + 97 "the %s list.", line, fn, type)); 98 } 99 return null; 100 } 101 resolvedAddressFromDatanodeID(DatanodeID id)102 static InetSocketAddress resolvedAddressFromDatanodeID(DatanodeID id) { 103 return new InetSocketAddress(id.getIpAddr(), id.getXferPort()); 104 } 105 getIncludes()106 synchronized HostSet getIncludes() { 107 return includes; 108 } 109 getExcludes()110 synchronized HostSet getExcludes() { 111 return excludes; 112 } 113 114 // If the includes list is empty, act as if everything is in the 115 // includes list. isIncluded(DatanodeID dn)116 synchronized boolean isIncluded(DatanodeID dn) { 117 return includes.isEmpty() || includes.match 118 (resolvedAddressFromDatanodeID(dn)); 119 } 120 isExcluded(DatanodeID dn)121 synchronized boolean isExcluded(DatanodeID dn) { 122 return excludes.match(resolvedAddressFromDatanodeID(dn)); 123 } 124 hasIncludes()125 synchronized boolean hasIncludes() { 126 return !includes.isEmpty(); 127 } 128 129 /** 130 * Read the includes and excludes lists from the named files. Any previous 131 * includes and excludes lists are discarded. 132 * @param includeFile the path to the new includes list 133 * @param excludeFile the path to the new excludes list 134 * @throws IOException thrown if there is a problem reading one of the files 135 */ refresh(String includeFile, String excludeFile)136 void refresh(String includeFile, String excludeFile) throws IOException { 137 HostSet newIncludes = readFile("included", includeFile); 138 HostSet newExcludes = readFile("excluded", excludeFile); 139 140 refresh(newIncludes, newExcludes); 141 } 142 143 /** 144 * Set the includes and excludes lists by the new HostSet instances. The 145 * old instances are discarded. 146 * @param newIncludes the new includes list 147 * @param newExcludes the new excludes list 148 */ 149 @VisibleForTesting refresh(HostSet newIncludes, HostSet newExcludes)150 void refresh(HostSet newIncludes, HostSet newExcludes) { 151 synchronized (this) { 152 includes = newIncludes; 153 excludes = newExcludes; 154 } 155 } 156 157 /** 158 * The HostSet allows efficient queries on matching wildcard addresses. 159 * <p/> 160 * For InetSocketAddress A and B with the same host address, 161 * we define a partial order between A and B, A <= B iff A.getPort() == B 162 * .getPort() || B.getPort() == 0. 163 */ 164 static class HostSet implements Iterable<InetSocketAddress> { 165 // Host -> lists of ports 166 private final Multimap<InetAddress, Integer> addrs = HashMultimap.create(); 167 168 /** 169 * The function that checks whether there exists an entry foo in the set 170 * so that foo <= addr. 171 */ matchedBy(InetSocketAddress addr)172 boolean matchedBy(InetSocketAddress addr) { 173 Collection<Integer> ports = addrs.get(addr.getAddress()); 174 return addr.getPort() == 0 ? !ports.isEmpty() : ports.contains(addr 175 .getPort()); 176 } 177 178 /** 179 * The function that checks whether there exists an entry foo in the set 180 * so that addr <= foo. 181 */ match(InetSocketAddress addr)182 boolean match(InetSocketAddress addr) { 183 int port = addr.getPort(); 184 Collection<Integer> ports = addrs.get(addr.getAddress()); 185 boolean exactMatch = ports.contains(port); 186 boolean genericMatch = ports.contains(0); 187 return exactMatch || genericMatch; 188 } 189 isEmpty()190 boolean isEmpty() { 191 return addrs.isEmpty(); 192 } 193 size()194 int size() { 195 return addrs.size(); 196 } 197 add(InetSocketAddress addr)198 void add(InetSocketAddress addr) { 199 Preconditions.checkArgument(!addr.isUnresolved()); 200 addrs.put(addr.getAddress(), addr.getPort()); 201 } 202 203 @Override iterator()204 public Iterator<InetSocketAddress> iterator() { 205 return new UnmodifiableIterator<InetSocketAddress>() { 206 private final Iterator<Map.Entry<InetAddress, 207 Integer>> it = addrs.entries().iterator(); 208 209 @Override 210 public boolean hasNext() { 211 return it.hasNext(); 212 } 213 214 @Override 215 public InetSocketAddress next() { 216 Map.Entry<InetAddress, Integer> e = it.next(); 217 return new InetSocketAddress(e.getKey(), e.getValue()); 218 } 219 }; 220 } 221 222 @Override toString()223 public String toString() { 224 StringBuilder sb = new StringBuilder("HostSet("); 225 Joiner.on(",").appendTo(sb, Iterators.transform(iterator(), 226 new Function<InetSocketAddress, String>() { 227 @Override 228 public String apply(@Nullable InetSocketAddress addr) { 229 assert addr != null; 230 return addr.getAddress().getHostAddress() + ":" + addr.getPort(); 231 } 232 })); 233 return sb.append(")").toString(); 234 } 235 } 236 } 237