1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.security; 19 20 import java.io.IOException; 21 import java.util.Collection; 22 import java.util.Collections; 23 import java.util.HashMap; 24 import java.util.List; 25 import java.util.Map; 26 import java.util.Set; 27 import java.util.concurrent.ExecutionException; 28 import java.util.concurrent.TimeUnit; 29 30 import com.google.common.annotations.VisibleForTesting; 31 import com.google.common.base.Ticker; 32 import com.google.common.cache.CacheBuilder; 33 import com.google.common.cache.Cache; 34 import com.google.common.cache.CacheLoader; 35 import com.google.common.cache.LoadingCache; 36 import org.apache.hadoop.HadoopIllegalArgumentException; 37 import org.apache.hadoop.classification.InterfaceAudience; 38 import org.apache.hadoop.classification.InterfaceAudience.Private; 39 import org.apache.hadoop.classification.InterfaceStability; 40 import org.apache.hadoop.conf.Configuration; 41 import org.apache.hadoop.fs.CommonConfigurationKeys; 42 import org.apache.hadoop.util.ReflectionUtils; 43 import org.apache.hadoop.util.StringUtils; 44 import org.apache.hadoop.util.Timer; 45 46 import org.apache.commons.logging.Log; 47 import org.apache.commons.logging.LogFactory; 48 49 /** 50 * A user-to-groups mapping service. 51 * 52 * {@link Groups} allows for server to get the various group memberships 53 * of a given user via the {@link #getGroups(String)} call, thus ensuring 54 * a consistent user-to-groups mapping and protects against vagaries of 55 * different mappings on servers and clients in a Hadoop cluster. 56 */ 57 @InterfaceAudience.LimitedPrivate({"HDFS", "MapReduce"}) 58 @InterfaceStability.Evolving 59 public class Groups { 60 private static final Log LOG = LogFactory.getLog(Groups.class); 61 62 private final GroupMappingServiceProvider impl; 63 64 private final LoadingCache<String, List<String>> cache; 65 private final Map<String, List<String>> staticUserToGroupsMap = 66 new HashMap<String, List<String>>(); 67 private final long cacheTimeout; 68 private final long negativeCacheTimeout; 69 private final long warningDeltaMs; 70 private final Timer timer; 71 private Set<String> negativeCache; 72 Groups(Configuration conf)73 public Groups(Configuration conf) { 74 this(conf, new Timer()); 75 } 76 Groups(Configuration conf, final Timer timer)77 public Groups(Configuration conf, final Timer timer) { 78 impl = 79 ReflectionUtils.newInstance( 80 conf.getClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING, 81 ShellBasedUnixGroupsMapping.class, 82 GroupMappingServiceProvider.class), 83 conf); 84 85 cacheTimeout = 86 conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS, 87 CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_SECS_DEFAULT) * 1000; 88 negativeCacheTimeout = 89 conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS, 90 CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_NEGATIVE_CACHE_SECS_DEFAULT) * 1000; 91 warningDeltaMs = 92 conf.getLong(CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS, 93 CommonConfigurationKeys.HADOOP_SECURITY_GROUPS_CACHE_WARN_AFTER_MS_DEFAULT); 94 parseStaticMapping(conf); 95 96 this.timer = timer; 97 this.cache = CacheBuilder.newBuilder() 98 .refreshAfterWrite(cacheTimeout, TimeUnit.MILLISECONDS) 99 .ticker(new TimerToTickerAdapter(timer)) 100 .expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS) 101 .build(new GroupCacheLoader()); 102 103 if(negativeCacheTimeout > 0) { 104 Cache<String, Boolean> tempMap = CacheBuilder.newBuilder() 105 .expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS) 106 .ticker(new TimerToTickerAdapter(timer)) 107 .build(); 108 negativeCache = Collections.newSetFromMap(tempMap.asMap()); 109 } 110 111 if(LOG.isDebugEnabled()) 112 LOG.debug("Group mapping impl=" + impl.getClass().getName() + 113 "; cacheTimeout=" + cacheTimeout + "; warningDeltaMs=" + 114 warningDeltaMs); 115 } 116 117 @VisibleForTesting getNegativeCache()118 Set<String> getNegativeCache() { 119 return negativeCache; 120 } 121 122 /* 123 * Parse the hadoop.user.group.static.mapping.overrides configuration to 124 * staticUserToGroupsMap 125 */ parseStaticMapping(Configuration conf)126 private void parseStaticMapping(Configuration conf) { 127 String staticMapping = conf.get( 128 CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES, 129 CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES_DEFAULT); 130 Collection<String> mappings = StringUtils.getStringCollection( 131 staticMapping, ";"); 132 for (String users : mappings) { 133 Collection<String> userToGroups = StringUtils.getStringCollection(users, 134 "="); 135 if (userToGroups.size() < 1 || userToGroups.size() > 2) { 136 throw new HadoopIllegalArgumentException("Configuration " 137 + CommonConfigurationKeys.HADOOP_USER_GROUP_STATIC_OVERRIDES 138 + " is invalid"); 139 } 140 String[] userToGroupsArray = userToGroups.toArray(new String[userToGroups 141 .size()]); 142 String user = userToGroupsArray[0]; 143 List<String> groups = Collections.emptyList(); 144 if (userToGroupsArray.length == 2) { 145 groups = (List<String>) StringUtils 146 .getStringCollection(userToGroupsArray[1]); 147 } 148 staticUserToGroupsMap.put(user, groups); 149 } 150 } 151 isNegativeCacheEnabled()152 private boolean isNegativeCacheEnabled() { 153 return negativeCacheTimeout > 0; 154 } 155 noGroupsForUser(String user)156 private IOException noGroupsForUser(String user) { 157 return new IOException("No groups found for user " + user); 158 } 159 160 /** 161 * Get the group memberships of a given user. 162 * If the user's group is not cached, this method may block. 163 * @param user User's name 164 * @return the group memberships of the user 165 * @throws IOException if user does not exist 166 */ getGroups(final String user)167 public List<String> getGroups(final String user) throws IOException { 168 // No need to lookup for groups of static users 169 List<String> staticMapping = staticUserToGroupsMap.get(user); 170 if (staticMapping != null) { 171 return staticMapping; 172 } 173 174 // Check the negative cache first 175 if (isNegativeCacheEnabled()) { 176 if (negativeCache.contains(user)) { 177 throw noGroupsForUser(user); 178 } 179 } 180 181 try { 182 return cache.get(user); 183 } catch (ExecutionException e) { 184 throw (IOException)e.getCause(); 185 } 186 } 187 188 /** 189 * Convert millisecond times from hadoop's timer to guava's nanosecond ticker. 190 */ 191 private static class TimerToTickerAdapter extends Ticker { 192 private Timer timer; 193 TimerToTickerAdapter(Timer timer)194 public TimerToTickerAdapter(Timer timer) { 195 this.timer = timer; 196 } 197 198 @Override read()199 public long read() { 200 final long NANOSECONDS_PER_MS = 1000000; 201 return timer.monotonicNow() * NANOSECONDS_PER_MS; 202 } 203 } 204 205 /** 206 * Deals with loading data into the cache. 207 */ 208 private class GroupCacheLoader extends CacheLoader<String, List<String>> { 209 /** 210 * This method will block if a cache entry doesn't exist, and 211 * any subsequent requests for the same user will wait on this 212 * request to return. If a user already exists in the cache, 213 * this will be run in the background. 214 * @param user key of cache 215 * @return List of groups belonging to user 216 * @throws IOException to prevent caching negative entries 217 */ 218 @Override load(String user)219 public List<String> load(String user) throws Exception { 220 List<String> groups = fetchGroupList(user); 221 222 if (groups.isEmpty()) { 223 if (isNegativeCacheEnabled()) { 224 negativeCache.add(user); 225 } 226 227 // We throw here to prevent Cache from retaining an empty group 228 throw noGroupsForUser(user); 229 } 230 231 return groups; 232 } 233 234 /** 235 * Queries impl for groups belonging to the user. This could involve I/O and take awhile. 236 */ fetchGroupList(String user)237 private List<String> fetchGroupList(String user) throws IOException { 238 long startMs = timer.monotonicNow(); 239 List<String> groupList = impl.getGroups(user); 240 long endMs = timer.monotonicNow(); 241 long deltaMs = endMs - startMs ; 242 UserGroupInformation.metrics.addGetGroups(deltaMs); 243 if (deltaMs > warningDeltaMs) { 244 LOG.warn("Potential performance problem: getGroups(user=" + user +") " + 245 "took " + deltaMs + " milliseconds."); 246 } 247 248 return groupList; 249 } 250 } 251 252 /** 253 * Refresh all user-to-groups mappings. 254 */ refresh()255 public void refresh() { 256 LOG.info("clearing userToGroupsMap cache"); 257 try { 258 impl.cacheGroupsRefresh(); 259 } catch (IOException e) { 260 LOG.warn("Error refreshing groups cache", e); 261 } 262 cache.invalidateAll(); 263 if(isNegativeCacheEnabled()) { 264 negativeCache.clear(); 265 } 266 } 267 268 /** 269 * Add groups to cache 270 * 271 * @param groups list of groups to add to cache 272 */ cacheGroupsAdd(List<String> groups)273 public void cacheGroupsAdd(List<String> groups) { 274 try { 275 impl.cacheGroupsAdd(groups); 276 } catch (IOException e) { 277 LOG.warn("Error caching groups", e); 278 } 279 } 280 281 private static Groups GROUPS = null; 282 283 /** 284 * Get the groups being used to map user-to-groups. 285 * @return the groups being used to map user-to-groups. 286 */ getUserToGroupsMappingService()287 public static Groups getUserToGroupsMappingService() { 288 return getUserToGroupsMappingService(new Configuration()); 289 } 290 291 /** 292 * Get the groups being used to map user-to-groups. 293 * @param conf 294 * @return the groups being used to map user-to-groups. 295 */ getUserToGroupsMappingService( Configuration conf)296 public static synchronized Groups getUserToGroupsMappingService( 297 Configuration conf) { 298 299 if(GROUPS == null) { 300 if(LOG.isDebugEnabled()) { 301 LOG.debug(" Creating new Groups object"); 302 } 303 GROUPS = new Groups(conf); 304 } 305 return GROUPS; 306 } 307 308 /** 309 * Create new groups used to map user-to-groups with loaded configuration. 310 * @param conf 311 * @return the groups being used to map user-to-groups. 312 */ 313 @Private 314 public static synchronized Groups getUserToGroupsMappingServiceWithLoadedConfiguration( Configuration conf)315 getUserToGroupsMappingServiceWithLoadedConfiguration( 316 Configuration conf) { 317 318 GROUPS = new Groups(conf); 319 return GROUPS; 320 } 321 } 322