1 /** 2 * 3 * Licensed to the Apache Software Foundation (ASF) under one 4 * or more contributor license agreements. See the NOTICE file 5 * distributed with this work for additional information 6 * regarding copyright ownership. The ASF licenses this file 7 * to you under the Apache License, Version 2.0 (the 8 * "License"); you may not use this file except in compliance 9 * with the License. You may obtain a copy of the License at 10 * 11 * http://www.apache.org/licenses/LICENSE-2.0 12 * 13 * Unless required by applicable law or agreed to in writing, software 14 * distributed under the License is distributed on an "AS IS" BASIS, 15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 * See the License for the specific language governing permissions and 17 * limitations under the License. 18 */ 19 20 package org.apache.hadoop.hbase.client; 21 22 import java.io.Closeable; 23 import java.io.IOException; 24 import java.util.ArrayList; 25 import java.util.List; 26 import java.util.NavigableMap; 27 import java.util.TreeMap; 28 29 import org.apache.commons.logging.Log; 30 import org.apache.commons.logging.LogFactory; 31 import org.apache.hadoop.hbase.classification.InterfaceAudience; 32 import org.apache.hadoop.conf.Configuration; 33 import org.apache.hadoop.hbase.HConstants; 34 import org.apache.hadoop.hbase.HRegionInfo; 35 import org.apache.hadoop.hbase.HRegionLocation; 36 import org.apache.hadoop.hbase.MetaTableAccessor; 37 import org.apache.hadoop.hbase.RegionLocations; 38 import org.apache.hadoop.hbase.ServerName; 39 import org.apache.hadoop.hbase.TableName; 40 import org.apache.hadoop.hbase.TableNotFoundException; 41 import org.apache.hadoop.hbase.util.Bytes; 42 import org.apache.hadoop.hbase.util.ExceptionUtil; 43 44 import com.google.common.annotations.VisibleForTesting; 45 46 /** 47 * Scanner class that contains the <code>hbase:meta</code> table scanning logic. 48 * Provided visitors will be called for each row. 49 * 50 * Although public visibility, this is not a public-facing API and may evolve in 51 * minor releases. 52 * 53 * <p> Note that during concurrent region splits, the scanner might not see 54 * hbase:meta changes across rows (for parent and daughter entries) consistently. 55 * see HBASE-5986, and {@link DefaultMetaScannerVisitor} for details. </p> 56 */ 57 @InterfaceAudience.Private 58 //TODO: merge this to MetaTableAccessor, get rid of it. 59 public class MetaScanner { 60 private static final Log LOG = LogFactory.getLog(MetaScanner.class); 61 /** 62 * Scans the meta table and calls a visitor on each RowResult and uses a empty 63 * start row value as table name. 64 * 65 * <p>Visible for testing. Use {@link 66 * #metaScan(Connection, MetaScannerVisitor, TableName)} instead. 67 * 68 * @param visitor A custom visitor 69 * @throws IOException e 70 */ 71 @VisibleForTesting // Do not use. Used by tests only and hbck. metaScan(Connection connection, MetaScannerVisitor visitor)72 public static void metaScan(Connection connection, 73 MetaScannerVisitor visitor) throws IOException { 74 metaScan(connection, visitor, null, null, Integer.MAX_VALUE); 75 } 76 77 /** 78 * Scans the meta table and calls a visitor on each RowResult. Uses a table 79 * name to locate meta regions. 80 * 81 * @param connection connection to use internally (null to use a new instance) 82 * @param visitor visitor object 83 * @param userTableName User table name in meta table to start scan at. Pass 84 * null if not interested in a particular table. 85 * @throws IOException e 86 */ metaScan(Connection connection, MetaScannerVisitor visitor, TableName userTableName)87 public static void metaScan(Connection connection, 88 MetaScannerVisitor visitor, TableName userTableName) throws IOException { 89 metaScan(connection, visitor, userTableName, null, Integer.MAX_VALUE, 90 TableName.META_TABLE_NAME); 91 } 92 93 /** 94 * Scans the meta table and calls a visitor on each RowResult. Uses a table 95 * name and a row name to locate meta regions. And it only scans at most 96 * <code>rowLimit</code> of rows. 97 * 98 * <p>Visible for testing. Use {@link 99 * #metaScan(Connection, MetaScannerVisitor, TableName)} instead. 100 * 101 * @param connection to scan on 102 * @param visitor Visitor object. 103 * @param userTableName User table name in meta table to start scan at. Pass 104 * null if not interested in a particular table. 105 * @param row Name of the row at the user table. The scan will start from 106 * the region row where the row resides. 107 * @param rowLimit Max of processed rows. If it is less than 0, it 108 * will be set to default value <code>Integer.MAX_VALUE</code>. 109 * @throws IOException e 110 */ 111 @VisibleForTesting // Do not use. Used by Master but by a method that is used testing. metaScan(Connection connection, MetaScannerVisitor visitor, TableName userTableName, byte[] row, int rowLimit)112 public static void metaScan(Connection connection, 113 MetaScannerVisitor visitor, TableName userTableName, byte[] row, 114 int rowLimit) 115 throws IOException { 116 metaScan(connection, visitor, userTableName, row, rowLimit, TableName 117 .META_TABLE_NAME); 118 } 119 120 /** 121 * Scans the meta table and calls a visitor on each RowResult. Uses a table 122 * name and a row name to locate meta regions. And it only scans at most 123 * <code>rowLimit</code> of rows. 124 * 125 * @param connection connection to use internally (null to use a new instance) 126 * @param visitor Visitor object. Closes the visitor before returning. 127 * @param tableName User table name in meta table to start scan at. Pass 128 * null if not interested in a particular table. 129 * @param row Name of the row at the user table. The scan will start from 130 * the region row where the row resides. 131 * @param rowLimit Max of processed rows. If it is less than 0, it 132 * will be set to default value <code>Integer.MAX_VALUE</code>. 133 * @param metaTableName Meta table to scan, root or meta. 134 * @throws IOException e 135 */ metaScan(Connection connection, final MetaScannerVisitor visitor, final TableName tableName, final byte[] row, final int rowLimit, final TableName metaTableName)136 static void metaScan(Connection connection, 137 final MetaScannerVisitor visitor, final TableName tableName, 138 final byte[] row, final int rowLimit, final TableName metaTableName) 139 throws IOException { 140 141 int rowUpperLimit = rowLimit > 0 ? rowLimit: Integer.MAX_VALUE; 142 // Calculate startrow for scan. 143 byte[] startRow; 144 // If the passed in 'connection' is 'managed' -- i.e. every second test uses 145 // an HTable or an HBaseAdmin with managed connections -- then doing 146 // connection.getTable will throw an exception saying you are NOT to use 147 // managed connections getting tables. Leaving this as it is for now. Will 148 // revisit when inclined to change all tests. User code probaby makes use of 149 // managed connections too so don't change it till post hbase 1.0. 150 try (Table metaTable = new HTable(TableName.META_TABLE_NAME, connection, null)) { 151 if (row != null) { 152 // Scan starting at a particular row in a particular table 153 Result startRowResult = getClosestRowOrBefore(metaTable, tableName, row, 154 connection.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS, 155 HConstants.DEFAULT_USE_META_REPLICAS)); 156 if (startRowResult == null) { 157 throw new TableNotFoundException("Cannot find row in " + metaTable.getName() + 158 " for table: " + tableName + ", row=" + Bytes.toStringBinary(row)); 159 } 160 HRegionInfo regionInfo = getHRegionInfo(startRowResult); 161 if (regionInfo == null) { 162 throw new IOException("HRegionInfo was null or empty in Meta for " + 163 tableName + ", row=" + Bytes.toStringBinary(row)); 164 } 165 byte[] rowBefore = regionInfo.getStartKey(); 166 startRow = HRegionInfo.createRegionName(tableName, rowBefore, HConstants.ZEROES, false); 167 } else if (tableName == null || tableName.getName().length == 0) { 168 // Full hbase:meta scan 169 startRow = HConstants.EMPTY_START_ROW; 170 } else { 171 // Scan hbase:meta for an entire table 172 startRow = HRegionInfo.createRegionName(tableName, HConstants.EMPTY_START_ROW, 173 HConstants.ZEROES, false); 174 } 175 final Scan scan = new Scan(startRow).addFamily(HConstants.CATALOG_FAMILY); 176 int scannerCaching = connection.getConfiguration() 177 .getInt(HConstants.HBASE_META_SCANNER_CACHING, 178 HConstants.DEFAULT_HBASE_META_SCANNER_CACHING); 179 if (connection.getConfiguration().getBoolean(HConstants.USE_META_REPLICAS, 180 HConstants.DEFAULT_USE_META_REPLICAS)) { 181 scan.setConsistency(Consistency.TIMELINE); 182 } 183 if (rowUpperLimit <= scannerCaching) { 184 scan.setSmall(true); 185 } 186 int rows = Math.min(rowLimit, scannerCaching); 187 scan.setCaching(rows); 188 if (LOG.isTraceEnabled()) { 189 LOG.trace("Scanning " + metaTableName.getNameAsString() + " starting at row=" + 190 Bytes.toStringBinary(startRow) + " for max=" + rowUpperLimit + " with caching=" + rows); 191 } 192 // Run the scan 193 try (ResultScanner resultScanner = metaTable.getScanner(scan)) { 194 Result result; 195 int processedRows = 0; 196 while ((result = resultScanner.next()) != null) { 197 if (visitor != null) { 198 if (!visitor.processRow(result)) break; 199 } 200 processedRows++; 201 if (processedRows >= rowUpperLimit) break; 202 } 203 } 204 } finally { 205 if (visitor != null) { 206 try { 207 visitor.close(); 208 } catch (Throwable t) { 209 ExceptionUtil.rethrowIfInterrupt(t); 210 LOG.debug("Got exception in closing the meta scanner visitor", t); 211 } 212 } 213 } 214 } 215 216 /** 217 * @return Get closest metatable region row to passed <code>row</code> 218 * @throws IOException 219 */ getClosestRowOrBefore(final Table metaTable, final TableName userTableName, final byte [] row, boolean useMetaReplicas)220 private static Result getClosestRowOrBefore(final Table metaTable, final TableName userTableName, 221 final byte [] row, boolean useMetaReplicas) 222 throws IOException { 223 byte[] searchRow = HRegionInfo.createRegionName(userTableName, row, HConstants.NINES, false); 224 Scan scan = Scan.createGetClosestRowOrBeforeReverseScan(searchRow); 225 if (useMetaReplicas) { 226 scan.setConsistency(Consistency.TIMELINE); 227 } 228 try (ResultScanner resultScanner = metaTable.getScanner(scan)) { 229 return resultScanner.next(); 230 } 231 } 232 233 /** 234 * Returns HRegionInfo object from the column 235 * HConstants.CATALOG_FAMILY:HConstants.REGIONINFO_QUALIFIER of the catalog 236 * table Result. 237 * @param data a Result object from the catalog table scan 238 * @return HRegionInfo or null 239 * @deprecated Use {@link org.apache.hadoop.hbase.MetaTableAccessor#getRegionLocations(Result)} 240 */ 241 @Deprecated getHRegionInfo(Result data)242 public static HRegionInfo getHRegionInfo(Result data) { 243 return HRegionInfo.getHRegionInfo(data); 244 } 245 246 /** 247 * Lists all of the regions currently in META. 248 * @param conf configuration 249 * @param connection to connect with 250 * @param offlined True if we are to include offlined regions, false and we'll 251 * leave out offlined regions from returned list. 252 * @return List of all user-space regions. 253 * @throws IOException 254 */ 255 @VisibleForTesting // And for hbck. listAllRegions(Configuration conf, Connection connection, final boolean offlined)256 public static List<HRegionInfo> listAllRegions(Configuration conf, Connection connection, 257 final boolean offlined) 258 throws IOException { 259 final List<HRegionInfo> regions = new ArrayList<HRegionInfo>(); 260 MetaScannerVisitor visitor = new MetaScannerVisitorBase() { 261 @Override 262 public boolean processRow(Result result) throws IOException { 263 if (result == null || result.isEmpty()) { 264 return true; 265 } 266 267 RegionLocations locations = MetaTableAccessor.getRegionLocations(result); 268 if (locations == null) return true; 269 for (HRegionLocation loc : locations.getRegionLocations()) { 270 if (loc != null) { 271 HRegionInfo regionInfo = loc.getRegionInfo(); 272 // If region offline AND we are not to include offlined regions, return. 273 if (regionInfo.isOffline() && !offlined) continue; 274 regions.add(regionInfo); 275 } 276 } 277 return true; 278 } 279 }; 280 metaScan(connection, visitor); 281 return regions; 282 } 283 284 /** 285 * Lists all of the table regions currently in META. 286 * @param conf 287 * @param offlined True if we are to include offlined regions, false and we'll 288 * leave out offlined regions from returned list. 289 * @return Map of all user-space regions to servers 290 * @throws IOException 291 * @deprecated Use {@link #allTableRegions(Connection, TableName)} instead 292 */ 293 @Deprecated allTableRegions(Configuration conf, Connection connection, final TableName tableName, boolean offlined)294 public static NavigableMap<HRegionInfo, ServerName> allTableRegions(Configuration conf, 295 Connection connection, final TableName tableName, boolean offlined) throws IOException { 296 return allTableRegions(connection, tableName); 297 } 298 299 /** 300 * Lists all of the table regions currently in META. 301 * @param connection 302 * @param tableName 303 * @return Map of all user-space regions to servers 304 * @throws IOException 305 */ allTableRegions( Connection connection, final TableName tableName)306 public static NavigableMap<HRegionInfo, ServerName> allTableRegions( 307 Connection connection, final TableName tableName) throws IOException { 308 final NavigableMap<HRegionInfo, ServerName> regions = 309 new TreeMap<HRegionInfo, ServerName>(); 310 MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { 311 @Override 312 public boolean processRowInternal(Result result) throws IOException { 313 RegionLocations locations = MetaTableAccessor.getRegionLocations(result); 314 if (locations == null) return true; 315 for (HRegionLocation loc : locations.getRegionLocations()) { 316 if (loc != null) { 317 HRegionInfo regionInfo = loc.getRegionInfo(); 318 regions.put(new UnmodifyableHRegionInfo(regionInfo), loc.getServerName()); 319 } 320 } 321 return true; 322 } 323 }; 324 metaScan(connection, visitor, tableName); 325 return regions; 326 } 327 328 /** 329 * Lists table regions and locations grouped by region range from META. 330 */ listTableRegionLocations(Configuration conf, Connection connection, final TableName tableName)331 public static List<RegionLocations> listTableRegionLocations(Configuration conf, 332 Connection connection, final TableName tableName) throws IOException { 333 final List<RegionLocations> regions = new ArrayList<RegionLocations>(); 334 MetaScannerVisitor visitor = new TableMetaScannerVisitor(tableName) { 335 @Override 336 public boolean processRowInternal(Result result) throws IOException { 337 RegionLocations locations = MetaTableAccessor.getRegionLocations(result); 338 if (locations == null) return true; 339 regions.add(locations); 340 return true; 341 } 342 }; 343 metaScan(connection, visitor, tableName); 344 return regions; 345 } 346 347 /** 348 * Visitor class called to process each row of the hbase:meta table 349 */ 350 public interface MetaScannerVisitor extends Closeable { 351 /** 352 * Visitor method that accepts a RowResult and the meta region location. 353 * Implementations can return false to stop the region's loop if it becomes 354 * unnecessary for some reason. 355 * 356 * @param rowResult result 357 * @return A boolean to know if it should continue to loop in the region 358 * @throws IOException e 359 */ processRow(Result rowResult)360 boolean processRow(Result rowResult) throws IOException; 361 } 362 363 public static abstract class MetaScannerVisitorBase implements MetaScannerVisitor { 364 @Override close()365 public void close() throws IOException { 366 } 367 } 368 369 /** 370 * A MetaScannerVisitor that skips offline regions and split parents 371 */ 372 public static abstract class DefaultMetaScannerVisitor 373 extends MetaScannerVisitorBase { 374 DefaultMetaScannerVisitor()375 public DefaultMetaScannerVisitor() { 376 super(); 377 } 378 processRowInternal(Result rowResult)379 public abstract boolean processRowInternal(Result rowResult) throws IOException; 380 381 @Override processRow(Result rowResult)382 public boolean processRow(Result rowResult) throws IOException { 383 HRegionInfo info = getHRegionInfo(rowResult); 384 if (info == null) { 385 return true; 386 } 387 388 //skip over offline and split regions 389 if (!(info.isOffline() || info.isSplit())) { 390 return processRowInternal(rowResult); 391 } 392 return true; 393 } 394 } 395 396 /** 397 * A MetaScannerVisitor for a table. Provides a consistent view of the table's 398 * hbase:meta entries during concurrent splits (see HBASE-5986 for details). This class 399 * does not guarantee ordered traversal of meta entries, and can block until the 400 * hbase:meta entries for daughters are available during splits. 401 */ 402 public static abstract class TableMetaScannerVisitor extends DefaultMetaScannerVisitor { 403 private TableName tableName; 404 TableMetaScannerVisitor(TableName tableName)405 public TableMetaScannerVisitor(TableName tableName) { 406 super(); 407 this.tableName = tableName; 408 } 409 410 @Override processRow(Result rowResult)411 public final boolean processRow(Result rowResult) throws IOException { 412 HRegionInfo info = getHRegionInfo(rowResult); 413 if (info == null) { 414 return true; 415 } 416 if (!(info.getTable().equals(tableName))) { 417 return false; 418 } 419 return super.processRow(rowResult); 420 } 421 } 422 } 423