1 /* 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 package org.apache.hadoop.hbase.client; 20 21 import java.nio.ByteBuffer; 22 import java.util.ArrayList; 23 import java.util.Arrays; 24 import java.util.HashMap; 25 import java.util.List; 26 import java.util.Map; 27 import java.util.NavigableMap; 28 import java.util.TreeMap; 29 import java.util.UUID; 30 31 import org.apache.hadoop.hbase.classification.InterfaceAudience; 32 import org.apache.hadoop.hbase.classification.InterfaceStability; 33 import org.apache.hadoop.hbase.Cell; 34 import org.apache.hadoop.hbase.CellScannable; 35 import org.apache.hadoop.hbase.CellScanner; 36 import org.apache.hadoop.hbase.CellUtil; 37 import org.apache.hadoop.hbase.HConstants; 38 import org.apache.hadoop.hbase.KeyValue; 39 import org.apache.hadoop.hbase.KeyValueUtil; 40 import org.apache.hadoop.hbase.Tag; 41 import org.apache.hadoop.hbase.exceptions.DeserializationException; 42 import org.apache.hadoop.hbase.io.HeapSize; 43 import org.apache.hadoop.hbase.protobuf.ProtobufUtil; 44 import org.apache.hadoop.hbase.security.access.AccessControlConstants; 45 import org.apache.hadoop.hbase.security.access.Permission; 46 import org.apache.hadoop.hbase.security.visibility.CellVisibility; 47 import org.apache.hadoop.hbase.security.visibility.VisibilityConstants; 48 import org.apache.hadoop.hbase.util.Bytes; 49 import org.apache.hadoop.hbase.util.ClassSize; 50 51 import com.google.common.collect.ArrayListMultimap; 52 import com.google.common.collect.ListMultimap; 53 import com.google.common.collect.Lists; 54 import com.google.common.io.ByteArrayDataInput; 55 import com.google.common.io.ByteArrayDataOutput; 56 import com.google.common.io.ByteStreams; 57 58 @InterfaceAudience.Public 59 @InterfaceStability.Evolving 60 public abstract class Mutation extends OperationWithAttributes implements Row, CellScannable, 61 HeapSize { 62 public static final long MUTATION_OVERHEAD = ClassSize.align( 63 // This 64 ClassSize.OBJECT + 65 // row + OperationWithAttributes.attributes 66 2 * ClassSize.REFERENCE + 67 // Timestamp 68 1 * Bytes.SIZEOF_LONG + 69 // durability 70 ClassSize.REFERENCE + 71 // familyMap 72 ClassSize.REFERENCE + 73 // familyMap 74 ClassSize.TREEMAP); 75 76 /** 77 * The attribute for storing the list of clusters that have consumed the change. 78 */ 79 private static final String CONSUMED_CLUSTER_IDS = "_cs.id"; 80 81 /** 82 * The attribute for storing TTL for the result of the mutation. 83 */ 84 private static final String OP_ATTRIBUTE_TTL = "_ttl"; 85 86 /** 87 * @deprecated this field is private as of HBase 2.0. 88 */ 89 @Deprecated 90 protected static final String RETURN_RESULTS = "_rr_"; 91 92 protected byte [] row = null; 93 protected long ts = HConstants.LATEST_TIMESTAMP; 94 protected Durability durability = Durability.USE_DEFAULT; 95 96 // A Map sorted by column family. 97 protected NavigableMap<byte [], List<Cell>> familyMap = 98 new TreeMap<byte [], List<Cell>>(Bytes.BYTES_COMPARATOR); 99 100 @Override cellScanner()101 public CellScanner cellScanner() { 102 return CellUtil.createCellScanner(getFamilyCellMap()); 103 } 104 105 /** 106 * Creates an empty list if one doesn't exist for the given column family 107 * or else it returns the associated list of Cell objects. 108 * 109 * @param family column family 110 * @return a list of Cell objects, returns an empty list if one doesn't exist. 111 */ getCellList(byte[] family)112 List<Cell> getCellList(byte[] family) { 113 List<Cell> list = this.familyMap.get(family); 114 if (list == null) { 115 list = new ArrayList<Cell>(); 116 } 117 return list; 118 } 119 120 /* 121 * Create a KeyValue with this objects row key and the Put identifier. 122 * 123 * @return a KeyValue with this objects row key and the Put identifier. 124 */ createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value)125 KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value) { 126 return new KeyValue(this.row, family, qualifier, ts, KeyValue.Type.Put, value); 127 } 128 129 /** 130 * Create a KeyValue with this objects row key and the Put identifier. 131 * @param family 132 * @param qualifier 133 * @param ts 134 * @param value 135 * @param tags - Specify the Tags as an Array {@link KeyValue.Tag} 136 * @return a KeyValue with this objects row key and the Put identifier. 137 */ createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags)138 KeyValue createPutKeyValue(byte[] family, byte[] qualifier, long ts, byte[] value, Tag[] tags) { 139 KeyValue kvWithTag = new KeyValue(this.row, family, qualifier, ts, value, tags); 140 return kvWithTag; 141 } 142 143 /* 144 * Create a KeyValue with this objects row key and the Put identifier. 145 * 146 * @return a KeyValue with this objects row key and the Put identifier. 147 */ createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value, Tag[] tags)148 KeyValue createPutKeyValue(byte[] family, ByteBuffer qualifier, long ts, ByteBuffer value, 149 Tag[] tags) { 150 return new KeyValue(this.row, 0, this.row == null ? 0 : this.row.length, 151 family, 0, family == null ? 0 : family.length, 152 qualifier, ts, KeyValue.Type.Put, value, tags != null ? Arrays.asList(tags) : null); 153 } 154 155 /** 156 * Compile the column family (i.e. schema) information 157 * into a Map. Useful for parsing and aggregation by debugging, 158 * logging, and administration tools. 159 * @return Map 160 */ 161 @Override getFingerprint()162 public Map<String, Object> getFingerprint() { 163 Map<String, Object> map = new HashMap<String, Object>(); 164 List<String> families = new ArrayList<String>(); 165 // ideally, we would also include table information, but that information 166 // is not stored in each Operation instance. 167 map.put("families", families); 168 for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) { 169 families.add(Bytes.toStringBinary(entry.getKey())); 170 } 171 return map; 172 } 173 174 /** 175 * Compile the details beyond the scope of getFingerprint (row, columns, 176 * timestamps, etc.) into a Map along with the fingerprinted information. 177 * Useful for debugging, logging, and administration tools. 178 * @param maxCols a limit on the number of columns output prior to truncation 179 * @return Map 180 */ 181 @Override toMap(int maxCols)182 public Map<String, Object> toMap(int maxCols) { 183 // we start with the fingerprint map and build on top of it. 184 Map<String, Object> map = getFingerprint(); 185 // replace the fingerprint's simple list of families with a 186 // map from column families to lists of qualifiers and kv details 187 Map<String, List<Map<String, Object>>> columns = 188 new HashMap<String, List<Map<String, Object>>>(); 189 map.put("families", columns); 190 map.put("row", Bytes.toStringBinary(this.row)); 191 int colCount = 0; 192 // iterate through all column families affected 193 for (Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) { 194 // map from this family to details for each cell affected within the family 195 List<Map<String, Object>> qualifierDetails = new ArrayList<Map<String, Object>>(); 196 columns.put(Bytes.toStringBinary(entry.getKey()), qualifierDetails); 197 colCount += entry.getValue().size(); 198 if (maxCols <= 0) { 199 continue; 200 } 201 // add details for each cell 202 for (Cell cell: entry.getValue()) { 203 if (--maxCols <= 0 ) { 204 continue; 205 } 206 Map<String, Object> cellMap = cellToStringMap(cell); 207 qualifierDetails.add(cellMap); 208 } 209 } 210 map.put("totalColumns", colCount); 211 // add the id if set 212 if (getId() != null) { 213 map.put("id", getId()); 214 } 215 // Add the TTL if set 216 // Long.MAX_VALUE is the default, and is interpreted to mean this attribute 217 // has not been set. 218 if (getTTL() != Long.MAX_VALUE) { 219 map.put("ttl", getTTL()); 220 } 221 return map; 222 } 223 cellToStringMap(Cell c)224 private static Map<String, Object> cellToStringMap(Cell c) { 225 Map<String, Object> stringMap = new HashMap<String, Object>(); 226 stringMap.put("qualifier", Bytes.toStringBinary(c.getQualifierArray(), c.getQualifierOffset(), 227 c.getQualifierLength())); 228 stringMap.put("timestamp", c.getTimestamp()); 229 stringMap.put("vlen", c.getValueLength()); 230 List<Tag> tags = Tag.asList(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength()); 231 if (tags != null) { 232 List<String> tagsString = new ArrayList<String>(); 233 for (Tag t : tags) { 234 tagsString.add((t.getType()) + ":" + Bytes.toStringBinary(t.getValue())); 235 } 236 stringMap.put("tag", tagsString); 237 } 238 return stringMap; 239 } 240 241 /** 242 * @deprecated Use {@link #getDurability()} instead. 243 * @return true if edits should be applied to WAL, false if not 244 */ 245 @Deprecated getWriteToWAL()246 public boolean getWriteToWAL() { 247 return this.durability != Durability.SKIP_WAL; 248 } 249 250 /** 251 * Set whether this Delete should be written to the WAL or not. 252 * Not writing the WAL means you may lose edits on server crash. 253 * This method will reset any changes made via {@link #setDurability(Durability)} 254 * @param write true if edits should be written to WAL, false if not 255 * @deprecated Use {@link #setDurability(Durability)} instead. 256 */ 257 @Deprecated setWriteToWAL(boolean write)258 public Mutation setWriteToWAL(boolean write) { 259 setDurability(write ? Durability.USE_DEFAULT : Durability.SKIP_WAL); 260 return this; 261 } 262 263 /** 264 * Set the durability for this mutation 265 * @param d 266 */ setDurability(Durability d)267 public Mutation setDurability(Durability d) { 268 this.durability = d; 269 return this; 270 } 271 272 /** Get the current durability */ getDurability()273 public Durability getDurability() { 274 return this.durability; 275 } 276 277 /** 278 * Method for retrieving the put's familyMap 279 * @return familyMap 280 */ getFamilyCellMap()281 public NavigableMap<byte [], List<Cell>> getFamilyCellMap() { 282 return this.familyMap; 283 } 284 285 /** 286 * Method for setting the put's familyMap 287 */ setFamilyCellMap(NavigableMap<byte [], List<Cell>> map)288 public Mutation setFamilyCellMap(NavigableMap<byte [], List<Cell>> map) { 289 // TODO: Shut this down or move it up to be a Constructor. Get new object rather than change 290 // this internal data member. 291 this.familyMap = map; 292 return this; 293 } 294 295 /** 296 * Method for retrieving the put's familyMap that is deprecated and inefficient. 297 * @return the map 298 * @deprecated use {@link #getFamilyCellMap()} instead. 299 */ 300 @Deprecated getFamilyMap()301 public NavigableMap<byte [], List<KeyValue>> getFamilyMap() { 302 TreeMap<byte[], List<KeyValue>> fm = 303 new TreeMap<byte[], List<KeyValue>>(Bytes.BYTES_COMPARATOR); 304 for (Map.Entry<byte[], List<Cell>> e : familyMap.entrySet()) { 305 List<KeyValue> kvl = new ArrayList<KeyValue>(e.getValue().size()); 306 for (Cell c : e.getValue()) { 307 kvl.add(KeyValueUtil.ensureKeyValue(c)); 308 } 309 fm.put(e.getKey(), kvl); 310 } 311 return fm; 312 } 313 314 /** 315 * Method for setting the put's familyMap that is deprecated and inefficient. 316 * @deprecated use {@link #setFamilyCellMap(NavigableMap)} instead. 317 */ 318 @Deprecated setFamilyMap(NavigableMap<byte [], List<KeyValue>> map)319 public Mutation setFamilyMap(NavigableMap<byte [], List<KeyValue>> map) { 320 TreeMap<byte[], List<Cell>> fm = new TreeMap<byte[], List<Cell>>(Bytes.BYTES_COMPARATOR); 321 for (Map.Entry<byte[], List<KeyValue>> e : map.entrySet()) { 322 fm.put(e.getKey(), Lists.<Cell>newArrayList(e.getValue())); 323 } 324 this.familyMap = fm; 325 return this; 326 } 327 328 /** 329 * Method to check if the familyMap is empty 330 * @return true if empty, false otherwise 331 */ isEmpty()332 public boolean isEmpty() { 333 return familyMap.isEmpty(); 334 } 335 336 /** 337 * Method for retrieving the delete's row 338 * @return row 339 */ 340 @Override getRow()341 public byte [] getRow() { 342 return this.row; 343 } 344 345 @Override compareTo(final Row d)346 public int compareTo(final Row d) { 347 return Bytes.compareTo(this.getRow(), d.getRow()); 348 } 349 350 /** 351 * Method for retrieving the timestamp 352 * @return timestamp 353 */ getTimeStamp()354 public long getTimeStamp() { 355 return this.ts; 356 } 357 358 /** 359 * Marks that the clusters with the given clusterIds have consumed the mutation 360 * @param clusterIds of the clusters that have consumed the mutation 361 */ setClusterIds(List<UUID> clusterIds)362 public Mutation setClusterIds(List<UUID> clusterIds) { 363 ByteArrayDataOutput out = ByteStreams.newDataOutput(); 364 out.writeInt(clusterIds.size()); 365 for (UUID clusterId : clusterIds) { 366 out.writeLong(clusterId.getMostSignificantBits()); 367 out.writeLong(clusterId.getLeastSignificantBits()); 368 } 369 setAttribute(CONSUMED_CLUSTER_IDS, out.toByteArray()); 370 return this; 371 } 372 373 /** 374 * @return the set of clusterIds that have consumed the mutation 375 */ getClusterIds()376 public List<UUID> getClusterIds() { 377 List<UUID> clusterIds = new ArrayList<UUID>(); 378 byte[] bytes = getAttribute(CONSUMED_CLUSTER_IDS); 379 if(bytes != null) { 380 ByteArrayDataInput in = ByteStreams.newDataInput(bytes); 381 int numClusters = in.readInt(); 382 for(int i=0; i<numClusters; i++){ 383 clusterIds.add(new UUID(in.readLong(), in.readLong())); 384 } 385 } 386 return clusterIds; 387 } 388 389 /** 390 * Sets the visibility expression associated with cells in this Mutation. 391 * It is illegal to set <code>CellVisibility</code> on <code>Delete</code> mutation. 392 * @param expression 393 */ setCellVisibility(CellVisibility expression)394 public Mutation setCellVisibility(CellVisibility expression) { 395 this.setAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY, ProtobufUtil 396 .toCellVisibility(expression).toByteArray()); 397 return this; 398 } 399 400 /** 401 * @return CellVisibility associated with cells in this Mutation. 402 * @throws DeserializationException 403 */ getCellVisibility()404 public CellVisibility getCellVisibility() throws DeserializationException { 405 byte[] cellVisibilityBytes = this.getAttribute(VisibilityConstants.VISIBILITY_LABELS_ATTR_KEY); 406 if (cellVisibilityBytes == null) return null; 407 return ProtobufUtil.toCellVisibility(cellVisibilityBytes); 408 } 409 410 /** 411 * Number of KeyValues carried by this Mutation. 412 * @return the total number of KeyValues 413 */ size()414 public int size() { 415 int size = 0; 416 for (List<Cell> cells : this.familyMap.values()) { 417 size += cells.size(); 418 } 419 return size; 420 } 421 422 /** 423 * @return the number of different families 424 */ numFamilies()425 public int numFamilies() { 426 return familyMap.size(); 427 } 428 429 /** 430 * @return Calculate what Mutation adds to class heap size. 431 */ 432 @Override heapSize()433 public long heapSize() { 434 long heapsize = MUTATION_OVERHEAD; 435 // Adding row 436 heapsize += ClassSize.align(ClassSize.ARRAY + this.row.length); 437 438 // Adding map overhead 439 heapsize += 440 ClassSize.align(this.familyMap.size() * ClassSize.MAP_ENTRY); 441 for(Map.Entry<byte [], List<Cell>> entry : this.familyMap.entrySet()) { 442 //Adding key overhead 443 heapsize += 444 ClassSize.align(ClassSize.ARRAY + entry.getKey().length); 445 446 //This part is kinds tricky since the JVM can reuse references if you 447 //store the same value, but have a good match with SizeOf at the moment 448 //Adding value overhead 449 heapsize += ClassSize.align(ClassSize.ARRAYLIST); 450 int size = entry.getValue().size(); 451 heapsize += ClassSize.align(ClassSize.ARRAY + 452 size * ClassSize.REFERENCE); 453 454 for(Cell cell : entry.getValue()) { 455 heapsize += CellUtil.estimatedHeapSizeOf(cell); 456 } 457 } 458 heapsize += getAttributeSize(); 459 heapsize += extraHeapSize(); 460 return ClassSize.align(heapsize); 461 } 462 463 /** 464 * @return The serialized ACL for this operation, or null if none 465 */ getACL()466 public byte[] getACL() { 467 return getAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL); 468 } 469 470 /** 471 * @param user User short name 472 * @param perms Permissions for the user 473 */ setACL(String user, Permission perms)474 public Mutation setACL(String user, Permission perms) { 475 setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL, 476 ProtobufUtil.toUsersAndPermissions(user, perms).toByteArray()); 477 return this; 478 } 479 480 /** 481 * @param perms A map of permissions for a user or users 482 */ setACL(Map<String, Permission> perms)483 public Mutation setACL(Map<String, Permission> perms) { 484 ListMultimap<String, Permission> permMap = ArrayListMultimap.create(); 485 for (Map.Entry<String, Permission> entry : perms.entrySet()) { 486 permMap.put(entry.getKey(), entry.getValue()); 487 } 488 setAttribute(AccessControlConstants.OP_ATTRIBUTE_ACL, 489 ProtobufUtil.toUsersAndPermissions(permMap).toByteArray()); 490 return this; 491 } 492 493 /** 494 * Return the TTL requested for the result of the mutation, in milliseconds. 495 * @return the TTL requested for the result of the mutation, in milliseconds, 496 * or Long.MAX_VALUE if unset 497 */ getTTL()498 public long getTTL() { 499 byte[] ttlBytes = getAttribute(OP_ATTRIBUTE_TTL); 500 if (ttlBytes != null) { 501 return Bytes.toLong(ttlBytes); 502 } 503 return Long.MAX_VALUE; 504 } 505 506 /** 507 * Set the TTL desired for the result of the mutation, in milliseconds. 508 * @param ttl the TTL desired for the result of the mutation, in milliseconds 509 * @return this 510 */ setTTL(long ttl)511 public Mutation setTTL(long ttl) { 512 setAttribute(OP_ATTRIBUTE_TTL, Bytes.toBytes(ttl)); 513 return this; 514 } 515 516 /** 517 * @return current value for returnResults 518 */ 519 // Used by Increment and Append only. 520 @InterfaceAudience.Private isReturnResults()521 protected boolean isReturnResults() { 522 byte[] v = getAttribute(RETURN_RESULTS); 523 return v == null ? true : Bytes.toBoolean(v); 524 } 525 526 @InterfaceAudience.Private 527 // Used by Increment and Append only. setReturnResults(boolean returnResults)528 protected Mutation setReturnResults(boolean returnResults) { 529 setAttribute(RETURN_RESULTS, Bytes.toBytes(returnResults)); 530 return this; 531 } 532 533 /** 534 * Subclasses should override this method to add the heap size of their own fields. 535 * @return the heap size to add (will be aligned). 536 */ extraHeapSize()537 protected long extraHeapSize(){ 538 return 0L; 539 } 540 541 542 /** 543 * @param row Row to check 544 * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or 545 * > {@link HConstants#MAX_ROW_LENGTH} 546 * @return <code>row</code> 547 */ checkRow(final byte [] row)548 static byte [] checkRow(final byte [] row) { 549 return checkRow(row, 0, row == null? 0: row.length); 550 } 551 552 /** 553 * @param row Row to check 554 * @param offset 555 * @param length 556 * @throws IllegalArgumentException Thrown if <code>row</code> is empty or null or 557 * > {@link HConstants#MAX_ROW_LENGTH} 558 * @return <code>row</code> 559 */ checkRow(final byte [] row, final int offset, final int length)560 static byte [] checkRow(final byte [] row, final int offset, final int length) { 561 if (row == null) { 562 throw new IllegalArgumentException("Row buffer is null"); 563 } 564 if (length == 0) { 565 throw new IllegalArgumentException("Row length is 0"); 566 } 567 if (length > HConstants.MAX_ROW_LENGTH) { 568 throw new IllegalArgumentException("Row length " + length + " is > " + 569 HConstants.MAX_ROW_LENGTH); 570 } 571 return row; 572 } 573 checkRow(ByteBuffer row)574 static void checkRow(ByteBuffer row) { 575 if (row == null) { 576 throw new IllegalArgumentException("Row buffer is null"); 577 } 578 if (row.remaining() == 0) { 579 throw new IllegalArgumentException("Row length is 0"); 580 } 581 if (row.remaining() > HConstants.MAX_ROW_LENGTH) { 582 throw new IllegalArgumentException("Row length " + row.remaining() + " is > " + 583 HConstants.MAX_ROW_LENGTH); 584 } 585 } 586 } 587