1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 package org.apache.hadoop.hbase.io.hfile; 19 20 import java.io.IOException; 21 import java.util.NavigableMap; 22 import java.util.NavigableSet; 23 import java.util.concurrent.ConcurrentSkipListMap; 24 import java.util.concurrent.ConcurrentSkipListSet; 25 26 import org.apache.hadoop.hbase.classification.InterfaceAudience; 27 import org.apache.hadoop.conf.Configuration; 28 import org.codehaus.jackson.JsonGenerationException; 29 import org.codehaus.jackson.annotate.JsonIgnoreProperties; 30 import org.codehaus.jackson.map.JsonMappingException; 31 import org.codehaus.jackson.map.ObjectMapper; 32 import org.codehaus.jackson.map.SerializationConfig; 33 34 import com.yammer.metrics.core.Histogram; 35 import com.yammer.metrics.core.MetricsRegistry; 36 import com.yammer.metrics.stats.Snapshot; 37 38 /** 39 * Utilty for aggregating counts in CachedBlocks and toString/toJSON CachedBlocks and BlockCaches. 40 * No attempt has been made at making this thread safe. 41 */ 42 @InterfaceAudience.Private 43 public class BlockCacheUtil { 44 /** 45 * Needed making histograms. 46 */ 47 private static final MetricsRegistry METRICS = new MetricsRegistry(); 48 49 /** 50 * Needed generating JSON. 51 */ 52 private static final ObjectMapper MAPPER = new ObjectMapper(); 53 static { MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false)54 MAPPER.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false); MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true)55 MAPPER.configure(SerializationConfig.Feature.FLUSH_AFTER_WRITE_VALUE, true); MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true)56 MAPPER.configure(SerializationConfig.Feature.INDENT_OUTPUT, true); 57 } 58 59 /** 60 * @param cb 61 * @return The block content as String. 62 */ toString(final CachedBlock cb, final long now)63 public static String toString(final CachedBlock cb, final long now) { 64 return "filename=" + cb.getFilename() + ", " + toStringMinusFileName(cb, now); 65 } 66 67 /** 68 * Little data structure to hold counts for a file. 69 * Used doing a toJSON. 70 */ 71 static class CachedBlockCountsPerFile { 72 private int count = 0; 73 private long size = 0; 74 private int countData = 0; 75 private long sizeData = 0; 76 private final String filename; 77 CachedBlockCountsPerFile(final String filename)78 CachedBlockCountsPerFile(final String filename) { 79 this.filename = filename; 80 } 81 getCount()82 public int getCount() { 83 return count; 84 } 85 getSize()86 public long getSize() { 87 return size; 88 } 89 getCountData()90 public int getCountData() { 91 return countData; 92 } 93 getSizeData()94 public long getSizeData() { 95 return sizeData; 96 } 97 getFilename()98 public String getFilename() { 99 return filename; 100 } 101 } 102 103 /** 104 * @param filename 105 * @param blocks 106 * @return A JSON String of <code>filename</code> and counts of <code>blocks</code> 107 * @throws JsonGenerationException 108 * @throws JsonMappingException 109 * @throws IOException 110 */ toJSON(final String filename, final NavigableSet<CachedBlock> blocks)111 public static String toJSON(final String filename, final NavigableSet<CachedBlock> blocks) 112 throws JsonGenerationException, JsonMappingException, IOException { 113 CachedBlockCountsPerFile counts = new CachedBlockCountsPerFile(filename); 114 for (CachedBlock cb: blocks) { 115 counts.count++; 116 counts.size += cb.getSize(); 117 BlockType bt = cb.getBlockType(); 118 if (bt != null && bt.isData()) { 119 counts.countData++; 120 counts.sizeData += cb.getSize(); 121 } 122 } 123 return MAPPER.writeValueAsString(counts); 124 } 125 126 /** 127 * @param cbsbf 128 * @return JSON string of <code>cbsf</code> aggregated 129 * @throws JsonGenerationException 130 * @throws JsonMappingException 131 * @throws IOException 132 */ toJSON(final CachedBlocksByFile cbsbf)133 public static String toJSON(final CachedBlocksByFile cbsbf) 134 throws JsonGenerationException, JsonMappingException, IOException { 135 return MAPPER.writeValueAsString(cbsbf); 136 } 137 138 /** 139 * @param bc 140 * @return JSON string of <code>bc</code> content. 141 * @throws JsonGenerationException 142 * @throws JsonMappingException 143 * @throws IOException 144 */ toJSON(final BlockCache bc)145 public static String toJSON(final BlockCache bc) 146 throws JsonGenerationException, JsonMappingException, IOException { 147 return MAPPER.writeValueAsString(bc); 148 } 149 150 /** 151 * @param cb 152 * @return The block content of <code>bc</code> as a String minus the filename. 153 */ toStringMinusFileName(final CachedBlock cb, final long now)154 public static String toStringMinusFileName(final CachedBlock cb, final long now) { 155 return "offset=" + cb.getOffset() + 156 ", size=" + cb.getSize() + 157 ", age=" + (now - cb.getCachedTime()) + 158 ", type=" + cb.getBlockType() + 159 ", priority=" + cb.getBlockPriority(); 160 } 161 162 /** 163 * Get a {@link CachedBlocksByFile} instance and load it up by iterating content in 164 * {@link BlockCache}. 165 * @param conf Used to read configurations 166 * @param bc Block Cache to iterate. 167 * @return Laoded up instance of CachedBlocksByFile 168 */ getLoadedCachedBlocksByFile(final Configuration conf, final BlockCache bc)169 public static CachedBlocksByFile getLoadedCachedBlocksByFile(final Configuration conf, 170 final BlockCache bc) { 171 CachedBlocksByFile cbsbf = new CachedBlocksByFile(conf); 172 for (CachedBlock cb: bc) { 173 if (cbsbf.update(cb)) break; 174 } 175 return cbsbf; 176 } 177 178 /** 179 * Use one of these to keep a running account of cached blocks by file. Throw it away when done. 180 * This is different than metrics in that it is stats on current state of a cache. 181 * See getLoadedCachedBlocksByFile 182 */ 183 @JsonIgnoreProperties({"cachedBlockStatsByFile"}) 184 public static class CachedBlocksByFile { 185 private int count; 186 private int dataBlockCount; 187 private long size; 188 private long dataSize; 189 private final long now = System.nanoTime(); 190 private final int max; 191 public static final int DEFAULT_MAX = 100000; 192 CachedBlocksByFile()193 CachedBlocksByFile() { 194 this(null); 195 } 196 CachedBlocksByFile(final Configuration c)197 CachedBlocksByFile(final Configuration c) { 198 this.max = c == null? DEFAULT_MAX: 199 c.getInt("hbase.ui.blockcache.by.file.max", DEFAULT_MAX); 200 } 201 202 /** 203 * Map by filename. use concurent utils because we want our Map and contained blocks sorted. 204 */ 205 private NavigableMap<String, NavigableSet<CachedBlock>> cachedBlockByFile = 206 new ConcurrentSkipListMap<String, NavigableSet<CachedBlock>>(); 207 Histogram age = METRICS.newHistogram(CachedBlocksByFile.class, "age"); 208 209 /** 210 * @param cb 211 * @return True if full.... if we won't be adding any more. 212 */ update(final CachedBlock cb)213 public boolean update(final CachedBlock cb) { 214 if (isFull()) return true; 215 NavigableSet<CachedBlock> set = this.cachedBlockByFile.get(cb.getFilename()); 216 if (set == null) { 217 set = new ConcurrentSkipListSet<CachedBlock>(); 218 this.cachedBlockByFile.put(cb.getFilename(), set); 219 } 220 set.add(cb); 221 this.size += cb.getSize(); 222 this.count++; 223 BlockType bt = cb.getBlockType(); 224 if (bt != null && bt.isData()) { 225 this.dataBlockCount++; 226 this.dataSize += cb.getSize(); 227 } 228 long age = this.now - cb.getCachedTime(); 229 this.age.update(age); 230 return false; 231 } 232 233 /** 234 * @return True if full; i.e. there are more items in the cache but we only loaded up 235 * the maximum set in configuration <code>hbase.ui.blockcache.by.file.max</code> 236 * (Default: DEFAULT_MAX). 237 */ isFull()238 public boolean isFull() { 239 return this.count >= this.max; 240 } 241 getCachedBlockStatsByFile()242 public NavigableMap<String, NavigableSet<CachedBlock>> getCachedBlockStatsByFile() { 243 return this.cachedBlockByFile; 244 } 245 246 /** 247 * @return count of blocks in the cache 248 */ getCount()249 public int getCount() { 250 return count; 251 } 252 getDataCount()253 public int getDataCount() { 254 return dataBlockCount; 255 } 256 257 /** 258 * @return size of blocks in the cache 259 */ getSize()260 public long getSize() { 261 return size; 262 } 263 264 /** 265 * @return Size of data. 266 */ getDataSize()267 public long getDataSize() { 268 return dataSize; 269 } 270 getAgeInCacheSnapshot()271 public AgeSnapshot getAgeInCacheSnapshot() { 272 return new AgeSnapshot(this.age); 273 } 274 275 @Override toString()276 public String toString() { 277 Snapshot snapshot = this.age.getSnapshot(); 278 return "count=" + count + ", dataBlockCount=" + this.dataBlockCount + ", size=" + size + 279 ", dataSize=" + getDataSize() + 280 ", mean age=" + this.age.mean() + ", stddev age=" + this.age.stdDev() + 281 ", min age=" + this.age.min() + ", max age=" + this.age.max() + 282 ", 95th percentile age=" + snapshot.get95thPercentile() + 283 ", 99th percentile age=" + snapshot.get99thPercentile(); 284 } 285 } 286 } 287