1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 package org.apache.hadoop.contrib.index.mapred; 20 21 import java.io.DataInput; 22 import java.io.DataOutput; 23 import java.io.IOException; 24 import java.util.ArrayList; 25 import java.util.Collections; 26 import java.util.StringTokenizer; 27 28 import org.apache.hadoop.io.Text; 29 import org.apache.hadoop.io.WritableComparable; 30 31 /** 32 * This class represents the metadata of a shard. Version is the version number 33 * of the entire index. Directory is the directory where this shard resides in. 34 * Generation is the Lucene index's generation. Version and generation are 35 * reserved for future use. 36 * 37 * Note: Currently the version number of the entire index is not used and 38 * defaults to -1. 39 */ 40 public class Shard implements WritableComparable { 41 42 // This method is copied from Path. normalizePath(String path)43 public static String normalizePath(String path) { 44 // remove double slashes & backslashes 45 path = path.replace("//", "/"); 46 path = path.replace("\\", "/"); 47 48 // trim trailing slash from non-root path (ignoring windows drive) 49 if (path.length() > 1 && path.endsWith("/")) { 50 path = path.substring(0, path.length() - 1); 51 } 52 53 return path; 54 } 55 setIndexShards(IndexUpdateConfiguration conf, Shard[] shards)56 public static void setIndexShards(IndexUpdateConfiguration conf, 57 Shard[] shards) { 58 StringBuilder shardsString = new StringBuilder(shards[0].toString()); 59 for (int i = 1; i < shards.length; i++) { 60 shardsString.append(","); 61 shardsString.append(shards[i].toString()); 62 } 63 conf.setIndexShards(shardsString.toString()); 64 } 65 getIndexShards(IndexUpdateConfiguration conf)66 public static Shard[] getIndexShards(IndexUpdateConfiguration conf) { 67 String shards = conf.getIndexShards(); 68 if (shards != null) { 69 ArrayList<Object> list = 70 Collections.list(new StringTokenizer(shards, ",")); 71 Shard[] result = new Shard[list.size()]; 72 for (int i = 0; i < list.size(); i++) { 73 result[i] = Shard.createShardFromString((String) list.get(i)); 74 } 75 return result; 76 } else { 77 return null; 78 } 79 } 80 81 // assume str is formatted correctly as a shard string createShardFromString(String str)82 private static Shard createShardFromString(String str) { 83 int first = str.indexOf("@"); 84 int second = str.indexOf("@", first + 1); 85 long version = Long.parseLong(str.substring(0, first)); 86 String dir = str.substring(first + 1, second); 87 long gen = Long.parseLong(str.substring(second + 1)); 88 return new Shard(version, dir, gen); 89 } 90 91 // index/shard version 92 // the shards in the same version of an index have the same version number 93 private long version; 94 private String dir; 95 private long gen; // Lucene's generation 96 97 /** 98 * Constructor. 99 */ Shard()100 public Shard() { 101 this.version = -1; 102 this.dir = null; 103 this.gen = -1; 104 } 105 106 /** 107 * Construct a shard from a versio number, a directory and a generation 108 * number. 109 * @param version the version number of the entire index 110 * @param dir the directory where this shard resides 111 * @param gen the generation of the Lucene instance 112 */ Shard(long version, String dir, long gen)113 public Shard(long version, String dir, long gen) { 114 this.version = version; 115 this.dir = normalizePath(dir); 116 this.gen = gen; 117 } 118 119 /** 120 * Construct using a shard object. 121 * @param shard the shard used by the constructor 122 */ Shard(Shard shard)123 public Shard(Shard shard) { 124 this.version = shard.version; 125 this.dir = shard.dir; 126 this.gen = shard.gen; 127 } 128 129 /** 130 * Get the version number of the entire index. 131 * @return the version number of the entire index 132 */ getVersion()133 public long getVersion() { 134 return version; 135 } 136 137 /** 138 * Get the directory where this shard resides. 139 * @return the directory where this shard resides 140 */ getDirectory()141 public String getDirectory() { 142 return dir; 143 } 144 145 /** 146 * Get the generation of the Lucene instance. 147 * @return the generation of the Lucene instance 148 */ getGeneration()149 public long getGeneration() { 150 return gen; 151 } 152 153 /* (non-Javadoc) 154 * @see java.lang.Object#toString() 155 */ toString()156 public String toString() { 157 return version + "@" + dir + "@" + gen; 158 } 159 160 // /////////////////////////////////// 161 // Writable 162 // /////////////////////////////////// 163 /* (non-Javadoc) 164 * @see org.apache.hadoop.io.Writable#write(java.io.DataOutput) 165 */ write(DataOutput out)166 public void write(DataOutput out) throws IOException { 167 out.writeLong(version); 168 Text.writeString(out, dir); 169 out.writeLong(gen); 170 } 171 172 /* (non-Javadoc) 173 * @see org.apache.hadoop.io.Writable#readFields(java.io.DataInput) 174 */ readFields(DataInput in)175 public void readFields(DataInput in) throws IOException { 176 version = in.readLong(); 177 dir = Text.readString(in); 178 gen = in.readLong(); 179 } 180 181 // /////////////////////////////////// 182 // Comparable 183 // /////////////////////////////////// 184 /* (non-Javadoc) 185 * @see java.lang.Comparable#compareTo(java.lang.Object) 186 */ compareTo(Object o)187 public int compareTo(Object o) { 188 return compareTo((Shard) o); 189 } 190 191 /** 192 * Compare to another shard. 193 * @param other another shard 194 * @return compare version first, then directory and finally generation 195 */ compareTo(Shard other)196 public int compareTo(Shard other) { 197 // compare version 198 if (version < other.version) { 199 return -1; 200 } else if (version > other.version) { 201 return 1; 202 } 203 // compare dir 204 int result = dir.compareTo(other.dir); 205 if (result != 0) { 206 return result; 207 } 208 // compare gen 209 if (gen < other.gen) { 210 return -1; 211 } else if (gen == other.gen) { 212 return 0; 213 } else { 214 return 1; 215 } 216 } 217 218 /* (non-Javadoc) 219 * @see java.lang.Object#equals(java.lang.Object) 220 */ equals(Object o)221 public boolean equals(Object o) { 222 if (this == o) { 223 return true; 224 } 225 if (!(o instanceof Shard)) { 226 return false; 227 } 228 Shard other = (Shard) o; 229 return version == other.version && dir.equals(other.dir) 230 && gen == other.gen; 231 } 232 233 /* (non-Javadoc) 234 * @see java.lang.Object#hashCode() 235 */ hashCode()236 public int hashCode() { 237 return (int) version ^ dir.hashCode() ^ (int) gen; 238 } 239 240 } 241