1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 package org.apache.hadoop.tools; 20 21 import org.apache.commons.logging.Log; 22 import org.apache.commons.logging.LogFactory; 23 import org.apache.hadoop.conf.Configuration; 24 import org.apache.hadoop.fs.FileStatus; 25 import org.apache.hadoop.fs.FileSystem; 26 import org.apache.hadoop.fs.Path; 27 import org.apache.hadoop.security.Credentials; 28 29 import java.io.IOException; 30 import java.util.List; 31 import java.util.ArrayList; 32 33 /** 34 * GlobbedCopyListing implements the CopyListing interface, to create the copy 35 * listing-file by "globbing" all specified source paths (wild-cards and all.) 36 */ 37 public class GlobbedCopyListing extends CopyListing { 38 private static final Log LOG = LogFactory.getLog(GlobbedCopyListing.class); 39 40 private final CopyListing simpleListing; 41 /** 42 * Constructor, to initialize the configuration. 43 * @param configuration The input Configuration object. 44 * @param credentials Credentials object on which the FS delegation tokens are cached. If null 45 * delegation token caching is skipped 46 */ GlobbedCopyListing(Configuration configuration, Credentials credentials)47 public GlobbedCopyListing(Configuration configuration, Credentials credentials) { 48 super(configuration, credentials); 49 simpleListing = new SimpleCopyListing(getConf(), credentials) ; 50 } 51 52 /** {@inheritDoc} */ 53 @Override validatePaths(DistCpOptions options)54 protected void validatePaths(DistCpOptions options) 55 throws IOException, InvalidInputException { 56 } 57 58 /** 59 * Implementation of CopyListing::buildListing(). 60 * Creates the copy listing by "globbing" all source-paths. 61 * @param pathToListingFile The location at which the copy-listing file 62 * is to be created. 63 * @param options Input Options for DistCp (indicating source/target paths.) 64 * @throws IOException 65 */ 66 @Override doBuildListing(Path pathToListingFile, DistCpOptions options)67 public void doBuildListing(Path pathToListingFile, 68 DistCpOptions options) throws IOException { 69 70 List<Path> globbedPaths = new ArrayList<Path>(); 71 if (options.getSourcePaths().isEmpty()) { 72 throw new InvalidInputException("Nothing to process. Source paths::EMPTY"); 73 } 74 75 for (Path p : options.getSourcePaths()) { 76 FileSystem fs = p.getFileSystem(getConf()); 77 FileStatus[] inputs = fs.globStatus(p); 78 79 if(inputs != null && inputs.length > 0) { 80 for (FileStatus onePath: inputs) { 81 globbedPaths.add(onePath.getPath()); 82 } 83 } else { 84 throw new InvalidInputException(p + " doesn't exist"); 85 } 86 } 87 88 DistCpOptions optionsGlobbed = new DistCpOptions(options); 89 optionsGlobbed.setSourcePaths(globbedPaths); 90 simpleListing.buildListing(pathToListingFile, optionsGlobbed); 91 } 92 93 /** {@inheritDoc} */ 94 @Override getBytesToCopy()95 protected long getBytesToCopy() { 96 return simpleListing.getBytesToCopy(); 97 } 98 99 /** {@inheritDoc} */ 100 @Override getNumberOfPaths()101 protected long getNumberOfPaths() { 102 return simpleListing.getNumberOfPaths(); 103 } 104 105 } 106