1 /**
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements.  See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership.  The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License.  You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18 
19 package org.apache.hadoop.tools;
20 
21 import org.apache.commons.logging.Log;
22 import org.apache.commons.logging.LogFactory;
23 import org.apache.hadoop.conf.Configuration;
24 import org.apache.hadoop.fs.FileStatus;
25 import org.apache.hadoop.fs.FileSystem;
26 import org.apache.hadoop.fs.Path;
27 import org.apache.hadoop.security.Credentials;
28 
29 import java.io.IOException;
30 import java.util.List;
31 import java.util.ArrayList;
32 
33 /**
34  * GlobbedCopyListing implements the CopyListing interface, to create the copy
35  * listing-file by "globbing" all specified source paths (wild-cards and all.)
36  */
37 public class GlobbedCopyListing extends CopyListing {
38   private static final Log LOG = LogFactory.getLog(GlobbedCopyListing.class);
39 
40   private final CopyListing simpleListing;
41   /**
42    * Constructor, to initialize the configuration.
43    * @param configuration The input Configuration object.
44    * @param credentials Credentials object on which the FS delegation tokens are cached. If null
45    * delegation token caching is skipped
46    */
GlobbedCopyListing(Configuration configuration, Credentials credentials)47   public GlobbedCopyListing(Configuration configuration, Credentials credentials) {
48     super(configuration, credentials);
49     simpleListing = new SimpleCopyListing(getConf(), credentials) ;
50   }
51 
52   /** {@inheritDoc} */
53   @Override
validatePaths(DistCpOptions options)54   protected void validatePaths(DistCpOptions options)
55       throws IOException, InvalidInputException {
56   }
57 
58   /**
59    * Implementation of CopyListing::buildListing().
60    * Creates the copy listing by "globbing" all source-paths.
61    * @param pathToListingFile The location at which the copy-listing file
62    *                           is to be created.
63    * @param options Input Options for DistCp (indicating source/target paths.)
64    * @throws IOException
65    */
66   @Override
doBuildListing(Path pathToListingFile, DistCpOptions options)67   public void doBuildListing(Path pathToListingFile,
68                              DistCpOptions options) throws IOException {
69 
70     List<Path> globbedPaths = new ArrayList<Path>();
71     if (options.getSourcePaths().isEmpty()) {
72       throw new InvalidInputException("Nothing to process. Source paths::EMPTY");
73     }
74 
75     for (Path p : options.getSourcePaths()) {
76       FileSystem fs = p.getFileSystem(getConf());
77       FileStatus[] inputs = fs.globStatus(p);
78 
79       if(inputs != null && inputs.length > 0) {
80         for (FileStatus onePath: inputs) {
81           globbedPaths.add(onePath.getPath());
82         }
83       } else {
84         throw new InvalidInputException(p + " doesn't exist");
85       }
86     }
87 
88     DistCpOptions optionsGlobbed = new DistCpOptions(options);
89     optionsGlobbed.setSourcePaths(globbedPaths);
90     simpleListing.buildListing(pathToListingFile, optionsGlobbed);
91   }
92 
93   /** {@inheritDoc} */
94   @Override
getBytesToCopy()95   protected long getBytesToCopy() {
96     return simpleListing.getBytesToCopy();
97   }
98 
99   /** {@inheritDoc} */
100   @Override
getNumberOfPaths()101   protected long getNumberOfPaths() {
102     return simpleListing.getNumberOfPaths();
103   }
104 
105 }
106