1 /**
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements.  See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership.  The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License.  You may obtain a copy of the License at
9  *
10  *     http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing, software
13  * distributed under the License is distributed on an "AS IS" BASIS,
14  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15  * See the License for the specific language governing permissions and
16  * limitations under the License.
17  */
18 
19 package org.apache.hadoop.tools.distcp2;
20 
21 import java.util.Iterator;
22 import java.util.NoSuchElementException;
23 
24 import org.apache.hadoop.conf.Configuration;
25 import org.apache.hadoop.fs.Path;
26 import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute;
27 import org.junit.Assert;
28 import org.junit.Test;
29 
30 public class TestOptionsParser {
31 
32   @Test
testParseIgnoreFailure()33   public void testParseIgnoreFailure() {
34     DistCpOptions options = OptionsParser.parse(new String[] {
35         "hdfs://localhost:8020/source/first",
36         "hdfs://localhost:8020/target/"});
37     Assert.assertFalse(options.shouldIgnoreFailures());
38 
39     options = OptionsParser.parse(new String[] {
40         "-i",
41         "hdfs://localhost:8020/source/first",
42         "hdfs://localhost:8020/target/"});
43     Assert.assertTrue(options.shouldIgnoreFailures());
44   }
45 
46   @Test
testParseOverwrite()47   public void testParseOverwrite() {
48     DistCpOptions options = OptionsParser.parse(new String[] {
49         "hdfs://localhost:8020/source/first",
50         "hdfs://localhost:8020/target/"});
51     Assert.assertFalse(options.shouldOverwrite());
52 
53     options = OptionsParser.parse(new String[] {
54         "-overwrite",
55         "hdfs://localhost:8020/source/first",
56         "hdfs://localhost:8020/target/"});
57     Assert.assertTrue(options.shouldOverwrite());
58 
59     try {
60       OptionsParser.parse(new String[] {
61           "-update",
62           "-overwrite",
63           "hdfs://localhost:8020/source/first",
64           "hdfs://localhost:8020/target/"});
65       Assert.fail("Update and overwrite aren't allowed together");
66     } catch (IllegalArgumentException ignore) {
67     }
68   }
69 
70   @Test
testLogPath()71   public void testLogPath() {
72     DistCpOptions options = OptionsParser.parse(new String[] {
73         "hdfs://localhost:8020/source/first",
74         "hdfs://localhost:8020/target/"});
75     Assert.assertNull(options.getLogPath());
76 
77     options = OptionsParser.parse(new String[] {
78         "-log",
79         "hdfs://localhost:8020/logs",
80         "hdfs://localhost:8020/source/first",
81         "hdfs://localhost:8020/target/"});
82     Assert.assertEquals(options.getLogPath(), new Path("hdfs://localhost:8020/logs"));
83   }
84 
85   @Test
testParseBlokcing()86   public void testParseBlokcing() {
87     DistCpOptions options = OptionsParser.parse(new String[] {
88         "hdfs://localhost:8020/source/first",
89         "hdfs://localhost:8020/target/"});
90     Assert.assertTrue(options.shouldBlock());
91 
92     options = OptionsParser.parse(new String[] {
93         "-async",
94         "hdfs://localhost:8020/source/first",
95         "hdfs://localhost:8020/target/"});
96     Assert.assertFalse(options.shouldBlock());
97   }
98 
99   @Test
testParsebandwidth()100   public void testParsebandwidth() {
101     DistCpOptions options = OptionsParser.parse(new String[] {
102         "hdfs://localhost:8020/source/first",
103         "hdfs://localhost:8020/target/"});
104     Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB);
105 
106     options = OptionsParser.parse(new String[] {
107         "-bandwidth",
108         "11",
109         "hdfs://localhost:8020/source/first",
110         "hdfs://localhost:8020/target/"});
111     Assert.assertEquals(options.getMapBandwidth(), 11);
112   }
113 
114   @Test(expected=IllegalArgumentException.class)
testParseNonPositiveBandwidth()115   public void testParseNonPositiveBandwidth() {
116     OptionsParser.parse(new String[] {
117         "-bandwidth",
118         "-11",
119         "hdfs://localhost:8020/source/first",
120         "hdfs://localhost:8020/target/"});
121   }
122 
123   @Test(expected=IllegalArgumentException.class)
testParseZeroBandwidth()124   public void testParseZeroBandwidth() {
125     OptionsParser.parse(new String[] {
126         "-bandwidth",
127         "0",
128         "hdfs://localhost:8020/source/first",
129         "hdfs://localhost:8020/target/"});
130   }
131 
132   @Test
testParseSkipCRC()133   public void testParseSkipCRC() {
134     DistCpOptions options = OptionsParser.parse(new String[] {
135         "hdfs://localhost:8020/source/first",
136         "hdfs://localhost:8020/target/"});
137     Assert.assertFalse(options.shouldSkipCRC());
138 
139     options = OptionsParser.parse(new String[] {
140         "-update",
141         "-skipcrccheck",
142         "hdfs://localhost:8020/source/first",
143         "hdfs://localhost:8020/target/"});
144     Assert.assertTrue(options.shouldSyncFolder());
145     Assert.assertTrue(options.shouldSkipCRC());
146   }
147 
148   @Test
testParseAtomicCommit()149   public void testParseAtomicCommit() {
150     DistCpOptions options = OptionsParser.parse(new String[] {
151         "hdfs://localhost:8020/source/first",
152         "hdfs://localhost:8020/target/"});
153     Assert.assertFalse(options.shouldAtomicCommit());
154 
155     options = OptionsParser.parse(new String[] {
156         "-atomic",
157         "hdfs://localhost:8020/source/first",
158         "hdfs://localhost:8020/target/"});
159     Assert.assertTrue(options.shouldAtomicCommit());
160 
161     try {
162       OptionsParser.parse(new String[] {
163           "-atomic",
164           "-update",
165           "hdfs://localhost:8020/source/first",
166           "hdfs://localhost:8020/target/"});
167       Assert.fail("Atomic and sync folders were allowed");
168     } catch (IllegalArgumentException ignore) { }
169   }
170 
171   @Test
testParseWorkPath()172   public void testParseWorkPath() {
173     DistCpOptions options = OptionsParser.parse(new String[] {
174         "hdfs://localhost:8020/source/first",
175         "hdfs://localhost:8020/target/"});
176     Assert.assertNull(options.getAtomicWorkPath());
177 
178     options = OptionsParser.parse(new String[] {
179         "-atomic",
180         "hdfs://localhost:8020/source/first",
181         "hdfs://localhost:8020/target/"});
182     Assert.assertNull(options.getAtomicWorkPath());
183 
184     options = OptionsParser.parse(new String[] {
185         "-atomic",
186         "-tmp",
187         "hdfs://localhost:8020/work",
188         "hdfs://localhost:8020/source/first",
189         "hdfs://localhost:8020/target/"});
190     Assert.assertEquals(options.getAtomicWorkPath(), new Path("hdfs://localhost:8020/work"));
191 
192     try {
193       OptionsParser.parse(new String[] {
194           "-tmp",
195           "hdfs://localhost:8020/work",
196           "hdfs://localhost:8020/source/first",
197           "hdfs://localhost:8020/target/"});
198       Assert.fail("work path was allowed without -atomic switch");
199     } catch (IllegalArgumentException ignore) {}
200   }
201 
202   @Test
testParseSyncFolders()203   public void testParseSyncFolders() {
204     DistCpOptions options = OptionsParser.parse(new String[] {
205         "hdfs://localhost:8020/source/first",
206         "hdfs://localhost:8020/target/"});
207     Assert.assertFalse(options.shouldSyncFolder());
208 
209     options = OptionsParser.parse(new String[] {
210         "-update",
211         "hdfs://localhost:8020/source/first",
212         "hdfs://localhost:8020/target/"});
213     Assert.assertTrue(options.shouldSyncFolder());
214   }
215 
216   @Test
testParseDeleteMissing()217   public void testParseDeleteMissing() {
218     DistCpOptions options = OptionsParser.parse(new String[] {
219         "hdfs://localhost:8020/source/first",
220         "hdfs://localhost:8020/target/"});
221     Assert.assertFalse(options.shouldDeleteMissing());
222 
223     options = OptionsParser.parse(new String[] {
224         "-update",
225         "-delete",
226         "hdfs://localhost:8020/source/first",
227         "hdfs://localhost:8020/target/"});
228     Assert.assertTrue(options.shouldSyncFolder());
229     Assert.assertTrue(options.shouldDeleteMissing());
230 
231     options = OptionsParser.parse(new String[] {
232         "-overwrite",
233         "-delete",
234         "hdfs://localhost:8020/source/first",
235         "hdfs://localhost:8020/target/"});
236     Assert.assertTrue(options.shouldOverwrite());
237     Assert.assertTrue(options.shouldDeleteMissing());
238 
239     try {
240       OptionsParser.parse(new String[] {
241           "-atomic",
242           "-delete",
243           "hdfs://localhost:8020/source/first",
244           "hdfs://localhost:8020/target/"});
245       Assert.fail("Atomic and delete folders were allowed");
246     } catch (IllegalArgumentException ignore) { }
247   }
248 
249   @Test
testParseSSLConf()250   public void testParseSSLConf() {
251     DistCpOptions options = OptionsParser.parse(new String[] {
252         "hdfs://localhost:8020/source/first",
253         "hdfs://localhost:8020/target/"});
254     Assert.assertNull(options.getSslConfigurationFile());
255 
256     options = OptionsParser.parse(new String[] {
257         "-mapredSslConf",
258         "/tmp/ssl-client.xml",
259         "hdfs://localhost:8020/source/first",
260         "hdfs://localhost:8020/target/"});
261     Assert.assertEquals(options.getSslConfigurationFile(), "/tmp/ssl-client.xml");
262   }
263 
264   @Test
testParseMaps()265   public void testParseMaps() {
266     DistCpOptions options = OptionsParser.parse(new String[] {
267         "hdfs://localhost:8020/source/first",
268         "hdfs://localhost:8020/target/"});
269     Assert.assertEquals(options.getMaxMaps(), DistCpConstants.DEFAULT_MAPS);
270 
271     options = OptionsParser.parse(new String[] {
272         "-m",
273         "1",
274         "hdfs://localhost:8020/source/first",
275         "hdfs://localhost:8020/target/"});
276     Assert.assertEquals(options.getMaxMaps(), 1);
277 
278     options = OptionsParser.parse(new String[] {
279         "-m",
280         "0",
281         "hdfs://localhost:8020/source/first",
282         "hdfs://localhost:8020/target/"});
283     Assert.assertEquals(options.getMaxMaps(), 1);
284 
285     try {
286       OptionsParser.parse(new String[] {
287           "-m",
288           "hello",
289           "hdfs://localhost:8020/source/first",
290           "hdfs://localhost:8020/target/"});
291       Assert.fail("Non numberic map parsed");
292     } catch (IllegalArgumentException ignore) { }
293 
294     try {
295       OptionsParser.parse(new String[] {
296           "-mapredXslConf",
297           "hdfs://localhost:8020/source/first",
298           "hdfs://localhost:8020/target/"});
299       Assert.fail("Non numberic map parsed");
300     } catch (IllegalArgumentException ignore) { }
301   }
302 
303   @Test
testSourceListing()304   public void testSourceListing() {
305     DistCpOptions options = OptionsParser.parse(new String[] {
306         "-f",
307         "hdfs://localhost:8020/source/first",
308         "hdfs://localhost:8020/target/"});
309     Assert.assertEquals(options.getSourceFileListing(),
310         new Path("hdfs://localhost:8020/source/first"));
311   }
312 
313   @Test
testSourceListingAndSourcePath()314   public void testSourceListingAndSourcePath() {
315     try {
316       OptionsParser.parse(new String[] {
317           "-f",
318           "hdfs://localhost:8020/source/first",
319           "hdfs://localhost:8020/source/first",
320           "hdfs://localhost:8020/target/"});
321       Assert.fail("Both source listing & source paths allowed");
322     } catch (IllegalArgumentException ignore) {}
323   }
324 
325   @Test
testMissingSourceInfo()326   public void testMissingSourceInfo() {
327     try {
328       OptionsParser.parse(new String[] {
329           "hdfs://localhost:8020/target/"});
330       Assert.fail("Neither source listing not source paths present");
331     } catch (IllegalArgumentException ignore) {}
332   }
333 
334   @Test
testMissingTarget()335   public void testMissingTarget() {
336     try {
337       OptionsParser.parse(new String[] {
338           "-f", "hdfs://localhost:8020/source"});
339       Assert.fail("Missing target allowed");
340     } catch (IllegalArgumentException ignore) {}
341   }
342 
343   @Test
testInvalidArgs()344   public void testInvalidArgs() {
345     try {
346       OptionsParser.parse(new String[] {
347           "-m", "-f", "hdfs://localhost:8020/source"});
348       Assert.fail("Missing map value");
349     } catch (IllegalArgumentException ignore) {}
350   }
351 
352   @Test
testToString()353   public void testToString() {
354     DistCpOptions option = new DistCpOptions(new Path("abc"), new Path("xyz"));
355     String val = "DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false, " +
356         "ignoreFailures=false, maxMaps=20, sslConfigurationFile='null', copyStrategy='uniformsize', " +
357         "sourceFileListing=abc, sourcePaths=null, targetPath=xyz}";
358     Assert.assertEquals(val, option.toString());
359     Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(),
360         DistCpOptionSwitch.ATOMIC_COMMIT.name());
361   }
362 
363   @Test
testCopyStrategy()364   public void testCopyStrategy() {
365     DistCpOptions options = OptionsParser.parse(new String[] {
366         "-strategy",
367         "dynamic",
368         "-f",
369         "hdfs://localhost:8020/source/first",
370         "hdfs://localhost:8020/target/"});
371     Assert.assertEquals(options.getCopyStrategy(), "dynamic");
372 
373     options = OptionsParser.parse(new String[] {
374         "-f",
375         "hdfs://localhost:8020/source/first",
376         "hdfs://localhost:8020/target/"});
377     Assert.assertEquals(options.getCopyStrategy(), DistCpConstants.UNIFORMSIZE);
378   }
379 
380   @Test
testTargetPath()381   public void testTargetPath() {
382     DistCpOptions options = OptionsParser.parse(new String[] {
383         "-f",
384         "hdfs://localhost:8020/source/first",
385         "hdfs://localhost:8020/target/"});
386     Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost:8020/target/"));
387   }
388 
389   @Test
testPreserve()390   public void testPreserve() {
391     DistCpOptions options = OptionsParser.parse(new String[] {
392         "-f",
393         "hdfs://localhost:8020/source/first",
394         "hdfs://localhost:8020/target/"});
395     Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE));
396     Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION));
397     Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION));
398     Assert.assertFalse(options.shouldPreserve(FileAttribute.USER));
399     Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP));
400 
401     options = OptionsParser.parse(new String[] {
402         "-p",
403         "-f",
404         "hdfs://localhost:8020/source/first",
405         "hdfs://localhost:8020/target/"});
406     Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE));
407     Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION));
408     Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
409     Assert.assertTrue(options.shouldPreserve(FileAttribute.USER));
410     Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
411 
412     options = OptionsParser.parse(new String[] {
413         "-p",
414         "hdfs://localhost:8020/source/first",
415         "hdfs://localhost:8020/target/"});
416     Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE));
417     Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION));
418     Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
419     Assert.assertTrue(options.shouldPreserve(FileAttribute.USER));
420     Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
421 
422     options = OptionsParser.parse(new String[] {
423         "-pbr",
424         "-f",
425         "hdfs://localhost:8020/source/first",
426         "hdfs://localhost:8020/target/"});
427     Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE));
428     Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION));
429     Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION));
430     Assert.assertFalse(options.shouldPreserve(FileAttribute.USER));
431     Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP));
432 
433     options = OptionsParser.parse(new String[] {
434         "-pbrgup",
435         "-f",
436         "hdfs://localhost:8020/source/first",
437         "hdfs://localhost:8020/target/"});
438     Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE));
439     Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION));
440     Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
441     Assert.assertTrue(options.shouldPreserve(FileAttribute.USER));
442     Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP));
443 
444     options = OptionsParser.parse(new String[] {
445         "-p",
446         "-f",
447         "hdfs://localhost:8020/source/first",
448         "hdfs://localhost:8020/target/"});
449     int i = 0;
450     Iterator<FileAttribute> attribIterator = options.preserveAttributes();
451     while (attribIterator.hasNext()) {
452       attribIterator.next();
453       i++;
454     }
455     Assert.assertEquals(i, 5);
456 
457     try {
458       OptionsParser.parse(new String[] {
459           "-pabc",
460           "-f",
461           "hdfs://localhost:8020/source/first",
462           "hdfs://localhost:8020/target"});
463       Assert.fail("Invalid preserve attribute");
464     }
465     catch (IllegalArgumentException ignore) {}
466     catch (NoSuchElementException ignore) {}
467 
468     options = OptionsParser.parse(new String[] {
469         "-f",
470         "hdfs://localhost:8020/source/first",
471         "hdfs://localhost:8020/target/"});
472     Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION));
473     options.preserve(FileAttribute.PERMISSION);
474     Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
475 
476     options.preserve(FileAttribute.PERMISSION);
477     Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION));
478   }
479 
480   @Test
testOptionsSwitchAddToConf()481   public void testOptionsSwitchAddToConf() {
482     Configuration conf = new Configuration();
483     Assert.assertNull(conf.get(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel()));
484     DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.ATOMIC_COMMIT);
485     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
486   }
487 
488   @Test
testOptionsAppendToConf()489   public void testOptionsAppendToConf() {
490     Configuration conf = new Configuration();
491     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
492     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
493     DistCpOptions options = OptionsParser.parse(new String[] {
494         "-atomic",
495         "-i",
496         "hdfs://localhost:8020/source/first",
497         "hdfs://localhost:8020/target/"});
498     options.appendToConf(conf);
499     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
500     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
501     Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1),
502         DistCpConstants.DEFAULT_BANDWIDTH_MB);
503 
504     conf = new Configuration();
505     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false));
506     Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false));
507     Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), null);
508     options = OptionsParser.parse(new String[] {
509         "-update",
510         "-delete",
511         "-pu",
512         "-bandwidth",
513         "11",
514         "hdfs://localhost:8020/source/first",
515         "hdfs://localhost:8020/target/"});
516     options.appendToConf(conf);
517     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false));
518     Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false));
519     Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), "U");
520     Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11);
521   }
522 }
523