1 /** 2 * Licensed to the Apache Software Foundation (ASF) under one 3 * or more contributor license agreements. See the NOTICE file 4 * distributed with this work for additional information 5 * regarding copyright ownership. The ASF licenses this file 6 * to you under the Apache License, Version 2.0 (the 7 * "License"); you may not use this file except in compliance 8 * with the License. You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 */ 18 19 package org.apache.hadoop.tools.distcp2; 20 21 import java.util.Iterator; 22 import java.util.NoSuchElementException; 23 24 import org.apache.hadoop.conf.Configuration; 25 import org.apache.hadoop.fs.Path; 26 import org.apache.hadoop.tools.distcp2.DistCpOptions.FileAttribute; 27 import org.junit.Assert; 28 import org.junit.Test; 29 30 public class TestOptionsParser { 31 32 @Test testParseIgnoreFailure()33 public void testParseIgnoreFailure() { 34 DistCpOptions options = OptionsParser.parse(new String[] { 35 "hdfs://localhost:8020/source/first", 36 "hdfs://localhost:8020/target/"}); 37 Assert.assertFalse(options.shouldIgnoreFailures()); 38 39 options = OptionsParser.parse(new String[] { 40 "-i", 41 "hdfs://localhost:8020/source/first", 42 "hdfs://localhost:8020/target/"}); 43 Assert.assertTrue(options.shouldIgnoreFailures()); 44 } 45 46 @Test testParseOverwrite()47 public void testParseOverwrite() { 48 DistCpOptions options = OptionsParser.parse(new String[] { 49 "hdfs://localhost:8020/source/first", 50 "hdfs://localhost:8020/target/"}); 51 Assert.assertFalse(options.shouldOverwrite()); 52 53 options = OptionsParser.parse(new String[] { 54 "-overwrite", 55 "hdfs://localhost:8020/source/first", 56 "hdfs://localhost:8020/target/"}); 57 Assert.assertTrue(options.shouldOverwrite()); 58 59 try { 60 OptionsParser.parse(new String[] { 61 "-update", 62 "-overwrite", 63 "hdfs://localhost:8020/source/first", 64 "hdfs://localhost:8020/target/"}); 65 Assert.fail("Update and overwrite aren't allowed together"); 66 } catch (IllegalArgumentException ignore) { 67 } 68 } 69 70 @Test testLogPath()71 public void testLogPath() { 72 DistCpOptions options = OptionsParser.parse(new String[] { 73 "hdfs://localhost:8020/source/first", 74 "hdfs://localhost:8020/target/"}); 75 Assert.assertNull(options.getLogPath()); 76 77 options = OptionsParser.parse(new String[] { 78 "-log", 79 "hdfs://localhost:8020/logs", 80 "hdfs://localhost:8020/source/first", 81 "hdfs://localhost:8020/target/"}); 82 Assert.assertEquals(options.getLogPath(), new Path("hdfs://localhost:8020/logs")); 83 } 84 85 @Test testParseBlokcing()86 public void testParseBlokcing() { 87 DistCpOptions options = OptionsParser.parse(new String[] { 88 "hdfs://localhost:8020/source/first", 89 "hdfs://localhost:8020/target/"}); 90 Assert.assertTrue(options.shouldBlock()); 91 92 options = OptionsParser.parse(new String[] { 93 "-async", 94 "hdfs://localhost:8020/source/first", 95 "hdfs://localhost:8020/target/"}); 96 Assert.assertFalse(options.shouldBlock()); 97 } 98 99 @Test testParsebandwidth()100 public void testParsebandwidth() { 101 DistCpOptions options = OptionsParser.parse(new String[] { 102 "hdfs://localhost:8020/source/first", 103 "hdfs://localhost:8020/target/"}); 104 Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB); 105 106 options = OptionsParser.parse(new String[] { 107 "-bandwidth", 108 "11", 109 "hdfs://localhost:8020/source/first", 110 "hdfs://localhost:8020/target/"}); 111 Assert.assertEquals(options.getMapBandwidth(), 11); 112 } 113 114 @Test(expected=IllegalArgumentException.class) testParseNonPositiveBandwidth()115 public void testParseNonPositiveBandwidth() { 116 OptionsParser.parse(new String[] { 117 "-bandwidth", 118 "-11", 119 "hdfs://localhost:8020/source/first", 120 "hdfs://localhost:8020/target/"}); 121 } 122 123 @Test(expected=IllegalArgumentException.class) testParseZeroBandwidth()124 public void testParseZeroBandwidth() { 125 OptionsParser.parse(new String[] { 126 "-bandwidth", 127 "0", 128 "hdfs://localhost:8020/source/first", 129 "hdfs://localhost:8020/target/"}); 130 } 131 132 @Test testParseSkipCRC()133 public void testParseSkipCRC() { 134 DistCpOptions options = OptionsParser.parse(new String[] { 135 "hdfs://localhost:8020/source/first", 136 "hdfs://localhost:8020/target/"}); 137 Assert.assertFalse(options.shouldSkipCRC()); 138 139 options = OptionsParser.parse(new String[] { 140 "-update", 141 "-skipcrccheck", 142 "hdfs://localhost:8020/source/first", 143 "hdfs://localhost:8020/target/"}); 144 Assert.assertTrue(options.shouldSyncFolder()); 145 Assert.assertTrue(options.shouldSkipCRC()); 146 } 147 148 @Test testParseAtomicCommit()149 public void testParseAtomicCommit() { 150 DistCpOptions options = OptionsParser.parse(new String[] { 151 "hdfs://localhost:8020/source/first", 152 "hdfs://localhost:8020/target/"}); 153 Assert.assertFalse(options.shouldAtomicCommit()); 154 155 options = OptionsParser.parse(new String[] { 156 "-atomic", 157 "hdfs://localhost:8020/source/first", 158 "hdfs://localhost:8020/target/"}); 159 Assert.assertTrue(options.shouldAtomicCommit()); 160 161 try { 162 OptionsParser.parse(new String[] { 163 "-atomic", 164 "-update", 165 "hdfs://localhost:8020/source/first", 166 "hdfs://localhost:8020/target/"}); 167 Assert.fail("Atomic and sync folders were allowed"); 168 } catch (IllegalArgumentException ignore) { } 169 } 170 171 @Test testParseWorkPath()172 public void testParseWorkPath() { 173 DistCpOptions options = OptionsParser.parse(new String[] { 174 "hdfs://localhost:8020/source/first", 175 "hdfs://localhost:8020/target/"}); 176 Assert.assertNull(options.getAtomicWorkPath()); 177 178 options = OptionsParser.parse(new String[] { 179 "-atomic", 180 "hdfs://localhost:8020/source/first", 181 "hdfs://localhost:8020/target/"}); 182 Assert.assertNull(options.getAtomicWorkPath()); 183 184 options = OptionsParser.parse(new String[] { 185 "-atomic", 186 "-tmp", 187 "hdfs://localhost:8020/work", 188 "hdfs://localhost:8020/source/first", 189 "hdfs://localhost:8020/target/"}); 190 Assert.assertEquals(options.getAtomicWorkPath(), new Path("hdfs://localhost:8020/work")); 191 192 try { 193 OptionsParser.parse(new String[] { 194 "-tmp", 195 "hdfs://localhost:8020/work", 196 "hdfs://localhost:8020/source/first", 197 "hdfs://localhost:8020/target/"}); 198 Assert.fail("work path was allowed without -atomic switch"); 199 } catch (IllegalArgumentException ignore) {} 200 } 201 202 @Test testParseSyncFolders()203 public void testParseSyncFolders() { 204 DistCpOptions options = OptionsParser.parse(new String[] { 205 "hdfs://localhost:8020/source/first", 206 "hdfs://localhost:8020/target/"}); 207 Assert.assertFalse(options.shouldSyncFolder()); 208 209 options = OptionsParser.parse(new String[] { 210 "-update", 211 "hdfs://localhost:8020/source/first", 212 "hdfs://localhost:8020/target/"}); 213 Assert.assertTrue(options.shouldSyncFolder()); 214 } 215 216 @Test testParseDeleteMissing()217 public void testParseDeleteMissing() { 218 DistCpOptions options = OptionsParser.parse(new String[] { 219 "hdfs://localhost:8020/source/first", 220 "hdfs://localhost:8020/target/"}); 221 Assert.assertFalse(options.shouldDeleteMissing()); 222 223 options = OptionsParser.parse(new String[] { 224 "-update", 225 "-delete", 226 "hdfs://localhost:8020/source/first", 227 "hdfs://localhost:8020/target/"}); 228 Assert.assertTrue(options.shouldSyncFolder()); 229 Assert.assertTrue(options.shouldDeleteMissing()); 230 231 options = OptionsParser.parse(new String[] { 232 "-overwrite", 233 "-delete", 234 "hdfs://localhost:8020/source/first", 235 "hdfs://localhost:8020/target/"}); 236 Assert.assertTrue(options.shouldOverwrite()); 237 Assert.assertTrue(options.shouldDeleteMissing()); 238 239 try { 240 OptionsParser.parse(new String[] { 241 "-atomic", 242 "-delete", 243 "hdfs://localhost:8020/source/first", 244 "hdfs://localhost:8020/target/"}); 245 Assert.fail("Atomic and delete folders were allowed"); 246 } catch (IllegalArgumentException ignore) { } 247 } 248 249 @Test testParseSSLConf()250 public void testParseSSLConf() { 251 DistCpOptions options = OptionsParser.parse(new String[] { 252 "hdfs://localhost:8020/source/first", 253 "hdfs://localhost:8020/target/"}); 254 Assert.assertNull(options.getSslConfigurationFile()); 255 256 options = OptionsParser.parse(new String[] { 257 "-mapredSslConf", 258 "/tmp/ssl-client.xml", 259 "hdfs://localhost:8020/source/first", 260 "hdfs://localhost:8020/target/"}); 261 Assert.assertEquals(options.getSslConfigurationFile(), "/tmp/ssl-client.xml"); 262 } 263 264 @Test testParseMaps()265 public void testParseMaps() { 266 DistCpOptions options = OptionsParser.parse(new String[] { 267 "hdfs://localhost:8020/source/first", 268 "hdfs://localhost:8020/target/"}); 269 Assert.assertEquals(options.getMaxMaps(), DistCpConstants.DEFAULT_MAPS); 270 271 options = OptionsParser.parse(new String[] { 272 "-m", 273 "1", 274 "hdfs://localhost:8020/source/first", 275 "hdfs://localhost:8020/target/"}); 276 Assert.assertEquals(options.getMaxMaps(), 1); 277 278 options = OptionsParser.parse(new String[] { 279 "-m", 280 "0", 281 "hdfs://localhost:8020/source/first", 282 "hdfs://localhost:8020/target/"}); 283 Assert.assertEquals(options.getMaxMaps(), 1); 284 285 try { 286 OptionsParser.parse(new String[] { 287 "-m", 288 "hello", 289 "hdfs://localhost:8020/source/first", 290 "hdfs://localhost:8020/target/"}); 291 Assert.fail("Non numberic map parsed"); 292 } catch (IllegalArgumentException ignore) { } 293 294 try { 295 OptionsParser.parse(new String[] { 296 "-mapredXslConf", 297 "hdfs://localhost:8020/source/first", 298 "hdfs://localhost:8020/target/"}); 299 Assert.fail("Non numberic map parsed"); 300 } catch (IllegalArgumentException ignore) { } 301 } 302 303 @Test testSourceListing()304 public void testSourceListing() { 305 DistCpOptions options = OptionsParser.parse(new String[] { 306 "-f", 307 "hdfs://localhost:8020/source/first", 308 "hdfs://localhost:8020/target/"}); 309 Assert.assertEquals(options.getSourceFileListing(), 310 new Path("hdfs://localhost:8020/source/first")); 311 } 312 313 @Test testSourceListingAndSourcePath()314 public void testSourceListingAndSourcePath() { 315 try { 316 OptionsParser.parse(new String[] { 317 "-f", 318 "hdfs://localhost:8020/source/first", 319 "hdfs://localhost:8020/source/first", 320 "hdfs://localhost:8020/target/"}); 321 Assert.fail("Both source listing & source paths allowed"); 322 } catch (IllegalArgumentException ignore) {} 323 } 324 325 @Test testMissingSourceInfo()326 public void testMissingSourceInfo() { 327 try { 328 OptionsParser.parse(new String[] { 329 "hdfs://localhost:8020/target/"}); 330 Assert.fail("Neither source listing not source paths present"); 331 } catch (IllegalArgumentException ignore) {} 332 } 333 334 @Test testMissingTarget()335 public void testMissingTarget() { 336 try { 337 OptionsParser.parse(new String[] { 338 "-f", "hdfs://localhost:8020/source"}); 339 Assert.fail("Missing target allowed"); 340 } catch (IllegalArgumentException ignore) {} 341 } 342 343 @Test testInvalidArgs()344 public void testInvalidArgs() { 345 try { 346 OptionsParser.parse(new String[] { 347 "-m", "-f", "hdfs://localhost:8020/source"}); 348 Assert.fail("Missing map value"); 349 } catch (IllegalArgumentException ignore) {} 350 } 351 352 @Test testToString()353 public void testToString() { 354 DistCpOptions option = new DistCpOptions(new Path("abc"), new Path("xyz")); 355 String val = "DistCpOptions{atomicCommit=false, syncFolder=false, deleteMissing=false, " + 356 "ignoreFailures=false, maxMaps=20, sslConfigurationFile='null', copyStrategy='uniformsize', " + 357 "sourceFileListing=abc, sourcePaths=null, targetPath=xyz}"; 358 Assert.assertEquals(val, option.toString()); 359 Assert.assertNotSame(DistCpOptionSwitch.ATOMIC_COMMIT.toString(), 360 DistCpOptionSwitch.ATOMIC_COMMIT.name()); 361 } 362 363 @Test testCopyStrategy()364 public void testCopyStrategy() { 365 DistCpOptions options = OptionsParser.parse(new String[] { 366 "-strategy", 367 "dynamic", 368 "-f", 369 "hdfs://localhost:8020/source/first", 370 "hdfs://localhost:8020/target/"}); 371 Assert.assertEquals(options.getCopyStrategy(), "dynamic"); 372 373 options = OptionsParser.parse(new String[] { 374 "-f", 375 "hdfs://localhost:8020/source/first", 376 "hdfs://localhost:8020/target/"}); 377 Assert.assertEquals(options.getCopyStrategy(), DistCpConstants.UNIFORMSIZE); 378 } 379 380 @Test testTargetPath()381 public void testTargetPath() { 382 DistCpOptions options = OptionsParser.parse(new String[] { 383 "-f", 384 "hdfs://localhost:8020/source/first", 385 "hdfs://localhost:8020/target/"}); 386 Assert.assertEquals(options.getTargetPath(), new Path("hdfs://localhost:8020/target/")); 387 } 388 389 @Test testPreserve()390 public void testPreserve() { 391 DistCpOptions options = OptionsParser.parse(new String[] { 392 "-f", 393 "hdfs://localhost:8020/source/first", 394 "hdfs://localhost:8020/target/"}); 395 Assert.assertFalse(options.shouldPreserve(FileAttribute.BLOCKSIZE)); 396 Assert.assertFalse(options.shouldPreserve(FileAttribute.REPLICATION)); 397 Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); 398 Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); 399 Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); 400 401 options = OptionsParser.parse(new String[] { 402 "-p", 403 "-f", 404 "hdfs://localhost:8020/source/first", 405 "hdfs://localhost:8020/target/"}); 406 Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); 407 Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); 408 Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); 409 Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); 410 Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); 411 412 options = OptionsParser.parse(new String[] { 413 "-p", 414 "hdfs://localhost:8020/source/first", 415 "hdfs://localhost:8020/target/"}); 416 Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); 417 Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); 418 Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); 419 Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); 420 Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); 421 422 options = OptionsParser.parse(new String[] { 423 "-pbr", 424 "-f", 425 "hdfs://localhost:8020/source/first", 426 "hdfs://localhost:8020/target/"}); 427 Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); 428 Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); 429 Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); 430 Assert.assertFalse(options.shouldPreserve(FileAttribute.USER)); 431 Assert.assertFalse(options.shouldPreserve(FileAttribute.GROUP)); 432 433 options = OptionsParser.parse(new String[] { 434 "-pbrgup", 435 "-f", 436 "hdfs://localhost:8020/source/first", 437 "hdfs://localhost:8020/target/"}); 438 Assert.assertTrue(options.shouldPreserve(FileAttribute.BLOCKSIZE)); 439 Assert.assertTrue(options.shouldPreserve(FileAttribute.REPLICATION)); 440 Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); 441 Assert.assertTrue(options.shouldPreserve(FileAttribute.USER)); 442 Assert.assertTrue(options.shouldPreserve(FileAttribute.GROUP)); 443 444 options = OptionsParser.parse(new String[] { 445 "-p", 446 "-f", 447 "hdfs://localhost:8020/source/first", 448 "hdfs://localhost:8020/target/"}); 449 int i = 0; 450 Iterator<FileAttribute> attribIterator = options.preserveAttributes(); 451 while (attribIterator.hasNext()) { 452 attribIterator.next(); 453 i++; 454 } 455 Assert.assertEquals(i, 5); 456 457 try { 458 OptionsParser.parse(new String[] { 459 "-pabc", 460 "-f", 461 "hdfs://localhost:8020/source/first", 462 "hdfs://localhost:8020/target"}); 463 Assert.fail("Invalid preserve attribute"); 464 } 465 catch (IllegalArgumentException ignore) {} 466 catch (NoSuchElementException ignore) {} 467 468 options = OptionsParser.parse(new String[] { 469 "-f", 470 "hdfs://localhost:8020/source/first", 471 "hdfs://localhost:8020/target/"}); 472 Assert.assertFalse(options.shouldPreserve(FileAttribute.PERMISSION)); 473 options.preserve(FileAttribute.PERMISSION); 474 Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); 475 476 options.preserve(FileAttribute.PERMISSION); 477 Assert.assertTrue(options.shouldPreserve(FileAttribute.PERMISSION)); 478 } 479 480 @Test testOptionsSwitchAddToConf()481 public void testOptionsSwitchAddToConf() { 482 Configuration conf = new Configuration(); 483 Assert.assertNull(conf.get(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel())); 484 DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.ATOMIC_COMMIT); 485 Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); 486 } 487 488 @Test testOptionsAppendToConf()489 public void testOptionsAppendToConf() { 490 Configuration conf = new Configuration(); 491 Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); 492 Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); 493 DistCpOptions options = OptionsParser.parse(new String[] { 494 "-atomic", 495 "-i", 496 "hdfs://localhost:8020/source/first", 497 "hdfs://localhost:8020/target/"}); 498 options.appendToConf(conf); 499 Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false)); 500 Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false)); 501 Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 502 DistCpConstants.DEFAULT_BANDWIDTH_MB); 503 504 conf = new Configuration(); 505 Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); 506 Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false)); 507 Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), null); 508 options = OptionsParser.parse(new String[] { 509 "-update", 510 "-delete", 511 "-pu", 512 "-bandwidth", 513 "11", 514 "hdfs://localhost:8020/source/first", 515 "hdfs://localhost:8020/target/"}); 516 options.appendToConf(conf); 517 Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false)); 518 Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.DELETE_MISSING.getConfigLabel(), false)); 519 Assert.assertEquals(conf.get(DistCpOptionSwitch.PRESERVE_STATUS.getConfigLabel()), "U"); 520 Assert.assertEquals(conf.getInt(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 11); 521 } 522 } 523