| /dports/devel/hadoop/hadoop-1.2.1/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/ |
| H A D | TestDistCacheEmulation.java | 81 FileSystem fs = FileSystem.get(jobConf); in validateDistCacheData()
184 JobConf jobConf = in runSetupGenerateDistCacheData() local
188 FileSystem fs = FileSystem.get(jobConf); in runSetupGenerateDistCacheData()
201 resetDistCacheConfigProperties(jobConf); in runSetupGenerateDistCacheData()
202 return jobConf; in runSetupGenerateDistCacheData()
230 JobConf jobConf = in testGenerateDistCacheData() local
260 sumOfFileSizes, jobConf.getLong( in validateSetupGenDC()
381 jobConf.setBoolean( in testDistCacheEmulationConfigurability()
427 6, jobConf.getStrings( in validateJobConfWithDCFiles()
448 JobConf jobConf = new JobConf(); in validateWithOutVisibilities() local
[all …]
|
| /dports/devel/hadoop/hadoop-1.2.1/src/contrib/index/src/java/org/apache/hadoop/contrib/index/mapred/ |
| H A D | IndexUpdater.java | 51 JobConf jobConf = in run() local
53 JobClient.runJob(jobConf); in run()
75 jobConf.setJobName(this.getClass().getName() + "_" in createJob()
79 FileInputFormat.setInputPaths(jobConf, inputPaths); in createJob()
80 FileOutputFormat.setOutputPath(jobConf, outputPath); in createJob()
82 jobConf.setNumMapTasks(numMapTasks); in createJob()
85 jobConf.setNumReduceTasks(shards.length); in createJob()
111 jobConf.setMapperClass(IndexUpdateMapper.class); in createJob()
113 jobConf.setCombinerClass(IndexUpdateCombiner.class); in createJob()
114 jobConf.setReducerClass(IndexUpdateReducer.class); in createJob()
[all …]
|
| /dports/databases/hbase/hbase-1.2.1/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/ |
| H A D | TestTableMapReduceUtil.java | 142 JobConf jobConf = new JobConf(cfg); in shouldNumberOfReduceTaskNotExceedNumberOfRegionsForGivenTable() local
149 jobConf.setNumReduceTasks(10); in shouldNumberOfReduceTaskNotExceedNumberOfRegionsForGivenTable()
159 JobConf jobConf = new JobConf(cfg); in shouldNumberOfMapTaskNotExceedNumberOfRegionsForGivenTable() local
164 jobConf.setNumMapTasks(10); in shouldNumberOfMapTaskNotExceedNumberOfRegionsForGivenTable()
174 JobConf jobConf = new JobConf(cfg); in shoudBeValidMapReduceEvaluation() local
177 jobConf.setNumReduceTasks(1); in shoudBeValidMapReduceEvaluation()
180 jobConf); in shoudBeValidMapReduceEvaluation()
186 if (jobConf != null) in shoudBeValidMapReduceEvaluation()
199 jobConf.setNumReduceTasks(2); in shoudBeValidMapReduceWithPartitionerEvaluation()
202 jobConf); in shoudBeValidMapReduceWithPartitionerEvaluation()
[all …]
|
| /dports/devel/hadoop2/hadoop-2.7.2-src/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/ |
| H A D | TestDistCacheEmulation.java | 89 String filesListFile = jobConf in validateDistCacheData()
91 FileSystem fs = FileSystem.get(jobConf); in validateDistCacheData()
207 FileSystem fs = FileSystem.get(jobConf); in runSetupGenerateDistCacheData()
221 resetDistCacheConfigProperties(jobConf); in runSetupGenerateDistCacheData()
222 return jobConf; in runSetupGenerateDistCacheData()
239 jobConf.setStrings("mapred.cache.files", ""); in resetDistCacheConfigProperties()
277 FileSystem fs = FileSystem.get(jobConf); in validateSetupGenDC()
389 FileSystem fs = FileSystem.get(jobConf); in testDistCacheEmulationConfigurability()
399 jobConf.setBoolean( in testDistCacheEmulationConfigurability()
419 JobConf jobConf = new JobConf(conf); in testDistCacheEmulator() local
[all …]
|
| /dports/devel/hadoop/hadoop-1.2.1/src/examples/org/apache/hadoop/examples/ |
| H A D | Sort.java | 77 jobConf.setJobName("sorter"); in run()
80 jobConf.setReducerClass(IdentityReducer.class); in run()
82 JobClient client = new JobClient(jobConf); in run()
137 jobConf.setNumReduceTasks(num_reduces); in run()
139 jobConf.setInputFormat(inputFormatClass); in run()
140 jobConf.setOutputFormat(outputFormatClass); in run()
142 jobConf.setOutputKeyClass(outputKeyClass); in run()
143 jobConf.setOutputValueClass(outputValueClass); in run()
165 DistributedCache.createSymlink(jobConf); in run()
172 FileOutputFormat.getOutputPath(jobConf) + in run()
[all …]
|
| H A D | Join.java | 72 jobConf.setJobName("join"); in run()
75 jobConf.setReducerClass(IdentityReducer.class); in run()
77 JobClient client = new JobClient(jobConf); in run()
129 jobConf.setNumMapTasks(num_maps); in run()
130 jobConf.setNumReduceTasks(num_reduces); in run()
137 FileOutputFormat.setOutputPath(jobConf, in run()
144 jobConf.setInputFormat(CompositeInputFormat.class); in run()
147 jobConf.setOutputFormat(outputFormatClass); in run()
149 jobConf.setOutputKeyClass(outputKeyClass); in run()
150 jobConf.setOutputValueClass(outputValueClass); in run()
[all …]
|
| H A D | PiEstimator.java | 249 jobConf.setOutputValueClass(LongWritable.class); in estimate()
252 jobConf.setMapperClass(PiMapper.class); in estimate()
253 jobConf.setNumMapTasks(numMaps); in estimate()
255 jobConf.setReducerClass(PiReducer.class); in estimate()
256 jobConf.setNumReduceTasks(1); in estimate()
260 jobConf.setSpeculativeExecution(false); in estimate()
265 FileInputFormat.setInputPaths(jobConf, inDir); in estimate()
268 final FileSystem fs = FileSystem.get(jobConf); in estimate()
284 fs, jobConf, file, in estimate()
297 JobClient.runJob(jobConf); in estimate()
[all …]
|
| /dports/devel/hadoop/hadoop-1.2.1/src/test/org/apache/hadoop/mapred/ |
| H A D | TestNodeGroupAwareTaskPlacement.java | 95 JobConf jobConf = mr.createJobConf(jobConfig); in launchJobAndTestCounters() local
163 JobConf jobConf = new JobConf(conf); in testTaskPlacement() local
196 nodeGroup3, hosts3, jobConf); in testTaskPlacement()
205 0, 0, 2, jobConf); in testTaskPlacement()
211 nodeGroup4, hosts4, jobConf); in testTaskPlacement()
224 0, 3, 0, jobConf); in testTaskPlacement()
238 jobConf.setJobName(jobName); in launchJob()
243 jobConf.setMapperClass(IdentityMapper.class); in launchJob()
247 jobConf.setNumMapTasks(numMaps); in launchJob()
248 jobConf.setNumReduceTasks(0); in launchJob()
[all …]
|
| H A D | MRBench.java | 144 JobConf jobConf = new JobConf(getConf()); in setupJob() local
145 jobConf.setJarByClass(MRBench.class); in setupJob()
151 jobConf.setOutputValueClass(UTF8.class); in setupJob()
157 jobConf.setJar(jarFile); in setupJob()
159 jobConf.setMapperClass(Map.class); in setupJob()
160 jobConf.setReducerClass(Reduce.class); in setupJob()
162 jobConf.setNumMapTasks(numMaps); in setupJob()
163 jobConf.setNumReduceTasks(numReduces); in setupJob()
164 jobConf in setupJob()
166 return jobConf; in setupJob()
[all …]
|
| H A D | TestInputPath.java | 29 JobConf jobConf = new JobConf(); in testInputPath() local
30 Path workingDir = jobConf.getWorkingDirectory(); in testInputPath()
34 FileInputFormat.setInputPaths(jobConf, path); in testInputPath()
46 FileInputFormat.setInputPaths(jobConf, path); in testInputPath()
47 paths = FileInputFormat.getInputPaths(jobConf); in testInputPath()
58 FileInputFormat.setInputPaths(jobConf, path); in testInputPath()
59 FileInputFormat.addInputPath(jobConf, path1); in testInputPath()
60 paths = FileInputFormat.getInputPaths(jobConf); in testInputPath()
66 paths = FileInputFormat.getInputPaths(jobConf); in testInputPath()
72 FileInputFormat.setInputPaths(jobConf, input); in testInputPath()
[all …]
|
| H A D | TestRackAwareTaskPlacement.java | 72 JobConf jobConf = mr.createJobConf(); in launchJobAndTestCounters() local
166 jobConf.setJobName(jobName); in launchJob()
169 FileInputFormat.setInputPaths(jobConf, inDir); in launchJob()
171 jobConf.setMapperClass(IdentityMapper.class); in launchJob()
172 jobConf.setReducerClass(IdentityReducer.class); in launchJob()
173 jobConf.setOutputKeyClass(BytesWritable.class); in launchJob()
174 jobConf.setOutputValueClass(BytesWritable.class); in launchJob()
175 jobConf.setNumMapTasks(numMaps); in launchJob()
176 jobConf.setNumReduceTasks(0); in launchJob()
177 jobConf.setJar("build/test/testjar/testjob.jar"); in launchJob()
[all …]
|
| H A D | TestCompressedEmptyMapOutputs.java | 75 jobConf.setJobName("null-sorter"); in runSort()
77 jobConf.setMapperClass(SinkMapper.class); in runSort()
78 jobConf.setReducerClass(IdentityReducer.class); in runSort()
80 jobConf.setNumReduceTasks(2); in runSort()
82 jobConf.setInputFormat(SequenceFileInputFormat.class); in runSort()
85 jobConf.setOutputKeyClass(BytesWritable.class); in runSort()
86 jobConf.setOutputValueClass(BytesWritable.class); in runSort()
88 FileInputFormat.setInputPaths(jobConf, sortInput); in runSort()
89 FileOutputFormat.setOutputPath(jobConf, sortOutput); in runSort()
92 jobConf.setCompressMapOutput(true); in runSort()
[all …]
|
| H A D | SortValidator.java | 334 jobConf.setMapperClass(Map.class); in checkRecords()
335 jobConf.setCombinerClass(Reduce.class); in checkRecords()
336 jobConf.setReducerClass(Reduce.class); in checkRecords()
338 jobConf.setNumMapTasks(noSortReduceTasks); in checkRecords()
339 jobConf.setNumReduceTasks(1); in checkRecords()
362 JobClient.runJob(jobConf); in checkRecords()
464 jobConf.setMapperClass(Map.class); in checkRecords()
465 jobConf.setReducerClass(Reduce.class); in checkRecords()
481 jobConf.setNumMapTasks(noMaps); in checkRecords()
482 jobConf.setNumReduceTasks(noReduces); in checkRecords()
[all …]
|
| H A D | TestJobEndNotifier.java | 133 JobConf jobConf = createTestJobConf( in testUriSubstitution() local
162 JobConf jobConf = createTestJobConf( in testRetryCount() local
192 JobConf jobConf = createTestJobConf( in testNotificationTimeout() local
197 jobConf.setJobEndNotificationURI(baseUrl + "jobend"); in testNotificationTimeout()
220 JobConf jobConf = createTestJobConf( in testLocalJobRunnerUriSubstitution() local
241 JobConf jobConf = createTestJobConf( in testLocalJobRunnerRetryCount() local
257 JobConf jobConf = new JobConf(conf); in createTestJobConf() local
258 jobConf.setInt("job.end.retry.attempts", retryAttempts); in createTestJobConf()
259 jobConf.set("job.end.retry.interval", "0"); in createTestJobConf()
260 jobConf.setJobEndNotificationURI(notificationUri); in createTestJobConf()
[all …]
|
| H A D | TestLimitTasksPerJobTaskScheduler.java | 27 protected JobConf jobConf; field in TestLimitTasksPerJobTaskScheduler
34 jobConf = new JobConf(); in setUp()
35 jobConf.setNumMapTasks(10); in setUp()
36 jobConf.setNumReduceTasks(10); in setUp()
39 scheduler.setConf(jobConf); in setUp()
56 jobConf.setLong(LimitTasksPerJobTaskScheduler.MAX_TASKS_PER_JOB_PROPERTY, in testMaxRunningTasksPerJob()
58 scheduler.setConf(jobConf); in testMaxRunningTasksPerJob()
59 TestJobQueueTaskScheduler.submitJobs(taskTrackerManager, jobConf, in testMaxRunningTasksPerJob()
91 jobConf.setLong(LimitTasksPerJobTaskScheduler.MAX_TASKS_PER_JOB_PROPERTY, in testMaxRunningTasksPerJobWithInterleavedTrackers()
93 scheduler.setConf(jobConf); in testMaxRunningTasksPerJobWithInterleavedTrackers()
[all …]
|
| /dports/devel/hadoop2/hadoop-2.7.2-src/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/ |
| H A D | TestMRFramework.java | 31 JobConf jobConf = new JobConf(); in testFramework() local
32 jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME); in testFramework()
33 jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME); in testFramework()
35 StreamUtil.isLocalJobTracker(jobConf)); in testFramework()
37 jobConf.set(JTConfig.JT_IPC_ADDRESS, MRConfig.LOCAL_FRAMEWORK_NAME); in testFramework()
38 jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME); in testFramework()
40 StreamUtil.isLocalJobTracker(jobConf)); in testFramework()
42 jobConf.set(JTConfig.JT_IPC_ADDRESS, "jthost:9090"); in testFramework()
43 jobConf.set(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME); in testFramework()
45 StreamUtil.isLocalJobTracker(jobConf)); in testFramework()
|
| /dports/devel/hadoop2/hadoop-2.7.2-src/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ |
| H A D | MRBench.java | 144 JobConf jobConf = new JobConf(getConf()); in setupJob() local
145 jobConf.setJarByClass(MRBench.class); in setupJob()
151 jobConf.setOutputValueClass(UTF8.class); in setupJob()
157 jobConf.setJar(jarFile); in setupJob()
159 jobConf.setMapperClass(Map.class); in setupJob()
160 jobConf.setReducerClass(Reduce.class); in setupJob()
162 jobConf.setNumMapTasks(numMaps); in setupJob()
163 jobConf.setNumReduceTasks(numReduces); in setupJob()
164 jobConf in setupJob()
166 return jobConf; in setupJob()
[all …]
|
| H A D | TestInputPath.java | 29 JobConf jobConf = new JobConf(); in testInputPath() local
30 Path workingDir = jobConf.getWorkingDirectory(); in testInputPath()
34 FileInputFormat.setInputPaths(jobConf, path); in testInputPath()
46 FileInputFormat.setInputPaths(jobConf, path); in testInputPath()
47 paths = FileInputFormat.getInputPaths(jobConf); in testInputPath()
58 FileInputFormat.setInputPaths(jobConf, path); in testInputPath()
59 FileInputFormat.addInputPath(jobConf, path1); in testInputPath()
60 paths = FileInputFormat.getInputPaths(jobConf); in testInputPath()
66 paths = FileInputFormat.getInputPaths(jobConf); in testInputPath()
72 FileInputFormat.setInputPaths(jobConf, input); in testInputPath()
[all …]
|
| H A D | SortValidator.java | 338 jobConf.setMapperClass(Map.class); in checkRecords()
339 jobConf.setCombinerClass(Reduce.class); in checkRecords()
340 jobConf.setReducerClass(Reduce.class); in checkRecords()
342 jobConf.setNumMapTasks(noSortReduceTasks); in checkRecords()
343 jobConf.setNumReduceTasks(1); in checkRecords()
367 JobClient.runJob(jobConf); in checkRecords()
483 jobConf.setReducerClass(Reduce.class); in checkRecords()
489 jobConf.getInt(MAPS_PER_HOST, 10); in checkRecords()
499 jobConf.setNumMapTasks(noMaps); in checkRecords()
500 jobConf.setNumReduceTasks(noReduces); in checkRecords()
[all …]
|
| H A D | TestMultipleLevelCaching.java | 153 JobConf jobConf = mr.createJobConf(); in launchJobAndTestCounters() local
173 jobConf.setJobName(jobName); in launchJob()
176 FileInputFormat.setInputPaths(jobConf, inDir); in launchJob()
177 FileOutputFormat.setOutputPath(jobConf, outputPath); in launchJob()
178 jobConf.setMapperClass(IdentityMapper.class); in launchJob()
179 jobConf.setReducerClass(IdentityReducer.class); in launchJob()
180 jobConf.setOutputKeyClass(BytesWritable.class); in launchJob()
181 jobConf.setOutputValueClass(BytesWritable.class); in launchJob()
182 jobConf.setNumMapTasks(numMaps); in launchJob()
183 jobConf.setNumReduceTasks(0); in launchJob()
[all …]
|
| /dports/devel/hadoop/hadoop-1.2.1/src/test/system/java/org/apache/hadoop/mapred/ |
| H A D | TestTaskChildsKilling.java | 104 JobConf jobConf = new JobConf(conf); in testProcessTreeCleanupOfKilledTask1() local
108 jobConf.setNumMapTasks(1); in testProcessTreeCleanupOfKilledTask1()
109 jobConf.setNumReduceTasks(0); in testProcessTreeCleanupOfKilledTask1()
110 jobConf.setMaxMapAttempts(1); in testProcessTreeCleanupOfKilledTask1()
216 JobConf jobConf = new JobConf(conf); in testProcessTreeCleanupOfKilledTask2() local
220 jobConf.setNumMapTasks(1); in testProcessTreeCleanupOfKilledTask2()
221 jobConf.setNumReduceTasks(0); in testProcessTreeCleanupOfKilledTask2()
324 jobConf.setNumMapTasks(1); in testProcessTreeCleanupOfFailedTask1()
325 jobConf.setNumReduceTasks(0); in testProcessTreeCleanupOfFailedTask1()
427 jobConf.setNumMapTasks(1); in testProcessTreeCleanupOfFailedTask2()
[all …]
|
| H A D | TestChildsKillingOfMemoryExceedsTask.java | 117 JobConf jobConf = new JobConf(conf); in testProcessTreeCleanupAfterJobKilled() local
118 jobConf.setJobName("String Appending"); in testProcessTreeCleanupAfterJobKilled()
121 jobConf.setNumMapTasks(1); in testProcessTreeCleanupAfterJobKilled()
122 jobConf.setNumReduceTasks(0); in testProcessTreeCleanupAfterJobKilled()
126 jobConf.setMemoryForMapTask(PER_TASK_LIMIT); in testProcessTreeCleanupAfterJobKilled()
132 RunningJob runJob = client.submitJob(jobConf); in testProcessTreeCleanupAfterJobKilled()
235 JobConf jobConf = new JobConf(conf); in testProcessTreeCleanupOfFailedTask() local
236 jobConf.setJobName("String Appending"); in testProcessTreeCleanupOfFailedTask()
239 jobConf.setNumMapTasks(1); in testProcessTreeCleanupOfFailedTask()
240 jobConf.setNumReduceTasks(0); in testProcessTreeCleanupOfFailedTask()
[all …]
|
| /dports/devel/hadoop/hadoop-1.2.1/src/mapred/org/apache/hadoop/mapred/ |
| H A D | ReduceTaskRunner.java | 54 public String getChildJavaOpts(JobConf jobConf, String defaultValue) { in getChildJavaOpts() argument
56 jobConf.get(JobConf.MAPRED_REDUCE_TASK_JAVA_OPTS, in getChildJavaOpts()
57 super.getChildJavaOpts(jobConf, in getChildJavaOpts()
59 String admin = jobConf.get(TaskRunner.MAPRED_REDUCE_ADMIN_JAVA_OPTS, in getChildJavaOpts()
65 public int getChildUlimit(JobConf jobConf) { in getChildUlimit() argument
66 return jobConf.getInt(JobConf.MAPRED_REDUCE_TASK_ULIMIT, in getChildUlimit()
67 super.getChildUlimit(jobConf)); in getChildUlimit()
71 public String getChildEnv(JobConf jobConf) { in getChildEnv() argument
72 return jobConf.get(JobConf.MAPRED_REDUCE_TASK_ENV, in getChildEnv()
73 super.getChildEnv(jobConf)); in getChildEnv()
|
| H A D | MapTaskRunner.java | 50 public String getChildJavaOpts(JobConf jobConf, String defaultValue) { in getChildJavaOpts() argument
52 jobConf.get(JobConf.MAPRED_MAP_TASK_JAVA_OPTS, in getChildJavaOpts()
53 super.getChildJavaOpts(jobConf, in getChildJavaOpts()
56 jobConf.get(TaskRunner.MAPRED_MAP_ADMIN_JAVA_OPTS, in getChildJavaOpts()
62 public int getChildUlimit(JobConf jobConf) { in getChildUlimit() argument
63 return jobConf.getInt(JobConf.MAPRED_MAP_TASK_ULIMIT, in getChildUlimit()
64 super.getChildUlimit(jobConf)); in getChildUlimit()
68 public String getChildEnv(JobConf jobConf) { in getChildEnv() argument
69 return jobConf.get(JobConf.MAPRED_MAP_TASK_ENV, super.getChildEnv(jobConf)); in getChildEnv()
|
| /dports/devel/hadoop/hadoop-1.2.1/src/mapred/org/apache/hadoop/mapred/lib/ |
| H A D | Chain.java | 134 jobConf = new JobConf(jobConf); in getChainElementConf()
137 jobConf.set(entry.getKey(), entry.getValue()); in getChainElementConf()
139 return jobConf; in getChainElementConf()
173 if (jobConf.getClass(prefix + CHAIN_REDUCER_CLASS, in addMapper()
180 int index = jobConf.getInt(prefix + CHAIN_MAPPER_SIZE, 0); in addMapper()
237 new DefaultStringifier<JobConf>(jobConf, JobConf.class); in addMapper()
239 jobConf.set(prefix + CHAIN_MAPPER_CONFIG + index, in addMapper()
247 jobConf.setInt(prefix + CHAIN_MAPPER_SIZE, index + 1); in addMapper()
306 jobConf.set(prefix + CHAIN_REDUCER_CONFIG, in setReducer()
319 public void configure(JobConf jobConf) { in configure() argument
[all …]
|