001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.hadoop.mapred.JobConf; 019 import org.apache.hadoop.mapred.RunningJob; 020 import org.apache.hadoop.mapred.pipes.Submitter; 021 import org.apache.hadoop.filecache.DistributedCache; 022 import org.apache.hadoop.fs.Path; 023 024 public class PipesMain extends MapReduceMain { 025 026 public static void main(String[] args) throws Exception { 027 run(PipesMain.class, args); 028 } 029 030 @Override 031 protected RunningJob submitJob(Configuration actionConf) throws Exception { 032 JobConf jobConf = new JobConf(); 033 034 String value = actionConf.get("oozie.pipes.map"); 035 if (value != null) { 036 jobConf.setBoolean("hadoop.pipes.java.mapper", true); 037 jobConf.set("mapred.mapper.class", value); 038 } 039 value = actionConf.get("oozie.pipes.reduce"); 040 if (value != null) { 041 jobConf.setBoolean("hadoop.pipes.java.reducer", true); 042 jobConf.set("mapred.reducer.class", value); 043 } 044 value = actionConf.get("oozie.pipes.inputformat"); 045 if (value != null) { 046 jobConf.setBoolean("hadoop.pipes.java.recordreader", true); 047 jobConf.set("mapred.input.format.class", value); 048 } 049 value = actionConf.get("oozie.pipes.partitioner"); 050 if (value != null) { 051 jobConf.set("mapred.partitioner.class", value); 052 } 053 value = actionConf.get("oozie.pipes.writer"); 054 if (value != null) { 055 jobConf.setBoolean("hadoop.pipes.java.recordwriter", true); 056 jobConf.set("mapred.output.format.class", value); 057 } 058 value = actionConf.get("oozie.pipes.program"); 059 if (value != null) { 060 jobConf.set("hadoop.pipes.executable", value); 061 if (value.contains("#")) { 062 DistributedCache.createSymlink(jobConf); 063 } 064 } 065 066 addActionConf(jobConf, actionConf); 067 068 //propagate delegation related props from launcher job to MR job 069 if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) { 070 jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION")); 071 } 072 073 return Submitter.jobSubmit(jobConf); 074 } 075 076 public static void setPipes(Configuration conf, String map, String reduce, String inputFormat, String partitioner, 077 String writer, String program, Path appPath) { 078 if (map != null) { 079 conf.set("oozie.pipes.map", map); 080 } 081 if (reduce != null) { 082 conf.set("oozie.pipes.reduce", reduce); 083 } 084 if (inputFormat != null) { 085 conf.set("oozie.pipes.inputformat", inputFormat); 086 } 087 if (partitioner != null) { 088 conf.set("oozie.pipes.partitioner", partitioner); 089 } 090 if (writer != null) { 091 conf.set("oozie.pipes.writer", writer); 092 } 093 if (program != null) { 094 Path path = null; 095 if (!program.startsWith("/")) { 096 path = new Path(appPath, program); 097 program = path.toString(); 098 } 099 conf.set("oozie.pipes.program", program); 100 101 } 102 } 103 104 }