001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.hadoop.mapred.JobConf;
019    import org.apache.hadoop.mapred.RunningJob;
020    import org.apache.hadoop.mapred.pipes.Submitter;
021    import org.apache.hadoop.filecache.DistributedCache;
022    import org.apache.hadoop.fs.Path;
023    
024    public class PipesMain extends MapReduceMain {
025    
026        public static void main(String[] args) throws Exception {
027            run(PipesMain.class, args);
028        }
029    
030        @Override
031        protected RunningJob submitJob(Configuration actionConf) throws Exception {
032            JobConf jobConf = new JobConf();
033    
034            String value = actionConf.get("oozie.pipes.map");
035            if (value != null) {
036                jobConf.setBoolean("hadoop.pipes.java.mapper", true);
037                jobConf.set("mapred.mapper.class", value);
038            }
039            value = actionConf.get("oozie.pipes.reduce");
040            if (value != null) {
041                jobConf.setBoolean("hadoop.pipes.java.reducer", true);
042                jobConf.set("mapred.reducer.class", value);
043            }
044            value = actionConf.get("oozie.pipes.inputformat");
045            if (value != null) {
046                jobConf.setBoolean("hadoop.pipes.java.recordreader", true);
047                jobConf.set("mapred.input.format.class", value);
048            }
049            value = actionConf.get("oozie.pipes.partitioner");
050            if (value != null) {
051                jobConf.set("mapred.partitioner.class", value);
052            }
053            value = actionConf.get("oozie.pipes.writer");
054            if (value != null) {
055                jobConf.setBoolean("hadoop.pipes.java.recordwriter", true);
056                jobConf.set("mapred.output.format.class", value);
057            }
058            value = actionConf.get("oozie.pipes.program");
059            if (value != null) {
060                jobConf.set("hadoop.pipes.executable", value);
061                if (value.contains("#")) {
062                    DistributedCache.createSymlink(jobConf);
063                }
064            }
065    
066            addActionConf(jobConf, actionConf);
067    
068            //propagate delegation related props from launcher job to MR job
069            if (System.getenv("HADOOP_TOKEN_FILE_LOCATION") != null) {
070                jobConf.set("mapreduce.job.credentials.binary", System.getenv("HADOOP_TOKEN_FILE_LOCATION"));
071            }
072    
073            return Submitter.jobSubmit(jobConf);
074        }
075    
076        public static void setPipes(Configuration conf, String map, String reduce, String inputFormat, String partitioner,
077                                    String writer, String program, Path appPath) {
078            if (map != null) {
079                conf.set("oozie.pipes.map", map);
080            }
081            if (reduce != null) {
082                conf.set("oozie.pipes.reduce", reduce);
083            }
084            if (inputFormat != null) {
085                conf.set("oozie.pipes.inputformat", inputFormat);
086            }
087            if (partitioner != null) {
088                conf.set("oozie.pipes.partitioner", partitioner);
089            }
090            if (writer != null) {
091                conf.set("oozie.pipes.writer", writer);
092            }
093            if (program != null) {
094                Path path = null;
095                if (!program.startsWith("/")) {
096                    path = new Path(appPath, program);
097                    program = path.toString();
098                }
099                conf.set("oozie.pipes.program", program);
100    
101            }
102        }
103    
104    }