001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.action.hadoop;
016    
017    import java.util.List;
018    
019    import org.apache.hadoop.conf.Configuration;
020    import org.apache.hadoop.fs.Path;
021    import org.apache.hadoop.mapred.Counters;
022    import org.apache.hadoop.mapred.JobClient;
023    import org.apache.hadoop.mapred.JobConf;
024    import org.apache.hadoop.mapred.JobID;
025    import org.apache.hadoop.mapred.RunningJob;
026    import org.apache.oozie.action.ActionExecutorException;
027    import org.apache.oozie.client.WorkflowAction;
028    import org.apache.oozie.util.XConfiguration;
029    import org.apache.oozie.util.XLog;
030    import org.apache.oozie.util.XmlUtils;
031    import org.jdom.Element;
032    import org.jdom.Namespace;
033    import org.json.simple.JSONObject;
034    
035    public class MapReduceActionExecutor extends JavaActionExecutor {
036    
037        public static final String HADOOP_COUNTERS = "hadoop.counters";
038        private XLog log = XLog.getLog(getClass());
039    
040        public MapReduceActionExecutor() {
041            super("map-reduce");
042        }
043    
044        @Override
045        protected List<Class> getLauncherClasses() {
046            List<Class> classes = super.getLauncherClasses();
047            classes.add(LauncherMain.class);
048            classes.add(MapReduceMain.class);
049            classes.add(StreamingMain.class);
050            classes.add(PipesMain.class);
051            return classes;
052        }
053    
054        @Override
055        protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
056            String mainClass;
057            Namespace ns = actionXml.getNamespace();
058            if (actionXml.getChild("streaming", ns) != null) {
059                mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, StreamingMain.class.getName());
060            }
061            else {
062                if (actionXml.getChild("pipes", ns) != null) {
063                    mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName());
064                }
065                else {
066                    mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName());
067                }
068            }
069            return mainClass;
070        }
071    
072        @Override
073        Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException {
074            super.setupLauncherConf(conf, actionXml, appPath, context);
075            conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true);
076            return conf;
077        }
078    
079        @Override
080        @SuppressWarnings("unchecked")
081        Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
082                throws ActionExecutorException {
083            Namespace ns = actionXml.getNamespace();
084            if (actionXml.getChild("streaming", ns) != null) {
085                Element streamingXml = actionXml.getChild("streaming", ns);
086                String mapper = streamingXml.getChildTextTrim("mapper", ns);
087                String reducer = streamingXml.getChildTextTrim("reducer", ns);
088                String recordReader = streamingXml.getChildTextTrim("record-reader", ns);
089                List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns);
090                String[] recordReaderMapping = new String[list.size()];
091                for (int i = 0; i < list.size(); i++) {
092                    recordReaderMapping[i] = list.get(i).getTextTrim();
093                }
094                list = (List<Element>) streamingXml.getChildren("env", ns);
095                String[] env = new String[list.size()];
096                for (int i = 0; i < list.size(); i++) {
097                    env[i] = list.get(i).getTextTrim();
098                }
099                StreamingMain.setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env);
100            }
101            else {
102                if (actionXml.getChild("pipes", ns) != null) {
103                    Element pipesXml = actionXml.getChild("pipes", ns);
104                    String map = pipesXml.getChildTextTrim("map", ns);
105                    String reduce = pipesXml.getChildTextTrim("reduce", ns);
106                    String inputFormat = pipesXml.getChildTextTrim("inputformat", ns);
107                    String partitioner = pipesXml.getChildTextTrim("partitioner", ns);
108                    String writer = pipesXml.getChildTextTrim("writer", ns);
109                    String program = pipesXml.getChildTextTrim("program", ns);
110                    PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program, appPath);
111                }
112            }
113            actionConf = super.setupActionConf(actionConf, context, actionXml, appPath);
114            return actionConf;
115        }
116    
117        @Override
118        public void end(Context context, WorkflowAction action) throws ActionExecutorException {
119            super.end(context, action);
120            JobClient jobClient = null;
121            boolean exception = false;
122            try {
123                if (action.getStatus() == WorkflowAction.Status.OK) {
124                    Element actionXml = XmlUtils.parseXml(action.getConf());
125                    Configuration conf = createBaseHadoopConf(context, actionXml);
126                    JobConf jobConf = new JobConf();
127                    XConfiguration.copy(conf, jobConf);
128                    jobClient = createJobClient(context, jobConf);
129                    RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalId()));
130                    if (runningJob == null) {
131                        throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002",
132                                                          "Unknown hadoop job [{0}] associated with action [{1}].  Failing this action!", action
133                                .getExternalId(), action.getId());
134                    }
135    
136                    // TODO this has to be done in a better way
137                    if (!runningJob.getJobName().startsWith("oozie:action:")) {
138                        throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR001",
139                                                          "ID swap should have happened in launcher job [{0}]", action.getExternalId());
140                    }
141                    Counters counters = runningJob.getCounters();
142                    if (counters != null) {
143                        JSONObject json = counterstoJson(counters);
144                        context.setVar(HADOOP_COUNTERS, json.toJSONString());
145                    }
146                    else {
147    
148                        context.setVar(HADOOP_COUNTERS, "");
149    
150                        XLog.getLog(getClass()).warn("Could not find Hadoop Counters for: [{0}]", action.getExternalId());
151                    }
152                }
153            }
154            catch (Exception ex) {
155                exception = true;
156                throw convertException(ex);
157            }
158            finally {
159                if (jobClient != null) {
160                    try {
161                        jobClient.close();
162                    }
163                    catch (Exception e) {
164                        if (exception) {
165                            log.error("JobClient error: ", e);
166                        }
167                        else {
168                            throw convertException(e);
169                        }
170                    }
171                }
172            }
173        }
174    
175        @SuppressWarnings("unchecked")
176        private JSONObject counterstoJson(Counters counters) {
177    
178            if (counters == null) {
179                return null;
180            }
181    
182            JSONObject groups = new JSONObject();
183            for (String gName : counters.getGroupNames()) {
184                JSONObject group = new JSONObject();
185                for (Counters.Counter counter : counters.getGroup(gName)) {
186                    String cName = counter.getName();
187                    Long cValue = counter.getCounter();
188                    group.put(cName, cValue);
189                }
190                groups.put(gName, group);
191            }
192            return groups;
193        }
194    
195    }