001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.action.hadoop; 016 017 import java.util.List; 018 019 import org.apache.hadoop.conf.Configuration; 020 import org.apache.hadoop.fs.Path; 021 import org.apache.hadoop.mapred.Counters; 022 import org.apache.hadoop.mapred.JobClient; 023 import org.apache.hadoop.mapred.JobConf; 024 import org.apache.hadoop.mapred.JobID; 025 import org.apache.hadoop.mapred.RunningJob; 026 import org.apache.oozie.action.ActionExecutorException; 027 import org.apache.oozie.client.WorkflowAction; 028 import org.apache.oozie.util.XConfiguration; 029 import org.apache.oozie.util.XLog; 030 import org.apache.oozie.util.XmlUtils; 031 import org.jdom.Element; 032 import org.jdom.Namespace; 033 import org.json.simple.JSONObject; 034 035 public class MapReduceActionExecutor extends JavaActionExecutor { 036 037 public static final String HADOOP_COUNTERS = "hadoop.counters"; 038 private XLog log = XLog.getLog(getClass()); 039 040 public MapReduceActionExecutor() { 041 super("map-reduce"); 042 } 043 044 @Override 045 protected List<Class> getLauncherClasses() { 046 List<Class> classes = super.getLauncherClasses(); 047 classes.add(LauncherMain.class); 048 classes.add(MapReduceMain.class); 049 classes.add(StreamingMain.class); 050 classes.add(PipesMain.class); 051 return classes; 052 } 053 054 @Override 055 protected String getLauncherMain(Configuration launcherConf, Element actionXml) { 056 String mainClass; 057 Namespace ns = actionXml.getNamespace(); 058 if (actionXml.getChild("streaming", ns) != null) { 059 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, StreamingMain.class.getName()); 060 } 061 else { 062 if (actionXml.getChild("pipes", ns) != null) { 063 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, PipesMain.class.getName()); 064 } 065 else { 066 mainClass = launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, MapReduceMain.class.getName()); 067 } 068 } 069 return mainClass; 070 } 071 072 @Override 073 Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context) throws ActionExecutorException { 074 super.setupLauncherConf(conf, actionXml, appPath, context); 075 conf.setBoolean("mapreduce.job.complete.cancel.delegation.tokens", true); 076 return conf; 077 } 078 079 @Override 080 @SuppressWarnings("unchecked") 081 Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath) 082 throws ActionExecutorException { 083 Namespace ns = actionXml.getNamespace(); 084 if (actionXml.getChild("streaming", ns) != null) { 085 Element streamingXml = actionXml.getChild("streaming", ns); 086 String mapper = streamingXml.getChildTextTrim("mapper", ns); 087 String reducer = streamingXml.getChildTextTrim("reducer", ns); 088 String recordReader = streamingXml.getChildTextTrim("record-reader", ns); 089 List<Element> list = (List<Element>) streamingXml.getChildren("record-reader-mapping", ns); 090 String[] recordReaderMapping = new String[list.size()]; 091 for (int i = 0; i < list.size(); i++) { 092 recordReaderMapping[i] = list.get(i).getTextTrim(); 093 } 094 list = (List<Element>) streamingXml.getChildren("env", ns); 095 String[] env = new String[list.size()]; 096 for (int i = 0; i < list.size(); i++) { 097 env[i] = list.get(i).getTextTrim(); 098 } 099 StreamingMain.setStreaming(actionConf, mapper, reducer, recordReader, recordReaderMapping, env); 100 } 101 else { 102 if (actionXml.getChild("pipes", ns) != null) { 103 Element pipesXml = actionXml.getChild("pipes", ns); 104 String map = pipesXml.getChildTextTrim("map", ns); 105 String reduce = pipesXml.getChildTextTrim("reduce", ns); 106 String inputFormat = pipesXml.getChildTextTrim("inputformat", ns); 107 String partitioner = pipesXml.getChildTextTrim("partitioner", ns); 108 String writer = pipesXml.getChildTextTrim("writer", ns); 109 String program = pipesXml.getChildTextTrim("program", ns); 110 PipesMain.setPipes(actionConf, map, reduce, inputFormat, partitioner, writer, program, appPath); 111 } 112 } 113 actionConf = super.setupActionConf(actionConf, context, actionXml, appPath); 114 return actionConf; 115 } 116 117 @Override 118 public void end(Context context, WorkflowAction action) throws ActionExecutorException { 119 super.end(context, action); 120 JobClient jobClient = null; 121 boolean exception = false; 122 try { 123 if (action.getStatus() == WorkflowAction.Status.OK) { 124 Element actionXml = XmlUtils.parseXml(action.getConf()); 125 Configuration conf = createBaseHadoopConf(context, actionXml); 126 JobConf jobConf = new JobConf(); 127 XConfiguration.copy(conf, jobConf); 128 jobClient = createJobClient(context, jobConf); 129 RunningJob runningJob = jobClient.getJob(JobID.forName(action.getExternalId())); 130 if (runningJob == null) { 131 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR002", 132 "Unknown hadoop job [{0}] associated with action [{1}]. Failing this action!", action 133 .getExternalId(), action.getId()); 134 } 135 136 // TODO this has to be done in a better way 137 if (!runningJob.getJobName().startsWith("oozie:action:")) { 138 throw new ActionExecutorException(ActionExecutorException.ErrorType.FAILED, "MR001", 139 "ID swap should have happened in launcher job [{0}]", action.getExternalId()); 140 } 141 Counters counters = runningJob.getCounters(); 142 if (counters != null) { 143 JSONObject json = counterstoJson(counters); 144 context.setVar(HADOOP_COUNTERS, json.toJSONString()); 145 } 146 else { 147 148 context.setVar(HADOOP_COUNTERS, ""); 149 150 XLog.getLog(getClass()).warn("Could not find Hadoop Counters for: [{0}]", action.getExternalId()); 151 } 152 } 153 } 154 catch (Exception ex) { 155 exception = true; 156 throw convertException(ex); 157 } 158 finally { 159 if (jobClient != null) { 160 try { 161 jobClient.close(); 162 } 163 catch (Exception e) { 164 if (exception) { 165 log.error("JobClient error: ", e); 166 } 167 else { 168 throw convertException(e); 169 } 170 } 171 } 172 } 173 } 174 175 @SuppressWarnings("unchecked") 176 private JSONObject counterstoJson(Counters counters) { 177 178 if (counters == null) { 179 return null; 180 } 181 182 JSONObject groups = new JSONObject(); 183 for (String gName : counters.getGroupNames()) { 184 JSONObject group = new JSONObject(); 185 for (Counters.Counter counter : counters.getGroup(gName)) { 186 String cName = counter.getName(); 187 Long cValue = counter.getCounter(); 188 group.put(cName, cValue); 189 } 190 groups.put(gName, group); 191 } 192 return groups; 193 } 194 195 }