001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.wf; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.oozie.service.WorkflowAppService; 019 import org.apache.oozie.util.XmlUtils; 020 import org.jdom.Element; 021 import org.jdom.Namespace; 022 import org.apache.oozie.client.XOozieClient; 023 import org.apache.oozie.command.CommandException; 024 025 import java.util.HashSet; 026 import java.util.Iterator; 027 import java.util.Map; 028 import java.util.Set; 029 030 public class SubmitMRXCommand extends SubmitHttpXCommand { 031 private static final Set<String> SKIPPED_CONFS = new HashSet<String>(); 032 033 public SubmitMRXCommand(Configuration conf, String authToken) { 034 super("submitMR", "submitMR", conf, authToken); 035 } 036 037 static { 038 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_USER); 039 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_UGI); 040 SKIPPED_CONFS.add(XOozieClient.JT); 041 SKIPPED_CONFS.add(XOozieClient.NN); 042 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_JT_KERBEROS_NAME); 043 SKIPPED_CONFS.add(WorkflowAppService.HADOOP_NN_KERBEROS_NAME); 044 } 045 046 private Element generateConfigurationSection(Configuration conf, Namespace ns) { 047 Element configuration = null; 048 Iterator<Map.Entry<String, String>> iter = conf.iterator(); 049 while (iter.hasNext()) { 050 Map.Entry<String, String> entry = iter.next(); 051 String name = entry.getKey(); 052 if (MANDATORY_OOZIE_CONFS.contains(name) || OPTIONAL_OOZIE_CONFS.contains(name) 053 || SKIPPED_CONFS.contains(name)) { 054 continue; 055 } 056 057 if (configuration == null) { 058 configuration = new Element("configuration", ns); 059 } 060 061 String value = entry.getValue(); 062 Element property = new Element("property", ns); 063 Element nameElement = new Element("name", ns); 064 nameElement.addContent(name != null ? name : ""); 065 property.addContent(nameElement); 066 Element valueElement = new Element("value", ns); 067 valueElement.addContent(value != null ? value : ""); 068 property.addContent(valueElement); 069 configuration.addContent(property); 070 } 071 072 return configuration; 073 } 074 075 private Element generateMRSection(Configuration conf, Namespace ns) { 076 Element mapreduce = new Element("map-reduce", ns); 077 Element jt = new Element("job-tracker", ns); 078 jt.addContent(conf.get(XOozieClient.JT)); 079 mapreduce.addContent(jt); 080 Element nn = new Element("name-node", ns); 081 nn.addContent(conf.get(XOozieClient.NN)); 082 mapreduce.addContent(nn); 083 084 if (conf.size() > MANDATORY_OOZIE_CONFS.size()) { // excluding JT, NN, 085 // LIBPATH 086 // configuration section 087 Element configuration = generateConfigurationSection(conf, ns); 088 if (configuration != null) { 089 mapreduce.addContent(configuration); 090 } 091 092 // file section 093 addFileSection(mapreduce, conf, ns); 094 095 // archive section 096 addArchiveSection(mapreduce, conf, ns); 097 } 098 099 return mapreduce; 100 } 101 102 /* 103 * (non-Javadoc) 104 * 105 * @see 106 * org.apache.oozie.command.wf.SubmitHttpCommand#getWorkflowXml(org.apache 107 * .hadoop.conf.Configuration) 108 */ 109 @Override 110 protected String getWorkflowXml(Configuration conf) { 111 for (String key : MANDATORY_OOZIE_CONFS) { 112 String value = conf.get(key); 113 if (value == null) { 114 throw new RuntimeException(key + " is not specified"); 115 } 116 } 117 118 Namespace ns = Namespace.getNamespace("uri:oozie:workflow:0.2"); 119 Element root = new Element("workflow-app", ns); 120 root.setAttribute("name", "oozie-mapreduce"); 121 122 Element start = new Element("start", ns); 123 start.setAttribute("to", "hadoop1"); 124 root.addContent(start); 125 126 Element action = new Element("action", ns); 127 action.setAttribute("name", "hadoop1"); 128 129 Element mapreduce = generateMRSection(conf, ns); 130 action.addContent(mapreduce); 131 132 Element ok = new Element("ok", ns); 133 ok.setAttribute("to", "end"); 134 action.addContent(ok); 135 136 Element error = new Element("error", ns); 137 error.setAttribute("to", "fail"); 138 action.addContent(error); 139 140 root.addContent(action); 141 142 Element kill = new Element("kill", ns); 143 kill.setAttribute("name", "fail"); 144 Element message = new Element("message", ns); 145 message.addContent("Map/Reduce failed, error message[${wf:errorMessage(wf:lastErrorNode())}]"); 146 kill.addContent(message); 147 root.addContent(kill); 148 149 Element end = new Element("end", ns); 150 end.setAttribute("name", "end"); 151 root.addContent(end); 152 153 return XmlUtils.prettyPrint(root).toString(); 154 } 155 156 @Override 157 protected String getEntityKey() { 158 return null; 159 } 160 161 @Override 162 protected boolean isLockRequired() { 163 return false; 164 } 165 166 @Override 167 protected void loadState() { 168 169 } 170 171 @Override 172 protected void verifyPrecondition() throws CommandException { 173 174 } 175 }