001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.coord; 016 017 import org.apache.hadoop.conf.Configuration; 018 019 import org.apache.oozie.client.CoordinatorAction; 020 import org.apache.oozie.client.OozieClient; 021 import org.apache.oozie.CoordinatorActionBean; 022 import org.apache.oozie.DagEngineException; 023 import org.apache.oozie.DagEngine; 024 import org.apache.oozie.ErrorCode; 025 import org.apache.oozie.WorkflowJobBean; 026 import org.apache.oozie.command.CommandException; 027 import org.apache.oozie.command.PreconditionException; 028 import org.apache.oozie.service.DagEngineService; 029 import org.apache.oozie.service.JPAService; 030 import org.apache.oozie.service.Services; 031 import org.apache.oozie.util.JobUtils; 032 import org.apache.oozie.util.LogUtils; 033 import org.apache.oozie.util.ParamChecker; 034 import org.apache.oozie.util.XLog; 035 import org.apache.oozie.util.XmlUtils; 036 import org.apache.oozie.util.XConfiguration; 037 import org.apache.oozie.util.db.SLADbOperations; 038 import org.apache.oozie.client.SLAEvent.SlaAppType; 039 import org.apache.oozie.client.SLAEvent.Status; 040 import org.apache.oozie.executor.jpa.JPAExecutorException; 041 import org.apache.oozie.executor.jpa.WorkflowJobGetJPAExecutor; 042 import org.apache.oozie.executor.jpa.WorkflowJobUpdateJPAExecutor; 043 044 import org.jdom.Element; 045 import org.jdom.JDOMException; 046 047 import java.io.IOException; 048 import java.io.StringReader; 049 050 public class CoordActionStartXCommand extends CoordinatorXCommand<Void> { 051 052 public static final String EL_ERROR = "EL_ERROR"; 053 public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR"; 054 public static final String COULD_NOT_START = "COULD_NOT_START"; 055 public static final String START_DATA_MISSING = "START_DATA_MISSING"; 056 public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING"; 057 058 private final XLog log = getLog(); 059 private String actionId = null; 060 private String user = null; 061 private String authToken = null; 062 private CoordinatorActionBean coordAction = null; 063 private JPAService jpaService = null; 064 065 public CoordActionStartXCommand(String id, String user, String token) { 066 //super("coord_action_start", "coord_action_start", 1, XLog.OPS); 067 super("coord_action_start", "coord_action_start", 1); 068 this.actionId = ParamChecker.notEmpty(id, "id"); 069 this.user = ParamChecker.notEmpty(user, "user"); 070 this.authToken = ParamChecker.notEmpty(token, "token"); 071 } 072 073 /** 074 * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from 075 * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract 076 * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf 077 * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf 078 * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table. 079 * Merge Action createdConf with actionXml to create new runConf with replaced variables 080 * 081 * @param action CoordinatorActionBean 082 * @return Configuration 083 * @throws CommandException 084 */ 085 private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException { 086 String createdConf = action.getCreatedConf(); 087 String actionXml = action.getActionXml(); 088 Element workflowProperties = null; 089 try { 090 workflowProperties = XmlUtils.parseXml(actionXml); 091 } 092 catch (JDOMException e1) { 093 log.warn("Configuration parse error in:" + actionXml); 094 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 095 } 096 // generate the 'runConf' for this action 097 // Step 1: runConf = createdConf 098 Configuration runConf = null; 099 try { 100 runConf = new XConfiguration(new StringReader(createdConf)); 101 } 102 catch (IOException e1) { 103 log.warn("Configuration parse error in:" + createdConf); 104 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 105 } 106 // Step 2: Merge local properties into runConf 107 // extract 'property' tags under 'configuration' block in the 108 // coordinator.xml (saved in actionxml column) 109 // convert Element to XConfiguration 110 Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace()) 111 .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration", 112 workflowProperties.getNamespace()); 113 if (configElement != null) { 114 String strConfig = XmlUtils.prettyPrint(configElement).toString(); 115 Configuration localConf; 116 try { 117 localConf = new XConfiguration(new StringReader(strConfig)); 118 } 119 catch (IOException e1) { 120 log.warn("Configuration parse error in:" + strConfig); 121 throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1); 122 } 123 124 // copy configuration properties in coordinator.xml to the runConf 125 XConfiguration.copy(localConf, runConf); 126 } 127 128 // Step 3: Extract value of 'app-path' in actionxml, and save it as a 129 // new property called 'oozie.wf.application.path' 130 // WF Engine requires the path to the workflow.xml to be saved under 131 // this property name 132 String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow", 133 workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue(); 134 runConf.set("oozie.wf.application.path", appPath); 135 return runConf; 136 } 137 138 @Override 139 protected Void execute() throws CommandException { 140 boolean makeFail = true; 141 String errCode = ""; 142 String errMsg = ""; 143 ParamChecker.notEmpty(user, "user"); 144 ParamChecker.notEmpty(authToken, "authToken"); 145 146 log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus()); 147 if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) { 148 // log.debug("getting.. job id: " + coordAction.getJobId()); 149 // create merged runConf to pass to WF Engine 150 Configuration runConf = mergeConfig(coordAction); 151 coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString()); 152 // log.debug("%%% merged runconf=" + 153 // XmlUtils.prettyPrint(runConf).toString()); 154 DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken); 155 try { 156 boolean startJob = true; 157 Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf())); 158 SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), Status.STARTED, 159 SlaAppType.COORDINATOR_ACTION, log); 160 161 // Normalize workflow appPath here; 162 JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf); 163 String wfId = dagEngine.submitJob(conf, startJob); 164 coordAction.setStatus(CoordinatorAction.Status.RUNNING); 165 coordAction.setExternalId(wfId); 166 coordAction.incrementAndGetPending(); 167 168 //store.updateCoordinatorAction(coordAction); 169 JPAService jpaService = Services.get().get(JPAService.class); 170 if (jpaService != null) { 171 log.debug("Updating WF record for WFID :" + wfId + " with parent id: " + actionId); 172 WorkflowJobBean wfJob = jpaService.execute(new WorkflowJobGetJPAExecutor(wfId)); 173 wfJob.setParentId(actionId); 174 jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob)); 175 jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionUpdateJPAExecutor(coordAction)); 176 } 177 else { 178 log.error(ErrorCode.E0610); 179 } 180 181 makeFail = false; 182 } 183 catch (DagEngineException dee) { 184 errMsg = dee.getMessage(); 185 errCode = "E1005"; 186 log.warn("can not create DagEngine for submitting jobs", dee); 187 } 188 catch (CommandException ce) { 189 errMsg = ce.getMessage(); 190 errCode = ce.getErrorCode().toString(); 191 log.warn("command exception occured ", ce); 192 } 193 catch (java.io.IOException ioe) { 194 errMsg = ioe.getMessage(); 195 errCode = "E1005"; 196 log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe); 197 } 198 catch (Exception ex) { 199 errMsg = ex.getMessage(); 200 errCode = "E1005"; 201 log.warn("can not create DagEngine for submitting jobs", ex); 202 } 203 finally { 204 if (makeFail == true) { // No DB exception occurs 205 log.warn("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg); 206 coordAction.setStatus(CoordinatorAction.Status.FAILED); 207 if (errMsg.length() > 254) { // Because table column size is 255 208 errMsg = errMsg.substring(0, 255); 209 } 210 coordAction.setErrorMessage(errMsg); 211 coordAction.setErrorCode(errCode); 212 213 JPAService jpaService = Services.get().get(JPAService.class); 214 if (jpaService != null) { 215 try { 216 jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionUpdateJPAExecutor(coordAction)); 217 } 218 catch (JPAExecutorException je) { 219 throw new CommandException(je); 220 } 221 } 222 else { 223 log.error(ErrorCode.E0610); 224 } 225 SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), Status.FAILED, 226 SlaAppType.COORDINATOR_ACTION, log); //Update SLA events 227 queue(new CoordActionReadyXCommand(coordAction.getJobId())); 228 } 229 } 230 } 231 return null; 232 } 233 234 @Override 235 protected String getEntityKey() { 236 return coordAction.getJobId(); 237 } 238 239 @Override 240 protected boolean isLockRequired() { 241 return true; 242 } 243 244 @Override 245 protected void loadState() throws CommandException { 246 247 } 248 249 @Override 250 protected void eagerLoadState() throws CommandException { 251 jpaService = Services.get().get(JPAService.class); 252 if (jpaService == null) { 253 throw new CommandException(ErrorCode.E0610); 254 } 255 256 try { 257 coordAction = jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionGetJPAExecutor(actionId)); 258 } 259 catch (JPAExecutorException je) { 260 throw new CommandException(je); 261 } 262 LogUtils.setLogInfo(coordAction, logInfo); 263 } 264 265 @Override 266 protected void verifyPrecondition() throws PreconditionException { 267 if (coordAction.getStatus() != CoordinatorAction.Status.SUBMITTED) { 268 throw new PreconditionException(ErrorCode.E1100); 269 } 270 } 271 }