001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.bundle; 016 017 import java.io.IOException; 018 import java.io.StringReader; 019 import java.util.Date; 020 import java.util.HashMap; 021 import java.util.List; 022 import java.util.Map; 023 import java.util.Map.Entry; 024 025 import org.apache.hadoop.conf.Configuration; 026 import org.apache.oozie.BundleActionBean; 027 import org.apache.oozie.BundleJobBean; 028 import org.apache.oozie.ErrorCode; 029 import org.apache.oozie.XException; 030 import org.apache.oozie.client.Job; 031 import org.apache.oozie.client.OozieClient; 032 import org.apache.oozie.command.CommandException; 033 import org.apache.oozie.command.PreconditionException; 034 import org.apache.oozie.command.StartTransitionXCommand; 035 import org.apache.oozie.command.coord.CoordSubmitXCommand; 036 import org.apache.oozie.executor.jpa.BundleActionGetJPAExecutor; 037 import org.apache.oozie.executor.jpa.BundleActionInsertJPAExecutor; 038 import org.apache.oozie.executor.jpa.BundleActionUpdateJPAExecutor; 039 import org.apache.oozie.executor.jpa.BundleJobGetJPAExecutor; 040 import org.apache.oozie.executor.jpa.BundleJobUpdateJPAExecutor; 041 import org.apache.oozie.executor.jpa.JPAExecutorException; 042 import org.apache.oozie.service.JPAService; 043 import org.apache.oozie.service.Services; 044 import org.apache.oozie.util.JobUtils; 045 import org.apache.oozie.util.LogUtils; 046 import org.apache.oozie.util.ParamChecker; 047 import org.apache.oozie.util.XConfiguration; 048 import org.apache.oozie.util.XmlUtils; 049 import org.jdom.Attribute; 050 import org.jdom.Element; 051 import org.jdom.JDOMException; 052 053 /** 054 * The command to start Bundle job 055 */ 056 public class BundleStartXCommand extends StartTransitionXCommand { 057 private final String jobId; 058 private BundleJobBean bundleJob; 059 private JPAService jpaService = null; 060 061 /** 062 * The constructor for class {@link BundleStartXCommand} 063 * 064 * @param jobId the bundle job id 065 */ 066 public BundleStartXCommand(String jobId) { 067 super("bundle_start", "bundle_start", 1); 068 this.jobId = ParamChecker.notEmpty(jobId, "jobId"); 069 } 070 071 /** 072 * The constructor for class {@link BundleStartXCommand} 073 * 074 * @param jobId the bundle job id 075 * @param dryrun true if dryrun is enable 076 */ 077 public BundleStartXCommand(String jobId, boolean dryrun) { 078 super("bundle_start", "bundle_start", 1, dryrun); 079 this.jobId = ParamChecker.notEmpty(jobId, "jobId"); 080 } 081 082 /* (non-Javadoc) 083 * @see org.apache.oozie.command.XCommand#getEntityKey() 084 */ 085 @Override 086 protected String getEntityKey() { 087 return jobId; 088 } 089 090 /* (non-Javadoc) 091 * @see org.apache.oozie.command.XCommand#isLockRequired() 092 */ 093 @Override 094 protected boolean isLockRequired() { 095 return true; 096 } 097 098 /* (non-Javadoc) 099 * @see org.apache.oozie.command.XCommand#verifyPrecondition() 100 */ 101 @Override 102 protected void verifyPrecondition() throws CommandException, PreconditionException { 103 eagerVerifyPrecondition(); 104 } 105 106 /* (non-Javadoc) 107 * @see org.apache.oozie.command.XCommand#eagerVerifyPrecondition() 108 */ 109 @Override 110 protected void eagerVerifyPrecondition() throws CommandException, PreconditionException { 111 if (bundleJob.getStatus() != Job.Status.PREP) { 112 String msg = "Bundle " + bundleJob.getId() + " is not in PREP status. It is in : " + bundleJob.getStatus(); 113 LOG.info(msg); 114 throw new PreconditionException(ErrorCode.E1100, msg); 115 } 116 } 117 /* (non-Javadoc) 118 * @see org.apache.oozie.command.XCommand#loadState() 119 */ 120 @Override 121 public void loadState() throws CommandException { 122 eagerLoadState(); 123 } 124 125 /* (non-Javadoc) 126 * @see org.apache.oozie.command.XCommand#eagerLoadState() 127 */ 128 @Override 129 public void eagerLoadState() throws CommandException { 130 try { 131 jpaService = Services.get().get(JPAService.class); 132 133 if (jpaService != null) { 134 this.bundleJob = jpaService.execute(new BundleJobGetJPAExecutor(jobId)); 135 LogUtils.setLogInfo(bundleJob, logInfo); 136 super.setJob(bundleJob); 137 138 } 139 else { 140 throw new CommandException(ErrorCode.E0610); 141 } 142 } 143 catch (XException ex) { 144 throw new CommandException(ex); 145 } 146 } 147 148 /* (non-Javadoc) 149 * @see org.apache.oozie.command.StartTransitionXCommand#StartChildren() 150 */ 151 @Override 152 public void StartChildren() throws CommandException { 153 LOG.debug("Started coord jobs for the bundle=[{0}]", jobId); 154 insertBundleActions(); 155 startCoordJobs(); 156 LOG.debug("Ended coord jobs for the bundle=[{0}]", jobId); 157 } 158 159 /* (non-Javadoc) 160 * @see org.apache.oozie.command.TransitionXCommand#notifyParent() 161 */ 162 @Override 163 public void notifyParent() { 164 } 165 166 /** 167 * Insert bundle actions 168 * 169 * @throws CommandException thrown if failed to create bundle actions 170 */ 171 @SuppressWarnings("unchecked") 172 private void insertBundleActions() throws CommandException { 173 if (bundleJob != null) { 174 Map<String, Boolean> map = new HashMap<String, Boolean>(); 175 try { 176 Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml()); 177 List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace()); 178 for (Element elem : coordElems) { 179 Attribute name = elem.getAttribute("name"); 180 Attribute critical = elem.getAttribute("critical"); 181 if (name != null) { 182 if (map.containsKey(name.getValue())) { 183 throw new CommandException(ErrorCode.E1304, name); 184 } 185 boolean isCritical = false; 186 if (critical != null && Boolean.parseBoolean(critical.getValue())) { 187 isCritical = true; 188 } 189 map.put(name.getValue(), isCritical); 190 } 191 else { 192 throw new CommandException(ErrorCode.E1305); 193 } 194 } 195 } 196 catch (JDOMException jex) { 197 throw new CommandException(ErrorCode.E1301, jex); 198 } 199 200 try { 201 // if there is no coordinator for this bundle, failed it. 202 if (map.isEmpty()) { 203 bundleJob.setStatus(Job.Status.FAILED); 204 bundleJob.resetPending(); 205 jpaService.execute(new BundleJobUpdateJPAExecutor(bundleJob)); 206 LOG.debug("No coord jobs for the bundle=[{0}], failed it!!", jobId); 207 throw new CommandException(ErrorCode.E1318, jobId); 208 } 209 210 for (Entry<String, Boolean> coordName : map.entrySet()) { 211 BundleActionBean action = createBundleAction(jobId, coordName.getKey(), coordName.getValue()); 212 213 jpaService.execute(new BundleActionInsertJPAExecutor(action)); 214 } 215 } 216 catch (JPAExecutorException je) { 217 throw new CommandException(je); 218 } 219 220 } 221 else { 222 throw new CommandException(ErrorCode.E0604, jobId); 223 } 224 } 225 226 private BundleActionBean createBundleAction(String jobId, String coordName, boolean isCritical) { 227 BundleActionBean action = new BundleActionBean(); 228 action.setBundleActionId(jobId + "_" + coordName); 229 action.setBundleId(jobId); 230 action.setCoordName(coordName); 231 action.setStatus(Job.Status.PREP); 232 action.setLastModifiedTime(new Date()); 233 if (isCritical) { 234 action.setCritical(); 235 } 236 else { 237 action.resetCritical(); 238 } 239 return action; 240 } 241 242 /** 243 * Start Coord Jobs 244 * 245 * @throws CommandException thrown if failed to start coord jobs 246 */ 247 @SuppressWarnings("unchecked") 248 private void startCoordJobs() throws CommandException { 249 if (bundleJob != null) { 250 try { 251 Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml()); 252 List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace()); 253 for (Element coordElem : coordElems) { 254 Attribute name = coordElem.getAttribute("name"); 255 Configuration coordConf = mergeConfig(coordElem); 256 coordConf.set(OozieClient.BUNDLE_ID, jobId); 257 258 queue(new CoordSubmitXCommand(coordConf, bundleJob.getAuthToken(), bundleJob.getId(), name.getValue())); 259 260 updateBundleAction(name.getValue()); 261 } 262 } 263 catch (JDOMException jex) { 264 throw new CommandException(ErrorCode.E1301, jex); 265 } 266 catch (JPAExecutorException je) { 267 throw new CommandException(je); 268 } 269 } 270 else { 271 throw new CommandException(ErrorCode.E0604, jobId); 272 } 273 } 274 275 private void updateBundleAction(String coordName) throws JPAExecutorException { 276 BundleActionBean action = jpaService.execute(new BundleActionGetJPAExecutor(jobId, coordName)); 277 action.incrementAndGetPending(); 278 action.setLastModifiedTime(new Date()); 279 jpaService.execute(new BundleActionUpdateJPAExecutor(action)); 280 } 281 282 /** 283 * Merge Bundle job config and the configuration from the coord job to pass 284 * to Coord Engine 285 * 286 * @param coordElem the coordinator configuration 287 * @return Configuration merged configuration 288 * @throws CommandException thrown if failed to merge configuration 289 */ 290 private Configuration mergeConfig(Element coordElem) throws CommandException { 291 String jobConf = bundleJob.getConf(); 292 // Step 1: runConf = jobConf 293 Configuration runConf = null; 294 try { 295 runConf = new XConfiguration(new StringReader(jobConf)); 296 } 297 catch (IOException e1) { 298 LOG.warn("Configuration parse error in:" + jobConf); 299 throw new CommandException(ErrorCode.E1306, e1.getMessage(), e1); 300 } 301 // Step 2: Merge local properties into runConf 302 // extract 'property' tags under 'configuration' block in the coordElem 303 // convert Element to XConfiguration 304 Element localConfigElement = coordElem.getChild("configuration", coordElem.getNamespace()); 305 306 if (localConfigElement != null) { 307 String strConfig = XmlUtils.prettyPrint(localConfigElement).toString(); 308 Configuration localConf; 309 try { 310 localConf = new XConfiguration(new StringReader(strConfig)); 311 } 312 catch (IOException e1) { 313 LOG.warn("Configuration parse error in:" + strConfig); 314 throw new CommandException(ErrorCode.E1307, e1.getMessage(), e1); 315 } 316 317 // copy configuration properties in the coordElem to the runConf 318 XConfiguration.copy(localConf, runConf); 319 } 320 321 // Step 3: Extract value of 'app-path' in coordElem, save it as a 322 // new property called 'oozie.coord.application.path', and normalize. 323 String appPath = coordElem.getChild("app-path", coordElem.getNamespace()).getValue(); 324 runConf.set(OozieClient.COORDINATOR_APP_PATH, appPath); 325 // Normalize coordinator appPath here; 326 try { 327 JobUtils.normalizeAppPath(runConf.get(OozieClient.USER_NAME), runConf.get(OozieClient.GROUP_NAME), runConf); 328 } 329 catch (IOException e) { 330 throw new CommandException(ErrorCode.E1001, runConf.get(OozieClient.COORDINATOR_APP_PATH)); 331 } 332 return runConf; 333 } 334 335 /* (non-Javadoc) 336 * @see org.apache.oozie.command.TransitionXCommand#getJob() 337 */ 338 @Override 339 public Job getJob() { 340 return bundleJob; 341 } 342 343 /* (non-Javadoc) 344 * @see org.apache.oozie.command.TransitionXCommand#updateJob() 345 */ 346 @Override 347 public void updateJob() throws CommandException { 348 try { 349 jpaService.execute(new BundleJobUpdateJPAExecutor(bundleJob)); 350 } 351 catch (JPAExecutorException je) { 352 throw new CommandException(je); 353 } 354 } 355 }