001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.bundle; 016 017 import java.io.IOException; 018 import java.io.InputStreamReader; 019 import java.io.Reader; 020 import java.io.StringReader; 021 import java.io.StringWriter; 022 import java.net.URI; 023 import java.net.URISyntaxException; 024 import java.util.Date; 025 import java.util.HashSet; 026 import java.util.List; 027 import java.util.Map; 028 import java.util.Set; 029 030 import javax.xml.transform.stream.StreamSource; 031 import javax.xml.validation.Validator; 032 033 import org.apache.hadoop.conf.Configuration; 034 import org.apache.hadoop.fs.FileSystem; 035 import org.apache.hadoop.fs.Path; 036 import org.apache.oozie.BundleJobBean; 037 import org.apache.oozie.ErrorCode; 038 import org.apache.oozie.client.Job; 039 import org.apache.oozie.client.OozieClient; 040 import org.apache.oozie.command.CommandException; 041 import org.apache.oozie.command.PreconditionException; 042 import org.apache.oozie.command.SubmitTransitionXCommand; 043 import org.apache.oozie.executor.jpa.BundleJobInsertJPAExecutor; 044 import org.apache.oozie.service.HadoopAccessorException; 045 import org.apache.oozie.service.HadoopAccessorService; 046 import org.apache.oozie.service.JPAService; 047 import org.apache.oozie.service.SchemaService; 048 import org.apache.oozie.service.Services; 049 import org.apache.oozie.service.UUIDService; 050 import org.apache.oozie.service.WorkflowAppService; 051 import org.apache.oozie.service.SchemaService.SchemaName; 052 import org.apache.oozie.service.UUIDService.ApplicationType; 053 import org.apache.oozie.util.DateUtils; 054 import org.apache.oozie.util.ELEvaluator; 055 import org.apache.oozie.util.IOUtils; 056 import org.apache.oozie.util.InstrumentUtils; 057 import org.apache.oozie.util.LogUtils; 058 import org.apache.oozie.util.ParamChecker; 059 import org.apache.oozie.util.PropertiesUtils; 060 import org.apache.oozie.util.XConfiguration; 061 import org.apache.oozie.util.XmlUtils; 062 import org.jdom.Attribute; 063 import org.jdom.Element; 064 import org.jdom.JDOMException; 065 import org.xml.sax.SAXException; 066 067 /** 068 * This Command will submit the bundle. 069 */ 070 public class BundleSubmitXCommand extends SubmitTransitionXCommand { 071 072 private Configuration conf; 073 private final String authToken; 074 public static final String CONFIG_DEFAULT = "bundle-config-default.xml"; 075 public static final String BUNDLE_XML_FILE = "bundle.xml"; 076 private final BundleJobBean bundleBean = new BundleJobBean(); 077 private String jobId; 078 private JPAService jpaService = null; 079 080 private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>(); 081 private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>(); 082 083 static { 084 String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY, 085 PropertiesUtils.HOUR, PropertiesUtils.MINUTE, PropertiesUtils.DAYS, PropertiesUtils.HOURS, 086 PropertiesUtils.MINUTES, PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, 087 PropertiesUtils.TB, PropertiesUtils.PB, PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, 088 PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN, PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS }; 089 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES); 090 091 String[] badDefaultProps = { PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI, 092 WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME }; 093 PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES); 094 PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES); 095 } 096 097 /** 098 * Constructor to create the bundle submit command. 099 * 100 * @param conf configuration for bundle job 101 * @param authToken to be used for authentication 102 */ 103 public BundleSubmitXCommand(Configuration conf, String authToken) { 104 super("bundle_submit", "bundle_submit", 1); 105 this.conf = ParamChecker.notNull(conf, "conf"); 106 this.authToken = ParamChecker.notEmpty(authToken, "authToken"); 107 } 108 109 /** 110 * Constructor to create the bundle submit command. 111 * 112 * @param dryrun true if dryrun is enable 113 * @param conf configuration for bundle job 114 * @param authToken to be used for authentication 115 */ 116 public BundleSubmitXCommand(boolean dryrun, Configuration conf, String authToken) { 117 this(conf, authToken); 118 this.dryrun = dryrun; 119 } 120 121 /* (non-Javadoc) 122 * @see org.apache.oozie.command.SubmitTransitionXCommand#submit() 123 */ 124 @Override 125 protected String submit() throws CommandException { 126 LOG.info("STARTED Bundle Submit"); 127 try { 128 InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation()); 129 130 XmlUtils.removeComments(this.bundleBean.getOrigJobXml().toString()); 131 // Resolving all variables in the job properties. 132 // This ensures the Hadoop Configuration semantics is preserved. 133 XConfiguration resolvedVarsConf = new XConfiguration(); 134 for (Map.Entry<String, String> entry : conf) { 135 resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey())); 136 } 137 conf = resolvedVarsConf; 138 139 String resolvedJobXml = resolvedVars(bundleBean.getOrigJobXml(), conf); 140 141 //verify the uniqueness of coord names 142 verifyCoordNameUnique(resolvedJobXml); 143 this.jobId = storeToDB(bundleBean, resolvedJobXml); 144 LogUtils.setLogInfo(bundleBean, logInfo); 145 146 if (dryrun) { 147 Date startTime = bundleBean.getStartTime(); 148 long startTimeMilli = startTime.getTime(); 149 long endTimeMilli = startTimeMilli + (3600 * 1000); 150 Date jobEndTime = bundleBean.getEndTime(); 151 Date endTime = new Date(endTimeMilli); 152 if (endTime.compareTo(jobEndTime) > 0) { 153 endTime = jobEndTime; 154 } 155 jobId = bundleBean.getId(); 156 LOG.info("[" + jobId + "]: Update status to PREP"); 157 bundleBean.setStatus(Job.Status.PREP); 158 try { 159 new XConfiguration(new StringReader(bundleBean.getConf())); 160 } 161 catch (IOException e1) { 162 LOG.warn("Configuration parse error. read from DB :" + bundleBean.getConf(), e1); 163 } 164 String output = bundleBean.getJobXml() + System.getProperty("line.separator"); 165 return output; 166 } 167 else { 168 if (bundleBean.getKickoffTime() == null) { 169 // If there is no KickOffTime, default kickoff is NOW. 170 LOG.debug("Since kickoff time is not defined for job id " + jobId 171 + ". Queuing and BundleStartXCommand immediately after submission"); 172 queue(new BundleStartXCommand(jobId)); 173 } 174 } 175 } 176 catch (Exception ex) { 177 throw new CommandException(ErrorCode.E1310, ex.getMessage(), ex); 178 } 179 LOG.info("ENDED Bundle Submit"); 180 return this.jobId; 181 } 182 183 /* (non-Javadoc) 184 * @see org.apache.oozie.command.TransitionXCommand#notifyParent() 185 */ 186 @Override 187 public void notifyParent() throws CommandException { 188 } 189 190 /* (non-Javadoc) 191 * @see org.apache.oozie.command.XCommand#getEntityKey() 192 */ 193 @Override 194 protected String getEntityKey() { 195 return null; 196 } 197 198 /* (non-Javadoc) 199 * @see org.apache.oozie.command.XCommand#isLockRequired() 200 */ 201 @Override 202 protected boolean isLockRequired() { 203 return false; 204 } 205 206 /* (non-Javadoc) 207 * @see org.apache.oozie.command.XCommand#loadState() 208 */ 209 @Override 210 protected void loadState() throws CommandException { 211 } 212 213 /* (non-Javadoc) 214 * @see org.apache.oozie.command.XCommand#verifyPrecondition() 215 */ 216 @Override 217 protected void verifyPrecondition() throws CommandException, PreconditionException { 218 } 219 220 /* (non-Javadoc) 221 * @see org.apache.oozie.command.XCommand#eagerLoadState() 222 */ 223 @Override 224 protected void eagerLoadState() throws CommandException { 225 super.eagerLoadState(); 226 jpaService = Services.get().get(JPAService.class); 227 if (jpaService == null) { 228 throw new CommandException(ErrorCode.E0610); 229 } 230 } 231 232 /* (non-Javadoc) 233 * @see org.apache.oozie.command.XCommand#eagerVerifyPrecondition() 234 */ 235 @Override 236 protected void eagerVerifyPrecondition() throws CommandException, PreconditionException { 237 try { 238 super.eagerVerifyPrecondition(); 239 mergeDefaultConfig(); 240 String appXml = readAndValidateXml(); 241 bundleBean.setOrigJobXml(appXml); 242 LOG.debug("jobXml after initial validation " + XmlUtils.prettyPrint(appXml).toString()); 243 } 244 catch (BundleJobException ex) { 245 LOG.warn("BundleJobException: ", ex); 246 throw new CommandException(ex); 247 } 248 catch (IllegalArgumentException iex) { 249 LOG.warn("IllegalArgumentException: ", iex); 250 throw new CommandException(ErrorCode.E1310, iex); 251 } 252 catch (Exception ex) { 253 LOG.warn("Exception: ", ex); 254 throw new CommandException(ErrorCode.E1310, ex); 255 } 256 } 257 258 /** 259 * Merge default configuration with user-defined configuration. 260 * 261 * @throws CommandException thrown if failed to merge configuration 262 */ 263 protected void mergeDefaultConfig() throws CommandException { 264 Path configDefault = null; 265 try { 266 String bundleAppPathStr = conf.get(OozieClient.BUNDLE_APP_PATH); 267 Path bundleAppPath = new Path(bundleAppPathStr); 268 String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME); 269 String group = ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME); 270 FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, bundleAppPath.toUri(), 271 new Configuration()); 272 273 // app path could be a directory 274 if (!fs.isFile(bundleAppPath)) { 275 configDefault = new Path(bundleAppPath, CONFIG_DEFAULT); 276 } else { 277 configDefault = new Path(bundleAppPath.getParent(), CONFIG_DEFAULT); 278 } 279 280 if (fs.exists(configDefault)) { 281 Configuration defaultConf = new XConfiguration(fs.open(configDefault)); 282 PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES); 283 XConfiguration.injectDefaults(defaultConf, conf); 284 } 285 else { 286 LOG.info("configDefault Doesn't exist " + configDefault); 287 } 288 PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES); 289 } 290 catch (IOException e) { 291 throw new CommandException(ErrorCode.E0702, e.getMessage() + " : Problem reading default config " 292 + configDefault, e); 293 } 294 catch (HadoopAccessorException e) { 295 throw new CommandException(e); 296 } 297 LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString()); 298 } 299 300 /** 301 * Read the application XML and validate against bundle Schema 302 * 303 * @return validated bundle XML 304 * @throws BundleJobException thrown if failed to read or validate xml 305 */ 306 private String readAndValidateXml() throws BundleJobException { 307 String appPath = ParamChecker.notEmpty(conf.get(OozieClient.BUNDLE_APP_PATH), OozieClient.BUNDLE_APP_PATH); 308 String bundleXml = readDefinition(appPath); 309 validateXml(bundleXml); 310 return bundleXml; 311 } 312 313 /** 314 * Read bundle definition. 315 * 316 * @param appPath application path. 317 * @param user user name. 318 * @param group group name. 319 * @param autToken authentication token. 320 * @return bundle definition. 321 * @throws BundleJobException thrown if the definition could not be read. 322 */ 323 protected String readDefinition(String appPath) throws BundleJobException { 324 String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME); 325 String group = ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME); 326 //Configuration confHadoop = CoordUtils.getHadoopConf(conf); 327 try { 328 URI uri = new URI(appPath); 329 LOG.debug("user =" + user + " group =" + group); 330 FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, uri, 331 new Configuration()); 332 Path appDefPath = null; 333 334 // app path could be a directory 335 Path path = new Path(uri.getPath()); 336 if (!fs.isFile(path)) { 337 appDefPath = new Path(path, BUNDLE_XML_FILE); 338 } else { 339 appDefPath = path; 340 } 341 342 Reader reader = new InputStreamReader(fs.open(appDefPath)); 343 StringWriter writer = new StringWriter(); 344 IOUtils.copyCharStream(reader, writer); 345 return writer.toString(); 346 } 347 catch (IOException ex) { 348 LOG.warn("IOException :" + XmlUtils.prettyPrint(conf), ex); 349 throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex); 350 } 351 catch (URISyntaxException ex) { 352 LOG.warn("URISyException :" + ex.getMessage()); 353 throw new BundleJobException(ErrorCode.E1302, appPath, ex.getMessage(), ex); 354 } 355 catch (HadoopAccessorException ex) { 356 throw new BundleJobException(ex); 357 } 358 catch (Exception ex) { 359 LOG.warn("Exception :", ex); 360 throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex); 361 } 362 } 363 364 /** 365 * Validate against Bundle XSD file 366 * 367 * @param xmlContent input bundle xml 368 * @throws BundleJobException thrown if failed to validate xml 369 */ 370 private void validateXml(String xmlContent) throws BundleJobException { 371 javax.xml.validation.Schema schema = Services.get().get(SchemaService.class).getSchema(SchemaName.BUNDLE); 372 Validator validator = schema.newValidator(); 373 try { 374 validator.validate(new StreamSource(new StringReader(xmlContent))); 375 } 376 catch (SAXException ex) { 377 LOG.warn("SAXException :", ex); 378 throw new BundleJobException(ErrorCode.E0701, ex.getMessage(), ex); 379 } 380 catch (IOException ex) { 381 LOG.warn("IOException :", ex); 382 throw new BundleJobException(ErrorCode.E0702, ex.getMessage(), ex); 383 } 384 } 385 386 /** 387 * Write a Bundle Job into database 388 * 389 * @param Bundle job bean 390 * @return job id 391 * @throws CommandException thrown if failed to store bundle job bean to db 392 */ 393 private String storeToDB(BundleJobBean bundleJob, String resolvedJobXml) throws CommandException { 394 try { 395 jobId = Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE); 396 397 bundleJob.setId(jobId); 398 bundleJob.setAuthToken(this.authToken); 399 bundleJob.setAppName(XmlUtils.parseXml(bundleBean.getOrigJobXml()).getAttributeValue("name")); 400 bundleJob.setAppName(bundleJob.getAppName()); 401 bundleJob.setAppPath(conf.get(OozieClient.BUNDLE_APP_PATH)); 402 // bundleJob.setStatus(BundleJob.Status.PREP); //This should be set in parent class. 403 bundleJob.setCreatedTime(new Date()); 404 bundleJob.setUser(conf.get(OozieClient.USER_NAME)); 405 bundleJob.setGroup(conf.get(OozieClient.GROUP_NAME)); 406 bundleJob.setConf(XmlUtils.prettyPrint(conf).toString()); 407 bundleJob.setJobXml(resolvedJobXml); 408 Element jobElement = XmlUtils.parseXml(resolvedJobXml); 409 Element controlsElement = jobElement.getChild("controls", jobElement.getNamespace()); 410 if (controlsElement != null) { 411 Element kickoffTimeElement = controlsElement.getChild("kick-off-time", jobElement.getNamespace()); 412 if (kickoffTimeElement != null && !kickoffTimeElement.getValue().isEmpty()) { 413 Date kickoffTime = DateUtils.parseDateUTC(kickoffTimeElement.getValue()); 414 bundleJob.setKickoffTime(kickoffTime); 415 } 416 } 417 bundleJob.setLastModifiedTime(new Date()); 418 419 if (!dryrun) { 420 jpaService.execute(new BundleJobInsertJPAExecutor(bundleJob)); 421 } 422 } 423 catch (Exception ex) { 424 throw new CommandException(ErrorCode.E1301, ex.getMessage(), ex); 425 } 426 return jobId; 427 } 428 429 /* (non-Javadoc) 430 * @see org.apache.oozie.command.TransitionXCommand#getJob() 431 */ 432 @Override 433 public Job getJob() { 434 return bundleBean; 435 } 436 437 /** 438 * Resolve job xml with conf 439 * 440 * @param bundleXml bundle job xml 441 * @param conf job configuration 442 * @return resolved job xml 443 * @throws BundleJobException thrown if failed to resolve variables 444 */ 445 private String resolvedVars(String bundleXml, Configuration conf) throws BundleJobException { 446 try { 447 ELEvaluator eval = createEvaluator(conf); 448 return eval.evaluate(bundleXml, String.class); 449 } 450 catch (Exception e) { 451 throw new BundleJobException(ErrorCode.E1004, e.getMessage(), e); 452 } 453 } 454 455 /** 456 * Create ELEvaluator 457 * 458 * @param conf job configuration 459 * @return ELEvaluator the evaluator for el function 460 * @throws BundleJobException thrown if failed to create evaluator 461 */ 462 public ELEvaluator createEvaluator(Configuration conf) throws BundleJobException { 463 ELEvaluator eval; 464 ELEvaluator.Context context; 465 try { 466 context = new ELEvaluator.Context(); 467 eval = new ELEvaluator(context); 468 for (Map.Entry<String, String> entry : conf) { 469 eval.setVariable(entry.getKey(), entry.getValue()); 470 } 471 } 472 catch (Exception e) { 473 throw new BundleJobException(ErrorCode.E1004, e.getMessage(), e); 474 } 475 return eval; 476 } 477 478 /** 479 * Verify the uniqueness of coordinator names 480 * 481 * @param resolved job xml 482 * @throws CommandException thrown if failed to verify the uniqueness of coordinator names 483 */ 484 @SuppressWarnings("unchecked") 485 private Void verifyCoordNameUnique(String resolvedJobXml) throws CommandException { 486 Set<String> set = new HashSet<String>(); 487 try { 488 Element bAppXml = XmlUtils.parseXml(resolvedJobXml); 489 List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace()); 490 for (Element elem : coordElems) { 491 Attribute name = elem.getAttribute("name"); 492 if (name != null) { 493 if (set.contains(name.getValue())) { 494 throw new CommandException(ErrorCode.E1304, name); 495 } 496 set.add(name.getValue()); 497 } 498 else { 499 throw new CommandException(ErrorCode.E1305); 500 } 501 } 502 } 503 catch (JDOMException jex) { 504 throw new CommandException(ErrorCode.E1301, jex); 505 } 506 507 return null; 508 } 509 510 /* (non-Javadoc) 511 * @see org.apache.oozie.command.TransitionXCommand#updateJob() 512 */ 513 @Override 514 public void updateJob() throws CommandException { 515 } 516 }