001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.command.coord; 016 017 import java.io.StringReader; 018 import java.util.Date; 019 import java.util.List; 020 021 import org.apache.hadoop.conf.Configuration; 022 import org.apache.oozie.CoordinatorActionBean; 023 import org.apache.oozie.ErrorCode; 024 import org.apache.oozie.client.CoordinatorAction; 025 import org.apache.oozie.command.CommandException; 026 import org.apache.oozie.coord.CoordELEvaluator; 027 import org.apache.oozie.coord.CoordELFunctions; 028 import org.apache.oozie.coord.CoordUtils; 029 import org.apache.oozie.coord.CoordinatorJobException; 030 import org.apache.oozie.coord.SyncCoordAction; 031 import org.apache.oozie.coord.TimeUnit; 032 import org.apache.oozie.service.Services; 033 import org.apache.oozie.service.UUIDService; 034 import org.apache.oozie.util.DateUtils; 035 import org.apache.oozie.util.ELEvaluator; 036 import org.apache.oozie.util.XConfiguration; 037 import org.apache.oozie.util.XmlUtils; 038 import org.jdom.Element; 039 040 public class CoordCommandUtils { 041 public static int CURRENT = 0; 042 public static int LATEST = 1; 043 public static int FUTURE = 2; 044 public static int UNEXPECTED = -1; 045 public static final String RESOLVED_UNRESOLVED_SEPARATOR = ";"; 046 047 /** 048 * parse a function like coord:latest(n)/future() and return the 'n'. 049 * <p/> 050 * @param function 051 * @param event 052 * @param appInst 053 * @param conf 054 * @param restArg 055 * @return int instanceNumber 056 * @throws Exception 057 */ 058 public static int getInstanceNumber(String function, Element event, SyncCoordAction appInst, Configuration conf, 059 StringBuilder restArg) throws Exception { 060 ELEvaluator eval = CoordELEvaluator 061 .createInstancesELEvaluator("coord-action-create-inst", event, appInst, conf); 062 String newFunc = CoordELFunctions.evalAndWrap(eval, function); 063 int funcType = getFuncType(newFunc); 064 if (funcType == CURRENT || funcType == LATEST) { 065 return parseOneArg(newFunc); 066 } 067 else { 068 return parseMoreArgs(newFunc, restArg); 069 } 070 } 071 072 private static int parseOneArg(String funcName) throws Exception { 073 int firstPos = funcName.indexOf("("); 074 int lastPos = funcName.lastIndexOf(")"); 075 if (firstPos >= 0 && lastPos > firstPos) { 076 String tmp = funcName.substring(firstPos + 1, lastPos).trim(); 077 if (tmp.length() > 0) { 078 return (int) Double.parseDouble(tmp); 079 } 080 } 081 throw new RuntimeException("Unformatted function :" + funcName); 082 } 083 084 private static int parseMoreArgs(String funcName, StringBuilder restArg) throws Exception { 085 int firstPos = funcName.indexOf("("); 086 int secondPos = funcName.lastIndexOf(","); 087 int lastPos = funcName.lastIndexOf(")"); 088 if (firstPos >= 0 && secondPos > firstPos) { 089 String tmp = funcName.substring(firstPos + 1, secondPos).trim(); 090 if (tmp.length() > 0) { 091 restArg.append(funcName.substring(secondPos + 1, lastPos).trim()); 092 return (int) Double.parseDouble(tmp); 093 } 094 } 095 throw new RuntimeException("Unformatted function :" + funcName); 096 } 097 098 /** 099 * @param EL function name 100 * @return type of EL function 101 */ 102 public static int getFuncType(String function) { 103 if (function.indexOf("current") >= 0) { 104 return CURRENT; 105 } 106 else if (function.indexOf("latest") >= 0) { 107 return LATEST; 108 } 109 else if (function.indexOf("future") >= 0) { 110 return FUTURE; 111 } 112 return UNEXPECTED; 113 // throw new RuntimeException("Unexpected instance name "+ function); 114 } 115 116 /** 117 * @param startInst: EL function name 118 * @param endInst: EL function name 119 * @throws CommandException if both are not the same function 120 */ 121 public static void checkIfBothSameType(String startInst, String endInst) throws CommandException { 122 if (getFuncType(startInst) != getFuncType(endInst)) { 123 throw new CommandException(ErrorCode.E1010, 124 " start-instance and end-instance both should be either latest or current or future\n" 125 + " start " + startInst + " and end " + endInst); 126 } 127 } 128 129 /** 130 * Resolve list of <instance> </instance> tags. 131 * 132 * @param event 133 * @param instances 134 * @param actionInst 135 * @param conf 136 * @param eval: ELEvalautor 137 * @throws Exception 138 */ 139 public static void resolveInstances(Element event, StringBuilder instances, SyncCoordAction actionInst, 140 Configuration conf, ELEvaluator eval) throws Exception { 141 for (Element eInstance : (List<Element>) event.getChildren("instance", event.getNamespace())) { 142 if (instances.length() > 0) { 143 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 144 } 145 instances.append(materializeInstance(event, eInstance.getTextTrim(), actionInst, conf, eval)); 146 } 147 event.removeChildren("instance", event.getNamespace()); 148 } 149 150 /** 151 * Resolve <start-instance> <end-insatnce> tag. Don't resolve any 152 * latest()/future() 153 * 154 * @param event 155 * @param instances 156 * @param appInst 157 * @param conf 158 * @param eval: ELEvalautor 159 * @throws Exception 160 */ 161 public static void resolveInstanceRange(Element event, StringBuilder instances, SyncCoordAction appInst, 162 Configuration conf, ELEvaluator eval) throws Exception { 163 Element eStartInst = event.getChild("start-instance", event.getNamespace()); 164 Element eEndInst = event.getChild("end-instance", event.getNamespace()); 165 if (eStartInst != null && eEndInst != null) { 166 String strStart = eStartInst.getTextTrim(); 167 String strEnd = eEndInst.getTextTrim(); 168 checkIfBothSameType(strStart, strEnd); 169 StringBuilder restArg = new StringBuilder(); // To store rest 170 // arguments for 171 // future 172 // function 173 int startIndex = getInstanceNumber(strStart, event, appInst, conf, restArg); 174 restArg.delete(0, restArg.length()); 175 int endIndex = getInstanceNumber(strEnd, event, appInst, conf, restArg); 176 if (startIndex > endIndex) { 177 throw new CommandException(ErrorCode.E1010, 178 " start-instance should be equal or earlier than the end-instance \n" 179 + XmlUtils.prettyPrint(event)); 180 } 181 int funcType = getFuncType(strStart); 182 if (funcType == CURRENT) { 183 // Everything could be resolved NOW. no latest() ELs 184 for (int i = endIndex; i >= startIndex; i--) { 185 String matInstance = materializeInstance(event, "${coord:current(" + i + ")}", appInst, conf, eval); 186 if (matInstance == null || matInstance.length() == 0) { 187 // Earlier than dataset's initial instance 188 break; 189 } 190 if (instances.length() > 0) { 191 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 192 } 193 instances.append(matInstance); 194 } 195 } 196 else { // latest(n)/future() EL is present 197 for (; startIndex <= endIndex; startIndex++) { 198 if (instances.length() > 0) { 199 instances.append(CoordELFunctions.INSTANCE_SEPARATOR); 200 } 201 if (funcType == LATEST) { 202 instances.append("${coord:latest(" + startIndex + ")}"); 203 } 204 else { // For future 205 instances.append("${coord:future(" + startIndex + ",'" + restArg + "')}"); 206 } 207 } 208 } 209 // Remove start-instance and end-instances 210 event.removeChild("start-instance", event.getNamespace()); 211 event.removeChild("end-instance", event.getNamespace()); 212 } 213 } 214 215 /** 216 * Materialize one instance like current(-2) 217 * 218 * @param event : <data-in> 219 * @param expr : instance like current(-1) 220 * @param appInst : application specific info 221 * @param conf 222 * @param evalInst :ELEvaluator 223 * @return materialized date string 224 * @throws Exception 225 */ 226 public static String materializeInstance(Element event, String expr, SyncCoordAction appInst, Configuration conf, 227 ELEvaluator evalInst) throws Exception { 228 if (event == null) { 229 return null; 230 } 231 // ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, 232 // appInst, conf); 233 return CoordELFunctions.evalAndWrap(evalInst, expr); 234 } 235 236 /** 237 * Create two new tags with <uris> and <unresolved-instances>. 238 * 239 * @param event 240 * @param instances 241 * @param dependencyList 242 * @throws Exception 243 */ 244 public static void separateResolvedAndUnresolved(Element event, StringBuilder instances, StringBuffer dependencyList) 245 throws Exception { 246 StringBuilder unresolvedInstances = new StringBuilder(); 247 StringBuilder urisWithDoneFlag = new StringBuilder(); 248 String uris = createEarlyURIs(event, instances.toString(), unresolvedInstances, urisWithDoneFlag); 249 if (uris.length() > 0) { 250 Element uriInstance = new Element("uris", event.getNamespace()); 251 uriInstance.addContent(uris); 252 event.getContent().add(1, uriInstance); 253 if (dependencyList.length() > 0) { 254 dependencyList.append(CoordELFunctions.INSTANCE_SEPARATOR); 255 } 256 dependencyList.append(urisWithDoneFlag); 257 } 258 if (unresolvedInstances.length() > 0) { 259 Element elemInstance = new Element("unresolved-instances", event.getNamespace()); 260 elemInstance.addContent(unresolvedInstances.toString()); 261 event.getContent().add(1, elemInstance); 262 } 263 } 264 265 /** 266 * The function create a list of URIs separated by "," using the instances 267 * time stamp and URI-template 268 * 269 * @param event : <data-in> event 270 * @param instances : List of time stamp separated by "," 271 * @param unresolvedInstances : list of instance with latest function 272 * @param urisWithDoneFlag : list of URIs with the done flag appended 273 * @return : list of URIs separated by ";" as a string. 274 * @throws Exception 275 */ 276 public static String createEarlyURIs(Element event, String instances, StringBuilder unresolvedInstances, 277 StringBuilder urisWithDoneFlag) throws Exception { 278 if (instances == null || instances.length() == 0) { 279 return ""; 280 } 281 String[] instanceList = instances.split(CoordELFunctions.INSTANCE_SEPARATOR); 282 StringBuilder uris = new StringBuilder(); 283 284 Element doneFlagElement = event.getChild("dataset", event.getNamespace()).getChild("done-flag", 285 event.getNamespace()); 286 String doneFlag = CoordUtils.getDoneFlag(doneFlagElement); 287 288 for (int i = 0; i < instanceList.length; i++) { 289 if(instanceList[i].trim().length() == 0) { 290 continue; 291 } 292 int funcType = getFuncType(instanceList[i]); 293 if (funcType == LATEST || funcType == FUTURE) { 294 if (unresolvedInstances.length() > 0) { 295 unresolvedInstances.append(CoordELFunctions.INSTANCE_SEPARATOR); 296 } 297 unresolvedInstances.append(instanceList[i]); 298 continue; 299 } 300 ELEvaluator eval = CoordELEvaluator.createURIELEvaluator(instanceList[i]); 301 if (uris.length() > 0) { 302 uris.append(CoordELFunctions.INSTANCE_SEPARATOR); 303 urisWithDoneFlag.append(CoordELFunctions.INSTANCE_SEPARATOR); 304 } 305 306 String uriPath = CoordELFunctions.evalAndWrap(eval, event.getChild("dataset", event.getNamespace()) 307 .getChild("uri-template", event.getNamespace()).getTextTrim()); 308 uris.append(uriPath); 309 if (doneFlag.length() > 0) { 310 uriPath += "/" + doneFlag; 311 } 312 urisWithDoneFlag.append(uriPath); 313 } 314 return uris.toString(); 315 } 316 317 /** 318 * @param eSla 319 * @param nominalTime 320 * @param conf 321 * @return boolean to determine whether the SLA element is present or not 322 * @throws CoordinatorJobException 323 */ 324 public static boolean materializeSLA(Element eSla, Date nominalTime, Configuration conf) 325 throws CoordinatorJobException { 326 if (eSla == null) { 327 // eAppXml.getNamespace("sla")); 328 return false; 329 } 330 try { 331 ELEvaluator evalSla = CoordELEvaluator.createSLAEvaluator(nominalTime, conf); 332 List<Element> elemList = eSla.getChildren(); 333 for (Element elem : elemList) { 334 String updated; 335 try { 336 updated = CoordELFunctions.evalAndWrap(evalSla, elem.getText().trim()); 337 } 338 catch (Exception e) { 339 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e); 340 } 341 elem.removeContent(); 342 elem.addContent(updated); 343 } 344 } 345 catch (Exception e) { 346 throw new CoordinatorJobException(ErrorCode.E1004, e.getMessage(), e); 347 } 348 return true; 349 } 350 351 /** 352 * Materialize one instance for specific nominal time. It includes: 1. 353 * Materialize data events (i.e. <data-in> and <data-out>) 2. Materialize 354 * data properties (i.e dataIn(<DS>) and dataOut(<DS>) 3. remove 'start' and 355 * 'end' tag 4. Add 'instance_number' and 'nominal-time' tag 356 * 357 * @param jobId coordinator job id 358 * @param dryrun true if it is dryrun 359 * @param eAction frequency unexploded-job 360 * @param nominalTime materialization time 361 * @param actualTime action actual time 362 * @param instanceCount instance numbers 363 * @param conf job configuration 364 * @param actionBean CoordinatorActionBean to materialize 365 * @return one materialized action for specific nominal time 366 * @throws Exception 367 */ 368 @SuppressWarnings("unchecked") 369 public static String materializeOneInstance(String jobId, boolean dryrun, Element eAction, Date nominalTime, 370 Date actualTime, int instanceCount, Configuration conf, CoordinatorActionBean actionBean) throws Exception { 371 String actionId = Services.get().get(UUIDService.class).generateChildId(jobId, instanceCount + ""); 372 SyncCoordAction appInst = new SyncCoordAction(); 373 appInst.setActionId(actionId); 374 appInst.setName(eAction.getAttributeValue("name")); 375 appInst.setNominalTime(nominalTime); 376 appInst.setActualTime(actualTime); 377 int frequency = Integer.parseInt(eAction.getAttributeValue("frequency")); 378 appInst.setFrequency(frequency); 379 appInst.setTimeUnit(TimeUnit.valueOf(eAction.getAttributeValue("freq_timeunit"))); 380 appInst.setTimeZone(DateUtils.getTimeZone(eAction.getAttributeValue("timezone"))); 381 appInst.setEndOfDuration(TimeUnit.valueOf(eAction.getAttributeValue("end_of_duration"))); 382 383 StringBuffer dependencyList = new StringBuffer(); 384 385 Element inputList = eAction.getChild("input-events", eAction.getNamespace()); 386 List<Element> dataInList = null; 387 if (inputList != null) { 388 dataInList = inputList.getChildren("data-in", eAction.getNamespace()); 389 materializeDataEvents(dataInList, appInst, conf, dependencyList); 390 } 391 392 Element outputList = eAction.getChild("output-events", eAction.getNamespace()); 393 List<Element> dataOutList = null; 394 if (outputList != null) { 395 dataOutList = outputList.getChildren("data-out", eAction.getNamespace()); 396 StringBuffer tmp = new StringBuffer(); 397 // no dependency checks 398 materializeDataEvents(dataOutList, appInst, conf, tmp); 399 } 400 401 eAction.removeAttribute("start"); 402 eAction.removeAttribute("end"); 403 eAction.setAttribute("instance-number", Integer.toString(instanceCount)); 404 eAction.setAttribute("action-nominal-time", DateUtils.formatDateUTC(nominalTime)); 405 eAction.setAttribute("action-actual-time", DateUtils.formatDateUTC(actualTime)); 406 407 boolean isSla = CoordCommandUtils.materializeSLA(eAction.getChild("action", eAction.getNamespace()).getChild( 408 "info", eAction.getNamespace("sla")), nominalTime, conf); 409 410 // Setting up action bean 411 actionBean.setCreatedConf(XmlUtils.prettyPrint(conf).toString()); 412 actionBean.setRunConf(XmlUtils.prettyPrint(conf).toString()); 413 actionBean.setCreatedTime(actualTime); 414 actionBean.setJobId(jobId); 415 actionBean.setId(actionId); 416 actionBean.setLastModifiedTime(new Date()); 417 actionBean.setStatus(CoordinatorAction.Status.WAITING); 418 actionBean.setActionNumber(instanceCount); 419 actionBean.setMissingDependencies(dependencyList.toString()); 420 actionBean.setNominalTime(nominalTime); 421 if (isSla == true) { 422 actionBean.setSlaXml(XmlUtils.prettyPrint( 423 eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"))) 424 .toString()); 425 } 426 427 // actionBean.setTrackerUri(trackerUri);//TOOD: 428 // actionBean.setConsoleUrl(consoleUrl); //TODO: 429 // actionBean.setType(type);//TODO: 430 // actionBean.setErrorInfo(errorCode, errorMessage); //TODO: 431 // actionBean.setExternalStatus(externalStatus);//TODO 432 if (!dryrun) { 433 return XmlUtils.prettyPrint(eAction).toString(); 434 } 435 else { 436 String action = XmlUtils.prettyPrint(eAction).toString(); 437 CoordActionInputCheckCommand coordActionInput = new CoordActionInputCheckCommand(actionBean.getId()); 438 StringBuilder actionXml = new StringBuilder(action); 439 StringBuilder existList = new StringBuilder(); 440 StringBuilder nonExistList = new StringBuilder(); 441 StringBuilder nonResolvedList = new StringBuilder(); 442 getResolvedList(actionBean.getMissingDependencies(), nonExistList, nonResolvedList); 443 Configuration actionConf = new XConfiguration(new StringReader(actionBean.getRunConf())); 444 coordActionInput.checkInput(actionXml, existList, nonExistList, actionConf); 445 return actionXml.toString(); 446 } 447 } 448 449 /** 450 * Materialize all <input-events>/<data-in> or <output-events>/<data-out> 451 * tags Create uris for resolved instances. Create unresolved instance for 452 * latest()/future(). 453 * 454 * @param events 455 * @param appInst 456 * @param conf 457 * @throws Exception 458 */ 459 public static void materializeDataEvents(List<Element> events, SyncCoordAction appInst, Configuration conf, 460 StringBuffer dependencyList) throws Exception { 461 462 if (events == null) { 463 return; 464 } 465 StringBuffer unresolvedList = new StringBuffer(); 466 for (Element event : events) { 467 StringBuilder instances = new StringBuilder(); 468 ELEvaluator eval = CoordELEvaluator.createInstancesELEvaluator(event, appInst, conf); 469 // Handle list of instance tag 470 resolveInstances(event, instances, appInst, conf, eval); 471 // Handle start-instance and end-instance 472 resolveInstanceRange(event, instances, appInst, conf, eval); 473 // Separate out the unresolved instances 474 separateResolvedAndUnresolved(event, instances, dependencyList); 475 String tmpUnresolved = event.getChildTextTrim("unresolved-instances", event.getNamespace()); 476 if (tmpUnresolved != null) { 477 if (unresolvedList.length() > 0) { 478 unresolvedList.append(CoordELFunctions.INSTANCE_SEPARATOR); 479 } 480 unresolvedList.append(tmpUnresolved); 481 } 482 } 483 if (unresolvedList.length() > 0) { 484 dependencyList.append(RESOLVED_UNRESOLVED_SEPARATOR); 485 dependencyList.append(unresolvedList); 486 } 487 return; 488 } 489 490 /** 491 * Get resolved string from missDepList 492 * 493 * @param missDepList 494 * @param resolved 495 * @param unresolved 496 * @return resolved string 497 */ 498 public static String getResolvedList(String missDepList, StringBuilder resolved, StringBuilder unresolved) { 499 if (missDepList != null) { 500 int index = missDepList.indexOf(RESOLVED_UNRESOLVED_SEPARATOR); 501 if (index < 0) { 502 resolved.append(missDepList); 503 } 504 else { 505 resolved.append(missDepList.substring(0, index)); 506 unresolved.append(missDepList.substring(index + 1)); 507 } 508 } 509 return resolved.toString(); 510 } 511 512 }