001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.coord; 016 017 import java.io.IOException; 018 import java.util.Calendar; 019 import java.util.Date; 020 import java.util.TimeZone; 021 022 import org.apache.hadoop.conf.Configuration; 023 import org.apache.hadoop.fs.Path; 024 025 import org.apache.oozie.client.OozieClient; 026 import org.apache.oozie.util.DateUtils; 027 import org.apache.oozie.util.ELEvaluator; 028 import org.apache.oozie.util.ParamChecker; 029 import org.apache.oozie.util.XLog; 030 import org.apache.oozie.service.HadoopAccessorException; 031 import org.apache.oozie.service.Services; 032 import org.apache.oozie.service.HadoopAccessorService; 033 034 /** 035 * This class implements the EL function related to coordinator 036 */ 037 038 public class CoordELFunctions { 039 final private static String DATASET = "oozie.coord.el.dataset.bean"; 040 final private static String COORD_ACTION = "oozie.coord.el.app.bean"; 041 final public static String CONFIGURATION = "oozie.coord.el.conf"; 042 // INSTANCE_SEPARATOR is used to separate multiple directories into one tag. 043 final public static String INSTANCE_SEPARATOR = "#"; 044 final public static String DIR_SEPARATOR = ","; 045 // TODO: in next release, support flexibility 046 private static String END_OF_OPERATION_INDICATOR_FILE = "_SUCCESS"; 047 048 /** 049 * Used in defining the frequency in 'day' unit. <p/> domain: <code> val > 0</code> and should be integer. 050 * 051 * @param val frequency in number of days. 052 * @return number of days and also set the frequency timeunit to "day" 053 */ 054 public static int ph1_coord_days(int val) { 055 val = ParamChecker.checkGTZero(val, "n"); 056 ELEvaluator eval = ELEvaluator.getCurrent(); 057 eval.setVariable("timeunit", TimeUnit.DAY); 058 eval.setVariable("endOfDuration", TimeUnit.NONE); 059 return val; 060 } 061 062 /** 063 * Used in defining the frequency in 'month' unit. <p/> domain: <code> val > 0</code> and should be integer. 064 * 065 * @param val frequency in number of months. 066 * @return number of months and also set the frequency timeunit to "month" 067 */ 068 public static int ph1_coord_months(int val) { 069 val = ParamChecker.checkGTZero(val, "n"); 070 ELEvaluator eval = ELEvaluator.getCurrent(); 071 eval.setVariable("timeunit", TimeUnit.MONTH); 072 eval.setVariable("endOfDuration", TimeUnit.NONE); 073 return val; 074 } 075 076 /** 077 * Used in defining the frequency in 'hour' unit. <p/> parameter value domain: <code> val > 0</code> and should 078 * be integer. 079 * 080 * @param val frequency in number of hours. 081 * @return number of minutes and also set the frequency timeunit to "minute" 082 */ 083 public static int ph1_coord_hours(int val) { 084 val = ParamChecker.checkGTZero(val, "n"); 085 ELEvaluator eval = ELEvaluator.getCurrent(); 086 eval.setVariable("timeunit", TimeUnit.MINUTE); 087 eval.setVariable("endOfDuration", TimeUnit.NONE); 088 return val * 60; 089 } 090 091 /** 092 * Used in defining the frequency in 'minute' unit. <p/> domain: <code> val > 0</code> and should be integer. 093 * 094 * @param val frequency in number of minutes. 095 * @return number of minutes and also set the frequency timeunit to "minute" 096 */ 097 public static int ph1_coord_minutes(int val) { 098 val = ParamChecker.checkGTZero(val, "n"); 099 ELEvaluator eval = ELEvaluator.getCurrent(); 100 eval.setVariable("timeunit", TimeUnit.MINUTE); 101 eval.setVariable("endOfDuration", TimeUnit.NONE); 102 return val; 103 } 104 105 /** 106 * Used in defining the frequency in 'day' unit and specify the "end of day" property. <p/> Every instance will 107 * start at 00:00 hour of each day. <p/> domain: <code> val > 0</code> and should be integer. 108 * 109 * @param val frequency in number of days. 110 * @return number of days and also set the frequency timeunit to "day" and end_of_duration flag to "day" 111 */ 112 public static int ph1_coord_endOfDays(int val) { 113 val = ParamChecker.checkGTZero(val, "n"); 114 ELEvaluator eval = ELEvaluator.getCurrent(); 115 eval.setVariable("timeunit", TimeUnit.DAY); 116 eval.setVariable("endOfDuration", TimeUnit.END_OF_DAY); 117 return val; 118 } 119 120 /** 121 * Used in defining the frequency in 'month' unit and specify the "end of month" property. <p/> Every instance will 122 * start at first day of each month at 00:00 hour. <p/> domain: <code> val > 0</code> and should be integer. 123 * 124 * @param val: frequency in number of months. 125 * @return number of months and also set the frequency timeunit to "month" and end_of_duration flag to "month" 126 */ 127 public static int ph1_coord_endOfMonths(int val) { 128 val = ParamChecker.checkGTZero(val, "n"); 129 ELEvaluator eval = ELEvaluator.getCurrent(); 130 eval.setVariable("timeunit", TimeUnit.MONTH); 131 eval.setVariable("endOfDuration", TimeUnit.END_OF_MONTH); 132 return val; 133 } 134 135 /** 136 * Calculate the difference of timezone offset in minutes between dataset and coordinator job. <p/> Depends on: <p/> 137 * 1. Timezone of both dataset and job <p/> 2. Action creation Time 138 * 139 * @return difference in minutes (DataSet TZ Offset - Application TZ offset) 140 */ 141 public static int ph2_coord_tzOffset() { 142 Date actionCreationTime = getActionCreationtime(); 143 TimeZone dsTZ = ParamChecker.notNull(getDatasetTZ(), "DatasetTZ"); 144 TimeZone jobTZ = ParamChecker.notNull(getJobTZ(), "JobTZ"); 145 // Apply the TZ into Calendar object 146 Calendar dsTime = Calendar.getInstance(dsTZ); 147 dsTime.setTime(actionCreationTime); 148 Calendar jobTime = Calendar.getInstance(jobTZ); 149 jobTime.setTime(actionCreationTime); 150 return (dsTime.get(Calendar.ZONE_OFFSET) - jobTime.get(Calendar.ZONE_OFFSET)) / (1000 * 60); 151 } 152 153 public static int ph3_coord_tzOffset() { 154 return ph2_coord_tzOffset(); 155 } 156 157 /** 158 * Returns the a date string while given a base date in 'strBaseDate', 159 * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH). 160 * 161 * @param strBaseDate -- base date 162 * @param offset -- any number 163 * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH 164 * @return date string 165 * @throws Exception 166 */ 167 public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception { 168 Calendar baseCalDate = DateUtils.getCalendar(strBaseDate); 169 StringBuilder buffer = new StringBuilder(); 170 baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset); 171 buffer.append(DateUtils.formatDateUTC(baseCalDate)); 172 return buffer.toString(); 173 } 174 175 public static String ph3_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception { 176 return ph2_coord_dateOffset(strBaseDate, offset, unit); 177 } 178 179 /** 180 * Determine the date-time in UTC of n-th future available dataset instance 181 * from nominal Time but not beyond the instance specified as 'instance. 182 * <p/> 183 * It depends on: 184 * <p/> 185 * 1. Data set frequency 186 * <p/> 187 * 2. Data set Time unit (day, month, minute) 188 * <p/> 189 * 3. Data set Time zone/DST 190 * <p/> 191 * 4. End Day/Month flag 192 * <p/> 193 * 5. Data set initial instance 194 * <p/> 195 * 6. Action Creation Time 196 * <p/> 197 * 7. Existence of dataset's directory 198 * 199 * @param n :instance count 200 * <p/> 201 * domain: n >= 0, n is integer 202 * @param instance: How many future instance it should check? value should 203 * be >=0 204 * @return date-time in UTC of the n-th instance 205 * <p/> 206 * @throws Exception 207 */ 208 public static String ph3_coord_future(int n, int instance) throws Exception { 209 ParamChecker.checkGEZero(n, "future:n"); 210 ParamChecker.checkGTZero(instance, "future:instance"); 211 if (isSyncDataSet()) {// For Sync Dataset 212 return coord_future_sync(n, instance); 213 } 214 else { 215 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 216 } 217 } 218 219 private static String coord_future_sync(int n, int instance) throws Exception { 220 ELEvaluator eval = ELEvaluator.getCurrent(); 221 String retVal = ""; 222 int datasetFrequency = (int) getDSFrequency();// in minutes 223 TimeUnit dsTimeUnit = getDSTimeUnit(); 224 int[] instCount = new int[1]; 225 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount); 226 if (nominalInstanceCal != null) { 227 Calendar initInstance = getInitialInstanceCal(); 228 nominalInstanceCal = (Calendar) initInstance.clone(); 229 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 230 231 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 232 if (ds == null) { 233 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 234 } 235 String uriTemplate = ds.getUriTemplate(); 236 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION); 237 if (conf == null) { 238 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION); 239 } 240 int available = 0, checkedInstance = 0; 241 boolean resolved = false; 242 String user = ParamChecker 243 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME); 244 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME), 245 OozieClient.GROUP_NAME); 246 String doneFlag = ds.getDoneFlag(); 247 while (instance >= checkedInstance) { 248 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal); 249 String uriPath = uriEval.evaluate(uriTemplate, String.class); 250 String pathWithDoneFlag = uriPath; 251 if (doneFlag.length() > 0) { 252 pathWithDoneFlag += "/" + doneFlag; 253 } 254 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) { 255 XLog.getLog(CoordELFunctions.class).debug("Found future(" + available + "): " + pathWithDoneFlag); 256 if (available == n) { 257 XLog.getLog(CoordELFunctions.class).debug("Found future File: " + pathWithDoneFlag); 258 resolved = true; 259 retVal = DateUtils.formatDateUTC(nominalInstanceCal); 260 eval.setVariable("resolved_path", uriPath); 261 break; 262 } 263 available++; 264 } 265 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), 266 // -datasetFrequency); 267 nominalInstanceCal = (Calendar) initInstance.clone(); 268 instCount[0]++; 269 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 270 checkedInstance++; 271 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 272 } 273 if (!resolved) { 274 // return unchanged future function with variable 'is_resolved' 275 // to 'false' 276 eval.setVariable("is_resolved", Boolean.FALSE); 277 retVal = "${coord:future(" + n + ", " + instance + ")}"; 278 } 279 else { 280 eval.setVariable("is_resolved", Boolean.TRUE); 281 } 282 } 283 else {// No feasible nominal time 284 eval.setVariable("is_resolved", Boolean.TRUE); 285 retVal = ""; 286 } 287 return retVal; 288 } 289 290 /** 291 * Return nominal time or Action Creation Time. 292 * <p/> 293 * 294 * @return coordinator action creation or materialization date time 295 * @throws Exception if unable to format the Date object to String 296 */ 297 public static String ph2_coord_nominalTime() throws Exception { 298 ELEvaluator eval = ELEvaluator.getCurrent(); 299 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 300 "Coordinator Action"); 301 return DateUtils.formatDateUTC(action.getNominalTime()); 302 } 303 304 public static String ph3_coord_nominalTime() throws Exception { 305 return ph2_coord_nominalTime(); 306 } 307 308 /** 309 * Convert from standard date-time formatting to a desired format. 310 * <p/> 311 * @param dateTimeStr - A timestamp in standard (ISO8601) format. 312 * @param format - A string representing the desired format. 313 * @return coordinator action creation or materialization date time 314 * @throws Exception if unable to format the Date object to String 315 */ 316 public static String ph2_coord_formatTime(String dateTimeStr, String format) 317 throws Exception { 318 Date dateTime = DateUtils.parseDateUTC(dateTimeStr); 319 return DateUtils.formatDateCustom(dateTime, format); 320 } 321 322 public static String ph3_coord_formatTime(String dateTimeStr, String format) 323 throws Exception { 324 return ph2_coord_formatTime(dateTimeStr, format); 325 } 326 327 /** 328 * Return Action Id. <p/> 329 * 330 * @return coordinator action Id 331 */ 332 public static String ph2_coord_actionId() throws Exception { 333 ELEvaluator eval = ELEvaluator.getCurrent(); 334 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 335 "Coordinator Action"); 336 return action.getActionId(); 337 } 338 339 public static String ph3_coord_actionId() throws Exception { 340 return ph2_coord_actionId(); 341 } 342 343 /** 344 * Return Job Name. <p/> 345 * 346 * @return coordinator name 347 */ 348 public static String ph2_coord_name() throws Exception { 349 ELEvaluator eval = ELEvaluator.getCurrent(); 350 SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION), 351 "Coordinator Action"); 352 return action.getName(); 353 } 354 355 public static String ph3_coord_name() throws Exception { 356 return ph2_coord_name(); 357 } 358 359 /** 360 * Return Action Start time. <p/> 361 * 362 * @return coordinator action start time 363 * @throws Exception if unable to format the Date object to String 364 */ 365 public static String ph2_coord_actualTime() throws Exception { 366 ELEvaluator eval = ELEvaluator.getCurrent(); 367 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 368 if (coordAction == null) { 369 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 370 } 371 return DateUtils.formatDateUTC(coordAction.getActualTime()); 372 } 373 374 public static String ph3_coord_actualTime() throws Exception { 375 return ph2_coord_actualTime(); 376 } 377 378 /** 379 * Used to specify a list of URI's that are used as input dir to the workflow job. <p/> Look for two evaluator-level 380 * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of 381 * URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something 382 * unresolved, this function will echo back the original function <p/> otherwise it sends the uris. 383 * 384 * @param dataInName : Datain name 385 * @return the list of URI's separated by INSTANCE_SEPARATOR <p/> if there are unresolved EL function (i.e. latest) 386 * , echo back <p/> the function without resolving the function. 387 */ 388 public static String ph3_coord_dataIn(String dataInName) { 389 String uris = ""; 390 ELEvaluator eval = ELEvaluator.getCurrent(); 391 uris = (String) eval.getVariable(".datain." + dataInName); 392 Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved"); 393 if (unresolved != null && unresolved.booleanValue() == true) { 394 return "${coord:dataIn('" + dataInName + "')}"; 395 } 396 return uris; 397 } 398 399 /** 400 * Used to specify a list of URI's that are output dir of the workflow job. <p/> Look for one evaluator-level 401 * variable <p/> dataout.<DATAOUT_NAME> <p/> It defines the current list of URI. <p/> otherwise it sends the uris. 402 * 403 * @param dataOutName : Dataout name 404 * @return the list of URI's separated by INSTANCE_SEPARATOR 405 */ 406 public static String ph3_coord_dataOut(String dataOutName) { 407 String uris = ""; 408 ELEvaluator eval = ELEvaluator.getCurrent(); 409 uris = (String) eval.getVariable(".dataout." + dataOutName); 410 return uris; 411 } 412 413 /** 414 * Determine the date-time in UTC of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency <p/> 2. 415 * Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. Data 416 * set initial instance <p/> 6. Action Creation Time 417 * 418 * @param n instance count domain: n is integer 419 * @return date-time in UTC of the n-th instance returns 'null' means n-th instance is earlier than Initial-Instance 420 * of DS 421 * @throws Exception 422 */ 423 public static String ph2_coord_current(int n) throws Exception { 424 if (isSyncDataSet()) { // For Sync Dataset 425 return coord_current_sync(n); 426 } 427 else { 428 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 429 } 430 } 431 432 /** 433 * Determine how many hours is on the date of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency 434 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. 435 * Data set initial instance <p/> 6. Action Creation Time 436 * 437 * @param n instance count <p/> domain: n is integer 438 * @return number of hours on that day <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS 439 * @throws Exception 440 */ 441 public static int ph2_coord_hoursInDay(int n) throws Exception { 442 int datasetFrequency = (int) getDSFrequency(); 443 // /Calendar nominalInstanceCal = 444 // getCurrentInstance(getActionCreationtime()); 445 Calendar nominalInstanceCal = getEffectiveNominalTime(); 446 if (nominalInstanceCal == null) { 447 return -1; 448 } 449 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n); 450 /* 451 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) 452 * { return -1; } 453 */ 454 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ 455 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 456 return DateUtils.hoursInDay(nominalInstanceCal); 457 } 458 459 public static int ph3_coord_hoursInDay(int n) throws Exception { 460 return ph2_coord_hoursInDay(n); 461 } 462 463 /** 464 * Calculate number of days in one month for n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency . 465 * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. 466 * Data set initial instance <p/> 6. Action Creation Time 467 * 468 * @param n instance count. domain: n is integer 469 * @return number of days in that month <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS 470 * @throws Exception 471 */ 472 public static int ph2_coord_daysInMonth(int n) throws Exception { 473 int datasetFrequency = (int) getDSFrequency();// in minutes 474 // Calendar nominalInstanceCal = 475 // getCurrentInstance(getActionCreationtime()); 476 Calendar nominalInstanceCal = getEffectiveNominalTime(); 477 if (nominalInstanceCal == null) { 478 return -1; 479 } 480 nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n); 481 /* 482 * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) 483 * { return -1; } 484 */ 485 nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ 486 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 487 return nominalInstanceCal.getActualMaximum(Calendar.DAY_OF_MONTH); 488 } 489 490 public static int ph3_coord_daysInMonth(int n) throws Exception { 491 return ph2_coord_daysInMonth(n); 492 } 493 494 /** 495 * Determine the date-time in UTC of n-th latest available dataset instance. <p/> It depends on: <p/> 1. Data set 496 * frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month 497 * flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of dataset's directory 498 * 499 * @param n :instance count <p/> domain: n > 0, n is integer 500 * @return date-time in UTC of the n-th instance <p/> returns 'null' means n-th instance is earlier than 501 * Initial-Instance of DS 502 * @throws Exception 503 */ 504 public static String ph3_coord_latest(int n) throws Exception { 505 if (n > 0) { 506 throw new IllegalArgumentException("paramter should be <= 0 but it is " + n); 507 } 508 if (isSyncDataSet()) {// For Sync Dataset 509 return coord_latest_sync(n); 510 } 511 else { 512 throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet"); 513 } 514 } 515 516 /** 517 * Configure an evaluator with data set and application specific information. <p/> Helper method of associating 518 * dataset and application object 519 * 520 * @param evaluator : to set variables 521 * @param ds : Data Set object 522 * @param coordAction : Application instance 523 */ 524 public static void configureEvaluator(ELEvaluator evaluator, SyncCoordDataset ds, SyncCoordAction coordAction) { 525 evaluator.setVariable(COORD_ACTION, coordAction); 526 evaluator.setVariable(DATASET, ds); 527 } 528 529 /** 530 * Helper method to wrap around with "${..}". <p/> 531 * 532 * 533 * @param eval :EL evaluator 534 * @param expr : expression to evaluate 535 * @return Resolved expression or echo back the same expression 536 * @throws Exception 537 */ 538 public static String evalAndWrap(ELEvaluator eval, String expr) throws Exception { 539 try { 540 eval.setVariable(".wrap", null); 541 String result = eval.evaluate(expr, String.class); 542 if (eval.getVariable(".wrap") != null) { 543 return "${" + result + "}"; 544 } 545 else { 546 return result; 547 } 548 } 549 catch (Exception e) { 550 throw new Exception("Unable to evaluate :" + expr + ":\n", e); 551 } 552 } 553 554 // Set of echo functions 555 556 public static String ph1_coord_current_echo(String n) { 557 return echoUnResolved("current", n); 558 } 559 560 public static String ph2_coord_current_echo(String n) { 561 return echoUnResolved("current", n); 562 } 563 564 public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) { 565 return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit); 566 } 567 568 public static String ph1_coord_formatTime_echo(String dateTime, String format) { 569 // Quote the dateTime value since it would contain a ':'. 570 return echoUnResolved("formatTime", "'"+dateTime+"'" + " , " + format); 571 } 572 573 public static String ph1_coord_latest_echo(String n) { 574 return echoUnResolved("latest", n); 575 } 576 577 public static String ph2_coord_latest_echo(String n) { 578 return ph1_coord_latest_echo(n); 579 } 580 581 public static String ph1_coord_future_echo(String n, String instance) { 582 return echoUnResolved("future", n + ", " + instance + ""); 583 } 584 585 public static String ph2_coord_future_echo(String n, String instance) { 586 return ph1_coord_future_echo(n, instance); 587 } 588 589 public static String ph1_coord_dataIn_echo(String n) { 590 ELEvaluator eval = ELEvaluator.getCurrent(); 591 String val = (String) eval.getVariable("oozie.dataname." + n); 592 if (val == null || val.equals("data-in") == false) { 593 XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid"); 594 throw new RuntimeException("data_in_name " + n + " is not valid"); 595 } 596 return echoUnResolved("dataIn", "'" + n + "'"); 597 } 598 599 public static String ph1_coord_dataOut_echo(String n) { 600 ELEvaluator eval = ELEvaluator.getCurrent(); 601 String val = (String) eval.getVariable("oozie.dataname." + n); 602 if (val == null || val.equals("data-out") == false) { 603 XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid"); 604 throw new RuntimeException("data_out_name " + n + " is not valid"); 605 } 606 return echoUnResolved("dataOut", "'" + n + "'"); 607 } 608 609 public static String ph1_coord_nominalTime_echo() { 610 return echoUnResolved("nominalTime", ""); 611 } 612 613 public static String ph1_coord_nominalTime_echo_wrap() { 614 // return "${coord:nominalTime()}"; // no resolution 615 return echoUnResolved("nominalTime", ""); 616 } 617 618 public static String ph1_coord_nominalTime_echo_fixed() { 619 return "2009-03-06T010:00"; // Dummy resolution 620 } 621 622 public static String ph1_coord_actualTime_echo_wrap() { 623 // return "${coord:actualTime()}"; // no resolution 624 return echoUnResolved("actualTime", ""); 625 } 626 627 public static String ph1_coord_actionId_echo() { 628 return echoUnResolved("actionId", ""); 629 } 630 631 public static String ph1_coord_name_echo() { 632 return echoUnResolved("name", ""); 633 } 634 635 // The following echo functions are not used in any phases yet 636 // They are here for future purpose. 637 public static String coord_minutes_echo(String n) { 638 return echoUnResolved("minutes", n); 639 } 640 641 public static String coord_hours_echo(String n) { 642 return echoUnResolved("hours", n); 643 } 644 645 public static String coord_days_echo(String n) { 646 return echoUnResolved("days", n); 647 } 648 649 public static String coord_endOfDay_echo(String n) { 650 return echoUnResolved("endOfDay", n); 651 } 652 653 public static String coord_months_echo(String n) { 654 return echoUnResolved("months", n); 655 } 656 657 public static String coord_endOfMonth_echo(String n) { 658 return echoUnResolved("endOfMonth", n); 659 } 660 661 public static String coord_actualTime_echo() { 662 return echoUnResolved("actualTime", ""); 663 } 664 665 // This echo function will always return "24" for validation only. 666 // This evaluation ****should not**** replace the original XML 667 // Create a temporary string and validate the function 668 // This is **required** for evaluating an expression like 669 // coord:HoursInDay(0) + 3 670 // actual evaluation will happen in phase 2 or phase 3. 671 public static String ph1_coord_hoursInDay_echo(String n) { 672 return "24"; 673 // return echoUnResolved("hoursInDay", n); 674 } 675 676 // This echo function will always return "30" for validation only. 677 // This evaluation ****should not**** replace the original XML 678 // Create a temporary string and validate the function 679 // This is **required** for evaluating an expression like 680 // coord:daysInMonth(0) + 3 681 // actual evaluation will happen in phase 2 or phase 3. 682 public static String ph1_coord_daysInMonth_echo(String n) { 683 // return echoUnResolved("daysInMonth", n); 684 return "30"; 685 } 686 687 // This echo function will always return "3" for validation only. 688 // This evaluation ****should not**** replace the original XML 689 // Create a temporary string and validate the function 690 // This is **required** for evaluating an expression like coord:tzOffset + 2 691 // actual evaluation will happen in phase 2 or phase 3. 692 public static String ph1_coord_tzOffset_echo() { 693 // return echoUnResolved("tzOffset", ""); 694 return "3"; 695 } 696 697 // Local methods 698 /** 699 * @param n 700 * @return n-th instance Date-Time from current instance for data-set <p/> return empty string ("") if the 701 * Action_Creation_time or the n-th instance <p/> is earlier than the Initial_Instance of dataset. 702 * @throws Exception 703 */ 704 private static String coord_current_sync(int n) throws Exception { 705 int datasetFrequency = getDSFrequency();// in minutes 706 TimeUnit dsTimeUnit = getDSTimeUnit(); 707 int[] instCount = new int[1];// used as pass by ref 708 Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount); 709 if (nominalInstanceCal == null) { 710 return ""; 711 } 712 nominalInstanceCal = getInitialInstanceCal(); 713 int absInstanceCount = instCount[0] + n; 714 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount); 715 716 if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) { 717 return ""; 718 } 719 String str = DateUtils.formatDateUTC(nominalInstanceCal); 720 return str; 721 } 722 723 /** 724 * @param offset 725 * @return n-th available latest instance Date-Time for SYNC data-set 726 * @throws Exception 727 */ 728 private static String coord_latest_sync(int offset) throws Exception { 729 if (offset > 0) { 730 throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0" 731 + offset); 732 } 733 ELEvaluator eval = ELEvaluator.getCurrent(); 734 String retVal = ""; 735 int datasetFrequency = (int) getDSFrequency();// in minutes 736 TimeUnit dsTimeUnit = getDSTimeUnit(); 737 int[] instCount = new int[1]; 738 Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount); 739 if (nominalInstanceCal != null) { 740 Calendar initInstance = getInitialInstanceCal(); 741 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 742 if (ds == null) { 743 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 744 } 745 String uriTemplate = ds.getUriTemplate(); 746 Configuration conf = (Configuration) eval.getVariable(CONFIGURATION); 747 if (conf == null) { 748 throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION); 749 } 750 int available = 0; 751 boolean resolved = false; 752 String user = ParamChecker 753 .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME); 754 String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME), 755 OozieClient.GROUP_NAME); 756 String doneFlag = ds.getDoneFlag(); 757 while (nominalInstanceCal.compareTo(initInstance) >= 0) { 758 ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal); 759 String uriPath = uriEval.evaluate(uriTemplate, String.class); 760 String pathWithDoneFlag = uriPath; 761 if (doneFlag.length() > 0) { 762 pathWithDoneFlag += "/" + doneFlag; 763 } 764 if (isPathAvailable(pathWithDoneFlag, user, group, conf)) { 765 XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + pathWithDoneFlag); 766 if (available == offset) { 767 XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag); 768 resolved = true; 769 retVal = DateUtils.formatDateUTC(nominalInstanceCal); 770 eval.setVariable("resolved_path", uriPath); 771 break; 772 } 773 774 available--; 775 } 776 // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), 777 // -datasetFrequency); 778 nominalInstanceCal = (Calendar) initInstance.clone(); 779 instCount[0]--; 780 nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency); 781 // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag()); 782 } 783 if (!resolved) { 784 // return unchanged latest function with variable 'is_resolved' 785 // to 'false' 786 eval.setVariable("is_resolved", Boolean.FALSE); 787 retVal = "${coord:latest(" + offset + ")}"; 788 } 789 else { 790 eval.setVariable("is_resolved", Boolean.TRUE); 791 } 792 } 793 else {// No feasible nominal time 794 eval.setVariable("is_resolved", Boolean.FALSE); 795 } 796 return retVal; 797 } 798 799 // TODO : Not an efficient way. In a loop environment, we could do something 800 // outside the loop 801 /** 802 * Check whether a URI path exists 803 * 804 * @param sPath 805 * @param conf 806 * @return 807 * @throws IOException 808 */ 809 810 private static boolean isPathAvailable(String sPath, String user, String group, Configuration conf) 811 throws IOException, HadoopAccessorException { 812 // sPath += "/" + END_OF_OPERATION_INDICATOR_FILE; 813 Path path = new Path(sPath); 814 return Services.get().get(HadoopAccessorService.class). 815 createFileSystem(user, group, path.toUri(), conf).exists(path); 816 } 817 818 /** 819 * @param tm 820 * @return a new Evaluator to be used for URI-template evaluation 821 */ 822 private static ELEvaluator getUriEvaluator(Calendar tm) { 823 ELEvaluator retEval = new ELEvaluator(); 824 retEval.setVariable("YEAR", tm.get(Calendar.YEAR)); 825 retEval.setVariable("MONTH", (tm.get(Calendar.MONTH) + 1) < 10 ? "0" + (tm.get(Calendar.MONTH) + 1) : (tm 826 .get(Calendar.MONTH) + 1)); 827 retEval.setVariable("DAY", tm.get(Calendar.DAY_OF_MONTH) < 10 ? "0" + tm.get(Calendar.DAY_OF_MONTH) : tm 828 .get(Calendar.DAY_OF_MONTH)); 829 retEval.setVariable("HOUR", tm.get(Calendar.HOUR_OF_DAY) < 10 ? "0" + tm.get(Calendar.HOUR_OF_DAY) : tm 830 .get(Calendar.HOUR_OF_DAY)); 831 retEval.setVariable("MINUTE", tm.get(Calendar.MINUTE) < 10 ? "0" + tm.get(Calendar.MINUTE) : tm 832 .get(Calendar.MINUTE)); 833 return retEval; 834 } 835 836 /** 837 * @return whether a data set is SYNCH or ASYNC 838 */ 839 private static boolean isSyncDataSet() { 840 ELEvaluator eval = ELEvaluator.getCurrent(); 841 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 842 if (ds == null) { 843 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 844 } 845 return ds.getType().equalsIgnoreCase("SYNC"); 846 } 847 848 /** 849 * Check whether a function should be resolved. 850 * 851 * @param functionName 852 * @param n 853 * @return null if the functionName needs to be resolved otherwise return the calling function unresolved. 854 */ 855 private static String checkIfResolved(String functionName, String n) { 856 ELEvaluator eval = ELEvaluator.getCurrent(); 857 String replace = (String) eval.getVariable("resolve_" + functionName); 858 if (replace == null || (replace != null && replace.equalsIgnoreCase("false"))) { // Don't 859 // resolve 860 // return "${coord:" + functionName + "(" + n +")}"; //Unresolved 861 eval.setVariable(".wrap", "true"); 862 return "coord:" + functionName + "(" + n + ")"; // Unresolved 863 } 864 return null; // Resolved it 865 } 866 867 private static String echoUnResolved(String functionName, String n) { 868 return echoUnResolvedPre(functionName, n, "coord:"); 869 } 870 871 private static String echoUnResolvedPre(String functionName, String n, String prefix) { 872 ELEvaluator eval = ELEvaluator.getCurrent(); 873 eval.setVariable(".wrap", "true"); 874 return prefix + functionName + "(" + n + ")"; // Unresolved 875 } 876 877 /** 878 * @return the initial instance of a DataSet in DATE 879 */ 880 private static Date getInitialInstance() { 881 return getInitialInstanceCal().getTime(); 882 // return ds.getInitInstance(); 883 } 884 885 /** 886 * @return the initial instance of a DataSet in Calendar 887 */ 888 private static Calendar getInitialInstanceCal() { 889 ELEvaluator eval = ELEvaluator.getCurrent(); 890 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 891 if (ds == null) { 892 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 893 } 894 Calendar effInitTS = Calendar.getInstance(); 895 effInitTS.setTime(ds.getInitInstance()); 896 effInitTS.setTimeZone(ds.getTimeZone()); 897 // To adjust EOD/EOM 898 DateUtils.moveToEnd(effInitTS, getDSEndOfFlag()); 899 return effInitTS; 900 // return ds.getInitInstance(); 901 } 902 903 /** 904 * @return Nominal or action creation Time when all the dependencies of an application instance are met. 905 */ 906 private static Date getActionCreationtime() { 907 ELEvaluator eval = ELEvaluator.getCurrent(); 908 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 909 if (coordAction == null) { 910 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 911 } 912 return coordAction.getNominalTime(); 913 } 914 915 /** 916 * @return Actual Time when all the dependencies of an application instance are met. 917 */ 918 private static Date getActualTime() { 919 ELEvaluator eval = ELEvaluator.getCurrent(); 920 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 921 if (coordAction == null) { 922 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 923 } 924 return coordAction.getActualTime(); 925 } 926 927 /** 928 * @return TimeZone for the application or job. 929 */ 930 private static TimeZone getJobTZ() { 931 ELEvaluator eval = ELEvaluator.getCurrent(); 932 SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION); 933 if (coordAction == null) { 934 throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION); 935 } 936 return coordAction.getTimeZone(); 937 } 938 939 /** 940 * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time) 941 * 942 * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of 943 * the dataset. 944 */ 945 private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) { 946 Date datasetInitialInstance = getInitialInstance(); 947 TimeUnit dsTimeUnit = getDSTimeUnit(); 948 TimeZone dsTZ = getDatasetTZ(); 949 // Convert Date to Calendar for corresponding TZ 950 Calendar current = Calendar.getInstance(); 951 current.setTime(datasetInitialInstance); 952 current.setTimeZone(dsTZ); 953 954 Calendar calEffectiveTime = Calendar.getInstance(); 955 calEffectiveTime.setTime(effectiveTime); 956 calEffectiveTime.setTimeZone(dsTZ); 957 instanceCount[0] = 0; 958 if (current.compareTo(calEffectiveTime) > 0) { 959 // Nominal Time < initial Instance 960 // TODO: getClass() call doesn't work from static method. 961 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+ 962 // current.getTime()); 963 return null; 964 } 965 Calendar origCurrent = (Calendar) current.clone(); 966 while (current.compareTo(calEffectiveTime) <= 0) { 967 current = (Calendar) origCurrent.clone(); 968 instanceCount[0]++; 969 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency()); 970 } 971 instanceCount[0]--; 972 973 current = (Calendar) origCurrent.clone(); 974 current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency()); 975 return current; 976 } 977 978 private static Calendar getEffectiveNominalTime() { 979 Date datasetInitialInstance = getInitialInstance(); 980 TimeZone dsTZ = getDatasetTZ(); 981 // Convert Date to Calendar for corresponding TZ 982 Calendar current = Calendar.getInstance(); 983 current.setTime(datasetInitialInstance); 984 current.setTimeZone(dsTZ); 985 986 Calendar calEffectiveTime = Calendar.getInstance(); 987 calEffectiveTime.setTime(getActionCreationtime()); 988 calEffectiveTime.setTimeZone(dsTZ); 989 if (current.compareTo(calEffectiveTime) > 0) { 990 // Nominal Time < initial Instance 991 // TODO: getClass() call doesn't work from static method. 992 // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+ 993 // current.getTime()); 994 return null; 995 } 996 return calEffectiveTime; 997 } 998 999 /** 1000 * @return dataset frequency in minutes 1001 */ 1002 private static int getDSFrequency() { 1003 ELEvaluator eval = ELEvaluator.getCurrent(); 1004 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1005 if (ds == null) { 1006 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1007 } 1008 return ds.getFrequency(); 1009 } 1010 1011 /** 1012 * @return dataset TimeUnit 1013 */ 1014 private static TimeUnit getDSTimeUnit() { 1015 ELEvaluator eval = ELEvaluator.getCurrent(); 1016 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1017 if (ds == null) { 1018 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1019 } 1020 return ds.getTimeUnit(); 1021 } 1022 1023 /** 1024 * @return dataset TimeZone 1025 */ 1026 private static TimeZone getDatasetTZ() { 1027 ELEvaluator eval = ELEvaluator.getCurrent(); 1028 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1029 if (ds == null) { 1030 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1031 } 1032 return ds.getTimeZone(); 1033 } 1034 1035 /** 1036 * @return dataset TimeUnit 1037 */ 1038 private static TimeUnit getDSEndOfFlag() { 1039 ELEvaluator eval = ELEvaluator.getCurrent(); 1040 SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET); 1041 if (ds == null) { 1042 throw new RuntimeException("Associated Dataset should be defined with key " + DATASET); 1043 } 1044 return ds.getEndOfDuration();// == null ? "": ds.getEndOfDuration(); 1045 } 1046 1047 /** 1048 * Return the user that submitted the coordinator job. 1049 * 1050 * @return the user that submitted the coordinator job. 1051 */ 1052 public static String coord_user() { 1053 ELEvaluator eval = ELEvaluator.getCurrent(); 1054 return (String) eval.getVariable(OozieClient.USER_NAME); 1055 } 1056 }