001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.coord;
016    
017    import java.io.IOException;
018    import java.util.Calendar;
019    import java.util.Date;
020    import java.util.TimeZone;
021    
022    import org.apache.hadoop.conf.Configuration;
023    import org.apache.hadoop.fs.Path;
024    
025    import org.apache.oozie.client.OozieClient;
026    import org.apache.oozie.util.DateUtils;
027    import org.apache.oozie.util.ELEvaluator;
028    import org.apache.oozie.util.ParamChecker;
029    import org.apache.oozie.util.XLog;
030    import org.apache.oozie.service.HadoopAccessorException;
031    import org.apache.oozie.service.Services;
032    import org.apache.oozie.service.HadoopAccessorService;
033    
034    /**
035     * This class implements the EL function related to coordinator
036     */
037    
038    public class CoordELFunctions {
039        final private static String DATASET = "oozie.coord.el.dataset.bean";
040        final private static String COORD_ACTION = "oozie.coord.el.app.bean";
041        final public static String CONFIGURATION = "oozie.coord.el.conf";
042        // INSTANCE_SEPARATOR is used to separate multiple directories into one tag.
043        final public static String INSTANCE_SEPARATOR = "#";
044        final public static String DIR_SEPARATOR = ",";
045        // TODO: in next release, support flexibility
046        private static String END_OF_OPERATION_INDICATOR_FILE = "_SUCCESS";
047    
048        /**
049         * Used in defining the frequency in 'day' unit. <p/> domain: <code> val &gt; 0</code> and should be integer.
050         *
051         * @param val frequency in number of days.
052         * @return number of days and also set the frequency timeunit to "day"
053         */
054        public static int ph1_coord_days(int val) {
055            val = ParamChecker.checkGTZero(val, "n");
056            ELEvaluator eval = ELEvaluator.getCurrent();
057            eval.setVariable("timeunit", TimeUnit.DAY);
058            eval.setVariable("endOfDuration", TimeUnit.NONE);
059            return val;
060        }
061    
062        /**
063         * Used in defining the frequency in 'month' unit. <p/> domain: <code> val &gt; 0</code> and should be integer.
064         *
065         * @param val frequency in number of months.
066         * @return number of months and also set the frequency timeunit to "month"
067         */
068        public static int ph1_coord_months(int val) {
069            val = ParamChecker.checkGTZero(val, "n");
070            ELEvaluator eval = ELEvaluator.getCurrent();
071            eval.setVariable("timeunit", TimeUnit.MONTH);
072            eval.setVariable("endOfDuration", TimeUnit.NONE);
073            return val;
074        }
075    
076        /**
077         * Used in defining the frequency in 'hour' unit. <p/> parameter value domain: <code> val &gt; 0</code> and should
078         * be integer.
079         *
080         * @param val frequency in number of hours.
081         * @return number of minutes and also set the frequency timeunit to "minute"
082         */
083        public static int ph1_coord_hours(int val) {
084            val = ParamChecker.checkGTZero(val, "n");
085            ELEvaluator eval = ELEvaluator.getCurrent();
086            eval.setVariable("timeunit", TimeUnit.MINUTE);
087            eval.setVariable("endOfDuration", TimeUnit.NONE);
088            return val * 60;
089        }
090    
091        /**
092         * Used in defining the frequency in 'minute' unit. <p/> domain: <code> val &gt; 0</code> and should be integer.
093         *
094         * @param val frequency in number of minutes.
095         * @return number of minutes and also set the frequency timeunit to "minute"
096         */
097        public static int ph1_coord_minutes(int val) {
098            val = ParamChecker.checkGTZero(val, "n");
099            ELEvaluator eval = ELEvaluator.getCurrent();
100            eval.setVariable("timeunit", TimeUnit.MINUTE);
101            eval.setVariable("endOfDuration", TimeUnit.NONE);
102            return val;
103        }
104    
105        /**
106         * Used in defining the frequency in 'day' unit and specify the "end of day" property. <p/> Every instance will
107         * start at 00:00 hour of each day. <p/> domain: <code> val &gt; 0</code> and should be integer.
108         *
109         * @param val frequency in number of days.
110         * @return number of days and also set the frequency timeunit to "day" and end_of_duration flag to "day"
111         */
112        public static int ph1_coord_endOfDays(int val) {
113            val = ParamChecker.checkGTZero(val, "n");
114            ELEvaluator eval = ELEvaluator.getCurrent();
115            eval.setVariable("timeunit", TimeUnit.DAY);
116            eval.setVariable("endOfDuration", TimeUnit.END_OF_DAY);
117            return val;
118        }
119    
120        /**
121         * Used in defining the frequency in 'month' unit and specify the "end of month" property. <p/> Every instance will
122         * start at first day of each month at 00:00 hour. <p/> domain: <code> val &gt; 0</code> and should be integer.
123         *
124         * @param val: frequency in number of months.
125         * @return number of months and also set the frequency timeunit to "month" and end_of_duration flag to "month"
126         */
127        public static int ph1_coord_endOfMonths(int val) {
128            val = ParamChecker.checkGTZero(val, "n");
129            ELEvaluator eval = ELEvaluator.getCurrent();
130            eval.setVariable("timeunit", TimeUnit.MONTH);
131            eval.setVariable("endOfDuration", TimeUnit.END_OF_MONTH);
132            return val;
133        }
134    
135        /**
136         * Calculate the difference of timezone offset in minutes between dataset and coordinator job. <p/> Depends on: <p/>
137         * 1. Timezone of both dataset and job <p/> 2. Action creation Time
138         *
139         * @return difference in minutes (DataSet TZ Offset - Application TZ offset)
140         */
141        public static int ph2_coord_tzOffset() {
142            Date actionCreationTime = getActionCreationtime();
143            TimeZone dsTZ = ParamChecker.notNull(getDatasetTZ(), "DatasetTZ");
144            TimeZone jobTZ = ParamChecker.notNull(getJobTZ(), "JobTZ");
145            // Apply the TZ into Calendar object
146            Calendar dsTime = Calendar.getInstance(dsTZ);
147            dsTime.setTime(actionCreationTime);
148            Calendar jobTime = Calendar.getInstance(jobTZ);
149            jobTime.setTime(actionCreationTime);
150            return (dsTime.get(Calendar.ZONE_OFFSET) - jobTime.get(Calendar.ZONE_OFFSET)) / (1000 * 60);
151        }
152    
153        public static int ph3_coord_tzOffset() {
154            return ph2_coord_tzOffset();
155        }
156    
157        /**
158         * Returns the a date string while given a base date in 'strBaseDate',
159         * offset and unit (e.g. DAY, MONTH, HOUR, MINUTE, MONTH).
160         *
161         * @param strBaseDate -- base date
162         * @param offset -- any number
163         * @param unit -- DAY, MONTH, HOUR, MINUTE, MONTH
164         * @return date string
165         * @throws Exception
166         */
167        public static String ph2_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
168            Calendar baseCalDate = DateUtils.getCalendar(strBaseDate);
169            StringBuilder buffer = new StringBuilder();
170            baseCalDate.add(TimeUnit.valueOf(unit).getCalendarUnit(), offset);
171            buffer.append(DateUtils.formatDateUTC(baseCalDate));
172            return buffer.toString();
173        }
174    
175        public static String ph3_coord_dateOffset(String strBaseDate, int offset, String unit) throws Exception {
176            return ph2_coord_dateOffset(strBaseDate, offset, unit);
177        }
178    
179        /**
180         * Determine the date-time in UTC of n-th future available dataset instance
181         * from nominal Time but not beyond the instance specified as 'instance.
182         * <p/>
183         * It depends on:
184         * <p/>
185         * 1. Data set frequency
186         * <p/>
187         * 2. Data set Time unit (day, month, minute)
188         * <p/>
189         * 3. Data set Time zone/DST
190         * <p/>
191         * 4. End Day/Month flag
192         * <p/>
193         * 5. Data set initial instance
194         * <p/>
195         * 6. Action Creation Time
196         * <p/>
197         * 7. Existence of dataset's directory
198         *
199         * @param n :instance count
200         *        <p/>
201         *        domain: n >= 0, n is integer
202         * @param instance: How many future instance it should check? value should
203         *        be >=0
204         * @return date-time in UTC of the n-th instance
205         *         <p/>
206         * @throws Exception
207         */
208        public static String ph3_coord_future(int n, int instance) throws Exception {
209            ParamChecker.checkGEZero(n, "future:n");
210            ParamChecker.checkGTZero(instance, "future:instance");
211            if (isSyncDataSet()) {// For Sync Dataset
212                return coord_future_sync(n, instance);
213            }
214            else {
215                throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
216            }
217        }
218    
219        private static String coord_future_sync(int n, int instance) throws Exception {
220            ELEvaluator eval = ELEvaluator.getCurrent();
221            String retVal = "";
222            int datasetFrequency = (int) getDSFrequency();// in minutes
223            TimeUnit dsTimeUnit = getDSTimeUnit();
224            int[] instCount = new int[1];
225            Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
226            if (nominalInstanceCal != null) {
227                Calendar initInstance = getInitialInstanceCal();
228                nominalInstanceCal = (Calendar) initInstance.clone();
229                nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
230    
231                SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
232                if (ds == null) {
233                    throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
234                }
235                String uriTemplate = ds.getUriTemplate();
236                Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
237                if (conf == null) {
238                    throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
239                }
240                int available = 0, checkedInstance = 0;
241                boolean resolved = false;
242                String user = ParamChecker
243                        .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
244                String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME),
245                        OozieClient.GROUP_NAME);
246                String doneFlag = ds.getDoneFlag();
247                while (instance >= checkedInstance) {
248                    ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
249                    String uriPath = uriEval.evaluate(uriTemplate, String.class);
250                    String pathWithDoneFlag = uriPath;
251                    if (doneFlag.length() > 0) {
252                        pathWithDoneFlag += "/" + doneFlag;
253                    }
254                    if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
255                        XLog.getLog(CoordELFunctions.class).debug("Found future(" + available + "): " + pathWithDoneFlag);
256                        if (available == n) {
257                            XLog.getLog(CoordELFunctions.class).debug("Found future File: " + pathWithDoneFlag);
258                            resolved = true;
259                            retVal = DateUtils.formatDateUTC(nominalInstanceCal);
260                            eval.setVariable("resolved_path", uriPath);
261                            break;
262                        }
263                        available++;
264                    }
265                    // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
266                    // -datasetFrequency);
267                    nominalInstanceCal = (Calendar) initInstance.clone();
268                    instCount[0]++;
269                    nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
270                    checkedInstance++;
271                    // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
272                }
273                if (!resolved) {
274                    // return unchanged future function with variable 'is_resolved'
275                    // to 'false'
276                    eval.setVariable("is_resolved", Boolean.FALSE);
277                    retVal = "${coord:future(" + n + ", " + instance + ")}";
278                }
279                else {
280                    eval.setVariable("is_resolved", Boolean.TRUE);
281                }
282            }
283            else {// No feasible nominal time
284                eval.setVariable("is_resolved", Boolean.TRUE);
285                retVal = "";
286            }
287            return retVal;
288        }
289    
290        /**
291         * Return nominal time or Action Creation Time.
292         * <p/>
293         *
294         * @return coordinator action creation or materialization date time
295         * @throws Exception if unable to format the Date object to String
296         */
297        public static String ph2_coord_nominalTime() throws Exception {
298            ELEvaluator eval = ELEvaluator.getCurrent();
299            SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
300                    "Coordinator Action");
301            return DateUtils.formatDateUTC(action.getNominalTime());
302        }
303    
304        public static String ph3_coord_nominalTime() throws Exception {
305            return ph2_coord_nominalTime();
306        }
307    
308        /**
309         * Convert from standard date-time formatting to a desired format.
310         * <p/>
311         * @param dateTimeStr - A timestamp in standard (ISO8601) format.
312         * @param format - A string representing the desired format.
313         * @return coordinator action creation or materialization date time
314         * @throws Exception if unable to format the Date object to String
315         */
316        public static String ph2_coord_formatTime(String dateTimeStr, String format)
317            throws Exception {
318            Date dateTime = DateUtils.parseDateUTC(dateTimeStr);
319            return DateUtils.formatDateCustom(dateTime, format);
320        }
321    
322        public static String ph3_coord_formatTime(String dateTimeStr, String format)
323            throws Exception {
324            return ph2_coord_formatTime(dateTimeStr, format);
325        }
326    
327        /**
328         * Return Action Id. <p/>
329         *
330         * @return coordinator action Id
331         */
332        public static String ph2_coord_actionId() throws Exception {
333            ELEvaluator eval = ELEvaluator.getCurrent();
334            SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
335                    "Coordinator Action");
336            return action.getActionId();
337        }
338    
339        public static String ph3_coord_actionId() throws Exception {
340            return ph2_coord_actionId();
341        }
342    
343        /**
344         * Return Job Name. <p/>
345         *
346         * @return coordinator name
347         */
348        public static String ph2_coord_name() throws Exception {
349            ELEvaluator eval = ELEvaluator.getCurrent();
350            SyncCoordAction action = ParamChecker.notNull((SyncCoordAction) eval.getVariable(COORD_ACTION),
351                    "Coordinator Action");
352            return action.getName();
353        }
354    
355        public static String ph3_coord_name() throws Exception {
356            return ph2_coord_name();
357        }
358    
359        /**
360         * Return Action Start time. <p/>
361         *
362         * @return coordinator action start time
363         * @throws Exception if unable to format the Date object to String
364         */
365        public static String ph2_coord_actualTime() throws Exception {
366            ELEvaluator eval = ELEvaluator.getCurrent();
367            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
368            if (coordAction == null) {
369                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
370            }
371            return DateUtils.formatDateUTC(coordAction.getActualTime());
372        }
373    
374        public static String ph3_coord_actualTime() throws Exception {
375            return ph2_coord_actualTime();
376        }
377    
378        /**
379         * Used to specify a list of URI's that are used as input dir to the workflow job. <p/> Look for two evaluator-level
380         * variables <p/> A) .datain.<DATAIN_NAME> B) .datain.<DATAIN_NAME>.unresolved <p/> A defines the current list of
381         * URI. <p/> B defines whether there are any unresolved EL-function (i.e latest) <p/> If there are something
382         * unresolved, this function will echo back the original function <p/> otherwise it sends the uris.
383         *
384         * @param dataInName : Datain name
385         * @return the list of URI's separated by INSTANCE_SEPARATOR <p/> if there are unresolved EL function (i.e. latest)
386         *         , echo back <p/> the function without resolving the function.
387         */
388        public static String ph3_coord_dataIn(String dataInName) {
389            String uris = "";
390            ELEvaluator eval = ELEvaluator.getCurrent();
391            uris = (String) eval.getVariable(".datain." + dataInName);
392            Boolean unresolved = (Boolean) eval.getVariable(".datain." + dataInName + ".unresolved");
393            if (unresolved != null && unresolved.booleanValue() == true) {
394                return "${coord:dataIn('" + dataInName + "')}";
395            }
396            return uris;
397        }
398    
399        /**
400         * Used to specify a list of URI's that are output dir of the workflow job. <p/> Look for one evaluator-level
401         * variable <p/> dataout.<DATAOUT_NAME> <p/> It defines the current list of URI. <p/> otherwise it sends the uris.
402         *
403         * @param dataOutName : Dataout name
404         * @return the list of URI's separated by INSTANCE_SEPARATOR
405         */
406        public static String ph3_coord_dataOut(String dataOutName) {
407            String uris = "";
408            ELEvaluator eval = ELEvaluator.getCurrent();
409            uris = (String) eval.getVariable(".dataout." + dataOutName);
410            return uris;
411        }
412    
413        /**
414         * Determine the date-time in UTC of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency <p/> 2.
415         * Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5. Data
416         * set initial instance <p/> 6. Action Creation Time
417         *
418         * @param n instance count domain: n is integer
419         * @return date-time in UTC of the n-th instance returns 'null' means n-th instance is earlier than Initial-Instance
420         *         of DS
421         * @throws Exception
422         */
423        public static String ph2_coord_current(int n) throws Exception {
424            if (isSyncDataSet()) { // For Sync Dataset
425                return coord_current_sync(n);
426            }
427            else {
428                throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
429            }
430        }
431    
432        /**
433         * Determine how many hours is on the date of n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency
434         * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
435         * Data set initial instance <p/> 6. Action Creation Time
436         *
437         * @param n instance count <p/> domain: n is integer
438         * @return number of hours on that day <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
439         * @throws Exception
440         */
441        public static int ph2_coord_hoursInDay(int n) throws Exception {
442            int datasetFrequency = (int) getDSFrequency();
443            // /Calendar nominalInstanceCal =
444            // getCurrentInstance(getActionCreationtime());
445            Calendar nominalInstanceCal = getEffectiveNominalTime();
446            if (nominalInstanceCal == null) {
447                return -1;
448            }
449            nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
450            /*
451             * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
452             * { return -1; }
453             */
454            nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
455            // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
456            return DateUtils.hoursInDay(nominalInstanceCal);
457        }
458    
459        public static int ph3_coord_hoursInDay(int n) throws Exception {
460            return ph2_coord_hoursInDay(n);
461        }
462    
463        /**
464         * Calculate number of days in one month for n-th dataset instance. <p/> It depends on: <p/> 1. Data set frequency .
465         * <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month flag <p/> 5.
466         * Data set initial instance <p/> 6. Action Creation Time
467         *
468         * @param n instance count. domain: n is integer
469         * @return number of days in that month <p/> returns -1 means n-th instance is earlier than Initial-Instance of DS
470         * @throws Exception
471         */
472        public static int ph2_coord_daysInMonth(int n) throws Exception {
473            int datasetFrequency = (int) getDSFrequency();// in minutes
474            // Calendar nominalInstanceCal =
475            // getCurrentInstance(getActionCreationtime());
476            Calendar nominalInstanceCal = getEffectiveNominalTime();
477            if (nominalInstanceCal == null) {
478                return -1;
479            }
480            nominalInstanceCal.add(getDSTimeUnit().getCalendarUnit(), datasetFrequency * n);
481            /*
482             * if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0)
483             * { return -1; }
484             */
485            nominalInstanceCal.setTimeZone(getDatasetTZ());// Use Dataset TZ
486            // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
487            return nominalInstanceCal.getActualMaximum(Calendar.DAY_OF_MONTH);
488        }
489    
490        public static int ph3_coord_daysInMonth(int n) throws Exception {
491            return ph2_coord_daysInMonth(n);
492        }
493    
494        /**
495         * Determine the date-time in UTC of n-th latest available dataset instance. <p/> It depends on: <p/> 1. Data set
496         * frequency <p/> 2. Data set Time unit (day, month, minute) <p/> 3. Data set Time zone/DST <p/> 4. End Day/Month
497         * flag <p/> 5. Data set initial instance <p/> 6. Action Creation Time <p/> 7. Existence of dataset's directory
498         *
499         * @param n :instance count <p/> domain: n > 0, n is integer
500         * @return date-time in UTC of the n-th instance <p/> returns 'null' means n-th instance is earlier than
501         *         Initial-Instance of DS
502         * @throws Exception
503         */
504        public static String ph3_coord_latest(int n) throws Exception {
505            if (n > 0) {
506                throw new IllegalArgumentException("paramter should be <= 0 but it is " + n);
507            }
508            if (isSyncDataSet()) {// For Sync Dataset
509                return coord_latest_sync(n);
510            }
511            else {
512                throw new UnsupportedOperationException("Asynchronous Dataset is not supported yet");
513            }
514        }
515    
516        /**
517         * Configure an evaluator with data set and application specific information. <p/> Helper method of associating
518         * dataset and application object
519         *
520         * @param evaluator : to set variables
521         * @param ds : Data Set object
522         * @param coordAction : Application instance
523         */
524        public static void configureEvaluator(ELEvaluator evaluator, SyncCoordDataset ds, SyncCoordAction coordAction) {
525            evaluator.setVariable(COORD_ACTION, coordAction);
526            evaluator.setVariable(DATASET, ds);
527        }
528    
529        /**
530         * Helper method to wrap around with "${..}". <p/>
531         *
532         *
533         * @param eval :EL evaluator
534         * @param expr : expression to evaluate
535         * @return Resolved expression or echo back the same expression
536         * @throws Exception
537         */
538        public static String evalAndWrap(ELEvaluator eval, String expr) throws Exception {
539            try {
540                eval.setVariable(".wrap", null);
541                String result = eval.evaluate(expr, String.class);
542                if (eval.getVariable(".wrap") != null) {
543                    return "${" + result + "}";
544                }
545                else {
546                    return result;
547                }
548            }
549            catch (Exception e) {
550                throw new Exception("Unable to evaluate :" + expr + ":\n", e);
551            }
552        }
553    
554        // Set of echo functions
555    
556        public static String ph1_coord_current_echo(String n) {
557            return echoUnResolved("current", n);
558        }
559    
560        public static String ph2_coord_current_echo(String n) {
561            return echoUnResolved("current", n);
562        }
563    
564        public static String ph1_coord_dateOffset_echo(String n, String offset, String unit) {
565            return echoUnResolved("dateOffset", n + " , " + offset + " , " + unit);
566        }
567    
568        public static String ph1_coord_formatTime_echo(String dateTime, String format) {
569            // Quote the dateTime value since it would contain a ':'.
570            return echoUnResolved("formatTime", "'"+dateTime+"'" + " , " + format);
571        }
572    
573        public static String ph1_coord_latest_echo(String n) {
574            return echoUnResolved("latest", n);
575        }
576    
577        public static String ph2_coord_latest_echo(String n) {
578            return ph1_coord_latest_echo(n);
579        }
580    
581        public static String ph1_coord_future_echo(String n, String instance) {
582            return echoUnResolved("future", n + ", " + instance + "");
583        }
584    
585        public static String ph2_coord_future_echo(String n, String instance) {
586            return ph1_coord_future_echo(n, instance);
587        }
588    
589        public static String ph1_coord_dataIn_echo(String n) {
590            ELEvaluator eval = ELEvaluator.getCurrent();
591            String val = (String) eval.getVariable("oozie.dataname." + n);
592            if (val == null || val.equals("data-in") == false) {
593                XLog.getLog(CoordELFunctions.class).error("data_in_name " + n + " is not valid");
594                throw new RuntimeException("data_in_name " + n + " is not valid");
595            }
596            return echoUnResolved("dataIn", "'" + n + "'");
597        }
598    
599        public static String ph1_coord_dataOut_echo(String n) {
600            ELEvaluator eval = ELEvaluator.getCurrent();
601            String val = (String) eval.getVariable("oozie.dataname." + n);
602            if (val == null || val.equals("data-out") == false) {
603                XLog.getLog(CoordELFunctions.class).error("data_out_name " + n + " is not valid");
604                throw new RuntimeException("data_out_name " + n + " is not valid");
605            }
606            return echoUnResolved("dataOut", "'" + n + "'");
607        }
608    
609        public static String ph1_coord_nominalTime_echo() {
610            return echoUnResolved("nominalTime", "");
611        }
612    
613        public static String ph1_coord_nominalTime_echo_wrap() {
614            // return "${coord:nominalTime()}"; // no resolution
615            return echoUnResolved("nominalTime", "");
616        }
617    
618        public static String ph1_coord_nominalTime_echo_fixed() {
619            return "2009-03-06T010:00"; // Dummy resolution
620        }
621    
622        public static String ph1_coord_actualTime_echo_wrap() {
623            // return "${coord:actualTime()}"; // no resolution
624            return echoUnResolved("actualTime", "");
625        }
626    
627        public static String ph1_coord_actionId_echo() {
628            return echoUnResolved("actionId", "");
629        }
630    
631        public static String ph1_coord_name_echo() {
632            return echoUnResolved("name", "");
633        }
634    
635        // The following echo functions are not used in any phases yet
636        // They are here for future purpose.
637        public static String coord_minutes_echo(String n) {
638            return echoUnResolved("minutes", n);
639        }
640    
641        public static String coord_hours_echo(String n) {
642            return echoUnResolved("hours", n);
643        }
644    
645        public static String coord_days_echo(String n) {
646            return echoUnResolved("days", n);
647        }
648    
649        public static String coord_endOfDay_echo(String n) {
650            return echoUnResolved("endOfDay", n);
651        }
652    
653        public static String coord_months_echo(String n) {
654            return echoUnResolved("months", n);
655        }
656    
657        public static String coord_endOfMonth_echo(String n) {
658            return echoUnResolved("endOfMonth", n);
659        }
660    
661        public static String coord_actualTime_echo() {
662            return echoUnResolved("actualTime", "");
663        }
664    
665        // This echo function will always return "24" for validation only.
666        // This evaluation ****should not**** replace the original XML
667        // Create a temporary string and validate the function
668        // This is **required** for evaluating an expression like
669        // coord:HoursInDay(0) + 3
670        // actual evaluation will happen in phase 2 or phase 3.
671        public static String ph1_coord_hoursInDay_echo(String n) {
672            return "24";
673            // return echoUnResolved("hoursInDay", n);
674        }
675    
676        // This echo function will always return "30" for validation only.
677        // This evaluation ****should not**** replace the original XML
678        // Create a temporary string and validate the function
679        // This is **required** for evaluating an expression like
680        // coord:daysInMonth(0) + 3
681        // actual evaluation will happen in phase 2 or phase 3.
682        public static String ph1_coord_daysInMonth_echo(String n) {
683            // return echoUnResolved("daysInMonth", n);
684            return "30";
685        }
686    
687        // This echo function will always return "3" for validation only.
688        // This evaluation ****should not**** replace the original XML
689        // Create a temporary string and validate the function
690        // This is **required** for evaluating an expression like coord:tzOffset + 2
691        // actual evaluation will happen in phase 2 or phase 3.
692        public static String ph1_coord_tzOffset_echo() {
693            // return echoUnResolved("tzOffset", "");
694            return "3";
695        }
696    
697        // Local methods
698        /**
699         * @param n
700         * @return n-th instance Date-Time from current instance for data-set <p/> return empty string ("") if the
701         *         Action_Creation_time or the n-th instance <p/> is earlier than the Initial_Instance of dataset.
702         * @throws Exception
703         */
704        private static String coord_current_sync(int n) throws Exception {
705            int datasetFrequency = getDSFrequency();// in minutes
706            TimeUnit dsTimeUnit = getDSTimeUnit();
707            int[] instCount = new int[1];// used as pass by ref
708            Calendar nominalInstanceCal = getCurrentInstance(getActionCreationtime(), instCount);
709            if (nominalInstanceCal == null) {
710                return "";
711            }
712            nominalInstanceCal = getInitialInstanceCal();
713            int absInstanceCount = instCount[0] + n;
714            nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), datasetFrequency * absInstanceCount);
715    
716            if (nominalInstanceCal.getTime().compareTo(getInitialInstance()) < 0) {
717                return "";
718            }
719            String str = DateUtils.formatDateUTC(nominalInstanceCal);
720            return str;
721        }
722    
723        /**
724         * @param offset
725         * @return n-th available latest instance Date-Time for SYNC data-set
726         * @throws Exception
727         */
728        private static String coord_latest_sync(int offset) throws Exception {
729            if (offset > 0) {
730                throw new RuntimeException("For latest there is no meaning " + "of positive instance. n should be <=0"
731                        + offset);
732            }
733            ELEvaluator eval = ELEvaluator.getCurrent();
734            String retVal = "";
735            int datasetFrequency = (int) getDSFrequency();// in minutes
736            TimeUnit dsTimeUnit = getDSTimeUnit();
737            int[] instCount = new int[1];
738            Calendar nominalInstanceCal = getCurrentInstance(getActualTime(), instCount);
739            if (nominalInstanceCal != null) {
740                Calendar initInstance = getInitialInstanceCal();
741                SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
742                if (ds == null) {
743                    throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
744                }
745                String uriTemplate = ds.getUriTemplate();
746                Configuration conf = (Configuration) eval.getVariable(CONFIGURATION);
747                if (conf == null) {
748                    throw new RuntimeException("Associated Configuration should be defined with key " + CONFIGURATION);
749                }
750                int available = 0;
751                boolean resolved = false;
752                String user = ParamChecker
753                        .notEmpty((String) eval.getVariable(OozieClient.USER_NAME), OozieClient.USER_NAME);
754                String group = ParamChecker.notEmpty((String) eval.getVariable(OozieClient.GROUP_NAME),
755                                                     OozieClient.GROUP_NAME);
756                String doneFlag = ds.getDoneFlag();
757                while (nominalInstanceCal.compareTo(initInstance) >= 0) {
758                    ELEvaluator uriEval = getUriEvaluator(nominalInstanceCal);
759                    String uriPath = uriEval.evaluate(uriTemplate, String.class);
760                    String pathWithDoneFlag = uriPath;
761                    if (doneFlag.length() > 0) {
762                        pathWithDoneFlag += "/" + doneFlag;
763                    }
764                    if (isPathAvailable(pathWithDoneFlag, user, group, conf)) {
765                        XLog.getLog(CoordELFunctions.class).debug("Found latest(" + available + "): " + pathWithDoneFlag);
766                        if (available == offset) {
767                            XLog.getLog(CoordELFunctions.class).debug("Found Latest File: " + pathWithDoneFlag);
768                            resolved = true;
769                            retVal = DateUtils.formatDateUTC(nominalInstanceCal);
770                            eval.setVariable("resolved_path", uriPath);
771                            break;
772                        }
773    
774                        available--;
775                    }
776                    // nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(),
777                    // -datasetFrequency);
778                    nominalInstanceCal = (Calendar) initInstance.clone();
779                    instCount[0]--;
780                    nominalInstanceCal.add(dsTimeUnit.getCalendarUnit(), instCount[0] * datasetFrequency);
781                    // DateUtils.moveToEnd(nominalInstanceCal, getDSEndOfFlag());
782                }
783                if (!resolved) {
784                    // return unchanged latest function with variable 'is_resolved'
785                    // to 'false'
786                    eval.setVariable("is_resolved", Boolean.FALSE);
787                    retVal = "${coord:latest(" + offset + ")}";
788                }
789                else {
790                    eval.setVariable("is_resolved", Boolean.TRUE);
791                }
792            }
793            else {// No feasible nominal time
794                eval.setVariable("is_resolved", Boolean.FALSE);
795            }
796            return retVal;
797        }
798    
799        // TODO : Not an efficient way. In a loop environment, we could do something
800        // outside the loop
801        /**
802         * Check whether a URI path exists
803         *
804         * @param sPath
805         * @param conf
806         * @return
807         * @throws IOException
808         */
809    
810        private static boolean isPathAvailable(String sPath, String user, String group, Configuration conf)
811                throws IOException, HadoopAccessorException {
812            // sPath += "/" + END_OF_OPERATION_INDICATOR_FILE;
813            Path path = new Path(sPath);
814            return Services.get().get(HadoopAccessorService.class).
815                    createFileSystem(user, group, path.toUri(), conf).exists(path);
816        }
817    
818        /**
819         * @param tm
820         * @return a new Evaluator to be used for URI-template evaluation
821         */
822        private static ELEvaluator getUriEvaluator(Calendar tm) {
823            ELEvaluator retEval = new ELEvaluator();
824            retEval.setVariable("YEAR", tm.get(Calendar.YEAR));
825            retEval.setVariable("MONTH", (tm.get(Calendar.MONTH) + 1) < 10 ? "0" + (tm.get(Calendar.MONTH) + 1) : (tm
826                    .get(Calendar.MONTH) + 1));
827            retEval.setVariable("DAY", tm.get(Calendar.DAY_OF_MONTH) < 10 ? "0" + tm.get(Calendar.DAY_OF_MONTH) : tm
828                    .get(Calendar.DAY_OF_MONTH));
829            retEval.setVariable("HOUR", tm.get(Calendar.HOUR_OF_DAY) < 10 ? "0" + tm.get(Calendar.HOUR_OF_DAY) : tm
830                    .get(Calendar.HOUR_OF_DAY));
831            retEval.setVariable("MINUTE", tm.get(Calendar.MINUTE) < 10 ? "0" + tm.get(Calendar.MINUTE) : tm
832                    .get(Calendar.MINUTE));
833            return retEval;
834        }
835    
836        /**
837         * @return whether a data set is SYNCH or ASYNC
838         */
839        private static boolean isSyncDataSet() {
840            ELEvaluator eval = ELEvaluator.getCurrent();
841            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
842            if (ds == null) {
843                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
844            }
845            return ds.getType().equalsIgnoreCase("SYNC");
846        }
847    
848        /**
849         * Check whether a function should be resolved.
850         *
851         * @param functionName
852         * @param n
853         * @return null if the functionName needs to be resolved otherwise return the calling function unresolved.
854         */
855        private static String checkIfResolved(String functionName, String n) {
856            ELEvaluator eval = ELEvaluator.getCurrent();
857            String replace = (String) eval.getVariable("resolve_" + functionName);
858            if (replace == null || (replace != null && replace.equalsIgnoreCase("false"))) { // Don't
859                // resolve
860                // return "${coord:" + functionName + "(" + n +")}"; //Unresolved
861                eval.setVariable(".wrap", "true");
862                return "coord:" + functionName + "(" + n + ")"; // Unresolved
863            }
864            return null; // Resolved it
865        }
866    
867        private static String echoUnResolved(String functionName, String n) {
868            return echoUnResolvedPre(functionName, n, "coord:");
869        }
870    
871        private static String echoUnResolvedPre(String functionName, String n, String prefix) {
872            ELEvaluator eval = ELEvaluator.getCurrent();
873            eval.setVariable(".wrap", "true");
874            return prefix + functionName + "(" + n + ")"; // Unresolved
875        }
876    
877        /**
878         * @return the initial instance of a DataSet in DATE
879         */
880        private static Date getInitialInstance() {
881            return getInitialInstanceCal().getTime();
882            // return ds.getInitInstance();
883        }
884    
885        /**
886         * @return the initial instance of a DataSet in Calendar
887         */
888        private static Calendar getInitialInstanceCal() {
889            ELEvaluator eval = ELEvaluator.getCurrent();
890            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
891            if (ds == null) {
892                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
893            }
894            Calendar effInitTS = Calendar.getInstance();
895            effInitTS.setTime(ds.getInitInstance());
896            effInitTS.setTimeZone(ds.getTimeZone());
897            // To adjust EOD/EOM
898            DateUtils.moveToEnd(effInitTS, getDSEndOfFlag());
899            return effInitTS;
900            // return ds.getInitInstance();
901        }
902    
903        /**
904         * @return Nominal or action creation Time when all the dependencies of an application instance are met.
905         */
906        private static Date getActionCreationtime() {
907            ELEvaluator eval = ELEvaluator.getCurrent();
908            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
909            if (coordAction == null) {
910                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
911            }
912            return coordAction.getNominalTime();
913        }
914    
915        /**
916         * @return Actual Time when all the dependencies of an application instance are met.
917         */
918        private static Date getActualTime() {
919            ELEvaluator eval = ELEvaluator.getCurrent();
920            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
921            if (coordAction == null) {
922                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
923            }
924            return coordAction.getActualTime();
925        }
926    
927        /**
928         * @return TimeZone for the application or job.
929         */
930        private static TimeZone getJobTZ() {
931            ELEvaluator eval = ELEvaluator.getCurrent();
932            SyncCoordAction coordAction = (SyncCoordAction) eval.getVariable(COORD_ACTION);
933            if (coordAction == null) {
934                throw new RuntimeException("Associated Application instance should be defined with key " + COORD_ACTION);
935            }
936            return coordAction.getTimeZone();
937        }
938    
939        /**
940         * Find the current instance based on effectiveTime (i.e Action_Creation_Time or Action_Start_Time)
941         *
942         * @return current instance i.e. current(0) returns null if effectiveTime is earlier than Initial Instance time of
943         *         the dataset.
944         */
945        private static Calendar getCurrentInstance(Date effectiveTime, int instanceCount[]) {
946            Date datasetInitialInstance = getInitialInstance();
947            TimeUnit dsTimeUnit = getDSTimeUnit();
948            TimeZone dsTZ = getDatasetTZ();
949            // Convert Date to Calendar for corresponding TZ
950            Calendar current = Calendar.getInstance();
951            current.setTime(datasetInitialInstance);
952            current.setTimeZone(dsTZ);
953    
954            Calendar calEffectiveTime = Calendar.getInstance();
955            calEffectiveTime.setTime(effectiveTime);
956            calEffectiveTime.setTimeZone(dsTZ);
957            instanceCount[0] = 0;
958            if (current.compareTo(calEffectiveTime) > 0) {
959                // Nominal Time < initial Instance
960                // TODO: getClass() call doesn't work from static method.
961                // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+
962                // current.getTime());
963                return null;
964            }
965            Calendar origCurrent = (Calendar) current.clone();
966            while (current.compareTo(calEffectiveTime) <= 0) {
967                current = (Calendar) origCurrent.clone();
968                instanceCount[0]++;
969                current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency());
970            }
971            instanceCount[0]--;
972    
973            current = (Calendar) origCurrent.clone();
974            current.add(dsTimeUnit.getCalendarUnit(), instanceCount[0] * getDSFrequency());
975            return current;
976        }
977    
978        private static Calendar getEffectiveNominalTime() {
979            Date datasetInitialInstance = getInitialInstance();
980            TimeZone dsTZ = getDatasetTZ();
981            // Convert Date to Calendar for corresponding TZ
982            Calendar current = Calendar.getInstance();
983            current.setTime(datasetInitialInstance);
984            current.setTimeZone(dsTZ);
985    
986            Calendar calEffectiveTime = Calendar.getInstance();
987            calEffectiveTime.setTime(getActionCreationtime());
988            calEffectiveTime.setTimeZone(dsTZ);
989            if (current.compareTo(calEffectiveTime) > 0) {
990                // Nominal Time < initial Instance
991                // TODO: getClass() call doesn't work from static method.
992                // XLog.getLog("CoordELFunction.class").warn("ACTION CREATED BEFORE INITIAL INSTACE "+
993                // current.getTime());
994                return null;
995            }
996            return calEffectiveTime;
997        }
998    
999        /**
1000         * @return dataset frequency in minutes
1001         */
1002        private static int getDSFrequency() {
1003            ELEvaluator eval = ELEvaluator.getCurrent();
1004            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1005            if (ds == null) {
1006                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1007            }
1008            return ds.getFrequency();
1009        }
1010    
1011        /**
1012         * @return dataset TimeUnit
1013         */
1014        private static TimeUnit getDSTimeUnit() {
1015            ELEvaluator eval = ELEvaluator.getCurrent();
1016            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1017            if (ds == null) {
1018                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1019            }
1020            return ds.getTimeUnit();
1021        }
1022    
1023        /**
1024         * @return dataset TimeZone
1025         */
1026        private static TimeZone getDatasetTZ() {
1027            ELEvaluator eval = ELEvaluator.getCurrent();
1028            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1029            if (ds == null) {
1030                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1031            }
1032            return ds.getTimeZone();
1033        }
1034    
1035        /**
1036         * @return dataset TimeUnit
1037         */
1038        private static TimeUnit getDSEndOfFlag() {
1039            ELEvaluator eval = ELEvaluator.getCurrent();
1040            SyncCoordDataset ds = (SyncCoordDataset) eval.getVariable(DATASET);
1041            if (ds == null) {
1042                throw new RuntimeException("Associated Dataset should be defined with key " + DATASET);
1043            }
1044            return ds.getEndOfDuration();// == null ? "": ds.getEndOfDuration();
1045        }
1046    
1047        /**
1048         * Return the user that submitted the coordinator job.
1049         *
1050         * @return the user that submitted the coordinator job.
1051         */
1052        public static String coord_user() {
1053            ELEvaluator eval = ELEvaluator.getCurrent();
1054            return (String) eval.getVariable(OozieClient.USER_NAME);
1055        }
1056    }