001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.coord;
016    
017    import java.io.IOException;
018    import java.io.StringReader;
019    import java.sql.Timestamp;
020    import java.util.Calendar;
021    import java.util.Date;
022    import java.util.TimeZone;
023    
024    import org.apache.hadoop.conf.Configuration;
025    import org.apache.oozie.CoordinatorActionBean;
026    import org.apache.oozie.CoordinatorJobBean;
027    import org.apache.oozie.ErrorCode;
028    import org.apache.oozie.client.CoordinatorJob;
029    import org.apache.oozie.client.Job;
030    import org.apache.oozie.client.SLAEvent.SlaAppType;
031    import org.apache.oozie.command.CommandException;
032    import org.apache.oozie.command.MaterializeTransitionXCommand;
033    import org.apache.oozie.command.PreconditionException;
034    import org.apache.oozie.command.bundle.BundleStatusUpdateXCommand;
035    import org.apache.oozie.coord.TimeUnit;
036    import org.apache.oozie.executor.jpa.CoordActionInsertJPAExecutor;
037    import org.apache.oozie.executor.jpa.CoordActionsActiveCountJPAExecutor;
038    import org.apache.oozie.executor.jpa.CoordJobGetJPAExecutor;
039    import org.apache.oozie.executor.jpa.CoordJobUpdateJPAExecutor;
040    import org.apache.oozie.executor.jpa.JPAExecutorException;
041    import org.apache.oozie.service.JPAService;
042    import org.apache.oozie.service.Service;
043    import org.apache.oozie.service.Services;
044    import org.apache.oozie.util.DateUtils;
045    import org.apache.oozie.util.Instrumentation;
046    import org.apache.oozie.util.LogUtils;
047    import org.apache.oozie.util.ParamChecker;
048    import org.apache.oozie.util.StatusUtils;
049    import org.apache.oozie.util.XConfiguration;
050    import org.apache.oozie.util.XmlUtils;
051    import org.apache.oozie.util.db.SLADbOperations;
052    import org.jdom.Element;
053    
054    /**
055     * Materialize actions for specified start and end time for coordinator job.
056     */
057    public class CoordMaterializeTransitionXCommand extends MaterializeTransitionXCommand {
058        private static final int LOOKAHEAD_WINDOW = 300; // We look ahead 5 minutes for materialization;
059        private JPAService jpaService = null;
060        private CoordinatorJobBean coordJob = null;
061        private String jobId = null;
062        private Date startMatdTime = null;
063        private Date endMatdTime = null;
064        private final int materializationWindow;
065        private int lastActionNumber = 1; // over-ride by DB value
066        private CoordinatorJob.Status prevStatus = null;
067        /**
068         * Default MAX timeout in minutes, after which coordinator input check will timeout
069         */
070        public static final String CONF_DEFAULT_MAX_TIMEOUT = Service.CONF_PREFIX + "coord.default.max.timeout";
071    
072        /**
073         * The constructor for class {@link CoordMaterializeTransitionXCommand}
074         *
075         * @param jobId coordinator job id
076         * @param materializationWindow materialization window to calculate end time
077         */
078        public CoordMaterializeTransitionXCommand(String jobId, int materializationWindow) {
079            super("coord_mater", "coord_mater", 1);
080            this.jobId = ParamChecker.notEmpty(jobId, "jobId");
081            this.materializationWindow = materializationWindow;
082        }
083    
084        /* (non-Javadoc)
085         * @see org.apache.oozie.command.MaterializeTransitionXCommand#transitToNext()
086         */
087        @Override
088        public void transitToNext() throws CommandException {
089        }
090    
091        /* (non-Javadoc)
092         * @see org.apache.oozie.command.TransitionXCommand#updateJob()
093         */
094        @Override
095        public void updateJob() throws CommandException {
096            try {
097                jpaService.execute(new CoordJobUpdateJPAExecutor(coordJob));
098            }
099            catch (JPAExecutorException jex) {
100                throw new CommandException(jex);
101            }
102        }
103    
104        /* (non-Javadoc)
105         * @see org.apache.oozie.command.XCommand#getEntityKey()
106         */
107        @Override
108        protected String getEntityKey() {
109            return jobId;
110        }
111    
112        @Override
113        protected boolean isLockRequired() {
114            return true;
115        }
116    
117        /* (non-Javadoc)
118         * @see org.apache.oozie.command.XCommand#loadState()
119         */
120        @Override
121        protected void loadState() throws CommandException {
122            jpaService = Services.get().get(JPAService.class);
123            if (jpaService == null) {
124                LOG.error(ErrorCode.E0610);
125            }
126    
127            try {
128                coordJob = jpaService.execute(new CoordJobGetJPAExecutor(jobId));
129                prevStatus = coordJob.getStatus();
130            }
131            catch (JPAExecutorException jex) {
132                throw new CommandException(jex);
133            }
134    
135            // calculate start materialize and end materialize time
136            calcMatdTime();
137    
138            LogUtils.setLogInfo(coordJob, logInfo);
139        }
140    
141        /**
142         * Calculate startMatdTime and endMatdTime from job's start time if next materialized time is null
143         *
144         * @throws CommandException thrown if failed to calculate startMatdTime and endMatdTime
145         */
146        protected void calcMatdTime() throws CommandException {
147            Timestamp startTime = coordJob.getNextMaterializedTimestamp();
148            if (startTime == null) {
149                startTime = coordJob.getStartTimestamp();
150            }
151            // calculate end time by adding materializationWindow to start time.
152            // need to convert materializationWindow from secs to milliseconds
153            long startTimeMilli = startTime.getTime();
154            long endTimeMilli = startTimeMilli + (materializationWindow * 1000);
155    
156            startMatdTime = DateUtils.toDate(new Timestamp(startTimeMilli));
157            endMatdTime = DateUtils.toDate(new Timestamp(endTimeMilli));
158            // if MaterializationWindow end time is greater than endTime
159            // for job, then set it to endTime of job
160            Date jobEndTime = coordJob.getEndTime();
161            if (endMatdTime.compareTo(jobEndTime) > 0) {
162                endMatdTime = jobEndTime;
163            }
164    
165            LOG.debug("Materializing coord job id=" + jobId + ", start=" + startMatdTime + ", end=" + endMatdTime
166                    + ", window=" + materializationWindow);
167        }
168    
169        /* (non-Javadoc)
170         * @see org.apache.oozie.command.XCommand#verifyPrecondition()
171         */
172        @Override
173        protected void verifyPrecondition() throws CommandException, PreconditionException {
174            if (!(coordJob.getStatus() == CoordinatorJobBean.Status.PREP || coordJob.getStatus() == CoordinatorJobBean.Status.RUNNING)) {
175                throw new PreconditionException(ErrorCode.E1100, "CoordMaterializeTransitionXCommand for jobId=" + jobId
176                        + " job is not in PREP or RUNNING but in " + coordJob.getStatus());
177            }
178    
179            if (coordJob.getNextMaterializedTimestamp() != null
180                    && coordJob.getNextMaterializedTimestamp().compareTo(coordJob.getEndTimestamp()) >= 0) {
181                throw new PreconditionException(ErrorCode.E1100, "CoordMaterializeTransitionXCommand for jobId=" + jobId
182                        + " job is already materialized");
183            }
184    
185            Timestamp startTime = coordJob.getNextMaterializedTimestamp();
186            if (startTime == null) {
187                startTime = coordJob.getStartTimestamp();
188    
189                if (startTime.after(new Timestamp(System.currentTimeMillis() + LOOKAHEAD_WINDOW * 1000))) {
190                    throw new PreconditionException(ErrorCode.E1100, "CoordMaterializeTransitionXCommand for jobId="
191                            + jobId + " job's start time is not reached yet - nothing to materialize");
192                }
193            }
194    
195            if (coordJob.getLastActionTime() != null && coordJob.getLastActionTime().compareTo(coordJob.getEndTime()) >= 0) {
196                throw new PreconditionException(ErrorCode.E1100, "ENDED Coordinator materialization for jobId = " + jobId
197                        + ", all actions have been materialized from start time = " + coordJob.getStartTime()
198                        + " to end time = " + coordJob.getEndTime() + ", job status = " + coordJob.getStatusStr());
199            }
200    
201            if (coordJob.getLastActionTime() != null && coordJob.getLastActionTime().compareTo(endMatdTime) >= 0) {
202                throw new PreconditionException(ErrorCode.E1100, "ENDED Coordinator materialization for jobId = " + jobId
203                        + ", action is *already* materialized for Materialization start time = " + startMatdTime
204                        + ", materialization end time = " + endMatdTime + ", job status = " + coordJob.getStatusStr());
205            }
206    
207            if (endMatdTime.after(coordJob.getEndTime())) {
208                throw new PreconditionException(ErrorCode.E1100, "ENDED Coordinator materialization for jobId = " + jobId
209                        + " materialization end time = " + endMatdTime + " surpasses coordinator job's end time = "
210                        + coordJob.getEndTime() + " job status = " + coordJob.getStatusStr());
211            }
212    
213            if (coordJob.getPauseTime() != null && !startMatdTime.before(coordJob.getPauseTime())) {
214                throw new PreconditionException(ErrorCode.E1100, "ENDED Coordinator materialization for jobId = " + jobId
215                        + ", materialization start time = " + startMatdTime
216                        + " is after or equal to coordinator job's pause time = " + coordJob.getPauseTime()
217                        + ", job status = " + coordJob.getStatusStr());
218            }
219    
220        }
221    
222        /* (non-Javadoc)
223         * @see org.apache.oozie.command.MaterializeTransitionXCommand#materialize()
224         */
225        @Override
226        protected void materialize() throws CommandException {
227            Instrumentation.Cron cron = new Instrumentation.Cron();
228            cron.start();
229            try {
230                materializeActions(false);
231                updateJobMaterializeInfo(coordJob);
232            }
233            catch (CommandException ex) {
234                LOG.warn("Exception occurs:" + ex.getMessage() + " Making the job failed ", ex);
235                coordJob.setStatus(Job.Status.FAILED);
236                coordJob.resetPending();
237            }
238            catch (Exception e) {
239                LOG.error("Excepion thrown :", e);
240                throw new CommandException(ErrorCode.E1001, e.getMessage(), e);
241            }
242            cron.stop();
243    
244        }
245    
246        /**
247         * Create action instances starting from "startMatdTime" to "endMatdTime" and store them into coord action table.
248         *
249         * @param dryrun if this is a dry run
250         * @throws Exception thrown if failed to materialize actions
251         */
252        protected String materializeActions(boolean dryrun) throws Exception {
253    
254            Configuration jobConf = null;
255            try {
256                jobConf = new XConfiguration(new StringReader(coordJob.getConf()));
257            }
258            catch (IOException ioe) {
259                LOG.warn("Configuration parse error. read from DB :" + coordJob.getConf(), ioe);
260                throw new CommandException(ErrorCode.E1005, ioe);
261            }
262    
263            String jobXml = coordJob.getJobXml();
264            Element eJob = XmlUtils.parseXml(jobXml);
265            TimeZone appTz = DateUtils.getTimeZone(coordJob.getTimeZone());
266            int frequency = coordJob.getFrequency();
267            TimeUnit freqTU = TimeUnit.valueOf(eJob.getAttributeValue("freq_timeunit"));
268            TimeUnit endOfFlag = TimeUnit.valueOf(eJob.getAttributeValue("end_of_duration"));
269            Calendar start = Calendar.getInstance(appTz);
270            start.setTime(startMatdTime);
271            DateUtils.moveToEnd(start, endOfFlag);
272            Calendar end = Calendar.getInstance(appTz);
273            end.setTime(endMatdTime);
274            lastActionNumber = coordJob.getLastActionNumber();
275            LOG.info("materialize actions for tz=" + appTz.getDisplayName() + ",\n start=" + start.getTime() + ", end="
276                    + end.getTime() + ",\n timeUnit " + freqTU.getCalendarUnit() + ",\n frequency :" + frequency + ":"
277                    + freqTU + ",\n lastActionNumber " + lastActionNumber);
278            // Keep the actual start time
279            Calendar origStart = Calendar.getInstance(appTz);
280            origStart.setTime(coordJob.getStartTimestamp());
281            // Move to the End of duration, if needed.
282            DateUtils.moveToEnd(origStart, endOfFlag);
283            // Cloning the start time to be used in loop iteration
284            Calendar effStart = (Calendar) origStart.clone();
285            // Move the time when the previous action finished
286            effStart.add(freqTU.getCalendarUnit(), lastActionNumber * frequency);
287    
288            StringBuilder actionStrings = new StringBuilder();
289            Date jobPauseTime = coordJob.getPauseTime();
290            Calendar pause = null;
291            if (jobPauseTime != null) {
292                pause = Calendar.getInstance(appTz);
293                pause.setTime(DateUtils.convertDateToTimestamp(jobPauseTime));
294            }
295    
296            String action = null;
297            JPAService jpaService = Services.get().get(JPAService.class);
298            int numWaitingActions = jpaService.execute(new CoordActionsActiveCountJPAExecutor(coordJob.getId()));
299            int maxActionToBeCreated = coordJob.getMatThrottling() - numWaitingActions;
300            LOG.debug("Coordinator job :" + coordJob.getId() + ", maxActionToBeCreated :" + maxActionToBeCreated
301                    + ", Mat_Throttle :" + coordJob.getMatThrottling() + ", numWaitingActions :" + numWaitingActions);
302    
303            while (effStart.compareTo(end) < 0 && maxActionToBeCreated-- > 0) {
304                if (pause != null && effStart.compareTo(pause) >= 0) {
305                    break;
306                }
307                CoordinatorActionBean actionBean = new CoordinatorActionBean();
308                lastActionNumber++;
309    
310                int timeout = coordJob.getTimeout();
311                LOG.debug("Materializing action for time=" + effStart.getTime() + ", lastactionnumber=" + lastActionNumber
312                        + " timeout=" + timeout + " minutes");
313                Date actualTime = new Date();
314                action = CoordCommandUtils.materializeOneInstance(jobId, dryrun, (Element) eJob.clone(),
315                        effStart.getTime(), actualTime, lastActionNumber, jobConf, actionBean);
316                actionBean.setTimeOut(timeout);
317    
318                if (!dryrun) {
319                    storeToDB(actionBean, action); // Storing to table
320                }
321                else {
322                    actionStrings.append("action for new instance");
323                    actionStrings.append(action);
324                }
325                // Restore the original start time
326                effStart = (Calendar) origStart.clone();
327                effStart.add(freqTU.getCalendarUnit(), lastActionNumber * frequency);
328            }
329    
330            endMatdTime = new Date(effStart.getTimeInMillis());
331            if (!dryrun) {
332                return action;
333            }
334            else {
335                return actionStrings.toString();
336            }
337        }
338    
339        private void storeToDB(CoordinatorActionBean actionBean, String actionXml) throws Exception {
340            LOG.debug("In storeToDB() coord action id = " + actionBean.getId() + ", size of actionXml = "
341                    + actionXml.length());
342            actionBean.setActionXml(actionXml);
343    
344            jpaService.execute(new CoordActionInsertJPAExecutor(actionBean));
345            writeActionRegistration(actionXml, actionBean);
346    
347            // TODO: time 100s should be configurable
348            queue(new CoordActionNotificationXCommand(actionBean), 100);
349            queue(new CoordActionInputCheckXCommand(actionBean.getId()), 100);
350        }
351    
352        private void writeActionRegistration(String actionXml, CoordinatorActionBean actionBean) throws Exception {
353            Element eAction = XmlUtils.parseXml(actionXml);
354            Element eSla = eAction.getChild("action", eAction.getNamespace()).getChild("info", eAction.getNamespace("sla"));
355            SLADbOperations.writeSlaRegistrationEvent(eSla, actionBean.getId(), SlaAppType.COORDINATOR_ACTION, coordJob
356                    .getUser(), coordJob.getGroup(), LOG);
357        }
358    
359        private void updateJobMaterializeInfo(CoordinatorJobBean job) throws CommandException {
360            job.setLastActionTime(endMatdTime);
361            job.setLastActionNumber(lastActionNumber);
362            // if the job endtime == action endtime, we don't need to materialize this job anymore
363            Date jobEndTime = job.getEndTime();
364    
365            LOG.info("[" + job.getId() + "]: Update status from " + job.getStatus() + " to RUNNING");
366            job.setStatus(Job.Status.RUNNING);
367            job.setPending();
368    
369            if (jobEndTime.compareTo(endMatdTime) <= 0) {
370                LOG.info("[" + job.getId() + "]: all actions have been materialized, job status = " + job.getStatus()
371                        + ", set pending to true");
372                // set doneMaterialization to true when materialization is done
373                job.setDoneMaterialization();
374            }
375            job.setStatus(StatusUtils.getStatus(job));
376            job.setNextMaterializedTime(endMatdTime);
377        }
378    
379        /* (non-Javadoc)
380         * @see org.apache.oozie.command.XCommand#getKey()
381         */
382        @Override
383        public String getKey() {
384            return getName() + "_" + jobId;
385        }
386    
387        /* (non-Javadoc)
388         * @see org.apache.oozie.command.TransitionXCommand#notifyParent()
389         */
390        @Override
391        public void notifyParent() throws CommandException {
392            // update bundle action only when status changes in coord job
393            if (this.coordJob.getBundleId() != null) {
394                if (!prevStatus.equals(coordJob.getStatus())) {
395                    BundleStatusUpdateXCommand bundleStatusUpdate = new BundleStatusUpdateXCommand(coordJob, prevStatus);
396                    bundleStatusUpdate.call();
397                }
398            }
399        }
400    }