001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie;
016    
017    import java.io.IOException;
018    import java.io.Writer;
019    import java.util.ArrayList;
020    import java.util.Date;
021    import java.util.HashMap;
022    import java.util.HashSet;
023    import java.util.List;
024    import java.util.Map;
025    import java.util.Set;
026    import java.util.StringTokenizer;
027    
028    import org.apache.hadoop.conf.Configuration;
029    import org.apache.oozie.client.CoordinatorJob;
030    import org.apache.oozie.client.OozieClient;
031    import org.apache.oozie.client.WorkflowJob;
032    import org.apache.oozie.command.CommandException;
033    import org.apache.oozie.command.coord.CoordActionInfoCommand;
034    import org.apache.oozie.command.coord.CoordActionInfoXCommand;
035    import org.apache.oozie.command.coord.CoordChangeCommand;
036    import org.apache.oozie.command.coord.CoordChangeXCommand;
037    import org.apache.oozie.command.coord.CoordJobCommand;
038    import org.apache.oozie.command.coord.CoordJobXCommand;
039    import org.apache.oozie.command.coord.CoordJobsCommand;
040    import org.apache.oozie.command.coord.CoordJobsXCommand;
041    import org.apache.oozie.command.coord.CoordKillCommand;
042    import org.apache.oozie.command.coord.CoordKillXCommand;
043    import org.apache.oozie.command.coord.CoordRerunCommand;
044    import org.apache.oozie.command.coord.CoordRerunXCommand;
045    import org.apache.oozie.command.coord.CoordResumeCommand;
046    import org.apache.oozie.command.coord.CoordResumeXCommand;
047    import org.apache.oozie.command.coord.CoordSubmitCommand;
048    import org.apache.oozie.command.coord.CoordSubmitXCommand;
049    import org.apache.oozie.command.coord.CoordSuspendCommand;
050    import org.apache.oozie.command.coord.CoordSuspendXCommand;
051    import org.apache.oozie.service.DagXLogInfoService;
052    import org.apache.oozie.service.Services;
053    import org.apache.oozie.service.XLogService;
054    import org.apache.oozie.util.ParamChecker;
055    import org.apache.oozie.util.XLog;
056    import org.apache.oozie.util.XLogStreamer;
057    
058    public class CoordinatorEngine extends BaseEngine {
059        private static boolean useXCommand = true;
060        private static XLog LOG = XLog.getLog(CoordinatorEngine.class);
061    
062        /**
063         * Create a system Coordinator engine, with no user and no group.
064         */
065        public CoordinatorEngine() {
066            if (Services.get().getConf().getBoolean(USE_XCOMMAND, true) == false) {
067                useXCommand = false;
068                LOG.debug("Oozie CoordinatorEngine is not using XCommands.");
069            }
070            else {
071                LOG.debug("Oozie CoordinatorEngine is using XCommands.");
072            }
073        }
074    
075        /**
076         * Create a Coordinator engine to perform operations on behave of a user.
077         *
078         * @param user user name.
079         * @param authToken the authentication token.
080         */
081        public CoordinatorEngine(String user, String authToken) {
082            this();
083            this.user = ParamChecker.notEmpty(user, "user");
084            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
085        }
086    
087        /*
088         * (non-Javadoc)
089         *
090         * @see org.apache.oozie.BaseEngine#getDefinition(java.lang.String)
091         */
092        @Override
093        public String getDefinition(String jobId) throws BaseEngineException {
094            CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId);
095            return job.getOrigJobXml();
096        }
097    
098        /**
099         * @param jobId
100         * @return CoordinatorJobBean
101         * @throws BaseEngineException
102         */
103        private CoordinatorJobBean getCoordJobWithNoActionInfo(String jobId) throws BaseEngineException {
104            try {
105                if (useXCommand) {
106                    return new CoordJobXCommand(jobId).call();
107                }
108                else {
109                    return new CoordJobCommand(jobId).call();
110                }
111            }
112            catch (CommandException ex) {
113                throw new BaseEngineException(ex);
114            }
115        }
116    
117        /**
118         * @param actionId
119         * @return CoordinatorActionBean
120         * @throws BaseEngineException
121         */
122        public CoordinatorActionBean getCoordAction(String actionId) throws BaseEngineException {
123            try {
124                if (useXCommand) {
125                    return new CoordActionInfoXCommand(actionId).call();
126                }
127                else {
128                    return new CoordActionInfoCommand(actionId).call();
129                }
130            }
131            catch (CommandException ex) {
132                throw new BaseEngineException(ex);
133            }
134        }
135    
136        /*
137         * (non-Javadoc)
138         *
139         * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String)
140         */
141        @Override
142        public CoordinatorJobBean getCoordJob(String jobId) throws BaseEngineException {
143            try {
144                if (useXCommand) {
145                    return new CoordJobXCommand(jobId).call();
146                }
147                else {
148                    return new CoordJobCommand(jobId).call();
149                }
150            }
151            catch (CommandException ex) {
152                throw new BaseEngineException(ex);
153            }
154        }
155    
156        /*
157         * (non-Javadoc)
158         *
159         * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String, int, int)
160         */
161        @Override
162        public CoordinatorJobBean getCoordJob(String jobId, int start, int length) throws BaseEngineException {
163            try {
164                if (useXCommand) {
165                    return new CoordJobXCommand(jobId, start, length).call();
166                }
167                else {
168                    return new CoordJobCommand(jobId, start, length).call();
169                }
170            }
171            catch (CommandException ex) {
172                throw new BaseEngineException(ex);
173            }
174        }
175    
176        /*
177         * (non-Javadoc)
178         *
179         * @see org.apache.oozie.BaseEngine#getJobIdForExternalId(java.lang.String)
180         */
181        @Override
182        public String getJobIdForExternalId(String externalId) throws CoordinatorEngineException {
183            return null;
184        }
185    
186        /*
187         * (non-Javadoc)
188         *
189         * @see org.apache.oozie.BaseEngine#kill(java.lang.String)
190         */
191        @Override
192        public void kill(String jobId) throws CoordinatorEngineException {
193            try {
194                if (useXCommand) {
195                    new CoordKillXCommand(jobId).call();
196                }
197                else {
198                    new CoordKillCommand(jobId).call();
199                }
200                LOG.info("User " + user + " killed the Coordinator job " + jobId);
201            }
202            catch (CommandException e) {
203                throw new CoordinatorEngineException(e);
204            }
205        }
206    
207        /* (non-Javadoc)
208         * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String)
209         */
210        @Override
211        public void change(String jobId, String changeValue) throws CoordinatorEngineException {
212            try {
213                if (useXCommand) {
214                    new CoordChangeXCommand(jobId, changeValue).call();
215                }
216                else {
217                    new CoordChangeCommand(jobId, changeValue).call();
218                }
219                LOG.info("User " + user + " changed the Coordinator job " + jobId + " to " + changeValue);
220            }
221            catch (CommandException e) {
222                throw new CoordinatorEngineException(e);
223            }
224        }
225    
226        @Override
227        @Deprecated
228        public void reRun(String jobId, Configuration conf) throws BaseEngineException {
229            throw new BaseEngineException(new XException(ErrorCode.E0301));
230        }
231    
232        /**
233         * Rerun coordinator actions for given rerunType
234         *
235         * @param jobId
236         * @param rerunType
237         * @param scope
238         * @param refresh
239         * @param noCleanup
240         * @throws BaseEngineException
241         */
242        public CoordinatorActionInfo reRun(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup)
243                throws BaseEngineException {
244            try {
245                if  (useXCommand) {
246                    return new CoordRerunXCommand(jobId, rerunType, scope, refresh, noCleanup).call();
247                }
248                else {
249                    return new CoordRerunCommand(jobId, rerunType, scope, refresh, noCleanup).call();
250                }
251            }
252            catch (CommandException ex) {
253                throw new BaseEngineException(ex);
254            }
255        }
256    
257        /*
258         * (non-Javadoc)
259         *
260         * @see org.apache.oozie.BaseEngine#resume(java.lang.String)
261         */
262        @Override
263        public void resume(String jobId) throws CoordinatorEngineException {
264            try {
265                if (useXCommand) {
266                    new CoordResumeXCommand(jobId).call();
267                }
268                else {
269                    new CoordResumeCommand(jobId).call();
270                }
271            }
272            catch (CommandException e) {
273                throw new CoordinatorEngineException(e);
274            }
275        }
276    
277        @Override
278        @Deprecated
279        public void start(String jobId) throws BaseEngineException {
280            throw new BaseEngineException(new XException(ErrorCode.E0301));
281        }
282    
283        /*
284         * (non-Javadoc)
285         *
286         * @see org.apache.oozie.BaseEngine#streamLog(java.lang.String,
287         * java.io.Writer)
288         */
289        @Override
290        public void streamLog(String jobId, Writer writer) throws IOException, BaseEngineException {
291            XLogStreamer.Filter filter = new XLogStreamer.Filter();
292            filter.setParameter(DagXLogInfoService.JOB, jobId);
293    
294            CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId);
295            Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), new Date(), writer);
296        }
297    
298        /*
299         * (non-Javadoc)
300         *
301         * @see
302         * org.apache.oozie.BaseEngine#submitJob(org.apache.hadoop.conf.Configuration
303         * , boolean)
304         */
305        @Override
306        public String submitJob(Configuration conf, boolean startJob) throws CoordinatorEngineException {
307            try {
308                String jobId;
309                if (useXCommand) {
310                    CoordSubmitXCommand submit = new CoordSubmitXCommand(conf, getAuthToken());
311                    jobId = submit.call();
312                }
313                else {
314                    CoordSubmitCommand submit = new CoordSubmitCommand(conf, getAuthToken());
315                    jobId = submit.call();
316                }
317                return jobId;
318            }
319            catch (CommandException ex) {
320                throw new CoordinatorEngineException(ex);
321            }
322        }
323    
324        /*
325         * (non-Javadoc)
326         *
327         * @see
328         * org.apache.oozie.BaseEngine#dryrunSubmit(org.apache.hadoop.conf.Configuration
329         * , boolean)
330         */
331        @Override
332        public String dryrunSubmit(Configuration conf, boolean startJob) throws CoordinatorEngineException {
333            try {
334                String jobId;
335                if (useXCommand) {
336                    CoordSubmitXCommand submit = new CoordSubmitXCommand(true, conf, getAuthToken());
337                    jobId = submit.call();
338                }
339                else {
340                    CoordSubmitCommand submit = new CoordSubmitCommand(true, conf, getAuthToken());
341                    jobId = submit.call();                
342                }
343                return jobId;
344            }
345            catch (CommandException ex) {
346                throw new CoordinatorEngineException(ex);
347            }
348        }
349    
350        /*
351         * (non-Javadoc)
352         *
353         * @see org.apache.oozie.BaseEngine#suspend(java.lang.String)
354         */
355        @Override
356        public void suspend(String jobId) throws CoordinatorEngineException {
357            try {
358                if (useXCommand) {
359                    new CoordSuspendXCommand(jobId).call();
360                }
361                else {
362                    new CoordSuspendCommand(jobId).call();
363                }
364            }
365            catch (CommandException e) {
366                throw new CoordinatorEngineException(e);
367            }
368    
369        }
370    
371        /*
372         * (non-Javadoc)
373         *
374         * @see org.apache.oozie.BaseEngine#getJob(java.lang.String)
375         */
376        @Override
377        public WorkflowJob getJob(String jobId) throws BaseEngineException {
378            throw new BaseEngineException(new XException(ErrorCode.E0301));
379        }
380    
381        /*
382         * (non-Javadoc)
383         *
384         * @see org.apache.oozie.BaseEngine#getJob(java.lang.String, int, int)
385         */
386        @Override
387        public WorkflowJob getJob(String jobId, int start, int length) throws BaseEngineException {
388            throw new BaseEngineException(new XException(ErrorCode.E0301));
389        }
390    
391        private static final Set<String> FILTER_NAMES = new HashSet<String>();
392    
393        static {
394            FILTER_NAMES.add(OozieClient.FILTER_USER);
395            FILTER_NAMES.add(OozieClient.FILTER_NAME);
396            FILTER_NAMES.add(OozieClient.FILTER_GROUP);
397            FILTER_NAMES.add(OozieClient.FILTER_STATUS);
398        }
399    
400        /**
401         * @param filterStr
402         * @param start
403         * @param len
404         * @return CoordinatorJobInfo
405         * @throws CoordinatorEngineException
406         */
407        public CoordinatorJobInfo getCoordJobs(String filterStr, int start, int len) throws CoordinatorEngineException {
408            Map<String, List<String>> filter = parseFilter(filterStr);
409    
410            try {
411                if (useXCommand) {
412                    return new CoordJobsXCommand(filter, start, len).call();
413                }
414                else {
415                    return new CoordJobsCommand(filter, start, len).call();
416                }
417            }
418            catch (CommandException ex) {
419                throw new CoordinatorEngineException(ex);
420            }
421        }
422    
423        /**
424         * @param filter
425         * @return Map<String, List<String>>
426         * @throws CoordinatorEngineException
427         */
428        private Map<String, List<String>> parseFilter(String filter) throws CoordinatorEngineException {
429            Map<String, List<String>> map = new HashMap<String, List<String>>();
430            if (filter != null) {
431                StringTokenizer st = new StringTokenizer(filter, ";");
432                while (st.hasMoreTokens()) {
433                    String token = st.nextToken();
434                    if (token.contains("=")) {
435                        String[] pair = token.split("=");
436                        if (pair.length != 2) {
437                            throw new CoordinatorEngineException(ErrorCode.E0420, filter,
438                                                                 "elements must be name=value pairs");
439                        }
440                        if (!FILTER_NAMES.contains(pair[0])) {
441                            throw new CoordinatorEngineException(ErrorCode.E0420, filter, XLog.format("invalid name [{0}]",
442                                                                                                      pair[0]));
443                        }
444                        if (pair[0].equals("status")) {
445                            try {
446                                CoordinatorJob.Status.valueOf(pair[1]);
447                            }
448                            catch (IllegalArgumentException ex) {
449                                throw new CoordinatorEngineException(ErrorCode.E0420, filter, XLog.format(
450                                        "invalid status [{0}]", pair[1]));
451                            }
452                        }
453                        List<String> list = map.get(pair[0]);
454                        if (list == null) {
455                            list = new ArrayList<String>();
456                            map.put(pair[0], list);
457                        }
458                        list.add(pair[1]);
459                    }
460                    else {
461                        throw new CoordinatorEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
462                    }
463                }
464            }
465            return map;
466        }
467    }