001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie;
016    
017    import java.io.IOException;
018    import java.io.Writer;
019    import java.util.ArrayList;
020    import java.util.Date;
021    import java.util.HashMap;
022    import java.util.HashSet;
023    import java.util.List;
024    import java.util.Map;
025    import java.util.Set;
026    import java.util.StringTokenizer;
027    
028    import org.apache.hadoop.conf.Configuration;
029    import org.apache.oozie.client.CoordinatorJob;
030    import org.apache.oozie.client.Job;
031    import org.apache.oozie.client.OozieClient;
032    import org.apache.oozie.client.WorkflowJob;
033    import org.apache.oozie.command.CommandException;
034    import org.apache.oozie.command.bundle.BundleJobChangeXCommand;
035    import org.apache.oozie.command.bundle.BundleJobResumeXCommand;
036    import org.apache.oozie.command.bundle.BundleJobSuspendXCommand;
037    import org.apache.oozie.command.bundle.BundleJobXCommand;
038    import org.apache.oozie.command.bundle.BundleJobsXCommand;
039    import org.apache.oozie.command.bundle.BundleKillXCommand;
040    import org.apache.oozie.command.bundle.BundleRerunXCommand;
041    import org.apache.oozie.command.bundle.BundleStartXCommand;
042    import org.apache.oozie.command.bundle.BundleSubmitXCommand;
043    import org.apache.oozie.service.DagXLogInfoService;
044    import org.apache.oozie.service.Services;
045    import org.apache.oozie.service.XLogService;
046    import org.apache.oozie.util.ParamChecker;
047    import org.apache.oozie.util.XLog;
048    import org.apache.oozie.util.XLogStreamer;
049    
050    public class BundleEngine extends BaseEngine {
051        /**
052         * Create a system Bundle engine, with no user and no group.
053         */
054        public BundleEngine() {
055        }
056    
057        /**
058         * Create a Bundle engine to perform operations on behave of a user.
059         *
060         * @param user user name.
061         * @param authToken the authentication token.
062         */
063        public BundleEngine(String user, String authToken) {
064            this.user = ParamChecker.notEmpty(user, "user");
065            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
066        }
067    
068        /* (non-Javadoc)
069         * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String)
070         */
071        @Override
072        public void change(String jobId, String changeValue) throws BundleEngineException {
073            try {
074                BundleJobChangeXCommand change = new BundleJobChangeXCommand(jobId, changeValue);
075                change.call();
076            }
077            catch (CommandException ex) {
078                throw new BundleEngineException(ex);
079            }
080        }
081    
082        /* (non-Javadoc)
083         * @see org.apache.oozie.BaseEngine#dryrunSubmit(org.apache.hadoop.conf.Configuration, boolean)
084         */
085        @Override
086        public String dryrunSubmit(Configuration conf, boolean startJob) throws BundleEngineException {
087            BundleSubmitXCommand submit = new BundleSubmitXCommand(true, conf, getAuthToken());
088            try {
089                String jobId = submit.call();
090                return jobId;
091            }
092            catch (CommandException ex) {
093                throw new BundleEngineException(ex);
094            }
095        }
096    
097        /* (non-Javadoc)
098         * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String)
099         */
100        @Override
101        public CoordinatorJob getCoordJob(String jobId) throws BundleEngineException {
102            throw new BundleEngineException(new XException(ErrorCode.E0301));
103        }
104    
105        public BundleJobBean getBundleJob(String jobId) throws BundleEngineException {
106            try {
107                return new BundleJobXCommand(jobId).call();
108            }
109            catch (CommandException ex) {
110                throw new BundleEngineException(ex);
111            }
112        }
113    
114        /* (non-Javadoc)
115         * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String, int, int)
116         */
117        @Override
118        public CoordinatorJob getCoordJob(String jobId, int start, int length) throws BundleEngineException {
119            throw new BundleEngineException(new XException(ErrorCode.E0301));
120        }
121    
122        /* (non-Javadoc)
123         * @see org.apache.oozie.BaseEngine#getDefinition(java.lang.String)
124         */
125        @Override
126        public String getDefinition(String jobId) throws BundleEngineException {
127            BundleJobBean job;
128            try {
129                job = new BundleJobXCommand(jobId).call();
130            }
131            catch (CommandException ex) {
132                throw new BundleEngineException(ex);
133            }
134            return job.getOrigJobXml();
135        }
136    
137        /* (non-Javadoc)
138         * @see org.apache.oozie.BaseEngine#getJob(java.lang.String)
139         */
140        @Override
141        public WorkflowJob getJob(String jobId) throws BundleEngineException {
142            throw new BundleEngineException(new XException(ErrorCode.E0301));
143        }
144    
145        /* (non-Javadoc)
146         * @see org.apache.oozie.BaseEngine#getJob(java.lang.String, int, int)
147         */
148        @Override
149        public WorkflowJob getJob(String jobId, int start, int length) throws BundleEngineException {
150            throw new BundleEngineException(new XException(ErrorCode.E0301));
151        }
152    
153        /* (non-Javadoc)
154         * @see org.apache.oozie.BaseEngine#getJobIdForExternalId(java.lang.String)
155         */
156        @Override
157        public String getJobIdForExternalId(String externalId) throws BundleEngineException {
158            return null;
159        }
160    
161        /* (non-Javadoc)
162         * @see org.apache.oozie.BaseEngine#kill(java.lang.String)
163         */
164        @Override
165        public void kill(String jobId) throws BundleEngineException {
166            try {
167                new BundleKillXCommand(jobId).call();
168            }
169            catch (CommandException e) {
170                throw new BundleEngineException(e);
171            }
172        }
173    
174        /* (non-Javadoc)
175         * @see org.apache.oozie.BaseEngine#reRun(java.lang.String, org.apache.hadoop.conf.Configuration)
176         */
177        @Override
178        @Deprecated
179        public void reRun(String jobId, Configuration conf) throws BundleEngineException {
180            throw new BundleEngineException(new XException(ErrorCode.E0301));
181        }
182    
183        /**
184         * Rerun Bundle actions for given rerunType
185         *
186         * @param jobId bundle job id
187         * @param coordScope the rerun scope for coordinator job names separated by ","
188         * @param dateScope the rerun scope for coordinator nominal times separated by ","
189         * @param refresh true if user wants to refresh input/outpur dataset urls
190         * @param noCleanup false if user wants to cleanup output events for given rerun actions
191         * @throws BaseEngineException thrown if failed to rerun
192         */
193        public void reRun(String jobId, String coordScope, String dateScope, boolean refresh, boolean noCleanup)
194                throws BaseEngineException {
195            try {
196                new BundleRerunXCommand(jobId, coordScope, dateScope, refresh, noCleanup).call();
197            }
198            catch (CommandException ex) {
199                throw new BaseEngineException(ex);
200            }
201        }
202    
203        /* (non-Javadoc)
204         * @see org.apache.oozie.BaseEngine#resume(java.lang.String)
205         */
206        @Override
207        public void resume(String jobId) throws BundleEngineException {
208            BundleJobResumeXCommand resume = new BundleJobResumeXCommand(jobId);
209            try {
210                resume.call();
211            }
212            catch (CommandException ex) {
213                throw new BundleEngineException(ex);
214            }
215        }
216    
217        /* (non-Javadoc)
218         * @see org.apache.oozie.BaseEngine#start(java.lang.String)
219         */
220        @Override
221        public void start(String jobId) throws BundleEngineException {
222            try {
223                new BundleStartXCommand(jobId).call();
224            }
225            catch (CommandException e) {
226                throw new BundleEngineException(e);
227            }
228        }
229    
230        /* (non-Javadoc)
231         * @see org.apache.oozie.BaseEngine#streamLog(java.lang.String, java.io.Writer)
232         */
233        @Override
234        public void streamLog(String jobId, Writer writer) throws IOException, BundleEngineException {
235            XLogStreamer.Filter filter = new XLogStreamer.Filter();
236            filter.setParameter(DagXLogInfoService.JOB, jobId);
237    
238            BundleJobBean job;
239            try {
240                job = new BundleJobXCommand(jobId).call();
241            }
242            catch (CommandException ex) {
243                throw new BundleEngineException(ex);
244            }
245    
246            Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), new Date(), writer);
247        }
248    
249        /* (non-Javadoc)
250         * @see org.apache.oozie.BaseEngine#submitJob(org.apache.hadoop.conf.Configuration, boolean)
251         */
252        @Override
253        public String submitJob(Configuration conf, boolean startJob) throws BundleEngineException {
254            try {
255                String jobId = new BundleSubmitXCommand(conf, getAuthToken()).call();
256    
257                if (startJob) {
258                    start(jobId);
259                }
260                return jobId;
261            }
262            catch (CommandException ex) {
263                throw new BundleEngineException(ex);
264            }
265        }
266    
267        /* (non-Javadoc)
268         * @see org.apache.oozie.BaseEngine#suspend(java.lang.String)
269         */
270        @Override
271        public void suspend(String jobId) throws BundleEngineException {
272            BundleJobSuspendXCommand suspend = new BundleJobSuspendXCommand(jobId);
273            try {
274                suspend.call();
275            }
276            catch (CommandException ex) {
277                throw new BundleEngineException(ex);
278            }
279        }
280    
281        private static final Set<String> FILTER_NAMES = new HashSet<String>();
282    
283        static {
284            FILTER_NAMES.add(OozieClient.FILTER_USER);
285            FILTER_NAMES.add(OozieClient.FILTER_NAME);
286            FILTER_NAMES.add(OozieClient.FILTER_GROUP);
287            FILTER_NAMES.add(OozieClient.FILTER_STATUS);
288        }
289    
290        /**
291         * Get bundle jobs
292         *
293         * @param filterStr the filter string
294         * @param start start location for paging
295         * @param len total length to get
296         * @return bundle job info
297         * @throws BundleEngineException thrown if failed to get bundle job info
298         */
299        public BundleJobInfo getBundleJobs(String filterStr, int start, int len) throws BundleEngineException {
300            Map<String, List<String>> filter = parseFilter(filterStr);
301    
302            try {
303                return new BundleJobsXCommand(filter, start, len).call();
304            }
305            catch (CommandException ex) {
306                throw new BundleEngineException(ex);
307            }
308        }
309    
310        /**
311         * Parse filter string to a map with key = filter name and values = filter values
312         *
313         * @param filter the filter string
314         * @return filter key and value map
315         * @throws CoordinatorEngineException thrown if failed to parse filter string
316         */
317        private Map<String, List<String>> parseFilter(String filter) throws BundleEngineException {
318            Map<String, List<String>> map = new HashMap<String, List<String>>();
319            if (filter != null) {
320                StringTokenizer st = new StringTokenizer(filter, ";");
321                while (st.hasMoreTokens()) {
322                    String token = st.nextToken();
323                    if (token.contains("=")) {
324                        String[] pair = token.split("=");
325                        if (pair.length != 2) {
326                            throw new BundleEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
327                        }
328                        if (!FILTER_NAMES.contains(pair[0])) {
329                            throw new BundleEngineException(ErrorCode.E0420, filter, XLog.format("invalid name [{0}]",
330                                    pair[0]));
331                        }
332                        if (pair[0].equals("status")) {
333                            try {
334                                Job.Status.valueOf(pair[1]);
335                            }
336                            catch (IllegalArgumentException ex) {
337                                throw new BundleEngineException(ErrorCode.E0420, filter, XLog.format(
338                                        "invalid status [{0}]", pair[1]));
339                            }
340                        }
341                        List<String> list = map.get(pair[0]);
342                        if (list == null) {
343                            list = new ArrayList<String>();
344                            map.put(pair[0], list);
345                        }
346                        list.add(pair[1]);
347                    }
348                    else {
349                        throw new BundleEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs");
350                    }
351                }
352            }
353            return map;
354        }
355    
356    }