001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.bundle;
016    
017    import java.io.IOException;
018    import java.io.StringReader;
019    import java.util.Date;
020    import java.util.HashMap;
021    import java.util.List;
022    import java.util.Map;
023    import java.util.Map.Entry;
024    
025    import org.apache.hadoop.conf.Configuration;
026    import org.apache.oozie.BundleActionBean;
027    import org.apache.oozie.BundleJobBean;
028    import org.apache.oozie.ErrorCode;
029    import org.apache.oozie.XException;
030    import org.apache.oozie.client.Job;
031    import org.apache.oozie.client.OozieClient;
032    import org.apache.oozie.command.CommandException;
033    import org.apache.oozie.command.PreconditionException;
034    import org.apache.oozie.command.StartTransitionXCommand;
035    import org.apache.oozie.command.coord.CoordSubmitXCommand;
036    import org.apache.oozie.executor.jpa.BundleActionGetJPAExecutor;
037    import org.apache.oozie.executor.jpa.BundleActionInsertJPAExecutor;
038    import org.apache.oozie.executor.jpa.BundleActionUpdateJPAExecutor;
039    import org.apache.oozie.executor.jpa.BundleJobGetJPAExecutor;
040    import org.apache.oozie.executor.jpa.BundleJobUpdateJPAExecutor;
041    import org.apache.oozie.executor.jpa.JPAExecutorException;
042    import org.apache.oozie.service.JPAService;
043    import org.apache.oozie.service.Services;
044    import org.apache.oozie.util.JobUtils;
045    import org.apache.oozie.util.LogUtils;
046    import org.apache.oozie.util.ParamChecker;
047    import org.apache.oozie.util.XConfiguration;
048    import org.apache.oozie.util.XmlUtils;
049    import org.jdom.Attribute;
050    import org.jdom.Element;
051    import org.jdom.JDOMException;
052    
053    /**
054     * The command to start Bundle job
055     */
056    public class BundleStartXCommand extends StartTransitionXCommand {
057        private final String jobId;
058        private BundleJobBean bundleJob;
059        private JPAService jpaService = null;
060    
061        /**
062         * The constructor for class {@link BundleStartXCommand}
063         *
064         * @param jobId the bundle job id
065         */
066        public BundleStartXCommand(String jobId) {
067            super("bundle_start", "bundle_start", 1);
068            this.jobId = ParamChecker.notEmpty(jobId, "jobId");
069        }
070    
071        /**
072         * The constructor for class {@link BundleStartXCommand}
073         *
074         * @param jobId the bundle job id
075         * @param dryrun true if dryrun is enable
076         */
077        public BundleStartXCommand(String jobId, boolean dryrun) {
078            super("bundle_start", "bundle_start", 1, dryrun);
079            this.jobId = ParamChecker.notEmpty(jobId, "jobId");
080        }
081    
082        /* (non-Javadoc)
083         * @see org.apache.oozie.command.XCommand#getEntityKey()
084         */
085        @Override
086        protected String getEntityKey() {
087            return jobId;
088        }
089    
090        /* (non-Javadoc)
091         * @see org.apache.oozie.command.XCommand#isLockRequired()
092         */
093        @Override
094        protected boolean isLockRequired() {
095            return true;
096        }
097    
098        /* (non-Javadoc)
099         * @see org.apache.oozie.command.XCommand#verifyPrecondition()
100         */
101        @Override
102        protected void verifyPrecondition() throws CommandException, PreconditionException {
103            eagerVerifyPrecondition();
104        }
105    
106        /* (non-Javadoc)
107         * @see org.apache.oozie.command.XCommand#eagerVerifyPrecondition()
108         */
109        @Override
110        protected void eagerVerifyPrecondition() throws CommandException, PreconditionException {
111            if (bundleJob.getStatus() != Job.Status.PREP) {
112                String msg = "Bundle " + bundleJob.getId() + " is not in PREP status. It is in : " + bundleJob.getStatus();
113                LOG.info(msg);
114                throw new PreconditionException(ErrorCode.E1100, msg);
115            }
116        }
117        /* (non-Javadoc)
118         * @see org.apache.oozie.command.XCommand#loadState()
119         */
120        @Override
121        public void loadState() throws CommandException {
122            eagerLoadState();
123        }
124    
125        /* (non-Javadoc)
126         * @see org.apache.oozie.command.XCommand#eagerLoadState()
127         */
128        @Override
129        public void eagerLoadState() throws CommandException {
130            try {
131                jpaService = Services.get().get(JPAService.class);
132    
133                if (jpaService != null) {
134                    this.bundleJob = jpaService.execute(new BundleJobGetJPAExecutor(jobId));
135                    LogUtils.setLogInfo(bundleJob, logInfo);
136                    super.setJob(bundleJob);
137    
138                }
139                else {
140                    throw new CommandException(ErrorCode.E0610);
141                }
142            }
143            catch (XException ex) {
144                throw new CommandException(ex);
145            }
146        }
147    
148        /* (non-Javadoc)
149         * @see org.apache.oozie.command.StartTransitionXCommand#StartChildren()
150         */
151        @Override
152        public void StartChildren() throws CommandException {
153            LOG.debug("Started coord jobs for the bundle=[{0}]", jobId);
154            insertBundleActions();
155            startCoordJobs();
156            LOG.debug("Ended coord jobs for the bundle=[{0}]", jobId);
157        }
158    
159        /* (non-Javadoc)
160         * @see org.apache.oozie.command.TransitionXCommand#notifyParent()
161         */
162        @Override
163        public void notifyParent() {
164        }
165    
166        /**
167         * Insert bundle actions
168         *
169         * @throws CommandException thrown if failed to create bundle actions
170         */
171        @SuppressWarnings("unchecked")
172        private void insertBundleActions() throws CommandException {
173            if (bundleJob != null) {
174                Map<String, Boolean> map = new HashMap<String, Boolean>();
175                try {
176                    Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml());
177                    List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
178                    for (Element elem : coordElems) {
179                        Attribute name = elem.getAttribute("name");
180                        Attribute critical = elem.getAttribute("critical");
181                        if (name != null) {
182                            if (map.containsKey(name.getValue())) {
183                                throw new CommandException(ErrorCode.E1304, name);
184                            }
185                            boolean isCritical = false;
186                            if (critical != null && Boolean.parseBoolean(critical.getValue())) {
187                                isCritical = true;
188                            }
189                            map.put(name.getValue(), isCritical);
190                        }
191                        else {
192                            throw new CommandException(ErrorCode.E1305);
193                        }
194                    }
195                }
196                catch (JDOMException jex) {
197                    throw new CommandException(ErrorCode.E1301, jex);
198                }
199    
200                try {
201                    // if there is no coordinator for this bundle, failed it.
202                    if (map.isEmpty()) {
203                        bundleJob.setStatus(Job.Status.FAILED);
204                        bundleJob.resetPending();
205                        jpaService.execute(new BundleJobUpdateJPAExecutor(bundleJob));
206                        LOG.debug("No coord jobs for the bundle=[{0}], failed it!!", jobId);
207                        throw new CommandException(ErrorCode.E1318, jobId);
208                    }
209    
210                    for (Entry<String, Boolean> coordName : map.entrySet()) {
211                        BundleActionBean action = createBundleAction(jobId, coordName.getKey(), coordName.getValue());
212    
213                        jpaService.execute(new BundleActionInsertJPAExecutor(action));
214                    }
215                }
216                catch (JPAExecutorException je) {
217                    throw new CommandException(je);
218                }
219    
220            }
221            else {
222                throw new CommandException(ErrorCode.E0604, jobId);
223            }
224        }
225    
226        private BundleActionBean createBundleAction(String jobId, String coordName, boolean isCritical) {
227            BundleActionBean action = new BundleActionBean();
228            action.setBundleActionId(jobId + "_" + coordName);
229            action.setBundleId(jobId);
230            action.setCoordName(coordName);
231            action.setStatus(Job.Status.PREP);
232            action.setLastModifiedTime(new Date());
233            if (isCritical) {
234                action.setCritical();
235            }
236            else {
237                action.resetCritical();
238            }
239            return action;
240        }
241    
242        /**
243         * Start Coord Jobs
244         *
245         * @throws CommandException thrown if failed to start coord jobs
246         */
247        @SuppressWarnings("unchecked")
248        private void startCoordJobs() throws CommandException {
249            if (bundleJob != null) {
250                try {
251                    Element bAppXml = XmlUtils.parseXml(bundleJob.getJobXml());
252                    List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
253                    for (Element coordElem : coordElems) {
254                        Attribute name = coordElem.getAttribute("name");
255                        Configuration coordConf = mergeConfig(coordElem);
256                        coordConf.set(OozieClient.BUNDLE_ID, jobId);
257    
258                        queue(new CoordSubmitXCommand(coordConf, bundleJob.getAuthToken(), bundleJob.getId(), name.getValue()));
259    
260                        updateBundleAction(name.getValue());
261                    }
262                }
263                catch (JDOMException jex) {
264                    throw new CommandException(ErrorCode.E1301, jex);
265                }
266                catch (JPAExecutorException je) {
267                    throw new CommandException(je);
268                }
269            }
270            else {
271                throw new CommandException(ErrorCode.E0604, jobId);
272            }
273        }
274    
275        private void updateBundleAction(String coordName) throws JPAExecutorException {
276            BundleActionBean action = jpaService.execute(new BundleActionGetJPAExecutor(jobId, coordName));
277            action.incrementAndGetPending();
278            action.setLastModifiedTime(new Date());
279            jpaService.execute(new BundleActionUpdateJPAExecutor(action));
280        }
281    
282        /**
283         * Merge Bundle job config and the configuration from the coord job to pass
284         * to Coord Engine
285         *
286         * @param coordElem the coordinator configuration
287         * @return Configuration merged configuration
288         * @throws CommandException thrown if failed to merge configuration
289         */
290        private Configuration mergeConfig(Element coordElem) throws CommandException {
291            String jobConf = bundleJob.getConf();
292            // Step 1: runConf = jobConf
293            Configuration runConf = null;
294            try {
295                runConf = new XConfiguration(new StringReader(jobConf));
296            }
297            catch (IOException e1) {
298                LOG.warn("Configuration parse error in:" + jobConf);
299                throw new CommandException(ErrorCode.E1306, e1.getMessage(), e1);
300            }
301            // Step 2: Merge local properties into runConf
302            // extract 'property' tags under 'configuration' block in the coordElem
303            // convert Element to XConfiguration
304            Element localConfigElement = coordElem.getChild("configuration", coordElem.getNamespace());
305    
306            if (localConfigElement != null) {
307                String strConfig = XmlUtils.prettyPrint(localConfigElement).toString();
308                Configuration localConf;
309                try {
310                    localConf = new XConfiguration(new StringReader(strConfig));
311                }
312                catch (IOException e1) {
313                    LOG.warn("Configuration parse error in:" + strConfig);
314                    throw new CommandException(ErrorCode.E1307, e1.getMessage(), e1);
315                }
316    
317                // copy configuration properties in the coordElem to the runConf
318                XConfiguration.copy(localConf, runConf);
319            }
320    
321            // Step 3: Extract value of 'app-path' in coordElem, save it as a
322            // new property called 'oozie.coord.application.path', and normalize.
323            String appPath = coordElem.getChild("app-path", coordElem.getNamespace()).getValue();
324            runConf.set(OozieClient.COORDINATOR_APP_PATH, appPath);
325            // Normalize coordinator appPath here;
326            try {
327                JobUtils.normalizeAppPath(runConf.get(OozieClient.USER_NAME), runConf.get(OozieClient.GROUP_NAME), runConf);
328            }
329            catch (IOException e) {
330                throw new CommandException(ErrorCode.E1001, runConf.get(OozieClient.COORDINATOR_APP_PATH));
331            }
332            return runConf;
333        }
334    
335        /* (non-Javadoc)
336         * @see org.apache.oozie.command.TransitionXCommand#getJob()
337         */
338        @Override
339        public Job getJob() {
340            return bundleJob;
341        }
342    
343        /* (non-Javadoc)
344         * @see org.apache.oozie.command.TransitionXCommand#updateJob()
345         */
346        @Override
347        public void updateJob() throws CommandException {
348            try {
349                jpaService.execute(new BundleJobUpdateJPAExecutor(bundleJob));
350            }
351            catch (JPAExecutorException je) {
352                throw new CommandException(je);
353            }
354        }
355    }