001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.bundle;
016    
017    import java.io.IOException;
018    import java.io.InputStreamReader;
019    import java.io.Reader;
020    import java.io.StringReader;
021    import java.io.StringWriter;
022    import java.net.URI;
023    import java.net.URISyntaxException;
024    import java.util.Date;
025    import java.util.HashSet;
026    import java.util.List;
027    import java.util.Map;
028    import java.util.Set;
029    
030    import javax.xml.transform.stream.StreamSource;
031    import javax.xml.validation.Validator;
032    
033    import org.apache.hadoop.conf.Configuration;
034    import org.apache.hadoop.fs.FileSystem;
035    import org.apache.hadoop.fs.Path;
036    import org.apache.oozie.BundleJobBean;
037    import org.apache.oozie.ErrorCode;
038    import org.apache.oozie.client.Job;
039    import org.apache.oozie.client.OozieClient;
040    import org.apache.oozie.command.CommandException;
041    import org.apache.oozie.command.PreconditionException;
042    import org.apache.oozie.command.SubmitTransitionXCommand;
043    import org.apache.oozie.executor.jpa.BundleJobInsertJPAExecutor;
044    import org.apache.oozie.service.HadoopAccessorException;
045    import org.apache.oozie.service.HadoopAccessorService;
046    import org.apache.oozie.service.JPAService;
047    import org.apache.oozie.service.SchemaService;
048    import org.apache.oozie.service.Services;
049    import org.apache.oozie.service.UUIDService;
050    import org.apache.oozie.service.WorkflowAppService;
051    import org.apache.oozie.service.SchemaService.SchemaName;
052    import org.apache.oozie.service.UUIDService.ApplicationType;
053    import org.apache.oozie.util.DateUtils;
054    import org.apache.oozie.util.ELEvaluator;
055    import org.apache.oozie.util.IOUtils;
056    import org.apache.oozie.util.InstrumentUtils;
057    import org.apache.oozie.util.LogUtils;
058    import org.apache.oozie.util.ParamChecker;
059    import org.apache.oozie.util.PropertiesUtils;
060    import org.apache.oozie.util.XConfiguration;
061    import org.apache.oozie.util.XmlUtils;
062    import org.jdom.Attribute;
063    import org.jdom.Element;
064    import org.jdom.JDOMException;
065    import org.xml.sax.SAXException;
066    
067    /**
068     * This Command will submit the bundle.
069     */
070    public class BundleSubmitXCommand extends SubmitTransitionXCommand {
071    
072        private Configuration conf;
073        private final String authToken;
074        public static final String CONFIG_DEFAULT = "bundle-config-default.xml";
075        public static final String BUNDLE_XML_FILE = "bundle.xml";
076        private final BundleJobBean bundleBean = new BundleJobBean();
077        private String jobId;
078        private JPAService jpaService = null;
079    
080        private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
081        private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
082    
083        static {
084            String[] badUserProps = { PropertiesUtils.YEAR, PropertiesUtils.MONTH, PropertiesUtils.DAY,
085                    PropertiesUtils.HOUR, PropertiesUtils.MINUTE, PropertiesUtils.DAYS, PropertiesUtils.HOURS,
086                    PropertiesUtils.MINUTES, PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB,
087                    PropertiesUtils.TB, PropertiesUtils.PB, PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN,
088                    PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN, PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
089            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
090    
091            String[] badDefaultProps = { PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
092                    WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME };
093            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
094            PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
095        }
096    
097        /**
098         * Constructor to create the bundle submit command.
099         *
100         * @param conf configuration for bundle job
101         * @param authToken to be used for authentication
102         */
103        public BundleSubmitXCommand(Configuration conf, String authToken) {
104            super("bundle_submit", "bundle_submit", 1);
105            this.conf = ParamChecker.notNull(conf, "conf");
106            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
107        }
108    
109        /**
110         * Constructor to create the bundle submit command.
111         *
112         * @param dryrun true if dryrun is enable
113         * @param conf configuration for bundle job
114         * @param authToken to be used for authentication
115         */
116        public BundleSubmitXCommand(boolean dryrun, Configuration conf, String authToken) {
117            this(conf, authToken);
118            this.dryrun = dryrun;
119        }
120    
121        /* (non-Javadoc)
122         * @see org.apache.oozie.command.SubmitTransitionXCommand#submit()
123         */
124        @Override
125        protected String submit() throws CommandException {
126            LOG.info("STARTED Bundle Submit");
127            try {
128                InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
129    
130                XmlUtils.removeComments(this.bundleBean.getOrigJobXml().toString());
131                // Resolving all variables in the job properties.
132                // This ensures the Hadoop Configuration semantics is preserved.
133                XConfiguration resolvedVarsConf = new XConfiguration();
134                for (Map.Entry<String, String> entry : conf) {
135                    resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
136                }
137                conf = resolvedVarsConf;
138    
139                String resolvedJobXml = resolvedVars(bundleBean.getOrigJobXml(), conf);
140    
141                //verify the uniqueness of coord names
142                verifyCoordNameUnique(resolvedJobXml);
143                this.jobId = storeToDB(bundleBean, resolvedJobXml);
144                LogUtils.setLogInfo(bundleBean, logInfo);
145    
146                if (dryrun) {
147                    Date startTime = bundleBean.getStartTime();
148                    long startTimeMilli = startTime.getTime();
149                    long endTimeMilli = startTimeMilli + (3600 * 1000);
150                    Date jobEndTime = bundleBean.getEndTime();
151                    Date endTime = new Date(endTimeMilli);
152                    if (endTime.compareTo(jobEndTime) > 0) {
153                        endTime = jobEndTime;
154                    }
155                    jobId = bundleBean.getId();
156                    LOG.info("[" + jobId + "]: Update status to PREP");
157                    bundleBean.setStatus(Job.Status.PREP);
158                    try {
159                        new XConfiguration(new StringReader(bundleBean.getConf()));
160                    }
161                    catch (IOException e1) {
162                        LOG.warn("Configuration parse error. read from DB :" + bundleBean.getConf(), e1);
163                    }
164                    String output = bundleBean.getJobXml() + System.getProperty("line.separator");
165                    return output;
166                }
167                else {
168                    if (bundleBean.getKickoffTime() == null) {
169                        // If there is no KickOffTime, default kickoff is NOW.
170                        LOG.debug("Since kickoff time is not defined for job id " + jobId
171                                + ". Queuing and BundleStartXCommand immediately after submission");
172                        queue(new BundleStartXCommand(jobId));
173                    }
174                }
175            }
176            catch (Exception ex) {
177                throw new CommandException(ErrorCode.E1310, ex.getMessage(), ex);
178            }
179            LOG.info("ENDED Bundle Submit");
180            return this.jobId;
181        }
182    
183        /* (non-Javadoc)
184         * @see org.apache.oozie.command.TransitionXCommand#notifyParent()
185         */
186        @Override
187        public void notifyParent() throws CommandException {
188        }
189    
190        /* (non-Javadoc)
191         * @see org.apache.oozie.command.XCommand#getEntityKey()
192         */
193        @Override
194        protected String getEntityKey() {
195            return null;
196        }
197    
198        /* (non-Javadoc)
199         * @see org.apache.oozie.command.XCommand#isLockRequired()
200         */
201        @Override
202        protected boolean isLockRequired() {
203            return false;
204        }
205    
206        /* (non-Javadoc)
207         * @see org.apache.oozie.command.XCommand#loadState()
208         */
209        @Override
210        protected void loadState() throws CommandException {
211        }
212    
213        /* (non-Javadoc)
214         * @see org.apache.oozie.command.XCommand#verifyPrecondition()
215         */
216        @Override
217        protected void verifyPrecondition() throws CommandException, PreconditionException {
218        }
219    
220        /* (non-Javadoc)
221         * @see org.apache.oozie.command.XCommand#eagerLoadState()
222         */
223        @Override
224        protected void eagerLoadState() throws CommandException {
225            super.eagerLoadState();
226            jpaService = Services.get().get(JPAService.class);
227            if (jpaService == null) {
228                throw new CommandException(ErrorCode.E0610);
229            }
230        }
231    
232        /* (non-Javadoc)
233         * @see org.apache.oozie.command.XCommand#eagerVerifyPrecondition()
234         */
235        @Override
236        protected void eagerVerifyPrecondition() throws CommandException, PreconditionException {
237            try {
238                super.eagerVerifyPrecondition();
239                mergeDefaultConfig();
240                String appXml = readAndValidateXml();
241                bundleBean.setOrigJobXml(appXml);
242                LOG.debug("jobXml after initial validation " + XmlUtils.prettyPrint(appXml).toString());
243            }
244            catch (BundleJobException ex) {
245                LOG.warn("BundleJobException:  ", ex);
246                throw new CommandException(ex);
247            }
248            catch (IllegalArgumentException iex) {
249                LOG.warn("IllegalArgumentException:  ", iex);
250                throw new CommandException(ErrorCode.E1310, iex);
251            }
252            catch (Exception ex) {
253                LOG.warn("Exception:  ", ex);
254                throw new CommandException(ErrorCode.E1310, ex);
255            }
256        }
257    
258        /**
259         * Merge default configuration with user-defined configuration.
260         *
261         * @throws CommandException thrown if failed to merge configuration
262         */
263        protected void mergeDefaultConfig() throws CommandException {
264            Path configDefault = null;
265            try {
266                String bundleAppPathStr = conf.get(OozieClient.BUNDLE_APP_PATH);
267                Path bundleAppPath = new Path(bundleAppPathStr);
268                String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
269                String group = ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
270                FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, bundleAppPath.toUri(),
271                        new Configuration());
272    
273                // app path could be a directory
274                if (!fs.isFile(bundleAppPath)) {
275                    configDefault = new Path(bundleAppPath, CONFIG_DEFAULT);
276                } else {
277                    configDefault = new Path(bundleAppPath.getParent(), CONFIG_DEFAULT);
278                }
279    
280                if (fs.exists(configDefault)) {
281                    Configuration defaultConf = new XConfiguration(fs.open(configDefault));
282                    PropertiesUtils.checkDisallowedProperties(defaultConf, DISALLOWED_DEFAULT_PROPERTIES);
283                    XConfiguration.injectDefaults(defaultConf, conf);
284                }
285                else {
286                    LOG.info("configDefault Doesn't exist " + configDefault);
287                }
288                PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
289            }
290            catch (IOException e) {
291                throw new CommandException(ErrorCode.E0702, e.getMessage() + " : Problem reading default config "
292                        + configDefault, e);
293            }
294            catch (HadoopAccessorException e) {
295                throw new CommandException(e);
296            }
297            LOG.debug("Merged CONF :" + XmlUtils.prettyPrint(conf).toString());
298        }
299    
300        /**
301         * Read the application XML and validate against bundle Schema
302         *
303         * @return validated bundle XML
304         * @throws BundleJobException thrown if failed to read or validate xml
305         */
306        private String readAndValidateXml() throws BundleJobException {
307            String appPath = ParamChecker.notEmpty(conf.get(OozieClient.BUNDLE_APP_PATH), OozieClient.BUNDLE_APP_PATH);
308            String bundleXml = readDefinition(appPath);
309            validateXml(bundleXml);
310            return bundleXml;
311        }
312    
313        /**
314         * Read bundle definition.
315         *
316         * @param appPath application path.
317         * @param user user name.
318         * @param group group name.
319         * @param autToken authentication token.
320         * @return bundle definition.
321         * @throws BundleJobException thrown if the definition could not be read.
322         */
323        protected String readDefinition(String appPath) throws BundleJobException {
324            String user = ParamChecker.notEmpty(conf.get(OozieClient.USER_NAME), OozieClient.USER_NAME);
325            String group = ParamChecker.notEmpty(conf.get(OozieClient.GROUP_NAME), OozieClient.GROUP_NAME);
326            //Configuration confHadoop = CoordUtils.getHadoopConf(conf);
327            try {
328                URI uri = new URI(appPath);
329                LOG.debug("user =" + user + " group =" + group);
330                FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, uri,
331                        new Configuration());
332                Path appDefPath = null;
333    
334                // app path could be a directory
335                Path path = new Path(uri.getPath());
336                if (!fs.isFile(path)) {
337                    appDefPath = new Path(path, BUNDLE_XML_FILE);
338                } else {
339                    appDefPath = path;
340                }
341    
342                Reader reader = new InputStreamReader(fs.open(appDefPath));
343                StringWriter writer = new StringWriter();
344                IOUtils.copyCharStream(reader, writer);
345                return writer.toString();
346            }
347            catch (IOException ex) {
348                LOG.warn("IOException :" + XmlUtils.prettyPrint(conf), ex);
349                throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex);
350            }
351            catch (URISyntaxException ex) {
352                LOG.warn("URISyException :" + ex.getMessage());
353                throw new BundleJobException(ErrorCode.E1302, appPath, ex.getMessage(), ex);
354            }
355            catch (HadoopAccessorException ex) {
356                throw new BundleJobException(ex);
357            }
358            catch (Exception ex) {
359                LOG.warn("Exception :", ex);
360                throw new BundleJobException(ErrorCode.E1301, ex.getMessage(), ex);
361            }
362        }
363    
364        /**
365         * Validate against Bundle XSD file
366         *
367         * @param xmlContent input bundle xml
368         * @throws BundleJobException thrown if failed to validate xml
369         */
370        private void validateXml(String xmlContent) throws BundleJobException {
371            javax.xml.validation.Schema schema = Services.get().get(SchemaService.class).getSchema(SchemaName.BUNDLE);
372            Validator validator = schema.newValidator();
373            try {
374                validator.validate(new StreamSource(new StringReader(xmlContent)));
375            }
376            catch (SAXException ex) {
377                LOG.warn("SAXException :", ex);
378                throw new BundleJobException(ErrorCode.E0701, ex.getMessage(), ex);
379            }
380            catch (IOException ex) {
381                LOG.warn("IOException :", ex);
382                throw new BundleJobException(ErrorCode.E0702, ex.getMessage(), ex);
383            }
384        }
385    
386        /**
387         * Write a Bundle Job into database
388         *
389         * @param Bundle job bean
390         * @return job id
391         * @throws CommandException thrown if failed to store bundle job bean to db
392         */
393        private String storeToDB(BundleJobBean bundleJob, String resolvedJobXml) throws CommandException {
394            try {
395                jobId = Services.get().get(UUIDService.class).generateId(ApplicationType.BUNDLE);
396    
397                bundleJob.setId(jobId);
398                bundleJob.setAuthToken(this.authToken);
399                bundleJob.setAppName(XmlUtils.parseXml(bundleBean.getOrigJobXml()).getAttributeValue("name"));
400                bundleJob.setAppName(bundleJob.getAppName());
401                bundleJob.setAppPath(conf.get(OozieClient.BUNDLE_APP_PATH));
402                // bundleJob.setStatus(BundleJob.Status.PREP); //This should be set in parent class.
403                bundleJob.setCreatedTime(new Date());
404                bundleJob.setUser(conf.get(OozieClient.USER_NAME));
405                bundleJob.setGroup(conf.get(OozieClient.GROUP_NAME));
406                bundleJob.setConf(XmlUtils.prettyPrint(conf).toString());
407                bundleJob.setJobXml(resolvedJobXml);
408                Element jobElement = XmlUtils.parseXml(resolvedJobXml);
409                Element controlsElement = jobElement.getChild("controls", jobElement.getNamespace());
410                if (controlsElement != null) {
411                    Element kickoffTimeElement = controlsElement.getChild("kick-off-time", jobElement.getNamespace());
412                    if (kickoffTimeElement != null && !kickoffTimeElement.getValue().isEmpty()) {
413                        Date kickoffTime = DateUtils.parseDateUTC(kickoffTimeElement.getValue());
414                        bundleJob.setKickoffTime(kickoffTime);
415                    }
416                }
417                bundleJob.setLastModifiedTime(new Date());
418    
419                if (!dryrun) {
420                    jpaService.execute(new BundleJobInsertJPAExecutor(bundleJob));
421                }
422            }
423            catch (Exception ex) {
424                throw new CommandException(ErrorCode.E1301, ex.getMessage(), ex);
425            }
426            return jobId;
427        }
428    
429        /* (non-Javadoc)
430         * @see org.apache.oozie.command.TransitionXCommand#getJob()
431         */
432        @Override
433        public Job getJob() {
434            return bundleBean;
435        }
436    
437        /**
438         * Resolve job xml with conf
439         *
440         * @param bundleXml bundle job xml
441         * @param conf job configuration
442         * @return resolved job xml
443         * @throws BundleJobException thrown if failed to resolve variables
444         */
445        private String resolvedVars(String bundleXml, Configuration conf) throws BundleJobException {
446            try {
447                ELEvaluator eval = createEvaluator(conf);
448                return eval.evaluate(bundleXml, String.class);
449            }
450            catch (Exception e) {
451                throw new BundleJobException(ErrorCode.E1004, e.getMessage(), e);
452            }
453        }
454    
455        /**
456         * Create ELEvaluator
457         *
458         * @param conf job configuration
459         * @return ELEvaluator the evaluator for el function
460         * @throws BundleJobException thrown if failed to create evaluator
461         */
462        public ELEvaluator createEvaluator(Configuration conf) throws BundleJobException {
463            ELEvaluator eval;
464            ELEvaluator.Context context;
465            try {
466                context = new ELEvaluator.Context();
467                eval = new ELEvaluator(context);
468                for (Map.Entry<String, String> entry : conf) {
469                    eval.setVariable(entry.getKey(), entry.getValue());
470                }
471            }
472            catch (Exception e) {
473                throw new BundleJobException(ErrorCode.E1004, e.getMessage(), e);
474            }
475            return eval;
476        }
477    
478        /**
479         * Verify the uniqueness of coordinator names
480         *
481         * @param resolved job xml
482         * @throws CommandException thrown if failed to verify the uniqueness of coordinator names
483         */
484        @SuppressWarnings("unchecked")
485        private Void verifyCoordNameUnique(String resolvedJobXml) throws CommandException {
486            Set<String> set = new HashSet<String>();
487            try {
488                Element bAppXml = XmlUtils.parseXml(resolvedJobXml);
489                List<Element> coordElems = bAppXml.getChildren("coordinator", bAppXml.getNamespace());
490                for (Element elem : coordElems) {
491                    Attribute name = elem.getAttribute("name");
492                    if (name != null) {
493                        if (set.contains(name.getValue())) {
494                            throw new CommandException(ErrorCode.E1304, name);
495                        }
496                        set.add(name.getValue());
497                    }
498                    else {
499                        throw new CommandException(ErrorCode.E1305);
500                    }
501                }
502            }
503            catch (JDOMException jex) {
504                throw new CommandException(ErrorCode.E1301, jex);
505            }
506    
507            return null;
508        }
509    
510        /* (non-Javadoc)
511         * @see org.apache.oozie.command.TransitionXCommand#updateJob()
512         */
513        @Override
514        public void updateJob() throws CommandException {
515        }
516    }