001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.wf;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.oozie.WorkflowJobBean;
019    import org.apache.oozie.ErrorCode;
020    import org.apache.oozie.service.JPAService;
021    import org.apache.oozie.service.WorkflowStoreService;
022    import org.apache.oozie.service.WorkflowAppService;
023    import org.apache.oozie.service.Services;
024    import org.apache.oozie.service.DagXLogInfoService;
025    import org.apache.oozie.util.InstrumentUtils;
026    import org.apache.oozie.util.LogUtils;
027    import org.apache.oozie.util.XLog;
028    import org.apache.oozie.util.ParamChecker;
029    import org.apache.oozie.util.XConfiguration;
030    import org.apache.oozie.util.XmlUtils;
031    import org.apache.oozie.command.CommandException;
032    import org.apache.oozie.executor.jpa.WorkflowJobInsertJPAExecutor;
033    import org.apache.oozie.store.StoreException;
034    import org.apache.oozie.workflow.WorkflowApp;
035    import org.apache.oozie.workflow.WorkflowException;
036    import org.apache.oozie.workflow.WorkflowInstance;
037    import org.apache.oozie.workflow.WorkflowLib;
038    import org.apache.oozie.util.PropertiesUtils;
039    import org.apache.oozie.client.OozieClient;
040    import org.apache.oozie.client.WorkflowJob;
041    import org.apache.oozie.client.XOozieClient;
042    import org.jdom.Element;
043    import org.jdom.Namespace;
044    
045    import java.util.Date;
046    import java.util.Map;
047    import java.util.Set;
048    import java.util.HashSet;
049    
050    public abstract class SubmitHttpXCommand extends WorkflowXCommand<String> {
051    
052        protected static final Set<String> MANDATORY_OOZIE_CONFS = new HashSet<String>();
053        protected static final Set<String> OPTIONAL_OOZIE_CONFS = new HashSet<String>();
054    
055        static {
056            MANDATORY_OOZIE_CONFS.add(XOozieClient.JT);
057            MANDATORY_OOZIE_CONFS.add(XOozieClient.NN);
058            MANDATORY_OOZIE_CONFS.add(OozieClient.LIBPATH);
059    
060            OPTIONAL_OOZIE_CONFS.add(XOozieClient.FILES);
061            OPTIONAL_OOZIE_CONFS.add(XOozieClient.ARCHIVES);
062        }
063    
064        private Configuration conf;
065        private String authToken;
066    
067        public SubmitHttpXCommand(String name, String type, Configuration conf, String authToken) {
068            super(name, type, 1);
069            this.conf = ParamChecker.notNull(conf, "conf");
070            this.authToken = ParamChecker.notEmpty(authToken, "authToken");
071        }
072    
073        private static final Set<String> DISALLOWED_DEFAULT_PROPERTIES = new HashSet<String>();
074        private static final Set<String> DISALLOWED_USER_PROPERTIES = new HashSet<String>();
075    
076        static {
077            String[] badUserProps = { PropertiesUtils.DAYS, PropertiesUtils.HOURS, PropertiesUtils.MINUTES,
078                    PropertiesUtils.KB, PropertiesUtils.MB, PropertiesUtils.GB, PropertiesUtils.TB, PropertiesUtils.PB,
079                    PropertiesUtils.RECORDS, PropertiesUtils.MAP_IN, PropertiesUtils.MAP_OUT, PropertiesUtils.REDUCE_IN,
080                    PropertiesUtils.REDUCE_OUT, PropertiesUtils.GROUPS };
081            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_USER_PROPERTIES);
082    
083            String[] badDefaultProps = { PropertiesUtils.HADOOP_USER, PropertiesUtils.HADOOP_UGI,
084                    WorkflowAppService.HADOOP_JT_KERBEROS_NAME, WorkflowAppService.HADOOP_NN_KERBEROS_NAME };
085            PropertiesUtils.createPropertySet(badUserProps, DISALLOWED_DEFAULT_PROPERTIES);
086            PropertiesUtils.createPropertySet(badDefaultProps, DISALLOWED_DEFAULT_PROPERTIES);
087        }
088    
089        /**
090         * Generate workflow xml from conf object
091         *
092         * @param conf the configuration object
093         * @return workflow xml def string representation
094         */
095        abstract protected String getWorkflowXml(Configuration conf);
096    
097        /* (non-Javadoc)
098         * @see org.apache.oozie.command.XCommand#execute()
099         */
100        @Override
101        protected String execute() throws CommandException {
102            InstrumentUtils.incrJobCounter(getName(), 1, getInstrumentation());
103            WorkflowAppService wps = Services.get().get(WorkflowAppService.class);
104            try {
105                XLog.Info.get().setParameter(DagXLogInfoService.TOKEN, conf.get(OozieClient.LOG_TOKEN));
106                String wfXml = getWorkflowXml(conf);
107                LOG.debug("workflow xml created on the server side is :\n");
108                LOG.debug(wfXml);
109                WorkflowApp app = wps.parseDef(wfXml);
110                XConfiguration protoActionConf = wps.createProtoActionConf(conf, authToken, false);
111                WorkflowLib workflowLib = Services.get().get(WorkflowStoreService.class).getWorkflowLibWithNoDB();
112    
113                PropertiesUtils.checkDisallowedProperties(conf, DISALLOWED_USER_PROPERTIES);
114    
115                // Resolving all variables in the job properties.
116                // This ensures the Hadoop Configuration semantics is preserved.
117                XConfiguration resolvedVarsConf = new XConfiguration();
118                for (Map.Entry<String, String> entry : conf) {
119                    resolvedVarsConf.set(entry.getKey(), conf.get(entry.getKey()));
120                }
121                conf = resolvedVarsConf;
122    
123                WorkflowInstance wfInstance;
124                try {
125                    wfInstance = workflowLib.createInstance(app, conf);
126                }
127                catch (WorkflowException e) {
128                    throw new StoreException(e);
129                }
130    
131                Configuration conf = wfInstance.getConf();
132    
133                WorkflowJobBean workflow = new WorkflowJobBean();
134                workflow.setId(wfInstance.getId());
135                workflow.setAppName(app.getName());
136                workflow.setAppPath(conf.get(OozieClient.APP_PATH));
137                workflow.setConf(XmlUtils.prettyPrint(conf).toString());
138                workflow.setProtoActionConf(protoActionConf.toXmlString());
139                workflow.setCreatedTime(new Date());
140                workflow.setLastModifiedTime(new Date());
141                workflow.setLogToken(conf.get(OozieClient.LOG_TOKEN, ""));
142                workflow.setStatus(WorkflowJob.Status.PREP);
143                workflow.setRun(0);
144                workflow.setUser(conf.get(OozieClient.USER_NAME));
145                workflow.setGroup(conf.get(OozieClient.GROUP_NAME));
146                workflow.setAuthToken(authToken);
147                workflow.setWorkflowInstance(wfInstance);
148                workflow.setExternalId(conf.get(OozieClient.EXTERNAL_ID));
149    
150                LogUtils.setLogInfo(workflow, logInfo);
151                JPAService jpaService = Services.get().get(JPAService.class);
152                if (jpaService != null) {
153                    jpaService.execute(new WorkflowJobInsertJPAExecutor(workflow));
154                }
155                else {
156                    LOG.error(ErrorCode.E0610);
157                    return null;
158                }
159    
160                return workflow.getId();
161            }
162            catch (WorkflowException ex) {
163                throw new CommandException(ex);
164            }
165            catch (Exception ex) {
166                throw new CommandException(ErrorCode.E0803, ex);
167            }
168        }
169    
170        static private void addSection(Element X, Namespace ns, String filesStr, String tagName) {
171            if (filesStr != null) {
172                String[] files = filesStr.split(",");
173                for (String f : files) {
174                    Element tagElement = new Element(tagName, ns);
175                    if (f.contains("#")) {
176                        tagElement.addContent(f);
177                    }
178                    else {
179                        String filename = f.substring(f.lastIndexOf("/") + 1, f.length());
180                        if (filename == null || filename.isEmpty()) {
181                            tagElement.addContent(f);
182                        }
183                        else {
184                            tagElement.addContent(f + "#" + filename);
185                        }
186                    }
187                    X.addContent(tagElement);
188                }
189            }
190        }
191    
192        /**
193         * Add file section in X.
194         *
195         * @param parent XML element to be appended
196         * @param conf Configuration object
197         * @param ns XML element namespace
198         */
199        static void addFileSection(Element X, Configuration conf, Namespace ns) {
200            String filesStr = conf.get(XOozieClient.FILES);
201            addSection(X, ns, filesStr, "file");
202        }
203    
204        /**
205         * Add archive section in X.
206         *
207         * @param parent XML element to be appended
208         * @param conf Configuration object
209         * @param ns XML element namespace
210         */
211        static void addArchiveSection(Element X, Configuration conf, Namespace ns) {
212            String archivesStr = conf.get(XOozieClient.ARCHIVES);
213            addSection(X, ns, archivesStr, "archive");
214        }
215    }