001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.service;
016    
017    import org.apache.hadoop.conf.Configuration;
018    import org.apache.hadoop.fs.FileStatus;
019    import org.apache.hadoop.fs.FileSystem;
020    import org.apache.hadoop.fs.Path;
021    import org.apache.hadoop.fs.PathFilter;
022    import org.apache.oozie.client.OozieClient;
023    import org.apache.oozie.workflow.WorkflowApp;
024    import org.apache.oozie.workflow.WorkflowException;
025    import org.apache.oozie.util.IOUtils;
026    import org.apache.oozie.util.XConfiguration;
027    import org.apache.oozie.util.XLog;
028    import org.apache.oozie.ErrorCode;
029    
030    import java.io.IOException;
031    import java.io.InputStreamReader;
032    import java.io.Reader;
033    import java.io.StringWriter;
034    import java.net.URI;
035    import java.net.URISyntaxException;
036    import java.util.ArrayList;
037    import java.util.List;
038    import java.util.Map;
039    
040    /**
041     * Service that provides application workflow definition reading from the path and creation of the proto configuration.
042     */
043    public abstract class WorkflowAppService implements Service {
044    
045        public static final String CONF_PREFIX = Service.CONF_PREFIX + "WorkflowAppService.";
046    
047        public static final String SYSTEM_LIB_PATH = CONF_PREFIX + "system.libpath";
048    
049        public static final String APP_LIB_PATH_LIST = "oozie.wf.application.lib";
050    
051        public static final String HADOOP_UGI = "hadoop.job.ugi";
052    
053        public static final String HADOOP_USER = "user.name";
054    
055        public static final String HADOOP_JT_KERBEROS_NAME = "mapreduce.jobtracker.kerberos.principal";
056    
057        public static final String HADOOP_NN_KERBEROS_NAME = "dfs.namenode.kerberos.principal";
058    
059        private Path systemLibPath;
060    
061        /**
062         * Initialize the workflow application service.
063         *
064         * @param services services instance.
065         */
066        public void init(Services services) {
067            String path = services.getConf().get(SYSTEM_LIB_PATH, " ");
068            if (path.trim().length() > 0) {
069                systemLibPath = new Path(path.trim());
070            }
071        }
072    
073        /**
074         * Destroy the workflow application service.
075         */
076        public void destroy() {
077        }
078    
079        /**
080         * Return the public interface for workflow application service.
081         *
082         * @return {@link WorkflowAppService}.
083         */
084        public Class<? extends Service> getInterface() {
085            return WorkflowAppService.class;
086        }
087    
088        /**
089         * Read workflow definition.
090         *
091         * @param appPath application path.
092         * @param user user name.
093         * @param group group name.
094         * @param autToken authentication token.
095         * @return workflow definition.
096         * @throws WorkflowException thrown if the definition could not be read.
097         */
098        protected String readDefinition(String appPath, String user, String group, String autToken)
099                throws WorkflowException {
100            try {
101                URI uri = new URI(appPath);
102                FileSystem fs = Services.get().get(HadoopAccessorService.class).
103                        createFileSystem(user, group, uri, new Configuration());
104    
105                // app path could be a directory
106                Path path = new Path(uri.getPath());
107                if (!fs.isFile(path)) {
108                    path = new Path(path, "workflow.xml");
109                }
110    
111                Reader reader = new InputStreamReader(fs.open(path));
112                StringWriter writer = new StringWriter();
113                IOUtils.copyCharStream(reader, writer);
114                return writer.toString();
115    
116            }
117            catch (IOException ex) {
118                throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
119            }
120            catch (URISyntaxException ex) {
121                throw new WorkflowException(ErrorCode.E0711, appPath, ex.getMessage(), ex);
122            }
123            catch (HadoopAccessorException ex) {
124                throw new WorkflowException(ex);
125            }
126            catch (Exception ex) {
127                throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex);
128            }
129        }
130    
131        /**
132         * Create proto configuration. <p/> The proto configuration includes the user,group and the paths which need to be
133         * added to distributed cache. These paths include .jar,.so and the resource file paths.
134         *
135         * @param jobConf job configuration.
136         * @param authToken authentication token.
137         * @param isWorkflowJob indicates if the job is a workflow job or not.
138         * @return proto configuration.
139         * @throws WorkflowException thrown if the proto action configuration could not be created.
140         */
141        public XConfiguration createProtoActionConf(Configuration jobConf, String authToken, boolean isWorkflowJob)
142                throws WorkflowException {
143            XConfiguration conf = new XConfiguration();
144            try {
145                String user = jobConf.get(OozieClient.USER_NAME);
146                String group = jobConf.get(OozieClient.GROUP_NAME);
147                String hadoopUgi = user + "," + group;
148    
149                conf.set(OozieClient.USER_NAME, user);
150                conf.set(OozieClient.GROUP_NAME, group);
151                conf.set(HADOOP_UGI, hadoopUgi);
152    
153                conf.set(HADOOP_JT_KERBEROS_NAME, jobConf.get(HADOOP_JT_KERBEROS_NAME));
154                conf.set(HADOOP_NN_KERBEROS_NAME, jobConf.get(HADOOP_NN_KERBEROS_NAME));
155    
156                URI uri = new URI(jobConf.get(OozieClient.APP_PATH));
157    
158                FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, uri, conf);
159    
160                Path appPath = new Path(uri.getPath());
161                XLog.getLog(getClass()).debug("jobConf.libPath = " + jobConf.get(OozieClient.LIBPATH));
162                XLog.getLog(getClass()).debug("jobConf.appPath = " + appPath);
163    
164                List<String> filePaths;
165                if (isWorkflowJob) {
166                    // app path could be a directory
167                    Path path = new Path(uri.getPath());
168                    if (!fs.isFile(path)) {
169                        filePaths = getLibFiles(fs, new Path(appPath + "/lib"));
170                    } else {
171                        filePaths = getLibFiles(fs, new Path(appPath.getParent(), "lib"));
172                    }
173                }
174                else {
175                    filePaths = new ArrayList<String>();
176                }
177    
178                if (jobConf.get(OozieClient.LIBPATH) != null) {
179                    Path libPath = new Path(jobConf.get(OozieClient.LIBPATH));
180                    List<String> libPaths = getLibFiles(fs, libPath);
181                    filePaths.addAll(libPaths);
182                }
183    
184                if (systemLibPath != null && jobConf.getBoolean(OozieClient.USE_SYSTEM_LIBPATH, false)) {
185                    List<String> libPaths = getLibFiles(fs, systemLibPath);
186                    filePaths.addAll(libPaths);
187                }
188    
189                conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()]));
190    
191                //Add all properties start with 'oozie.'
192                for (Map.Entry<String, String> entry : jobConf) {
193                    if (entry.getKey().startsWith("oozie.")) {
194                        String name = entry.getKey();
195                        String value = entry.getValue();
196                        conf.set(name, value);
197                    }
198                }
199                return conf;
200            }
201            catch (IOException ex) {
202                throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
203            }
204            catch (URISyntaxException ex) {
205                throw new WorkflowException(ErrorCode.E0711, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex);
206            }
207            catch (HadoopAccessorException ex) {
208                throw new WorkflowException(ex);
209            }
210            catch (Exception ex) {
211                throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH),
212                                            ex.getMessage(), ex);
213            }
214        }
215    
216        /**
217         * Parse workflow definition.
218         *
219         * @param jobConf job configuration.
220         * @param authToken authentication token.
221         * @return workflow application.
222         * @throws WorkflowException thrown if the workflow application could not be parsed.
223         */
224        public abstract WorkflowApp parseDef(Configuration jobConf, String authToken) throws WorkflowException;
225    
226        /**
227         * Parse workflow definition.
228         * @param wfXml workflow.
229         * @return workflow application.
230         * @throws WorkflowException thrown if the workflow application could not be parsed.
231         */
232        public abstract WorkflowApp parseDef(String wfXml) throws WorkflowException;
233    
234        /**
235         * Get all library paths.
236         *
237         * @param fs file system object.
238         * @param libPath hdfs library path.
239         * @return list of paths.
240         * @throws IOException thrown if the lib paths could not be obtained.
241         */
242        private List<String> getLibFiles(FileSystem fs, Path libPath) throws IOException {
243            List<String> libPaths = new ArrayList<String>();
244            if (fs.exists(libPath)) {
245                FileStatus[] files = fs.listStatus(libPath, new NoPathFilter());
246    
247                for (FileStatus file : files) {
248                    libPaths.add(file.getPath().toUri().getPath().trim());
249                }
250            }
251            else {
252                XLog.getLog(getClass()).warn("libpath [{0}] does not exists", libPath);
253            }
254            return libPaths;
255        }
256    
257        /*
258         * Filter class doing no filtering.
259         * We dont need define this class, but seems fs.listStatus() is not working properly without this.
260         * So providing this dummy no filtering Filter class.
261         */
262        private class NoPathFilter implements PathFilter {
263            @Override
264            public boolean accept(Path path) {
265                return true;
266            }
267        }
268    }