001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie.service; 016 017 import org.apache.hadoop.conf.Configuration; 018 import org.apache.hadoop.fs.FileStatus; 019 import org.apache.hadoop.fs.FileSystem; 020 import org.apache.hadoop.fs.Path; 021 import org.apache.hadoop.fs.PathFilter; 022 import org.apache.oozie.client.OozieClient; 023 import org.apache.oozie.workflow.WorkflowApp; 024 import org.apache.oozie.workflow.WorkflowException; 025 import org.apache.oozie.util.IOUtils; 026 import org.apache.oozie.util.XConfiguration; 027 import org.apache.oozie.util.XLog; 028 import org.apache.oozie.ErrorCode; 029 030 import java.io.IOException; 031 import java.io.InputStreamReader; 032 import java.io.Reader; 033 import java.io.StringWriter; 034 import java.net.URI; 035 import java.net.URISyntaxException; 036 import java.util.ArrayList; 037 import java.util.List; 038 import java.util.Map; 039 040 /** 041 * Service that provides application workflow definition reading from the path and creation of the proto configuration. 042 */ 043 public abstract class WorkflowAppService implements Service { 044 045 public static final String CONF_PREFIX = Service.CONF_PREFIX + "WorkflowAppService."; 046 047 public static final String SYSTEM_LIB_PATH = CONF_PREFIX + "system.libpath"; 048 049 public static final String APP_LIB_PATH_LIST = "oozie.wf.application.lib"; 050 051 public static final String HADOOP_UGI = "hadoop.job.ugi"; 052 053 public static final String HADOOP_USER = "user.name"; 054 055 public static final String HADOOP_JT_KERBEROS_NAME = "mapreduce.jobtracker.kerberos.principal"; 056 057 public static final String HADOOP_NN_KERBEROS_NAME = "dfs.namenode.kerberos.principal"; 058 059 private Path systemLibPath; 060 061 /** 062 * Initialize the workflow application service. 063 * 064 * @param services services instance. 065 */ 066 public void init(Services services) { 067 String path = services.getConf().get(SYSTEM_LIB_PATH, " "); 068 if (path.trim().length() > 0) { 069 systemLibPath = new Path(path.trim()); 070 } 071 } 072 073 /** 074 * Destroy the workflow application service. 075 */ 076 public void destroy() { 077 } 078 079 /** 080 * Return the public interface for workflow application service. 081 * 082 * @return {@link WorkflowAppService}. 083 */ 084 public Class<? extends Service> getInterface() { 085 return WorkflowAppService.class; 086 } 087 088 /** 089 * Read workflow definition. 090 * 091 * @param appPath application path. 092 * @param user user name. 093 * @param group group name. 094 * @param autToken authentication token. 095 * @return workflow definition. 096 * @throws WorkflowException thrown if the definition could not be read. 097 */ 098 protected String readDefinition(String appPath, String user, String group, String autToken) 099 throws WorkflowException { 100 try { 101 URI uri = new URI(appPath); 102 FileSystem fs = Services.get().get(HadoopAccessorService.class). 103 createFileSystem(user, group, uri, new Configuration()); 104 105 // app path could be a directory 106 Path path = new Path(uri.getPath()); 107 if (!fs.isFile(path)) { 108 path = new Path(path, "workflow.xml"); 109 } 110 111 Reader reader = new InputStreamReader(fs.open(path)); 112 StringWriter writer = new StringWriter(); 113 IOUtils.copyCharStream(reader, writer); 114 return writer.toString(); 115 116 } 117 catch (IOException ex) { 118 throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex); 119 } 120 catch (URISyntaxException ex) { 121 throw new WorkflowException(ErrorCode.E0711, appPath, ex.getMessage(), ex); 122 } 123 catch (HadoopAccessorException ex) { 124 throw new WorkflowException(ex); 125 } 126 catch (Exception ex) { 127 throw new WorkflowException(ErrorCode.E0710, ex.getMessage(), ex); 128 } 129 } 130 131 /** 132 * Create proto configuration. <p/> The proto configuration includes the user,group and the paths which need to be 133 * added to distributed cache. These paths include .jar,.so and the resource file paths. 134 * 135 * @param jobConf job configuration. 136 * @param authToken authentication token. 137 * @param isWorkflowJob indicates if the job is a workflow job or not. 138 * @return proto configuration. 139 * @throws WorkflowException thrown if the proto action configuration could not be created. 140 */ 141 public XConfiguration createProtoActionConf(Configuration jobConf, String authToken, boolean isWorkflowJob) 142 throws WorkflowException { 143 XConfiguration conf = new XConfiguration(); 144 try { 145 String user = jobConf.get(OozieClient.USER_NAME); 146 String group = jobConf.get(OozieClient.GROUP_NAME); 147 String hadoopUgi = user + "," + group; 148 149 conf.set(OozieClient.USER_NAME, user); 150 conf.set(OozieClient.GROUP_NAME, group); 151 conf.set(HADOOP_UGI, hadoopUgi); 152 153 conf.set(HADOOP_JT_KERBEROS_NAME, jobConf.get(HADOOP_JT_KERBEROS_NAME)); 154 conf.set(HADOOP_NN_KERBEROS_NAME, jobConf.get(HADOOP_NN_KERBEROS_NAME)); 155 156 URI uri = new URI(jobConf.get(OozieClient.APP_PATH)); 157 158 FileSystem fs = Services.get().get(HadoopAccessorService.class).createFileSystem(user, group, uri, conf); 159 160 Path appPath = new Path(uri.getPath()); 161 XLog.getLog(getClass()).debug("jobConf.libPath = " + jobConf.get(OozieClient.LIBPATH)); 162 XLog.getLog(getClass()).debug("jobConf.appPath = " + appPath); 163 164 List<String> filePaths; 165 if (isWorkflowJob) { 166 // app path could be a directory 167 Path path = new Path(uri.getPath()); 168 if (!fs.isFile(path)) { 169 filePaths = getLibFiles(fs, new Path(appPath + "/lib")); 170 } else { 171 filePaths = getLibFiles(fs, new Path(appPath.getParent(), "lib")); 172 } 173 } 174 else { 175 filePaths = new ArrayList<String>(); 176 } 177 178 if (jobConf.get(OozieClient.LIBPATH) != null) { 179 Path libPath = new Path(jobConf.get(OozieClient.LIBPATH)); 180 List<String> libPaths = getLibFiles(fs, libPath); 181 filePaths.addAll(libPaths); 182 } 183 184 if (systemLibPath != null && jobConf.getBoolean(OozieClient.USE_SYSTEM_LIBPATH, false)) { 185 List<String> libPaths = getLibFiles(fs, systemLibPath); 186 filePaths.addAll(libPaths); 187 } 188 189 conf.setStrings(APP_LIB_PATH_LIST, filePaths.toArray(new String[filePaths.size()])); 190 191 //Add all properties start with 'oozie.' 192 for (Map.Entry<String, String> entry : jobConf) { 193 if (entry.getKey().startsWith("oozie.")) { 194 String name = entry.getKey(); 195 String value = entry.getValue(); 196 conf.set(name, value); 197 } 198 } 199 return conf; 200 } 201 catch (IOException ex) { 202 throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex); 203 } 204 catch (URISyntaxException ex) { 205 throw new WorkflowException(ErrorCode.E0711, jobConf.get(OozieClient.APP_PATH), ex.getMessage(), ex); 206 } 207 catch (HadoopAccessorException ex) { 208 throw new WorkflowException(ex); 209 } 210 catch (Exception ex) { 211 throw new WorkflowException(ErrorCode.E0712, jobConf.get(OozieClient.APP_PATH), 212 ex.getMessage(), ex); 213 } 214 } 215 216 /** 217 * Parse workflow definition. 218 * 219 * @param jobConf job configuration. 220 * @param authToken authentication token. 221 * @return workflow application. 222 * @throws WorkflowException thrown if the workflow application could not be parsed. 223 */ 224 public abstract WorkflowApp parseDef(Configuration jobConf, String authToken) throws WorkflowException; 225 226 /** 227 * Parse workflow definition. 228 * @param wfXml workflow. 229 * @return workflow application. 230 * @throws WorkflowException thrown if the workflow application could not be parsed. 231 */ 232 public abstract WorkflowApp parseDef(String wfXml) throws WorkflowException; 233 234 /** 235 * Get all library paths. 236 * 237 * @param fs file system object. 238 * @param libPath hdfs library path. 239 * @return list of paths. 240 * @throws IOException thrown if the lib paths could not be obtained. 241 */ 242 private List<String> getLibFiles(FileSystem fs, Path libPath) throws IOException { 243 List<String> libPaths = new ArrayList<String>(); 244 if (fs.exists(libPath)) { 245 FileStatus[] files = fs.listStatus(libPath, new NoPathFilter()); 246 247 for (FileStatus file : files) { 248 libPaths.add(file.getPath().toUri().getPath().trim()); 249 } 250 } 251 else { 252 XLog.getLog(getClass()).warn("libpath [{0}] does not exists", libPath); 253 } 254 return libPaths; 255 } 256 257 /* 258 * Filter class doing no filtering. 259 * We dont need define this class, but seems fs.listStatus() is not working properly without this. 260 * So providing this dummy no filtering Filter class. 261 */ 262 private class NoPathFilter implements PathFilter { 263 @Override 264 public boolean accept(Path path) { 265 return true; 266 } 267 } 268 }