001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie; 016 017 import java.io.IOException; 018 import java.io.Writer; 019 import java.util.ArrayList; 020 import java.util.Date; 021 import java.util.HashMap; 022 import java.util.HashSet; 023 import java.util.List; 024 import java.util.Map; 025 import java.util.Set; 026 import java.util.StringTokenizer; 027 028 import org.apache.hadoop.conf.Configuration; 029 import org.apache.oozie.client.CoordinatorJob; 030 import org.apache.oozie.client.Job; 031 import org.apache.oozie.client.OozieClient; 032 import org.apache.oozie.client.WorkflowJob; 033 import org.apache.oozie.command.CommandException; 034 import org.apache.oozie.command.bundle.BundleJobChangeXCommand; 035 import org.apache.oozie.command.bundle.BundleJobResumeXCommand; 036 import org.apache.oozie.command.bundle.BundleJobSuspendXCommand; 037 import org.apache.oozie.command.bundle.BundleJobXCommand; 038 import org.apache.oozie.command.bundle.BundleJobsXCommand; 039 import org.apache.oozie.command.bundle.BundleKillXCommand; 040 import org.apache.oozie.command.bundle.BundleRerunXCommand; 041 import org.apache.oozie.command.bundle.BundleStartXCommand; 042 import org.apache.oozie.command.bundle.BundleSubmitXCommand; 043 import org.apache.oozie.service.DagXLogInfoService; 044 import org.apache.oozie.service.Services; 045 import org.apache.oozie.service.XLogService; 046 import org.apache.oozie.util.ParamChecker; 047 import org.apache.oozie.util.XLog; 048 import org.apache.oozie.util.XLogStreamer; 049 050 public class BundleEngine extends BaseEngine { 051 /** 052 * Create a system Bundle engine, with no user and no group. 053 */ 054 public BundleEngine() { 055 } 056 057 /** 058 * Create a Bundle engine to perform operations on behave of a user. 059 * 060 * @param user user name. 061 * @param authToken the authentication token. 062 */ 063 public BundleEngine(String user, String authToken) { 064 this.user = ParamChecker.notEmpty(user, "user"); 065 this.authToken = ParamChecker.notEmpty(authToken, "authToken"); 066 } 067 068 /* (non-Javadoc) 069 * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String) 070 */ 071 @Override 072 public void change(String jobId, String changeValue) throws BundleEngineException { 073 try { 074 BundleJobChangeXCommand change = new BundleJobChangeXCommand(jobId, changeValue); 075 change.call(); 076 } 077 catch (CommandException ex) { 078 throw new BundleEngineException(ex); 079 } 080 } 081 082 /* (non-Javadoc) 083 * @see org.apache.oozie.BaseEngine#dryrunSubmit(org.apache.hadoop.conf.Configuration, boolean) 084 */ 085 @Override 086 public String dryrunSubmit(Configuration conf, boolean startJob) throws BundleEngineException { 087 BundleSubmitXCommand submit = new BundleSubmitXCommand(true, conf, getAuthToken()); 088 try { 089 String jobId = submit.call(); 090 return jobId; 091 } 092 catch (CommandException ex) { 093 throw new BundleEngineException(ex); 094 } 095 } 096 097 /* (non-Javadoc) 098 * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String) 099 */ 100 @Override 101 public CoordinatorJob getCoordJob(String jobId) throws BundleEngineException { 102 throw new BundleEngineException(new XException(ErrorCode.E0301)); 103 } 104 105 public BundleJobBean getBundleJob(String jobId) throws BundleEngineException { 106 try { 107 return new BundleJobXCommand(jobId).call(); 108 } 109 catch (CommandException ex) { 110 throw new BundleEngineException(ex); 111 } 112 } 113 114 /* (non-Javadoc) 115 * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String, int, int) 116 */ 117 @Override 118 public CoordinatorJob getCoordJob(String jobId, int start, int length) throws BundleEngineException { 119 throw new BundleEngineException(new XException(ErrorCode.E0301)); 120 } 121 122 /* (non-Javadoc) 123 * @see org.apache.oozie.BaseEngine#getDefinition(java.lang.String) 124 */ 125 @Override 126 public String getDefinition(String jobId) throws BundleEngineException { 127 BundleJobBean job; 128 try { 129 job = new BundleJobXCommand(jobId).call(); 130 } 131 catch (CommandException ex) { 132 throw new BundleEngineException(ex); 133 } 134 return job.getOrigJobXml(); 135 } 136 137 /* (non-Javadoc) 138 * @see org.apache.oozie.BaseEngine#getJob(java.lang.String) 139 */ 140 @Override 141 public WorkflowJob getJob(String jobId) throws BundleEngineException { 142 throw new BundleEngineException(new XException(ErrorCode.E0301)); 143 } 144 145 /* (non-Javadoc) 146 * @see org.apache.oozie.BaseEngine#getJob(java.lang.String, int, int) 147 */ 148 @Override 149 public WorkflowJob getJob(String jobId, int start, int length) throws BundleEngineException { 150 throw new BundleEngineException(new XException(ErrorCode.E0301)); 151 } 152 153 /* (non-Javadoc) 154 * @see org.apache.oozie.BaseEngine#getJobIdForExternalId(java.lang.String) 155 */ 156 @Override 157 public String getJobIdForExternalId(String externalId) throws BundleEngineException { 158 return null; 159 } 160 161 /* (non-Javadoc) 162 * @see org.apache.oozie.BaseEngine#kill(java.lang.String) 163 */ 164 @Override 165 public void kill(String jobId) throws BundleEngineException { 166 try { 167 new BundleKillXCommand(jobId).call(); 168 } 169 catch (CommandException e) { 170 throw new BundleEngineException(e); 171 } 172 } 173 174 /* (non-Javadoc) 175 * @see org.apache.oozie.BaseEngine#reRun(java.lang.String, org.apache.hadoop.conf.Configuration) 176 */ 177 @Override 178 @Deprecated 179 public void reRun(String jobId, Configuration conf) throws BundleEngineException { 180 throw new BundleEngineException(new XException(ErrorCode.E0301)); 181 } 182 183 /** 184 * Rerun Bundle actions for given rerunType 185 * 186 * @param jobId bundle job id 187 * @param coordScope the rerun scope for coordinator job names separated by "," 188 * @param dateScope the rerun scope for coordinator nominal times separated by "," 189 * @param refresh true if user wants to refresh input/outpur dataset urls 190 * @param noCleanup false if user wants to cleanup output events for given rerun actions 191 * @throws BaseEngineException thrown if failed to rerun 192 */ 193 public void reRun(String jobId, String coordScope, String dateScope, boolean refresh, boolean noCleanup) 194 throws BaseEngineException { 195 try { 196 new BundleRerunXCommand(jobId, coordScope, dateScope, refresh, noCleanup).call(); 197 } 198 catch (CommandException ex) { 199 throw new BaseEngineException(ex); 200 } 201 } 202 203 /* (non-Javadoc) 204 * @see org.apache.oozie.BaseEngine#resume(java.lang.String) 205 */ 206 @Override 207 public void resume(String jobId) throws BundleEngineException { 208 BundleJobResumeXCommand resume = new BundleJobResumeXCommand(jobId); 209 try { 210 resume.call(); 211 } 212 catch (CommandException ex) { 213 throw new BundleEngineException(ex); 214 } 215 } 216 217 /* (non-Javadoc) 218 * @see org.apache.oozie.BaseEngine#start(java.lang.String) 219 */ 220 @Override 221 public void start(String jobId) throws BundleEngineException { 222 try { 223 new BundleStartXCommand(jobId).call(); 224 } 225 catch (CommandException e) { 226 throw new BundleEngineException(e); 227 } 228 } 229 230 /* (non-Javadoc) 231 * @see org.apache.oozie.BaseEngine#streamLog(java.lang.String, java.io.Writer) 232 */ 233 @Override 234 public void streamLog(String jobId, Writer writer) throws IOException, BundleEngineException { 235 XLogStreamer.Filter filter = new XLogStreamer.Filter(); 236 filter.setParameter(DagXLogInfoService.JOB, jobId); 237 238 BundleJobBean job; 239 try { 240 job = new BundleJobXCommand(jobId).call(); 241 } 242 catch (CommandException ex) { 243 throw new BundleEngineException(ex); 244 } 245 246 Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), new Date(), writer); 247 } 248 249 /* (non-Javadoc) 250 * @see org.apache.oozie.BaseEngine#submitJob(org.apache.hadoop.conf.Configuration, boolean) 251 */ 252 @Override 253 public String submitJob(Configuration conf, boolean startJob) throws BundleEngineException { 254 try { 255 String jobId = new BundleSubmitXCommand(conf, getAuthToken()).call(); 256 257 if (startJob) { 258 start(jobId); 259 } 260 return jobId; 261 } 262 catch (CommandException ex) { 263 throw new BundleEngineException(ex); 264 } 265 } 266 267 /* (non-Javadoc) 268 * @see org.apache.oozie.BaseEngine#suspend(java.lang.String) 269 */ 270 @Override 271 public void suspend(String jobId) throws BundleEngineException { 272 BundleJobSuspendXCommand suspend = new BundleJobSuspendXCommand(jobId); 273 try { 274 suspend.call(); 275 } 276 catch (CommandException ex) { 277 throw new BundleEngineException(ex); 278 } 279 } 280 281 private static final Set<String> FILTER_NAMES = new HashSet<String>(); 282 283 static { 284 FILTER_NAMES.add(OozieClient.FILTER_USER); 285 FILTER_NAMES.add(OozieClient.FILTER_NAME); 286 FILTER_NAMES.add(OozieClient.FILTER_GROUP); 287 FILTER_NAMES.add(OozieClient.FILTER_STATUS); 288 } 289 290 /** 291 * Get bundle jobs 292 * 293 * @param filterStr the filter string 294 * @param start start location for paging 295 * @param len total length to get 296 * @return bundle job info 297 * @throws BundleEngineException thrown if failed to get bundle job info 298 */ 299 public BundleJobInfo getBundleJobs(String filterStr, int start, int len) throws BundleEngineException { 300 Map<String, List<String>> filter = parseFilter(filterStr); 301 302 try { 303 return new BundleJobsXCommand(filter, start, len).call(); 304 } 305 catch (CommandException ex) { 306 throw new BundleEngineException(ex); 307 } 308 } 309 310 /** 311 * Parse filter string to a map with key = filter name and values = filter values 312 * 313 * @param filter the filter string 314 * @return filter key and value map 315 * @throws CoordinatorEngineException thrown if failed to parse filter string 316 */ 317 private Map<String, List<String>> parseFilter(String filter) throws BundleEngineException { 318 Map<String, List<String>> map = new HashMap<String, List<String>>(); 319 if (filter != null) { 320 StringTokenizer st = new StringTokenizer(filter, ";"); 321 while (st.hasMoreTokens()) { 322 String token = st.nextToken(); 323 if (token.contains("=")) { 324 String[] pair = token.split("="); 325 if (pair.length != 2) { 326 throw new BundleEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs"); 327 } 328 if (!FILTER_NAMES.contains(pair[0])) { 329 throw new BundleEngineException(ErrorCode.E0420, filter, XLog.format("invalid name [{0}]", 330 pair[0])); 331 } 332 if (pair[0].equals("status")) { 333 try { 334 Job.Status.valueOf(pair[1]); 335 } 336 catch (IllegalArgumentException ex) { 337 throw new BundleEngineException(ErrorCode.E0420, filter, XLog.format( 338 "invalid status [{0}]", pair[1])); 339 } 340 } 341 List<String> list = map.get(pair[0]); 342 if (list == null) { 343 list = new ArrayList<String>(); 344 map.put(pair[0], list); 345 } 346 list.add(pair[1]); 347 } 348 else { 349 throw new BundleEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs"); 350 } 351 } 352 } 353 return map; 354 } 355 356 }