001 /** 002 * Copyright (c) 2010 Yahoo! Inc. All rights reserved. 003 * Licensed under the Apache License, Version 2.0 (the "License"); 004 * you may not use this file except in compliance with the License. 005 * You may obtain a copy of the License at 006 * 007 * http://www.apache.org/licenses/LICENSE-2.0 008 * 009 * Unless required by applicable law or agreed to in writing, software 010 * distributed under the License is distributed on an "AS IS" BASIS, 011 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 012 * See the License for the specific language governing permissions and 013 * limitations under the License. See accompanying LICENSE file. 014 */ 015 package org.apache.oozie; 016 017 import java.io.IOException; 018 import java.io.Writer; 019 import java.util.ArrayList; 020 import java.util.Date; 021 import java.util.HashMap; 022 import java.util.HashSet; 023 import java.util.List; 024 import java.util.Map; 025 import java.util.Set; 026 import java.util.StringTokenizer; 027 028 import org.apache.hadoop.conf.Configuration; 029 import org.apache.oozie.client.CoordinatorJob; 030 import org.apache.oozie.client.OozieClient; 031 import org.apache.oozie.client.WorkflowJob; 032 import org.apache.oozie.command.CommandException; 033 import org.apache.oozie.command.coord.CoordActionInfoCommand; 034 import org.apache.oozie.command.coord.CoordActionInfoXCommand; 035 import org.apache.oozie.command.coord.CoordChangeCommand; 036 import org.apache.oozie.command.coord.CoordChangeXCommand; 037 import org.apache.oozie.command.coord.CoordJobCommand; 038 import org.apache.oozie.command.coord.CoordJobXCommand; 039 import org.apache.oozie.command.coord.CoordJobsCommand; 040 import org.apache.oozie.command.coord.CoordJobsXCommand; 041 import org.apache.oozie.command.coord.CoordKillCommand; 042 import org.apache.oozie.command.coord.CoordKillXCommand; 043 import org.apache.oozie.command.coord.CoordRerunCommand; 044 import org.apache.oozie.command.coord.CoordRerunXCommand; 045 import org.apache.oozie.command.coord.CoordResumeCommand; 046 import org.apache.oozie.command.coord.CoordResumeXCommand; 047 import org.apache.oozie.command.coord.CoordSubmitCommand; 048 import org.apache.oozie.command.coord.CoordSubmitXCommand; 049 import org.apache.oozie.command.coord.CoordSuspendCommand; 050 import org.apache.oozie.command.coord.CoordSuspendXCommand; 051 import org.apache.oozie.service.DagXLogInfoService; 052 import org.apache.oozie.service.Services; 053 import org.apache.oozie.service.XLogService; 054 import org.apache.oozie.util.ParamChecker; 055 import org.apache.oozie.util.XLog; 056 import org.apache.oozie.util.XLogStreamer; 057 058 public class CoordinatorEngine extends BaseEngine { 059 private static boolean useXCommand = true; 060 private static XLog LOG = XLog.getLog(CoordinatorEngine.class); 061 062 /** 063 * Create a system Coordinator engine, with no user and no group. 064 */ 065 public CoordinatorEngine() { 066 if (Services.get().getConf().getBoolean(USE_XCOMMAND, true) == false) { 067 useXCommand = false; 068 LOG.debug("Oozie CoordinatorEngine is not using XCommands."); 069 } 070 else { 071 LOG.debug("Oozie CoordinatorEngine is using XCommands."); 072 } 073 } 074 075 /** 076 * Create a Coordinator engine to perform operations on behave of a user. 077 * 078 * @param user user name. 079 * @param authToken the authentication token. 080 */ 081 public CoordinatorEngine(String user, String authToken) { 082 this(); 083 this.user = ParamChecker.notEmpty(user, "user"); 084 this.authToken = ParamChecker.notEmpty(authToken, "authToken"); 085 } 086 087 /* 088 * (non-Javadoc) 089 * 090 * @see org.apache.oozie.BaseEngine#getDefinition(java.lang.String) 091 */ 092 @Override 093 public String getDefinition(String jobId) throws BaseEngineException { 094 CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId); 095 return job.getOrigJobXml(); 096 } 097 098 /** 099 * @param jobId 100 * @return CoordinatorJobBean 101 * @throws BaseEngineException 102 */ 103 private CoordinatorJobBean getCoordJobWithNoActionInfo(String jobId) throws BaseEngineException { 104 try { 105 if (useXCommand) { 106 return new CoordJobXCommand(jobId).call(); 107 } 108 else { 109 return new CoordJobCommand(jobId).call(); 110 } 111 } 112 catch (CommandException ex) { 113 throw new BaseEngineException(ex); 114 } 115 } 116 117 /** 118 * @param actionId 119 * @return CoordinatorActionBean 120 * @throws BaseEngineException 121 */ 122 public CoordinatorActionBean getCoordAction(String actionId) throws BaseEngineException { 123 try { 124 if (useXCommand) { 125 return new CoordActionInfoXCommand(actionId).call(); 126 } 127 else { 128 return new CoordActionInfoCommand(actionId).call(); 129 } 130 } 131 catch (CommandException ex) { 132 throw new BaseEngineException(ex); 133 } 134 } 135 136 /* 137 * (non-Javadoc) 138 * 139 * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String) 140 */ 141 @Override 142 public CoordinatorJobBean getCoordJob(String jobId) throws BaseEngineException { 143 try { 144 if (useXCommand) { 145 return new CoordJobXCommand(jobId).call(); 146 } 147 else { 148 return new CoordJobCommand(jobId).call(); 149 } 150 } 151 catch (CommandException ex) { 152 throw new BaseEngineException(ex); 153 } 154 } 155 156 /* 157 * (non-Javadoc) 158 * 159 * @see org.apache.oozie.BaseEngine#getCoordJob(java.lang.String, int, int) 160 */ 161 @Override 162 public CoordinatorJobBean getCoordJob(String jobId, int start, int length) throws BaseEngineException { 163 try { 164 if (useXCommand) { 165 return new CoordJobXCommand(jobId, start, length).call(); 166 } 167 else { 168 return new CoordJobCommand(jobId, start, length).call(); 169 } 170 } 171 catch (CommandException ex) { 172 throw new BaseEngineException(ex); 173 } 174 } 175 176 /* 177 * (non-Javadoc) 178 * 179 * @see org.apache.oozie.BaseEngine#getJobIdForExternalId(java.lang.String) 180 */ 181 @Override 182 public String getJobIdForExternalId(String externalId) throws CoordinatorEngineException { 183 return null; 184 } 185 186 /* 187 * (non-Javadoc) 188 * 189 * @see org.apache.oozie.BaseEngine#kill(java.lang.String) 190 */ 191 @Override 192 public void kill(String jobId) throws CoordinatorEngineException { 193 try { 194 if (useXCommand) { 195 new CoordKillXCommand(jobId).call(); 196 } 197 else { 198 new CoordKillCommand(jobId).call(); 199 } 200 LOG.info("User " + user + " killed the Coordinator job " + jobId); 201 } 202 catch (CommandException e) { 203 throw new CoordinatorEngineException(e); 204 } 205 } 206 207 /* (non-Javadoc) 208 * @see org.apache.oozie.BaseEngine#change(java.lang.String, java.lang.String) 209 */ 210 @Override 211 public void change(String jobId, String changeValue) throws CoordinatorEngineException { 212 try { 213 if (useXCommand) { 214 new CoordChangeXCommand(jobId, changeValue).call(); 215 } 216 else { 217 new CoordChangeCommand(jobId, changeValue).call(); 218 } 219 LOG.info("User " + user + " changed the Coordinator job " + jobId + " to " + changeValue); 220 } 221 catch (CommandException e) { 222 throw new CoordinatorEngineException(e); 223 } 224 } 225 226 @Override 227 @Deprecated 228 public void reRun(String jobId, Configuration conf) throws BaseEngineException { 229 throw new BaseEngineException(new XException(ErrorCode.E0301)); 230 } 231 232 /** 233 * Rerun coordinator actions for given rerunType 234 * 235 * @param jobId 236 * @param rerunType 237 * @param scope 238 * @param refresh 239 * @param noCleanup 240 * @throws BaseEngineException 241 */ 242 public CoordinatorActionInfo reRun(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup) 243 throws BaseEngineException { 244 try { 245 if (useXCommand) { 246 return new CoordRerunXCommand(jobId, rerunType, scope, refresh, noCleanup).call(); 247 } 248 else { 249 return new CoordRerunCommand(jobId, rerunType, scope, refresh, noCleanup).call(); 250 } 251 } 252 catch (CommandException ex) { 253 throw new BaseEngineException(ex); 254 } 255 } 256 257 /* 258 * (non-Javadoc) 259 * 260 * @see org.apache.oozie.BaseEngine#resume(java.lang.String) 261 */ 262 @Override 263 public void resume(String jobId) throws CoordinatorEngineException { 264 try { 265 if (useXCommand) { 266 new CoordResumeXCommand(jobId).call(); 267 } 268 else { 269 new CoordResumeCommand(jobId).call(); 270 } 271 } 272 catch (CommandException e) { 273 throw new CoordinatorEngineException(e); 274 } 275 } 276 277 @Override 278 @Deprecated 279 public void start(String jobId) throws BaseEngineException { 280 throw new BaseEngineException(new XException(ErrorCode.E0301)); 281 } 282 283 /* 284 * (non-Javadoc) 285 * 286 * @see org.apache.oozie.BaseEngine#streamLog(java.lang.String, 287 * java.io.Writer) 288 */ 289 @Override 290 public void streamLog(String jobId, Writer writer) throws IOException, BaseEngineException { 291 XLogStreamer.Filter filter = new XLogStreamer.Filter(); 292 filter.setParameter(DagXLogInfoService.JOB, jobId); 293 294 CoordinatorJobBean job = getCoordJobWithNoActionInfo(jobId); 295 Services.get().get(XLogService.class).streamLog(filter, job.getCreatedTime(), new Date(), writer); 296 } 297 298 /* 299 * (non-Javadoc) 300 * 301 * @see 302 * org.apache.oozie.BaseEngine#submitJob(org.apache.hadoop.conf.Configuration 303 * , boolean) 304 */ 305 @Override 306 public String submitJob(Configuration conf, boolean startJob) throws CoordinatorEngineException { 307 try { 308 String jobId; 309 if (useXCommand) { 310 CoordSubmitXCommand submit = new CoordSubmitXCommand(conf, getAuthToken()); 311 jobId = submit.call(); 312 } 313 else { 314 CoordSubmitCommand submit = new CoordSubmitCommand(conf, getAuthToken()); 315 jobId = submit.call(); 316 } 317 return jobId; 318 } 319 catch (CommandException ex) { 320 throw new CoordinatorEngineException(ex); 321 } 322 } 323 324 /* 325 * (non-Javadoc) 326 * 327 * @see 328 * org.apache.oozie.BaseEngine#dryrunSubmit(org.apache.hadoop.conf.Configuration 329 * , boolean) 330 */ 331 @Override 332 public String dryrunSubmit(Configuration conf, boolean startJob) throws CoordinatorEngineException { 333 try { 334 String jobId; 335 if (useXCommand) { 336 CoordSubmitXCommand submit = new CoordSubmitXCommand(true, conf, getAuthToken()); 337 jobId = submit.call(); 338 } 339 else { 340 CoordSubmitCommand submit = new CoordSubmitCommand(true, conf, getAuthToken()); 341 jobId = submit.call(); 342 } 343 return jobId; 344 } 345 catch (CommandException ex) { 346 throw new CoordinatorEngineException(ex); 347 } 348 } 349 350 /* 351 * (non-Javadoc) 352 * 353 * @see org.apache.oozie.BaseEngine#suspend(java.lang.String) 354 */ 355 @Override 356 public void suspend(String jobId) throws CoordinatorEngineException { 357 try { 358 if (useXCommand) { 359 new CoordSuspendXCommand(jobId).call(); 360 } 361 else { 362 new CoordSuspendCommand(jobId).call(); 363 } 364 } 365 catch (CommandException e) { 366 throw new CoordinatorEngineException(e); 367 } 368 369 } 370 371 /* 372 * (non-Javadoc) 373 * 374 * @see org.apache.oozie.BaseEngine#getJob(java.lang.String) 375 */ 376 @Override 377 public WorkflowJob getJob(String jobId) throws BaseEngineException { 378 throw new BaseEngineException(new XException(ErrorCode.E0301)); 379 } 380 381 /* 382 * (non-Javadoc) 383 * 384 * @see org.apache.oozie.BaseEngine#getJob(java.lang.String, int, int) 385 */ 386 @Override 387 public WorkflowJob getJob(String jobId, int start, int length) throws BaseEngineException { 388 throw new BaseEngineException(new XException(ErrorCode.E0301)); 389 } 390 391 private static final Set<String> FILTER_NAMES = new HashSet<String>(); 392 393 static { 394 FILTER_NAMES.add(OozieClient.FILTER_USER); 395 FILTER_NAMES.add(OozieClient.FILTER_NAME); 396 FILTER_NAMES.add(OozieClient.FILTER_GROUP); 397 FILTER_NAMES.add(OozieClient.FILTER_STATUS); 398 } 399 400 /** 401 * @param filterStr 402 * @param start 403 * @param len 404 * @return CoordinatorJobInfo 405 * @throws CoordinatorEngineException 406 */ 407 public CoordinatorJobInfo getCoordJobs(String filterStr, int start, int len) throws CoordinatorEngineException { 408 Map<String, List<String>> filter = parseFilter(filterStr); 409 410 try { 411 if (useXCommand) { 412 return new CoordJobsXCommand(filter, start, len).call(); 413 } 414 else { 415 return new CoordJobsCommand(filter, start, len).call(); 416 } 417 } 418 catch (CommandException ex) { 419 throw new CoordinatorEngineException(ex); 420 } 421 } 422 423 /** 424 * @param filter 425 * @return Map<String, List<String>> 426 * @throws CoordinatorEngineException 427 */ 428 private Map<String, List<String>> parseFilter(String filter) throws CoordinatorEngineException { 429 Map<String, List<String>> map = new HashMap<String, List<String>>(); 430 if (filter != null) { 431 StringTokenizer st = new StringTokenizer(filter, ";"); 432 while (st.hasMoreTokens()) { 433 String token = st.nextToken(); 434 if (token.contains("=")) { 435 String[] pair = token.split("="); 436 if (pair.length != 2) { 437 throw new CoordinatorEngineException(ErrorCode.E0420, filter, 438 "elements must be name=value pairs"); 439 } 440 if (!FILTER_NAMES.contains(pair[0])) { 441 throw new CoordinatorEngineException(ErrorCode.E0420, filter, XLog.format("invalid name [{0}]", 442 pair[0])); 443 } 444 if (pair[0].equals("status")) { 445 try { 446 CoordinatorJob.Status.valueOf(pair[1]); 447 } 448 catch (IllegalArgumentException ex) { 449 throw new CoordinatorEngineException(ErrorCode.E0420, filter, XLog.format( 450 "invalid status [{0}]", pair[1])); 451 } 452 } 453 List<String> list = map.get(pair[0]); 454 if (list == null) { 455 list = new ArrayList<String>(); 456 map.put(pair[0], list); 457 } 458 list.add(pair[1]); 459 } 460 else { 461 throw new CoordinatorEngineException(ErrorCode.E0420, filter, "elements must be name=value pairs"); 462 } 463 } 464 } 465 return map; 466 } 467 }