001    /**
002     * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
003     * Licensed under the Apache License, Version 2.0 (the "License");
004     * you may not use this file except in compliance with the License.
005     * You may obtain a copy of the License at
006     *
007     *   http://www.apache.org/licenses/LICENSE-2.0
008     *
009     *  Unless required by applicable law or agreed to in writing, software
010     *  distributed under the License is distributed on an "AS IS" BASIS,
011     *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
012     *  See the License for the specific language governing permissions and
013     *  limitations under the License. See accompanying LICENSE file.
014     */
015    package org.apache.oozie.command.coord;
016    
017    import org.apache.hadoop.conf.Configuration;
018    
019    import org.apache.oozie.client.CoordinatorAction;
020    import org.apache.oozie.client.OozieClient;
021    import org.apache.oozie.CoordinatorActionBean;
022    import org.apache.oozie.DagEngineException;
023    import org.apache.oozie.DagEngine;
024    import org.apache.oozie.ErrorCode;
025    import org.apache.oozie.WorkflowJobBean;
026    import org.apache.oozie.command.CommandException;
027    import org.apache.oozie.command.PreconditionException;
028    import org.apache.oozie.service.DagEngineService;
029    import org.apache.oozie.service.JPAService;
030    import org.apache.oozie.service.Services;
031    import org.apache.oozie.util.JobUtils;
032    import org.apache.oozie.util.LogUtils;
033    import org.apache.oozie.util.ParamChecker;
034    import org.apache.oozie.util.XLog;
035    import org.apache.oozie.util.XmlUtils;
036    import org.apache.oozie.util.XConfiguration;
037    import org.apache.oozie.util.db.SLADbOperations;
038    import org.apache.oozie.client.SLAEvent.SlaAppType;
039    import org.apache.oozie.client.SLAEvent.Status;
040    import org.apache.oozie.executor.jpa.JPAExecutorException;
041    import org.apache.oozie.executor.jpa.WorkflowJobGetJPAExecutor;
042    import org.apache.oozie.executor.jpa.WorkflowJobUpdateJPAExecutor;
043    
044    import org.jdom.Element;
045    import org.jdom.JDOMException;
046    
047    import java.io.IOException;
048    import java.io.StringReader;
049    
050    public class CoordActionStartXCommand extends CoordinatorXCommand<Void> {
051    
052        public static final String EL_ERROR = "EL_ERROR";
053        public static final String EL_EVAL_ERROR = "EL_EVAL_ERROR";
054        public static final String COULD_NOT_START = "COULD_NOT_START";
055        public static final String START_DATA_MISSING = "START_DATA_MISSING";
056        public static final String EXEC_DATA_MISSING = "EXEC_DATA_MISSING";
057    
058        private final XLog log = getLog();
059        private String actionId = null;
060        private String user = null;
061        private String authToken = null;
062        private CoordinatorActionBean coordAction = null;
063        private JPAService jpaService = null;
064    
065        public CoordActionStartXCommand(String id, String user, String token) {
066            //super("coord_action_start", "coord_action_start", 1, XLog.OPS);
067            super("coord_action_start", "coord_action_start", 1);
068            this.actionId = ParamChecker.notEmpty(id, "id");
069            this.user = ParamChecker.notEmpty(user, "user");
070            this.authToken = ParamChecker.notEmpty(token, "token");
071        }
072    
073        /**
074         * Create config to pass to WF Engine 1. Get createdConf from coord_actions table 2. Get actionXml from
075         * coord_actions table. Extract all 'property' tags and merge createdConf (overwrite duplicate keys). 3. Extract
076         * 'app-path' from actionXML. Create a new property called 'oozie.wf.application.path' and merge with createdConf
077         * (overwrite duplicate keys) 4. Read contents of config-default.xml in workflow directory. 5. Merge createdConf
078         * with config-default.xml (overwrite duplicate keys). 6. Results is runConf which is saved in coord_actions table.
079         * Merge Action createdConf with actionXml to create new runConf with replaced variables
080         *
081         * @param action CoordinatorActionBean
082         * @return Configuration
083         * @throws CommandException
084         */
085        private Configuration mergeConfig(CoordinatorActionBean action) throws CommandException {
086            String createdConf = action.getCreatedConf();
087            String actionXml = action.getActionXml();
088            Element workflowProperties = null;
089            try {
090                workflowProperties = XmlUtils.parseXml(actionXml);
091            }
092            catch (JDOMException e1) {
093                log.warn("Configuration parse error in:" + actionXml);
094                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
095            }
096            // generate the 'runConf' for this action
097            // Step 1: runConf = createdConf
098            Configuration runConf = null;
099            try {
100                runConf = new XConfiguration(new StringReader(createdConf));
101            }
102            catch (IOException e1) {
103                log.warn("Configuration parse error in:" + createdConf);
104                throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
105            }
106            // Step 2: Merge local properties into runConf
107            // extract 'property' tags under 'configuration' block in the
108            // coordinator.xml (saved in actionxml column)
109            // convert Element to XConfiguration
110            Element configElement = (Element) workflowProperties.getChild("action", workflowProperties.getNamespace())
111                    .getChild("workflow", workflowProperties.getNamespace()).getChild("configuration",
112                                                                                      workflowProperties.getNamespace());
113            if (configElement != null) {
114                String strConfig = XmlUtils.prettyPrint(configElement).toString();
115                Configuration localConf;
116                try {
117                    localConf = new XConfiguration(new StringReader(strConfig));
118                }
119                catch (IOException e1) {
120                    log.warn("Configuration parse error in:" + strConfig);
121                    throw new CommandException(ErrorCode.E1005, e1.getMessage(), e1);
122                }
123    
124                // copy configuration properties in coordinator.xml to the runConf
125                XConfiguration.copy(localConf, runConf);
126            }
127    
128            // Step 3: Extract value of 'app-path' in actionxml, and save it as a
129            // new property called 'oozie.wf.application.path'
130            // WF Engine requires the path to the workflow.xml to be saved under
131            // this property name
132            String appPath = workflowProperties.getChild("action", workflowProperties.getNamespace()).getChild("workflow",
133                                                                                                               workflowProperties.getNamespace()).getChild("app-path", workflowProperties.getNamespace()).getValue();
134            runConf.set("oozie.wf.application.path", appPath);
135            return runConf;
136        }
137    
138        @Override
139        protected Void execute() throws CommandException {
140            boolean makeFail = true;
141            String errCode = "";
142            String errMsg = "";
143            ParamChecker.notEmpty(user, "user");
144            ParamChecker.notEmpty(authToken, "authToken");
145    
146            log.debug("actionid=" + actionId + ", status=" + coordAction.getStatus());
147            if (coordAction.getStatus() == CoordinatorAction.Status.SUBMITTED) {
148                // log.debug("getting.. job id: " + coordAction.getJobId());
149                // create merged runConf to pass to WF Engine
150                Configuration runConf = mergeConfig(coordAction);
151                coordAction.setRunConf(XmlUtils.prettyPrint(runConf).toString());
152                // log.debug("%%% merged runconf=" +
153                // XmlUtils.prettyPrint(runConf).toString());
154                DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(user, authToken);
155                try {
156                    boolean startJob = true;
157                    Configuration conf = new XConfiguration(new StringReader(coordAction.getRunConf()));
158                    SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), Status.STARTED,
159                                                    SlaAppType.COORDINATOR_ACTION, log);
160    
161                    // Normalize workflow appPath here;
162                    JobUtils.normalizeAppPath(conf.get(OozieClient.USER_NAME), conf.get(OozieClient.GROUP_NAME), conf);
163                    String wfId = dagEngine.submitJob(conf, startJob);
164                    coordAction.setStatus(CoordinatorAction.Status.RUNNING);
165                    coordAction.setExternalId(wfId);
166                    coordAction.incrementAndGetPending();
167    
168                    //store.updateCoordinatorAction(coordAction);
169                    JPAService jpaService = Services.get().get(JPAService.class);
170                    if (jpaService != null) {
171                        log.debug("Updating WF record for WFID :" + wfId + " with parent id: " + actionId);
172                        WorkflowJobBean wfJob = jpaService.execute(new WorkflowJobGetJPAExecutor(wfId));
173                        wfJob.setParentId(actionId);
174                        jpaService.execute(new WorkflowJobUpdateJPAExecutor(wfJob));
175                        jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionUpdateJPAExecutor(coordAction));
176                    }
177                    else {
178                        log.error(ErrorCode.E0610);
179                    }
180    
181                    makeFail = false;
182                }
183                catch (DagEngineException dee) {
184                    errMsg = dee.getMessage();
185                    errCode = "E1005";
186                    log.warn("can not create DagEngine for submitting jobs", dee);
187                }
188                catch (CommandException ce) {
189                    errMsg = ce.getMessage();
190                    errCode = ce.getErrorCode().toString();
191                    log.warn("command exception occured ", ce);
192                }
193                catch (java.io.IOException ioe) {
194                    errMsg = ioe.getMessage();
195                    errCode = "E1005";
196                    log.warn("Configuration parse error. read from DB :" + coordAction.getRunConf(), ioe);
197                }
198                catch (Exception ex) {
199                    errMsg = ex.getMessage();
200                    errCode = "E1005";
201                    log.warn("can not create DagEngine for submitting jobs", ex);
202                }
203                finally {
204                    if (makeFail == true) { // No DB exception occurs
205                        log.warn("Failing the action " + coordAction.getId() + ". Because " + errCode + " : " + errMsg);
206                        coordAction.setStatus(CoordinatorAction.Status.FAILED);
207                        if (errMsg.length() > 254) { // Because table column size is 255
208                            errMsg = errMsg.substring(0, 255);
209                        }
210                        coordAction.setErrorMessage(errMsg);
211                        coordAction.setErrorCode(errCode);
212    
213                        JPAService jpaService = Services.get().get(JPAService.class);
214                        if (jpaService != null) {
215                            try {
216                                jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionUpdateJPAExecutor(coordAction));
217                            }
218                            catch (JPAExecutorException je) {
219                                throw new CommandException(je);
220                            }
221                        }
222                        else {
223                            log.error(ErrorCode.E0610);
224                        }
225                        SLADbOperations.writeStausEvent(coordAction.getSlaXml(), coordAction.getId(), Status.FAILED,
226                                SlaAppType.COORDINATOR_ACTION, log); //Update SLA events
227                        queue(new CoordActionReadyXCommand(coordAction.getJobId()));
228                    }
229                }
230            }
231            return null;
232        }
233    
234        @Override
235        protected String getEntityKey() {
236            return coordAction.getJobId();
237        }
238    
239        @Override
240        protected boolean isLockRequired() {
241            return true;
242        }
243    
244        @Override
245        protected void loadState() throws CommandException {
246    
247        }
248    
249        @Override
250        protected void eagerLoadState() throws CommandException {
251            jpaService = Services.get().get(JPAService.class);
252            if (jpaService == null) {
253                throw new CommandException(ErrorCode.E0610);
254            }
255    
256            try {
257                coordAction = jpaService.execute(new org.apache.oozie.executor.jpa.CoordActionGetJPAExecutor(actionId));
258            }
259            catch (JPAExecutorException je) {
260                throw new CommandException(je);
261            }
262            LogUtils.setLogInfo(coordAction, logInfo);
263        }
264    
265        @Override
266        protected void verifyPrecondition() throws PreconditionException {
267            if (coordAction.getStatus() != CoordinatorAction.Status.SUBMITTED) {
268                throw new PreconditionException(ErrorCode.E1100);
269            }
270        }
271    }