001    /*
002     * CSHApplication.java
003     *
004     * Created on July 10, 2003, 11:22 AM
005     *
006     * This file is part of the STAR Scheduler.
007     * Copyright (c) 2002-2006 STAR Collaboration - Brookhaven National Laboratory
008     *
009     * STAR Scheduler is free software; you can redistribute it and/or modify
010     * it under the terms of the GNU General Public License as published by
011     * the Free Software Foundation; either version 2 of the License, or
012     * (at your option) any later version.
013     *
014     * STAR Scheduler is distributed in the hope that it will be useful,
015     * but WITHOUT ANY WARRANTY; without even the implied warranty of
016     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
017     * GNU General Public License for more details.
018     *
019     * You should have received a copy of the GNU General Public License
020     * along with STAR Scheduler; if not, write to the Free Software
021     * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
022     */
023    package gov.bnl.star.offline.scheduler.Dispatchers.lsf;
024    
025    
026    
027    import gov.bnl.star.offline.scheduler.*;
028    import gov.bnl.star.offline.scheduler.request.Request;
029    import gov.bnl.star.offline.scheduler.catalog.PhysicalFile;
030    //import gov.bnl.star.offline.scheduler.util.FileListToolkit;
031    //import gov.bnl.star.offline.scheduler.util.FilesystemToolkit;
032    //import gov.bnl.star.offline.scheduler.util.VariablesToolkit;
033    //import gov.bnl.star.offline.scheduler.util.ConfigToolkit;
034    import gov.bnl.star.offline.scheduler.util.*;
035    import java.text.SimpleDateFormat;
036     import java.util.Date;
037    
038    
039    import gov.bnl.star.offline.scheduler.util.sandbox.*;
040    import gov.bnl.star.offline.scheduler.informationService.InformationService;
041     
042    import java.io.File;
043    import java.io.FileOutputStream;
044    import java.io.PrintStream;
045    
046    import java.util.*;
047    import java.util.logging.Level;
048    import org.apache.log4j.Logger;
049    
050    
051    /**
052     * Application module for csh.
053     * <p/>
054     * This module defines the interface of a csh task, and prepare the task
055     * information in the following environment variables:
056     * <p/>
057     * $JOBID - the unique job id given by the scheduler<br>
058     * $FILESLIST - the file name of a text file containing the file list assigned to
059     * the job<br>
060     * $INPUTFILECOUNT - the number of files assigned to the job<br>
061     * $INPUTFILExx - the input files associated to the job<br>
062     * <p/>
063     *
064     * @author Gabriele Carcassi, Jerome Lauret, Levente Hajdu
065     * @version 1.0 2002/12/26
066     */
067    public class CSHApplication implements java.io.Serializable {
068        static private Logger log = Logger.getLogger(CSHApplication.class.getName());
069        private Request request;
070        private Job job;
071        private String scratchDir = "Not_defined_in_config_file";
072        private String submissionCommand;
073        private Map variables;
074        private boolean variablesSet = false; 
075    
076        /**
077         * Holds value of property recurseCopy.
078         */
079        private boolean recurseCopy = false;
080    
081        /**
082         * Holds the value for the property of EnvVariableLimit
083         */
084        private int EnvVariableLimit;
085        private boolean EnvVariableLimitNotReached = true; //Set false if in "any" job the EnvVariableLimit was reached
086        
087        /**
088         * Holds value of property copyScript.
089         */
090        private String copyScript;
091    
092        /**
093         * Holds value of property storageScript.
094         */
095        private String storageScript;
096    
097        /**
098         * Holds value of property registerScript.
099         */
100        private String registerScript;
101    
102        /**
103         * Creates a new instance of CSHApplication
104         */
105        public CSHApplication() {
106        }
107    
108        /** Returns the CSH application module. In the future, the application should
109         * be taken directly from the component library. This method is provided
110         * for compatibility with the old framework.
111         * @return the CSH application module
112         */
113    //    public static CSHApplication getInstance() {
114    //       // System.out.println("getting copy for ComponentLibrary");
115    //        //called aboat 11 times/dispach 
116    //        return (CSHApplication) ComponentLibrary.getInstance().getComponent("CSHApplication");
117    //    }
118        
119        
120        
121        /**
122         * Prepares the task, by creating the script.
123         */
124        public void prepareJob() {
125            try {
126    
127                PrintStream inputFileList = new PrintStream(new FileOutputStream(new File(getInputFileListName())));
128                createInputFileList(inputFileList);
129            } catch (Exception e) {
130                log.fatal("Couldn't create the file list", e);
131                throw new RuntimeException("Couldn't create the file list " + getInputFileListName() + ": " + e.getMessage());
132            }
133    
134            try {
135                PrintStream cshScript = new PrintStream(new FileOutputStream(new File(getCSHScriptName())));
136                createCSHScript(cshScript);
137                makeScriptFileExecutable(getCSHScriptName());
138    
139                cshScript.close(); //close the CSH stream, or else the job will not run
140    
141    
142            } catch (Exception e) {
143                log.fatal("Couldn't create the script", e);
144                e.printStackTrace();
145                throw new RuntimeException("Couldn't create the script " + getCSHScriptName() + ": " + e.getMessage() );
146            }
147        }
148    
149        /**
150         * Returns the command line to execute the task set by <CODE>setJob</CODE>.
151         *
152         * @return Command line to execute the task
153         */
154        public String getCommandLine() {
155            return getCSHScriptName();
156        }
157    
158        //Hold the chmod string from the config file
159        private String makeExecutableCommand;
160    
161        /**
162         * Returns the chmod command from the config file
163         */
164        public String getMakeExecutableCommand() {
165            return makeExecutableCommand;
166        }
167    
168        public void setMakeExecutableCommand(String makeExecutableCommand) {
169            this.makeExecutableCommand = makeExecutableCommand;
170        }
171    
172    
173        /* Calls chmod to make the script executable. */
174        private void makeScriptFileExecutable(String fileName) {
175            // Hack to make the Unit tests work on windows.
176            // chmod doesn't exist on Windows (if you don't have cygwin)
177            if (System.getProperty("os.name").startsWith("Win")) return;
178    
179            try {
180                // TODO: chmod should be a variable. This routine
181                // itself should be OS-flavor independent.
182                
183                String chmod = getMakeExecutableCommand() + fileName;
184                //String chmod = "/bin/chmod +x " + fileName;
185                
186                log.info("Executing \"" + chmod + "\"");
187    
188                java.lang.Process shell = Runtime.getRuntime().exec(chmod);
189                shell.waitFor();
190            } catch (Exception e) {
191                log.fatal("Couldn't make the script executable", e);
192                throw new RuntimeException("Couldn't make the script " + fileName +
193                        " executable", e);
194            }
195        }
196    
197        /**
198         * Sets the task the application will have to prepare.
199         *
200         * @param request the request that originated the task
201         * @param job     the job to be executed
202         */
203        public void setJob(Request request, Job job) {
204            this.request = request;
205            this.job = job;
206            scratchDir = null;
207            submissionCommand = null;
208            variables = null;
209            variablesSet = false;
210            this.scratchDir =  ConfigToolkit.findProgram("$SCRATCH", job); 
211    
212            /*
213            if (job.getAccessPoint().getSite().getProgramLocations() == null)
214                System.out.println("\nCan't find \"ProgramLocations\" block in " + job.getAccessPoint().getSite().getSiteName() + "(Site) -->" + job.getQueueObj().getBatchSystem().getName() + "(BatchSystem) block in config file $SCRATCH could not be set !. Please contact your SUMS systems administrator to modify the config file.");
215            else {
216                if (job.getAccessPoint().getSite().getProgramLocations().containsKey("$SCRATCH"))
217                    this.scratchDir = (String) job.getAccessPoint().getSite().getProgramLocations().get("$SCRATCH");
218                else
219                    System.out.println("\nThe \"$SCRATCH\" value could not be found in the ProgramLocations map of site=\"" + job.getAccessPoint().getSite().getSiteName() + "\". Please contact your SUMS systems administrator to modify the config file.");
220            }
221    
222             */
223        
224            
225        }
226    
227        /** Sets the scratch directory for the temporary output files of the job.
228         * @param scratchDir the temporary output directory
229         */
230        /*
231        public void setScratchDir(String scratchDir) {
232            this.scratchDir = scratchDir;
233        } */
234    
235        /**
236         * Sets the submittion command used to submit the job. This will be included in the
237         * script for resubmission.
238         *
239         * @param submissionCommand the command line to submit the job to the batch system
240         */
241        public void setSubmissionCommand(String submissionCommand) {
242            this.submissionCommand = submissionCommand;
243        }
244    
245        /**
246         * Generates the csh script to be submitted through LSF.
247         *
248         * @param out the stream on which to print the script
249         */
250        private void createCSHScript(PrintStream out) {
251            // Sets the shell that will interpret the script
252            
253            ////////////test area////////////////
254            Map config = (Map) ComponentLibrary.getInstance().getComponent("registerScript");
255    
256            out.println("#!/bin/csh");
257    
258            out.println("# -------------------------------------- ");
259            out.println("# Script generated  at " + new Date() +
260                    " by the STAR scheduler and submitted with");
261            out.println("# " + submissionCommand);
262            out.println("# -------------------------------------- ");
263            
264            out.println("\n" + getInitializationScript());
265    
266            Map env = getEnvironmentVariables();
267            Iterator variables = env.keySet().iterator();
268            Iterator FilesNameVariable = env.keySet().iterator();
269    
270            // Sets all the environment variable.
271            out.println();
272            out.println("# Preparing environment variables");
273    
274            String varName, varValue;
275    
276    
277            int counter = 0;
278            String padding = "";
279            while (variables.hasNext()) { //First pass prints all the environment variables that do not start with "INPUTFILE" 
280                varName = (String) variables.next();
281                varValue = (String) env.get(varName);
282                if ((!varName.startsWith("INPUTFILE")) || (varName.matches("INPUTFILECOUNT"))) {
283                    out.println(padding + "setenv " + varName + " " + varValue);
284                }
285            }
286    
287            while (FilesNameVariable.hasNext()) { //Second pass prints all the environment variables that start with "INPUTFILE"
288                varName = (String) FilesNameVariable.next();
289                varValue = (String) env.get(varName);
290                if (varName.startsWith("INPUTFILE") && (!varName.matches("INPUTFILECOUNT"))) {
291                    if (counter == EnvVariableLimit) {
292                        out.println("#");
293                        out.println("# Note: The configuration requested a limitation of the");
294                        out.println("# number of INPUTFILE(n) environemnt variables at " + EnvVariableLimit);
295                        out.println("# The rest of the variables will be commented.");
296                        out.println("#");
297                        padding = "# ";
298                        
299                        if(EnvVariableLimitNotReached){ //Print the warning to the user only one time then flip EnvVariableLimitNotReached to false 
300                            System.out.println("\n\n***************************************");
301                            System.out.println("The configuration requested a limitation of the number of INPUTFILE(n) environment variables at " + EnvVariableLimit + ", the rest will be commented out.\n");
302                            System.out.println("***************************************\n");
303                            EnvVariableLimitNotReached = false;
304                        }   
305                    }
306                    counter++;
307                    out.println(padding + "setenv " + varName + " " + varValue);
308                }
309            }
310            
311            
312                
313            
314              String mkdir = ConfigToolkit.findProgram("mkdir", job);
315    //        String mkdir; //test for mkdir it the configfile if the program location table if it is not there just use the defalt
316    //        if(job.getAccessPoint().getSite().getProgramLocations().containsKey("mkdir")){
317    //            mkdir = (String) job.getAccessPoint().getSite().getProgramLocations().get("mkdir");
318    //        }
319    //        else mkdir = "mkdir -p";
320            
321            
322            
323            // Creates the scratch directory
324            out.println();
325            out.println("# Creating the scratch directory, return failure status");
326            out.println(mkdir + " -p $SCRATCH");
327            out.println("set STS=$status");
328            out.println("if (! -d $SCRATCH) then");
329            out.println("    echo \"Scheduler:: Failed to create $SCRATCH on $HOST\"");
330            out.println("    exit $STS");
331            out.println("endif");
332            
333    
334            if(request.getSandbox() != null){
335                out.println();
336                out.println(importPakages());
337            }
338          
339            
340            out.println();
341            out.println("#Note: The default directory in which jobs start has be fix to $SCRATCH");
342            out.println("cd $SCRATCH"); //Note: This line sets the startup directory to $SCRATCH. It resolves the problem of different batch systems starting in different directories. 
343            
344    
345            out.println("echo \"--------------------------------\"");
346            out.println("echo \'" + gov.bnl.star.offline.scheduler.SchedulerCommandLine.getProgramName() + " we are starting in $SCRATCH :\' `/bin/pwd`");
347            out.println("echo \"--------------------------------\"");
348            out.println("/bin/ls -l");
349    
350            // soft linking to home dir removed for now because sandbox will replace this command 
351            //out.println("ln -s " + FilesystemToolkit.getCurrentDirectory() + "/* ."); //Note: This line sets the startup directory to $SCRATCH. It resolves the problem of different batch systems starting in different directories. 
352            
353    
354            // Adds the command to be executed
355            out.println();
356            out.println("###################################################");
357            //out.println("/bin/csh -s ___EOD"+job.getJobID()+"__");
358            out.println("# User command BEGIN ----------------------------->");
359            
360            //Let the RDLApplicationResolver try to resolve the test, if it can not use the back up
361            //if(request.getCommand() != null)
362            
363            
364            if (job.getQueueObj() != null && request.getCommand() == null) {
365                if (job.getQueueObj().getRDLApplicationResolver() != null)
366                    out.println(job.getQueueObj().getRDLApplicationResolver().resolve(request.getApplication(), request.getTask()));
367                else
368                    out.println(request.getCommand());
369            } else
370                out.println(request.getCommand());
371    
372    
373            out.println("# <------------------------------User command BEGIN");
374            //out.println("___EOD"+job.getJobID()+"__");
375            out.println("###################################################");
376    
377            // Copy output files
378            out.println();
379            out.println("# Copy output files (if any where specified)");
380            out.print(generateScriptFragment(request.getOutputList()));
381            // Delete the scratch directory
382            out.println();
383            out.println("# Delete the scratch directory");
384            out.println("/bin/rm -fr $SCRATCH");
385        }
386    
387        String generateScriptFragment(List outputList) {
388            StringBuffer buffer = new StringBuffer();
389    
390            String scriptsString = " ";
391            for (int nOutput = 0; nOutput < outputList.size(); nOutput++) {
392                OutputFile output = (OutputFile) outputList.get(nOutput);
393                if (output.getActionList().size() != 0) {
394                    for (int nOutputAction = 0; nOutputAction < output.getActionList().size(); nOutputAction++) {
395                        String script = ((OutputAction) output.getActionList().get(nOutputAction)).getScriptName();
396                        if (scriptsString.indexOf(script) == -1)
397                            scriptsString = scriptsString.concat(script + " ");
398                    }
399                }
400            }
401            
402            //TODO think about how this can be moved to the config file
403            
404            //copy output actions sripts to the local site
405            /*
406            if (scriptsString.length() != 1) {
407                //This only should be used for pdsf, but its hard coded for testing now
408                buffer.append("setenv X509_CERT_DIR /auto/pkg/globus-common/certificates \n");
409                buffer.append("module use /usr/local/pkg/HRM/pkg/modules/rh8 \n");
410                buffer.append("module load hrm/1.2.8a \n");
411                buffer.append("module load globus/2.4.3 orbacus \n");
412                buffer.append("module load rrs \n");
413                buffer.append("module load gcc/2.95.3-thread \n");
414                buffer.append("foreach file (" + scriptsString + ")\n" +
415                        "   globus-url-copy -p 25 gsiftp://" + ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName() + getHelperScripts() + "$file  file:$SCRATCH/$file \n" +
416                        "   chmod +x $file\n" +
417                        "end \n");
418            } */
419            
420    
421            for (int nOutput = 0; nOutput < outputList.size(); nOutput++) {
422    
423                OutputFile output = (OutputFile) outputList.get(nOutput);
424                if (output.getActionList().size() != 0) {
425    
426                    //buffer.append("setenv OUTPUTFILE").append(nOutput).append(" file:/$SCRATCH/").append(output.getFromScratch()).append("\n");
427                    
428                  
429                    for (int nOutputAction = 0; nOutputAction < output.getActionList().size(); nOutputAction++) {
430    
431    
432                        // if (output.getActionList().get(nOutputAction) instanceof OutputCopyAction) {
433                        //     buffer.append(generateScriptFragment(nOutput, (OutputCopyAction) output.getActionList().get(nOutputAction)));
434                        // } else if (output.getActionList().get(nOutputAction) instanceof OutputRegisterAction) {
435                        //     buffer.append(generateScriptFragment(nOutput, (OutputRegisterAction) output.getActionList().get(nOutputAction)));
436                        // } else {
437                        //     log.severe("Unsupported OutputAction time found: " + output.getActionList().get(nOutputAction).getClass().getName());
438                        //     throw new RuntimeException("Unsupported OutputAction time found");
439                        // }
440                        
441                        OutputAction outputAction = (OutputAction) output.getActionList().get(nOutputAction);
442                        buffer.append(outputAction.makeScriptString(output.getFromScratch(), ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName()));
443    
444                        //System.out.println(" " + outputAction.getScriptName());
445                        //outputAction.getScriptName();
446    
447                    }
448                } else {
449    
450                    //This code copys files back from FromScratch at the uers request //the path has should be added
451                    //String gridCopy = "globus-url-copy file://$SCRATCH/".concat(output.getFromScratch()).concat(" ").concat("gsiftp://").concat(getGatekeepr()).concat("/").concat(output.getToURL().getPath()).concat("\n");
452                    String gridCopy = FillCopyCommand(getGridCopyCommand(), output.getFromScratch(), output.getToURL().getPath(), true) + "\n";
453                    String cpCopy = FillCopyCommand(getCopyCommand(), output.getFromScratch(), output.getToURL().getPath(), false) + "\n";
454                    if (recurseCopy) cpCopy = FillCopyCommand(getRecursiveCopyCommand(), output.getFromScratch(), output.getToURL().getPath(), false) + "\n";
455                    
456                    
457                    
458                    
459                    /* This code was replaced with the code below. With the configuration file reshape, the type of queue (local/grid) can be determined by looking at the user�s local site and the queues local site.
460                    if(output.getToURL().toString().toLowerCase().startsWith("gsiftp:")){
461                        buffer.append(gridCopy);
462                    }
463                    //else if the user starts his request with file: then use cp
464                    else if(output.getToURL().toString().toLowerCase().startsWith("file:")){
465                        buffer.append(cpCopy);
466                    } 
467                    // else if the jobs is sent to a queue that has Implementation = "grid" than use globus-url-copy
468                    else if (((Queue) job.getQueueObj()).getImplementation().startsWith("grid")){
469                        buffer.append(gridCopy);
470                    }
471                    else{ //just use cp if all else fails
472                        buffer.append(cpCopy);
473                    } 
474                    */
475                    
476                    
477                    if (ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())) {
478                        buffer.append(cpCopy);
479                    } else {
480                        buffer.append(gridCopy);
481                    }
482    
483    
484                }
485            }
486            return buffer.toString();
487        }
488    
489        String generateScriptFragment(int nOutput, OutputCopyAction action) {
490            /*
491          StringBuffer fragment = new StringBuffer();
492             if (action.getFileRef() != null) {
493                 if (action.getStorageService() != null) {
494                     fragment.append("setenv OUTPUTFILE").append(nOutput).append('_').
495                     append(action.getFileRef()).append(" `").append(getStorageScript()).
496                     append(" \"").append(action.getStorageService()).append("\" `").
497                     append(action.getURI().getPath()).append("\n");
498    
499                 } else {
500                     fragment.append("setenv OUTPUTFILE").append(nOutput).append('_').
501                     append(action.getFileRef()).append(" ").
502                     append(action.getURI()).append("\n");
503                 }
504                 fragment.append(getCopyScript()).append(" $OUTPUTFILE").append(nOutput).
505                 append(" $OUTPUTFILE").append(nOutput).append('_').append(action.getFileRef()).
506                 append("\n");
507             } else {
508                 fragment.append(getCopyScript()).append(" $OUTPUTFILE").append(nOutput);
509                 if (action.getStorageService() != null) {
510                     fragment.append(" `").append(getStorageScript()).append(" \"").append(action.getStorageService()).append("\" `").append(action.getURI().getPath()).append("\n");
511                 } else {
512                     fragment.append(" ").append(action.getURI()).append("\n");
513                 }
514             }
515             return fragment.toString();
516             */ return " ";
517        }
518    
519        String generateScriptFragment(int nOutput, OutputRegisterAction action) {
520            /*
521            StringBuffer fragment = new StringBuffer();
522            fragment.append(getRegisterScript()).append(" $OUTPUTFILE").append(nOutput);
523            if (action.getFileRef() != null) {
524                fragment.append('_'). append(action.getFileRef());
525            }
526            int queryStart = action.getURI().getSchemeSpecificPart().indexOf('?');
527            queryStart++;
528            fragment.append(" '").append(action.getURI().getSchemeSpecificPart().substring(queryStart)).append("'\n");
529            return fragment.toString();
530             */ return " ";
531        }
532    
533        /**
534         * Generates the input file list. It consists of full path names, one line for each
535         * one.
536         *
537         * @param out the stream on which to print the file list
538         */
539        private void createInputFileList(PrintStream out) {        
540            //printJobInputList(job.getInput());
541            
542            variables = new TreeMap();
543            
544            if ("paths".equals(request.getFileListType())) {
545                FileListToolkit.createPathsFileList(out, job.getInput(), variables);
546            } else if ("rootd".equals(request.getFileListType())) {
547                FileListToolkit.createLocalRemoteFileList(out, job.getInput(), variables, false);
548            } else if ("xrootd".equals(request.getFileListType())) {
549                FileListToolkit.createLocalRemoteFileList(out, job.getInput(), variables, true);
550            }
551        }
552        private void printJobInputList(List jobList){
553            System.out.println("-----------------------------------------------");
554            for (int i = 0; i < jobList.size(); i++) {
555              PhysicalFile physicalFile = (PhysicalFile) jobList.get(i);
556              System.out.println("File n."+i+ "is:"+physicalFile.getStorage()+"::"+physicalFile.getPath()+"::"+physicalFile.getFilename());
557              System.out.println("\n");
558          }
559            System.out.println("------------------------------------------------\n");
560        }
561    
562        /**
563         * Returns the path and name for the csh script.
564         *
565         * @return the csh script path and name
566         */
567        public String getCSHScriptName() {
568            
569            return FilesystemToolkit.getCurrentDirectory() + "/sched" + job.getJobID() + ".csh";
570            
571        }
572    
573    
574        /**
575         * Returns the name for the csh script (without the path).
576         *
577         * @return the csh script name (without the path)
578         */
579        public String getCSHScriptFileName() {
580            /*
581           The path is being dropped because different drivers for mounting the PanFS path are reporting the path differently. Instated we will put �cd $path� before the command we execute.
582           Example:
583           /pdirect/star+institutions/data05/scratch/lbhajdu/
584           /star/data05/scratch/lbhajdu/
585           */
586            return "sched" + job.getJobID() + ".csh";
587        }
588    
589    
590        /**
591         * Returns the name for the input file list file.
592         *
593         * @return the input file list name
594         */
595        public String getInputFileListName() {
596            return FilesystemToolkit.getCurrentDirectory() + "/sched" + job.getJobID() + ".list";
597        }
598    
599    
600        private Map getEnvironmentVariables() {
601                    
602            if (variables == null) variables = new TreeMap();  //This can still be null if there is no filelist
603            if(variables.containsKey("JOBID")) return variables;  //if added one time do not add then again 
604            
605            variables.put("JOBID", job.getJobID());
606            variables.put("SUMS_name", "\"" + request.getName() + "\"" );
607            variables.put("SUMS_USER", System.getProperty("user.name"));
608            variables.put("SUMS_AUTHENTICATED_USER", "\"" + request.getAuthenticatedUser() + "\"");
609            variables.put("REQUESTID", job.getRequestID());
610            //variables.put("PROCESSID", job.getProcessID()); //has been renamed to $JOBINDEX
611            variables.put("JOBINDEX", job.getProcessID());
612            
613            
614            try { //add the dir of the location the user is submimtting from
615                    java.net.InetAddress localMachine = java.net.InetAddress.getLocalHost();
616                    variables.put("SUBMITTINGDIRECTORY",  FilesystemToolkit.getCurrentDirectory());
617                    variables.put("SUBMITTINGNODE",  localMachine.getHostName() ); 
618            }catch (java.net.UnknownHostException uhe) { 
619                    variables.put("SUBMITTINGDIRECTORY",  FilesystemToolkit.getCurrentDirectory());
620                    variables.put("SUBMITTINGNODE",  "UnknownHostException" ); 
621            }
622           
623    
624            SimpleDateFormat sdf = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss" );
625            sdf.setTimeZone(java.util.TimeZone.getTimeZone("GMT"));
626            variables.put("SUBMIT_TIME", "\"" + sdf.format( new Date()) + "\"" );
627            // enviroment variable used for ROOT to load particular labreries for xrootd client
628            if ("xrootd".equals(request.getFileListType())){
629                variables.put("USEXROOTD","1");
630            }
631    
632            //variables.put("SUBMIT_TIME", "\"" + (new Date()).toString() + "\"" );
633            //SimpleDateFormat dateFormat = new SimpleDateFormat( "yyyy-MM-dd" ); 
634            //dateFormat.format( new Date() );
635            
636            
637            if(request.getNProcesses()!= -1){
638                variables.put("SUMS_nProcesses",  String.valueOf(request.getNProcesses()) );
639            }
640            else{
641                variables.put("SUMS_nProcesses", String.valueOf(request.getJobs().size()) );
642            }
643            
644            
645            
646            
647    
648            // FIXME: scratch dir should be passed in a nicer way
649            variables.put("SCRATCH", scratchDir);
650            
651            variables.put("FILELIST", getInputFileListName());
652    
653            List inputList = job.getInput();
654            variables.put("INPUTFILECOUNT", Integer.toString(inputList.size()));
655    
656              //This is now done in FileListToolkit 
657    //        for (int nFile = 0; nFile < inputList.size(); nFile++) {
658    //            PhysicalFile file = (PhysicalFile) inputList.get(nFile);
659    //            variables.put("INPUTFILE" + nFile, file.getPath() + "/" + file.getFilename());
660    //        }
661            
662           
663            //If there is only one input file per csh make an Environment Variable called FILEBASENAME
664            if (inputList.size() == 1) {
665                PhysicalFile file = (PhysicalFile) inputList.get(0);
666                if (!(file.getFilename().indexOf(".") == -1))
667                    variables.put("FILEBASENAME", file.getFilename().substring(0, file.getFilename().indexOf(".")));
668                else
669                    variables.put("FILEBASENAME", file.getFilename());
670            } else { //All other times FILEBASENAME will be set to this value
671                //variables.put("FILEBASENAME", "FILEBASENAME.NOT.INITIALIZED." + job.getJobID());
672                variables.put("FILEBASENAME", "\"\"");  //Blank is better because people may use the variable FILEBASENAME. (JL, LH)
673            }
674            
675    
676    //        String APP = ComponentLibrary.getInstance().getProgramLocation(job.getAccessPoint().getSite(), "$APP");
677    //        if(APP != null){
678    //            APP = APP.trim();
679    //            if(APP.trim() != "") variables.put("APP", APP);
680    //        }
681            
682    
683        
684    /////////////////////////////////////////////used for setting $APP////////////////////////////////////////////////        
685            String APP = null; //Try to get $data from the IS.
686            if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){ //should only be done on the grid
687                InformationService informationService = job.getAccessPoint().getSite().getInformationService();
688                if(informationService != null){
689                    log.debug("Trying to get $APPA from IS (information service).");
690                    APP = informationService.getAppDir();
691                    log.debug("The value is: " + ((APP == null) ? "null" : APP));
692                }
693            }
694            
695            String APPFromComponentLibrary = ComponentLibrary.getInstance().getProgramLocation(job.getAccessPoint().getSite(), "$APP");
696            log.debug("Trying to get $APP from ProgramLocations table. The value is: " + ((APPFromComponentLibrary == null) ? "null" : APPFromComponentLibrary) + "this value is only used by OSG (Open Science Grid) Grid jobs");
697            if(APPFromComponentLibrary != null){
698                APP = APPFromComponentLibrary.trim();
699                log.debug("This final $data value will be \"" + APP + "\".\nComponentLibrary value overwrites all others.");
700            }
701      
702            if(APP != null){
703                APP = APP.trim();
704                if(APP.trim().length() > 0) variables.put("APP", APP);
705            }
706            else log.debug("There was no value for $APP. It will not be set.");
707            
708           
709    /////////////////////////////////////////////used for setting $DATA////////////////////////////////////////////////        
710            String DATA = null; //Try to get $data from the IS.
711            if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){ //should only be done on the grid
712                InformationService informationService = job.getAccessPoint().getSite().getInformationService();
713                if(informationService != null){
714                    DATA = informationService.getDataDir();
715                    log.debug("Trying to get $DATA from IS (information service). The value is: " + ((DATA == null) ? "null" : DATA) );
716                }
717            }
718            
719            String DATAFromComponentLibrary = ComponentLibrary.getInstance().getProgramLocation(job.getAccessPoint().getSite(), "$DATA");
720            log.debug("Trying to get $DATA from ProgramLocations table. The value is: " + ((DATA == null) ? "null" : DATA.trim()) + "this value is only used by OSG (Open Science Grid) Grid jobs");
721            if(DATAFromComponentLibrary != null){
722                DATA = DATAFromComponentLibrary;
723                log.debug("This final $data value will be \"" + DATA + "\".\nComponentLibrary value overwrites all others.");
724            }
725      
726            if(DATA != null){
727                DATA = DATA.trim();
728                if(DATA.trim().length() > 0) variables.put("DATA", DATA);
729            }
730            else log.debug("There was no value for $DATA. It will not be set.");
731    /////////////////////////////////////////////////Set items only used when the job is a grid job//////////////////////////////////////////////////////
732    
733            if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){ //should only be done on the grid
734                InformationService informationService = job.getAccessPoint().getSite().getInformationService();
735                if(informationService != null){
736                    
737                    String gridDir = informationService.getGKGlobusDir();
738                    if(gridDir != null){
739                        variables.put("GKGlobusDir", "\"" + gridDir + "\"");
740                    }
741                    else log.error("\"GKGlobusDir\" will not be displayed because returnned value was null");
742                    String WNClient = informationService.getWNClient();
743                    if(WNClient != null){
744                        variables.put("WNClient", "\"" + WNClient + "\"");
745                    }
746                    else log.error("\"WNClient\" will not be displayed because returnned value was null");
747      
748                }
749                else log.error("\"GKGlobusDir\" will not be displayed because no iinformationService object for the site could be recovered");
750            }
751            else log.debug("\"GKGlobusDir\" no set becuase this is not a grid job");
752            
753    //////////////////////////////////////////////        
754    
755    
756            return variables;
757        }
758    
759        /**
760         * Returns the file for standard input redirection.
761         *
762         * @return stdin file
763         */
764        public String getStdin() {
765            if (job.getStdin() != null) {
766                return VariablesToolkit.substituteVariables(job.getStdin().getPath(), this.getEnvironmentVariables());
767                //CSHApplication.getInstance().getEnvironmentVariables());
768            }
769    
770            return null;
771        }
772    
773        /**
774         * Returns the file for standard output redirection.
775         *
776         * @return stdout file
777         */
778        public String getStdout() {
779            if (job.getStdout() != null) {
780                if (job.getStdout() == Request.discard) {
781                    return "/dev/null";
782                }
783    
784                return VariablesToolkit.substituteVariables(job.getStdout().getPath(), this.getEnvironmentVariables());
785                // CSHApplication.getInstance().getEnvironmentVariables());
786            }
787    
788            return null;
789        }
790    
791        /**
792         * Returns the file for standard error redirection.
793         *
794         * @return stderr file
795         */
796        public String getStderr() {
797            if (job.getStderr() != null) {
798                if (job.getStderr() == Request.discard) {
799                    return "/dev/null";
800                }
801    
802                return VariablesToolkit.substituteVariables(job.getStderr().getPath(), this.getEnvironmentVariables());
803                //CSHApplication.getInstance().getEnvironmentVariables());
804            }
805    
806            return null;
807        }
808    
809        /**
810         * Returns the job name to be assigned to the batch system.
811         *
812         * @return the job name to be used by the batch system
813         */
814        public String getJobName() {
815            return request.getName();
816        }
817    
818        /**
819         * Getter for property recurseCopy.
820         *
821         * @return Value of property recurseCopy.
822         */
823        public boolean isRecurseCopy() {
824            return this.recurseCopy;
825        }
826    
827        /**
828         * Setter for property recurseCopy.
829         *
830         * @param recurseCopy New value of property recurseCopy.
831         */
832        public void setRecurseCopy(boolean recurseCopy) {
833            this.recurseCopy = recurseCopy;
834        }
835    
836    
837        /**
838         * Getter for property EnvVariableLimit.
839         *
840         * @return Value of property EnvVariableLimit.
841         */
842        public int getEnvVariableLimit() {
843            return this.EnvVariableLimit;
844        }
845    
846        /**
847         * Setter for property EnvVariableLimit.
848         *
849         * @param EnvVariableLimit New value of property EnvVariableLimit.
850         */
851        public void setEnvVariableLimit(int EnvVariableLimit) {
852            this.EnvVariableLimit = EnvVariableLimit;
853        }
854    
855    
856        /**
857         * Getter for property copyScript.
858         *
859         * @return Value of property copyScript.
860         */
861        public String getCopyScript() {
862            return this.copyScript;
863        }
864    
865        /**
866         * Setter for property copyScript.
867         *
868         * @param copyScript New value of property copyScript.
869         */
870        public void setCopyScript(String copyScript) {
871            this.copyScript = copyScript;
872        }
873    
874        /**
875         * Getter for property storageScript.
876         *
877         * @return Value of property storageScript.
878         */
879        public String getStorageScript() {
880            return this.storageScript;
881        }
882    
883        /**
884         * Setter for property storageScript.
885         *
886         * @param storageScript New value of property storageScript.
887         */
888        public void setStorageScript(String storageScript) {
889            this.storageScript = storageScript;
890        }
891    
892        /**
893         * Getter for property registerScript.
894         *
895         * @return Value of property registerScript.
896         */
897        public String getRegisterScript() {
898            return this.registerScript;
899        }
900    
901        /**
902         * Setter for property registerScript.
903         *
904         * @param registerScript New value of property registerScript.
905         */
906        public void setRegisterScript(String registerScript) {
907            this.registerScript = registerScript;
908        }
909    
910    
911        private String gridCopyCommand = "(gridCopyCommand not set in config file file!)";
912    
913        /**
914         * sets the command for copying files back, when the user asks for <output fromScratch="/path/x" toURL="gsiftp:/path/x"/>
915         * use (fromScratch) for source and (toURL) for destination.
916         * Example: globus-url-copy file://$SCRATCH/(fromScratch) gsiftp://stragrid04.star.bnl.gov/(toURL)
917         */
918        public void setGridCopyCommand(String gridCopyCommand) {
919            this.gridCopyCommand = gridCopyCommand;
920        }
921    
922        public String getGridCopyCommand() {
923            return gridCopyCommand;
924        }
925    
926        private String CopyCommand = "(CopyCommand not set in config file file!)";
927    
928        /**
929         * sets the command for copying files back, when the user asks for <output fromScratch=x toURL="file:/path/"/>
930         * use (fromScratch) for source and (toURL) for destination.
931         * Example: /bin/cp $SCRATCH/(fromScratch) (toURL)
932         */
933        public void setCopyCommand(String CopyCommand) {
934            this.CopyCommand = CopyCommand;
935        }
936    
937        public String getCopyCommand() {
938            return CopyCommand;
939        }
940    
941        private String recursiveCopyCommand = "(recursiveCopyCommand not set in config file file!)";
942    
943        /**
944         * sets the command for copying files back, when the user asks for <output fromScratch=x toURL="file:/path/"/>
945         * use (fromScratch) for source and (toURL) for destination.
946         * Example: /bin/cp -r $SCRATCH/(fromScratch) (toURL)
947         */
948        public void setRecursiveCopyCommand(String recursiveCopyCommand) {
949            this.recursiveCopyCommand = recursiveCopyCommand;
950        }
951    
952        public String getRecursiveCopyCommand() {
953            return recursiveCopyCommand;
954        }
955    
956        /**
957         * replaces the (fromScratch) and the (toURL) in the copy command with the true path
958         */
959        public String FillCopyCommand(String command, String fromScratch, String toURL, boolean isGrid) {
960    
961            if (!isGrid) {
962                if (fromScratch.matches("^\\Q./\\E.*"))
963                    fromScratch = fromScratch.replaceAll("(^\\Q.\\E)(.*)", "`pwd`$2");
964                else
965                    fromScratch = "$SCRATCH/".concat(fromScratch);
966            }
967    
968            StringBuffer buffer = new StringBuffer();
969            buffer.append(command);
970            while (buffer.indexOf("(fromScratch)") != -1) {
971                buffer.replace(buffer.indexOf("(fromScratch)"), (buffer.indexOf("(fromScratch)") + "(fromScratch)".length()), fromScratch);
972            }
973    
974            while (buffer.indexOf("(toURL)") != -1) {
975                buffer.replace(buffer.indexOf("(toURL)"), (buffer.indexOf("(toURL)") + "(toURL)".length()), toURL);
976            }
977    
978    
979            while (buffer.indexOf("(copyBackGateKeeper)") != -1) {
980                //ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName();
981                buffer.replace(buffer.indexOf("(copyBackGateKeeper)"), (buffer.indexOf("(copyBackGateKeeper)") + "(copyBackGateKeeper)".length()), ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName());
982            }
983            
984            //This did not working, it would fail with "Illegal group reference" when fromScratch or toURL had a '$' in it.
985            //return command.replaceAll("\\(fromScratch\\)", fromScratch).replaceAll("\\(toURL\\)", toURL); 
986            
987            return buffer.toString();
988    
989        }
990    
991    
992        private String helperScripts = "[helper_script_path_not_set]";
993    
994        public String getHelperScripts() {
995            return helperScripts;
996        }
997    
998        public void setHelperScripts(String helperScripts) {
999            this.helperScripts = helperScripts;
1000        }
1001        
1002        private String initializationScript = "";
1003        public String getInitializationScript(){return this.initializationScript; }
1004        public void setInitializationScript(String initializationScript){this.initializationScript = initializationScript;    }
1005        
1006    
1007        public String importPakages(){
1008            String importPakagesText = "\n#import packages \n";
1009            
1010            Sandbox sandbox = request.getSandbox();
1011    
1012            //Add warnings about special sandbox grid requirements here:
1013            if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){
1014                
1015                if(sandbox.getClass().getName().matches(".*ZipSandbox$")){
1016                    if (! getEnvironmentVariables().containsKey("DATA")){
1017                        String Warning = "Warning: Scheduler has no configuration information for $DATA at site " + job.getAccessPoint().getSite().getSiteName() + ". If $DATA is not set in the environment of the worker node the sandbox will fail. For more information please contact your scheduler administrator.";
1018                        System.out.println("\n\n" + Warning + "\n");
1019                        log.error(Warning);
1020                    } 
1021                }        
1022            }
1023            
1024            
1025            
1026            importPakagesText = importPakagesText.concat(sandbox.InitializationCommands()); 
1027    
1028            List packages = sandbox.getPackages();
1029            
1030            for(int i = 0; i != packages.size(); i++){
1031                SandboxPackage sandboxPackage = (SandboxPackage) packages.get(i);
1032    
1033                if(! sandbox.PackageExists(sandboxPackage)){ //This to see if the package dose not exist
1034                    //make the package
1035                    sandbox.MakePackage(sandboxPackage, request);
1036                }
1037    
1038                importPakagesText = importPakagesText.concat(sandbox.installCommand(sandboxPackage, job));
1039            }
1040                
1041            return importPakagesText;
1042        }
1043        
1044    
1045    }