001 /* 002 * CSHApplication.java 003 * 004 * Created on July 10, 2003, 11:22 AM 005 * 006 * This file is part of the STAR Scheduler. 007 * Copyright (c) 2002-2006 STAR Collaboration - Brookhaven National Laboratory 008 * 009 * STAR Scheduler is free software; you can redistribute it and/or modify 010 * it under the terms of the GNU General Public License as published by 011 * the Free Software Foundation; either version 2 of the License, or 012 * (at your option) any later version. 013 * 014 * STAR Scheduler is distributed in the hope that it will be useful, 015 * but WITHOUT ANY WARRANTY; without even the implied warranty of 016 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 017 * GNU General Public License for more details. 018 * 019 * You should have received a copy of the GNU General Public License 020 * along with STAR Scheduler; if not, write to the Free Software 021 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 022 */ 023 package gov.bnl.star.offline.scheduler.Dispatchers.lsf; 024 025 026 027 import gov.bnl.star.offline.scheduler.*; 028 import gov.bnl.star.offline.scheduler.request.Request; 029 import gov.bnl.star.offline.scheduler.catalog.PhysicalFile; 030 //import gov.bnl.star.offline.scheduler.util.FileListToolkit; 031 //import gov.bnl.star.offline.scheduler.util.FilesystemToolkit; 032 //import gov.bnl.star.offline.scheduler.util.VariablesToolkit; 033 //import gov.bnl.star.offline.scheduler.util.ConfigToolkit; 034 import gov.bnl.star.offline.scheduler.util.*; 035 import java.text.SimpleDateFormat; 036 import java.util.Date; 037 038 039 import gov.bnl.star.offline.scheduler.util.sandbox.*; 040 import gov.bnl.star.offline.scheduler.informationService.InformationService; 041 042 import java.io.File; 043 import java.io.FileOutputStream; 044 import java.io.PrintStream; 045 046 import java.util.*; 047 import java.util.logging.Level; 048 import org.apache.log4j.Logger; 049 050 051 /** 052 * Application module for csh. 053 * <p/> 054 * This module defines the interface of a csh task, and prepare the task 055 * information in the following environment variables: 056 * <p/> 057 * $JOBID - the unique job id given by the scheduler<br> 058 * $FILESLIST - the file name of a text file containing the file list assigned to 059 * the job<br> 060 * $INPUTFILECOUNT - the number of files assigned to the job<br> 061 * $INPUTFILExx - the input files associated to the job<br> 062 * <p/> 063 * 064 * @author Gabriele Carcassi, Jerome Lauret, Levente Hajdu 065 * @version 1.0 2002/12/26 066 */ 067 public class CSHApplication implements java.io.Serializable { 068 static private Logger log = Logger.getLogger(CSHApplication.class.getName()); 069 private Request request; 070 private Job job; 071 private String scratchDir = "Not_defined_in_config_file"; 072 private String submissionCommand; 073 private Map variables; 074 private boolean variablesSet = false; 075 076 /** 077 * Holds value of property recurseCopy. 078 */ 079 private boolean recurseCopy = false; 080 081 /** 082 * Holds the value for the property of EnvVariableLimit 083 */ 084 private int EnvVariableLimit; 085 private boolean EnvVariableLimitNotReached = true; //Set false if in "any" job the EnvVariableLimit was reached 086 087 /** 088 * Holds value of property copyScript. 089 */ 090 private String copyScript; 091 092 /** 093 * Holds value of property storageScript. 094 */ 095 private String storageScript; 096 097 /** 098 * Holds value of property registerScript. 099 */ 100 private String registerScript; 101 102 /** 103 * Creates a new instance of CSHApplication 104 */ 105 public CSHApplication() { 106 } 107 108 /** Returns the CSH application module. In the future, the application should 109 * be taken directly from the component library. This method is provided 110 * for compatibility with the old framework. 111 * @return the CSH application module 112 */ 113 // public static CSHApplication getInstance() { 114 // // System.out.println("getting copy for ComponentLibrary"); 115 // //called aboat 11 times/dispach 116 // return (CSHApplication) ComponentLibrary.getInstance().getComponent("CSHApplication"); 117 // } 118 119 120 121 /** 122 * Prepares the task, by creating the script. 123 */ 124 public void prepareJob() { 125 try { 126 127 PrintStream inputFileList = new PrintStream(new FileOutputStream(new File(getInputFileListName()))); 128 createInputFileList(inputFileList); 129 } catch (Exception e) { 130 log.fatal("Couldn't create the file list", e); 131 throw new RuntimeException("Couldn't create the file list " + getInputFileListName() + ": " + e.getMessage()); 132 } 133 134 try { 135 PrintStream cshScript = new PrintStream(new FileOutputStream(new File(getCSHScriptName()))); 136 createCSHScript(cshScript); 137 makeScriptFileExecutable(getCSHScriptName()); 138 139 cshScript.close(); //close the CSH stream, or else the job will not run 140 141 142 } catch (Exception e) { 143 log.fatal("Couldn't create the script", e); 144 e.printStackTrace(); 145 throw new RuntimeException("Couldn't create the script " + getCSHScriptName() + ": " + e.getMessage() ); 146 } 147 } 148 149 /** 150 * Returns the command line to execute the task set by <CODE>setJob</CODE>. 151 * 152 * @return Command line to execute the task 153 */ 154 public String getCommandLine() { 155 return getCSHScriptName(); 156 } 157 158 //Hold the chmod string from the config file 159 private String makeExecutableCommand; 160 161 /** 162 * Returns the chmod command from the config file 163 */ 164 public String getMakeExecutableCommand() { 165 return makeExecutableCommand; 166 } 167 168 public void setMakeExecutableCommand(String makeExecutableCommand) { 169 this.makeExecutableCommand = makeExecutableCommand; 170 } 171 172 173 /* Calls chmod to make the script executable. */ 174 private void makeScriptFileExecutable(String fileName) { 175 // Hack to make the Unit tests work on windows. 176 // chmod doesn't exist on Windows (if you don't have cygwin) 177 if (System.getProperty("os.name").startsWith("Win")) return; 178 179 try { 180 // TODO: chmod should be a variable. This routine 181 // itself should be OS-flavor independent. 182 183 String chmod = getMakeExecutableCommand() + fileName; 184 //String chmod = "/bin/chmod +x " + fileName; 185 186 log.info("Executing \"" + chmod + "\""); 187 188 java.lang.Process shell = Runtime.getRuntime().exec(chmod); 189 shell.waitFor(); 190 } catch (Exception e) { 191 log.fatal("Couldn't make the script executable", e); 192 throw new RuntimeException("Couldn't make the script " + fileName + 193 " executable", e); 194 } 195 } 196 197 /** 198 * Sets the task the application will have to prepare. 199 * 200 * @param request the request that originated the task 201 * @param job the job to be executed 202 */ 203 public void setJob(Request request, Job job) { 204 this.request = request; 205 this.job = job; 206 scratchDir = null; 207 submissionCommand = null; 208 variables = null; 209 variablesSet = false; 210 this.scratchDir = ConfigToolkit.findProgram("$SCRATCH", job); 211 212 /* 213 if (job.getAccessPoint().getSite().getProgramLocations() == null) 214 System.out.println("\nCan't find \"ProgramLocations\" block in " + job.getAccessPoint().getSite().getSiteName() + "(Site) -->" + job.getQueueObj().getBatchSystem().getName() + "(BatchSystem) block in config file $SCRATCH could not be set !. Please contact your SUMS systems administrator to modify the config file."); 215 else { 216 if (job.getAccessPoint().getSite().getProgramLocations().containsKey("$SCRATCH")) 217 this.scratchDir = (String) job.getAccessPoint().getSite().getProgramLocations().get("$SCRATCH"); 218 else 219 System.out.println("\nThe \"$SCRATCH\" value could not be found in the ProgramLocations map of site=\"" + job.getAccessPoint().getSite().getSiteName() + "\". Please contact your SUMS systems administrator to modify the config file."); 220 } 221 222 */ 223 224 225 } 226 227 /** Sets the scratch directory for the temporary output files of the job. 228 * @param scratchDir the temporary output directory 229 */ 230 /* 231 public void setScratchDir(String scratchDir) { 232 this.scratchDir = scratchDir; 233 } */ 234 235 /** 236 * Sets the submittion command used to submit the job. This will be included in the 237 * script for resubmission. 238 * 239 * @param submissionCommand the command line to submit the job to the batch system 240 */ 241 public void setSubmissionCommand(String submissionCommand) { 242 this.submissionCommand = submissionCommand; 243 } 244 245 /** 246 * Generates the csh script to be submitted through LSF. 247 * 248 * @param out the stream on which to print the script 249 */ 250 private void createCSHScript(PrintStream out) { 251 // Sets the shell that will interpret the script 252 253 ////////////test area//////////////// 254 Map config = (Map) ComponentLibrary.getInstance().getComponent("registerScript"); 255 256 out.println("#!/bin/csh"); 257 258 out.println("# -------------------------------------- "); 259 out.println("# Script generated at " + new Date() + 260 " by the STAR scheduler and submitted with"); 261 out.println("# " + submissionCommand); 262 out.println("# -------------------------------------- "); 263 264 out.println("\n" + getInitializationScript()); 265 266 Map env = getEnvironmentVariables(); 267 Iterator variables = env.keySet().iterator(); 268 Iterator FilesNameVariable = env.keySet().iterator(); 269 270 // Sets all the environment variable. 271 out.println(); 272 out.println("# Preparing environment variables"); 273 274 String varName, varValue; 275 276 277 int counter = 0; 278 String padding = ""; 279 while (variables.hasNext()) { //First pass prints all the environment variables that do not start with "INPUTFILE" 280 varName = (String) variables.next(); 281 varValue = (String) env.get(varName); 282 if ((!varName.startsWith("INPUTFILE")) || (varName.matches("INPUTFILECOUNT"))) { 283 out.println(padding + "setenv " + varName + " " + varValue); 284 } 285 } 286 287 while (FilesNameVariable.hasNext()) { //Second pass prints all the environment variables that start with "INPUTFILE" 288 varName = (String) FilesNameVariable.next(); 289 varValue = (String) env.get(varName); 290 if (varName.startsWith("INPUTFILE") && (!varName.matches("INPUTFILECOUNT"))) { 291 if (counter == EnvVariableLimit) { 292 out.println("#"); 293 out.println("# Note: The configuration requested a limitation of the"); 294 out.println("# number of INPUTFILE(n) environemnt variables at " + EnvVariableLimit); 295 out.println("# The rest of the variables will be commented."); 296 out.println("#"); 297 padding = "# "; 298 299 if(EnvVariableLimitNotReached){ //Print the warning to the user only one time then flip EnvVariableLimitNotReached to false 300 System.out.println("\n\n***************************************"); 301 System.out.println("The configuration requested a limitation of the number of INPUTFILE(n) environment variables at " + EnvVariableLimit + ", the rest will be commented out.\n"); 302 System.out.println("***************************************\n"); 303 EnvVariableLimitNotReached = false; 304 } 305 } 306 counter++; 307 out.println(padding + "setenv " + varName + " " + varValue); 308 } 309 } 310 311 312 313 314 String mkdir = ConfigToolkit.findProgram("mkdir", job); 315 // String mkdir; //test for mkdir it the configfile if the program location table if it is not there just use the defalt 316 // if(job.getAccessPoint().getSite().getProgramLocations().containsKey("mkdir")){ 317 // mkdir = (String) job.getAccessPoint().getSite().getProgramLocations().get("mkdir"); 318 // } 319 // else mkdir = "mkdir -p"; 320 321 322 323 // Creates the scratch directory 324 out.println(); 325 out.println("# Creating the scratch directory, return failure status"); 326 out.println(mkdir + " -p $SCRATCH"); 327 out.println("set STS=$status"); 328 out.println("if (! -d $SCRATCH) then"); 329 out.println(" echo \"Scheduler:: Failed to create $SCRATCH on $HOST\""); 330 out.println(" exit $STS"); 331 out.println("endif"); 332 333 334 if(request.getSandbox() != null){ 335 out.println(); 336 out.println(importPakages()); 337 } 338 339 340 out.println(); 341 out.println("#Note: The default directory in which jobs start has be fix to $SCRATCH"); 342 out.println("cd $SCRATCH"); //Note: This line sets the startup directory to $SCRATCH. It resolves the problem of different batch systems starting in different directories. 343 344 345 out.println("echo \"--------------------------------\""); 346 out.println("echo \'" + gov.bnl.star.offline.scheduler.SchedulerCommandLine.getProgramName() + " we are starting in $SCRATCH :\' `/bin/pwd`"); 347 out.println("echo \"--------------------------------\""); 348 out.println("/bin/ls -l"); 349 350 // soft linking to home dir removed for now because sandbox will replace this command 351 //out.println("ln -s " + FilesystemToolkit.getCurrentDirectory() + "/* ."); //Note: This line sets the startup directory to $SCRATCH. It resolves the problem of different batch systems starting in different directories. 352 353 354 // Adds the command to be executed 355 out.println(); 356 out.println("###################################################"); 357 //out.println("/bin/csh -s ___EOD"+job.getJobID()+"__"); 358 out.println("# User command BEGIN ----------------------------->"); 359 360 //Let the RDLApplicationResolver try to resolve the test, if it can not use the back up 361 //if(request.getCommand() != null) 362 363 364 if (job.getQueueObj() != null && request.getCommand() == null) { 365 if (job.getQueueObj().getRDLApplicationResolver() != null) 366 out.println(job.getQueueObj().getRDLApplicationResolver().resolve(request.getApplication(), request.getTask())); 367 else 368 out.println(request.getCommand()); 369 } else 370 out.println(request.getCommand()); 371 372 373 out.println("# <------------------------------User command BEGIN"); 374 //out.println("___EOD"+job.getJobID()+"__"); 375 out.println("###################################################"); 376 377 // Copy output files 378 out.println(); 379 out.println("# Copy output files (if any where specified)"); 380 out.print(generateScriptFragment(request.getOutputList())); 381 // Delete the scratch directory 382 out.println(); 383 out.println("# Delete the scratch directory"); 384 out.println("/bin/rm -fr $SCRATCH"); 385 } 386 387 String generateScriptFragment(List outputList) { 388 StringBuffer buffer = new StringBuffer(); 389 390 String scriptsString = " "; 391 for (int nOutput = 0; nOutput < outputList.size(); nOutput++) { 392 OutputFile output = (OutputFile) outputList.get(nOutput); 393 if (output.getActionList().size() != 0) { 394 for (int nOutputAction = 0; nOutputAction < output.getActionList().size(); nOutputAction++) { 395 String script = ((OutputAction) output.getActionList().get(nOutputAction)).getScriptName(); 396 if (scriptsString.indexOf(script) == -1) 397 scriptsString = scriptsString.concat(script + " "); 398 } 399 } 400 } 401 402 //TODO think about how this can be moved to the config file 403 404 //copy output actions sripts to the local site 405 /* 406 if (scriptsString.length() != 1) { 407 //This only should be used for pdsf, but its hard coded for testing now 408 buffer.append("setenv X509_CERT_DIR /auto/pkg/globus-common/certificates \n"); 409 buffer.append("module use /usr/local/pkg/HRM/pkg/modules/rh8 \n"); 410 buffer.append("module load hrm/1.2.8a \n"); 411 buffer.append("module load globus/2.4.3 orbacus \n"); 412 buffer.append("module load rrs \n"); 413 buffer.append("module load gcc/2.95.3-thread \n"); 414 buffer.append("foreach file (" + scriptsString + ")\n" + 415 " globus-url-copy -p 25 gsiftp://" + ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName() + getHelperScripts() + "$file file:$SCRATCH/$file \n" + 416 " chmod +x $file\n" + 417 "end \n"); 418 } */ 419 420 421 for (int nOutput = 0; nOutput < outputList.size(); nOutput++) { 422 423 OutputFile output = (OutputFile) outputList.get(nOutput); 424 if (output.getActionList().size() != 0) { 425 426 //buffer.append("setenv OUTPUTFILE").append(nOutput).append(" file:/$SCRATCH/").append(output.getFromScratch()).append("\n"); 427 428 429 for (int nOutputAction = 0; nOutputAction < output.getActionList().size(); nOutputAction++) { 430 431 432 // if (output.getActionList().get(nOutputAction) instanceof OutputCopyAction) { 433 // buffer.append(generateScriptFragment(nOutput, (OutputCopyAction) output.getActionList().get(nOutputAction))); 434 // } else if (output.getActionList().get(nOutputAction) instanceof OutputRegisterAction) { 435 // buffer.append(generateScriptFragment(nOutput, (OutputRegisterAction) output.getActionList().get(nOutputAction))); 436 // } else { 437 // log.severe("Unsupported OutputAction time found: " + output.getActionList().get(nOutputAction).getClass().getName()); 438 // throw new RuntimeException("Unsupported OutputAction time found"); 439 // } 440 441 OutputAction outputAction = (OutputAction) output.getActionList().get(nOutputAction); 442 buffer.append(outputAction.makeScriptString(output.getFromScratch(), ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName())); 443 444 //System.out.println(" " + outputAction.getScriptName()); 445 //outputAction.getScriptName(); 446 447 } 448 } else { 449 450 //This code copys files back from FromScratch at the uers request //the path has should be added 451 //String gridCopy = "globus-url-copy file://$SCRATCH/".concat(output.getFromScratch()).concat(" ").concat("gsiftp://").concat(getGatekeepr()).concat("/").concat(output.getToURL().getPath()).concat("\n"); 452 String gridCopy = FillCopyCommand(getGridCopyCommand(), output.getFromScratch(), output.getToURL().getPath(), true) + "\n"; 453 String cpCopy = FillCopyCommand(getCopyCommand(), output.getFromScratch(), output.getToURL().getPath(), false) + "\n"; 454 if (recurseCopy) cpCopy = FillCopyCommand(getRecursiveCopyCommand(), output.getFromScratch(), output.getToURL().getPath(), false) + "\n"; 455 456 457 458 459 /* This code was replaced with the code below. With the configuration file reshape, the type of queue (local/grid) can be determined by looking at the user�s local site and the queues local site. 460 if(output.getToURL().toString().toLowerCase().startsWith("gsiftp:")){ 461 buffer.append(gridCopy); 462 } 463 //else if the user starts his request with file: then use cp 464 else if(output.getToURL().toString().toLowerCase().startsWith("file:")){ 465 buffer.append(cpCopy); 466 } 467 // else if the jobs is sent to a queue that has Implementation = "grid" than use globus-url-copy 468 else if (((Queue) job.getQueueObj()).getImplementation().startsWith("grid")){ 469 buffer.append(gridCopy); 470 } 471 else{ //just use cp if all else fails 472 buffer.append(cpCopy); 473 } 474 */ 475 476 477 if (ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())) { 478 buffer.append(cpCopy); 479 } else { 480 buffer.append(gridCopy); 481 } 482 483 484 } 485 } 486 return buffer.toString(); 487 } 488 489 String generateScriptFragment(int nOutput, OutputCopyAction action) { 490 /* 491 StringBuffer fragment = new StringBuffer(); 492 if (action.getFileRef() != null) { 493 if (action.getStorageService() != null) { 494 fragment.append("setenv OUTPUTFILE").append(nOutput).append('_'). 495 append(action.getFileRef()).append(" `").append(getStorageScript()). 496 append(" \"").append(action.getStorageService()).append("\" `"). 497 append(action.getURI().getPath()).append("\n"); 498 499 } else { 500 fragment.append("setenv OUTPUTFILE").append(nOutput).append('_'). 501 append(action.getFileRef()).append(" "). 502 append(action.getURI()).append("\n"); 503 } 504 fragment.append(getCopyScript()).append(" $OUTPUTFILE").append(nOutput). 505 append(" $OUTPUTFILE").append(nOutput).append('_').append(action.getFileRef()). 506 append("\n"); 507 } else { 508 fragment.append(getCopyScript()).append(" $OUTPUTFILE").append(nOutput); 509 if (action.getStorageService() != null) { 510 fragment.append(" `").append(getStorageScript()).append(" \"").append(action.getStorageService()).append("\" `").append(action.getURI().getPath()).append("\n"); 511 } else { 512 fragment.append(" ").append(action.getURI()).append("\n"); 513 } 514 } 515 return fragment.toString(); 516 */ return " "; 517 } 518 519 String generateScriptFragment(int nOutput, OutputRegisterAction action) { 520 /* 521 StringBuffer fragment = new StringBuffer(); 522 fragment.append(getRegisterScript()).append(" $OUTPUTFILE").append(nOutput); 523 if (action.getFileRef() != null) { 524 fragment.append('_'). append(action.getFileRef()); 525 } 526 int queryStart = action.getURI().getSchemeSpecificPart().indexOf('?'); 527 queryStart++; 528 fragment.append(" '").append(action.getURI().getSchemeSpecificPart().substring(queryStart)).append("'\n"); 529 return fragment.toString(); 530 */ return " "; 531 } 532 533 /** 534 * Generates the input file list. It consists of full path names, one line for each 535 * one. 536 * 537 * @param out the stream on which to print the file list 538 */ 539 private void createInputFileList(PrintStream out) { 540 //printJobInputList(job.getInput()); 541 542 variables = new TreeMap(); 543 544 if ("paths".equals(request.getFileListType())) { 545 FileListToolkit.createPathsFileList(out, job.getInput(), variables); 546 } else if ("rootd".equals(request.getFileListType())) { 547 FileListToolkit.createLocalRemoteFileList(out, job.getInput(), variables, false); 548 } else if ("xrootd".equals(request.getFileListType())) { 549 FileListToolkit.createLocalRemoteFileList(out, job.getInput(), variables, true); 550 } 551 } 552 private void printJobInputList(List jobList){ 553 System.out.println("-----------------------------------------------"); 554 for (int i = 0; i < jobList.size(); i++) { 555 PhysicalFile physicalFile = (PhysicalFile) jobList.get(i); 556 System.out.println("File n."+i+ "is:"+physicalFile.getStorage()+"::"+physicalFile.getPath()+"::"+physicalFile.getFilename()); 557 System.out.println("\n"); 558 } 559 System.out.println("------------------------------------------------\n"); 560 } 561 562 /** 563 * Returns the path and name for the csh script. 564 * 565 * @return the csh script path and name 566 */ 567 public String getCSHScriptName() { 568 569 return FilesystemToolkit.getCurrentDirectory() + "/sched" + job.getJobID() + ".csh"; 570 571 } 572 573 574 /** 575 * Returns the name for the csh script (without the path). 576 * 577 * @return the csh script name (without the path) 578 */ 579 public String getCSHScriptFileName() { 580 /* 581 The path is being dropped because different drivers for mounting the PanFS path are reporting the path differently. Instated we will put �cd $path� before the command we execute. 582 Example: 583 /pdirect/star+institutions/data05/scratch/lbhajdu/ 584 /star/data05/scratch/lbhajdu/ 585 */ 586 return "sched" + job.getJobID() + ".csh"; 587 } 588 589 590 /** 591 * Returns the name for the input file list file. 592 * 593 * @return the input file list name 594 */ 595 public String getInputFileListName() { 596 return FilesystemToolkit.getCurrentDirectory() + "/sched" + job.getJobID() + ".list"; 597 } 598 599 600 private Map getEnvironmentVariables() { 601 602 if (variables == null) variables = new TreeMap(); //This can still be null if there is no filelist 603 if(variables.containsKey("JOBID")) return variables; //if added one time do not add then again 604 605 variables.put("JOBID", job.getJobID()); 606 variables.put("SUMS_name", "\"" + request.getName() + "\"" ); 607 variables.put("SUMS_USER", System.getProperty("user.name")); 608 variables.put("SUMS_AUTHENTICATED_USER", "\"" + request.getAuthenticatedUser() + "\""); 609 variables.put("REQUESTID", job.getRequestID()); 610 //variables.put("PROCESSID", job.getProcessID()); //has been renamed to $JOBINDEX 611 variables.put("JOBINDEX", job.getProcessID()); 612 613 614 try { //add the dir of the location the user is submimtting from 615 java.net.InetAddress localMachine = java.net.InetAddress.getLocalHost(); 616 variables.put("SUBMITTINGDIRECTORY", FilesystemToolkit.getCurrentDirectory()); 617 variables.put("SUBMITTINGNODE", localMachine.getHostName() ); 618 }catch (java.net.UnknownHostException uhe) { 619 variables.put("SUBMITTINGDIRECTORY", FilesystemToolkit.getCurrentDirectory()); 620 variables.put("SUBMITTINGNODE", "UnknownHostException" ); 621 } 622 623 624 SimpleDateFormat sdf = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss" ); 625 sdf.setTimeZone(java.util.TimeZone.getTimeZone("GMT")); 626 variables.put("SUBMIT_TIME", "\"" + sdf.format( new Date()) + "\"" ); 627 // enviroment variable used for ROOT to load particular labreries for xrootd client 628 if ("xrootd".equals(request.getFileListType())){ 629 variables.put("USEXROOTD","1"); 630 } 631 632 //variables.put("SUBMIT_TIME", "\"" + (new Date()).toString() + "\"" ); 633 //SimpleDateFormat dateFormat = new SimpleDateFormat( "yyyy-MM-dd" ); 634 //dateFormat.format( new Date() ); 635 636 637 if(request.getNProcesses()!= -1){ 638 variables.put("SUMS_nProcesses", String.valueOf(request.getNProcesses()) ); 639 } 640 else{ 641 variables.put("SUMS_nProcesses", String.valueOf(request.getJobs().size()) ); 642 } 643 644 645 646 647 648 // FIXME: scratch dir should be passed in a nicer way 649 variables.put("SCRATCH", scratchDir); 650 651 variables.put("FILELIST", getInputFileListName()); 652 653 List inputList = job.getInput(); 654 variables.put("INPUTFILECOUNT", Integer.toString(inputList.size())); 655 656 //This is now done in FileListToolkit 657 // for (int nFile = 0; nFile < inputList.size(); nFile++) { 658 // PhysicalFile file = (PhysicalFile) inputList.get(nFile); 659 // variables.put("INPUTFILE" + nFile, file.getPath() + "/" + file.getFilename()); 660 // } 661 662 663 //If there is only one input file per csh make an Environment Variable called FILEBASENAME 664 if (inputList.size() == 1) { 665 PhysicalFile file = (PhysicalFile) inputList.get(0); 666 if (!(file.getFilename().indexOf(".") == -1)) 667 variables.put("FILEBASENAME", file.getFilename().substring(0, file.getFilename().indexOf("."))); 668 else 669 variables.put("FILEBASENAME", file.getFilename()); 670 } else { //All other times FILEBASENAME will be set to this value 671 //variables.put("FILEBASENAME", "FILEBASENAME.NOT.INITIALIZED." + job.getJobID()); 672 variables.put("FILEBASENAME", "\"\""); //Blank is better because people may use the variable FILEBASENAME. (JL, LH) 673 } 674 675 676 // String APP = ComponentLibrary.getInstance().getProgramLocation(job.getAccessPoint().getSite(), "$APP"); 677 // if(APP != null){ 678 // APP = APP.trim(); 679 // if(APP.trim() != "") variables.put("APP", APP); 680 // } 681 682 683 684 /////////////////////////////////////////////used for setting $APP//////////////////////////////////////////////// 685 String APP = null; //Try to get $data from the IS. 686 if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){ //should only be done on the grid 687 InformationService informationService = job.getAccessPoint().getSite().getInformationService(); 688 if(informationService != null){ 689 log.debug("Trying to get $APPA from IS (information service)."); 690 APP = informationService.getAppDir(); 691 log.debug("The value is: " + ((APP == null) ? "null" : APP)); 692 } 693 } 694 695 String APPFromComponentLibrary = ComponentLibrary.getInstance().getProgramLocation(job.getAccessPoint().getSite(), "$APP"); 696 log.debug("Trying to get $APP from ProgramLocations table. The value is: " + ((APPFromComponentLibrary == null) ? "null" : APPFromComponentLibrary) + "this value is only used by OSG (Open Science Grid) Grid jobs"); 697 if(APPFromComponentLibrary != null){ 698 APP = APPFromComponentLibrary.trim(); 699 log.debug("This final $data value will be \"" + APP + "\".\nComponentLibrary value overwrites all others."); 700 } 701 702 if(APP != null){ 703 APP = APP.trim(); 704 if(APP.trim().length() > 0) variables.put("APP", APP); 705 } 706 else log.debug("There was no value for $APP. It will not be set."); 707 708 709 /////////////////////////////////////////////used for setting $DATA//////////////////////////////////////////////// 710 String DATA = null; //Try to get $data from the IS. 711 if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){ //should only be done on the grid 712 InformationService informationService = job.getAccessPoint().getSite().getInformationService(); 713 if(informationService != null){ 714 DATA = informationService.getDataDir(); 715 log.debug("Trying to get $DATA from IS (information service). The value is: " + ((DATA == null) ? "null" : DATA) ); 716 } 717 } 718 719 String DATAFromComponentLibrary = ComponentLibrary.getInstance().getProgramLocation(job.getAccessPoint().getSite(), "$DATA"); 720 log.debug("Trying to get $DATA from ProgramLocations table. The value is: " + ((DATA == null) ? "null" : DATA.trim()) + "this value is only used by OSG (Open Science Grid) Grid jobs"); 721 if(DATAFromComponentLibrary != null){ 722 DATA = DATAFromComponentLibrary; 723 log.debug("This final $data value will be \"" + DATA + "\".\nComponentLibrary value overwrites all others."); 724 } 725 726 if(DATA != null){ 727 DATA = DATA.trim(); 728 if(DATA.trim().length() > 0) variables.put("DATA", DATA); 729 } 730 else log.debug("There was no value for $DATA. It will not be set."); 731 /////////////////////////////////////////////////Set items only used when the job is a grid job////////////////////////////////////////////////////// 732 733 if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){ //should only be done on the grid 734 InformationService informationService = job.getAccessPoint().getSite().getInformationService(); 735 if(informationService != null){ 736 737 String gridDir = informationService.getGKGlobusDir(); 738 if(gridDir != null){ 739 variables.put("GKGlobusDir", "\"" + gridDir + "\""); 740 } 741 else log.error("\"GKGlobusDir\" will not be displayed because returnned value was null"); 742 String WNClient = informationService.getWNClient(); 743 if(WNClient != null){ 744 variables.put("WNClient", "\"" + WNClient + "\""); 745 } 746 else log.error("\"WNClient\" will not be displayed because returnned value was null"); 747 748 } 749 else log.error("\"GKGlobusDir\" will not be displayed because no iinformationService object for the site could be recovered"); 750 } 751 else log.debug("\"GKGlobusDir\" no set becuase this is not a grid job"); 752 753 ////////////////////////////////////////////// 754 755 756 return variables; 757 } 758 759 /** 760 * Returns the file for standard input redirection. 761 * 762 * @return stdin file 763 */ 764 public String getStdin() { 765 if (job.getStdin() != null) { 766 return VariablesToolkit.substituteVariables(job.getStdin().getPath(), this.getEnvironmentVariables()); 767 //CSHApplication.getInstance().getEnvironmentVariables()); 768 } 769 770 return null; 771 } 772 773 /** 774 * Returns the file for standard output redirection. 775 * 776 * @return stdout file 777 */ 778 public String getStdout() { 779 if (job.getStdout() != null) { 780 if (job.getStdout() == Request.discard) { 781 return "/dev/null"; 782 } 783 784 return VariablesToolkit.substituteVariables(job.getStdout().getPath(), this.getEnvironmentVariables()); 785 // CSHApplication.getInstance().getEnvironmentVariables()); 786 } 787 788 return null; 789 } 790 791 /** 792 * Returns the file for standard error redirection. 793 * 794 * @return stderr file 795 */ 796 public String getStderr() { 797 if (job.getStderr() != null) { 798 if (job.getStderr() == Request.discard) { 799 return "/dev/null"; 800 } 801 802 return VariablesToolkit.substituteVariables(job.getStderr().getPath(), this.getEnvironmentVariables()); 803 //CSHApplication.getInstance().getEnvironmentVariables()); 804 } 805 806 return null; 807 } 808 809 /** 810 * Returns the job name to be assigned to the batch system. 811 * 812 * @return the job name to be used by the batch system 813 */ 814 public String getJobName() { 815 return request.getName(); 816 } 817 818 /** 819 * Getter for property recurseCopy. 820 * 821 * @return Value of property recurseCopy. 822 */ 823 public boolean isRecurseCopy() { 824 return this.recurseCopy; 825 } 826 827 /** 828 * Setter for property recurseCopy. 829 * 830 * @param recurseCopy New value of property recurseCopy. 831 */ 832 public void setRecurseCopy(boolean recurseCopy) { 833 this.recurseCopy = recurseCopy; 834 } 835 836 837 /** 838 * Getter for property EnvVariableLimit. 839 * 840 * @return Value of property EnvVariableLimit. 841 */ 842 public int getEnvVariableLimit() { 843 return this.EnvVariableLimit; 844 } 845 846 /** 847 * Setter for property EnvVariableLimit. 848 * 849 * @param EnvVariableLimit New value of property EnvVariableLimit. 850 */ 851 public void setEnvVariableLimit(int EnvVariableLimit) { 852 this.EnvVariableLimit = EnvVariableLimit; 853 } 854 855 856 /** 857 * Getter for property copyScript. 858 * 859 * @return Value of property copyScript. 860 */ 861 public String getCopyScript() { 862 return this.copyScript; 863 } 864 865 /** 866 * Setter for property copyScript. 867 * 868 * @param copyScript New value of property copyScript. 869 */ 870 public void setCopyScript(String copyScript) { 871 this.copyScript = copyScript; 872 } 873 874 /** 875 * Getter for property storageScript. 876 * 877 * @return Value of property storageScript. 878 */ 879 public String getStorageScript() { 880 return this.storageScript; 881 } 882 883 /** 884 * Setter for property storageScript. 885 * 886 * @param storageScript New value of property storageScript. 887 */ 888 public void setStorageScript(String storageScript) { 889 this.storageScript = storageScript; 890 } 891 892 /** 893 * Getter for property registerScript. 894 * 895 * @return Value of property registerScript. 896 */ 897 public String getRegisterScript() { 898 return this.registerScript; 899 } 900 901 /** 902 * Setter for property registerScript. 903 * 904 * @param registerScript New value of property registerScript. 905 */ 906 public void setRegisterScript(String registerScript) { 907 this.registerScript = registerScript; 908 } 909 910 911 private String gridCopyCommand = "(gridCopyCommand not set in config file file!)"; 912 913 /** 914 * sets the command for copying files back, when the user asks for <output fromScratch="/path/x" toURL="gsiftp:/path/x"/> 915 * use (fromScratch) for source and (toURL) for destination. 916 * Example: globus-url-copy file://$SCRATCH/(fromScratch) gsiftp://stragrid04.star.bnl.gov/(toURL) 917 */ 918 public void setGridCopyCommand(String gridCopyCommand) { 919 this.gridCopyCommand = gridCopyCommand; 920 } 921 922 public String getGridCopyCommand() { 923 return gridCopyCommand; 924 } 925 926 private String CopyCommand = "(CopyCommand not set in config file file!)"; 927 928 /** 929 * sets the command for copying files back, when the user asks for <output fromScratch=x toURL="file:/path/"/> 930 * use (fromScratch) for source and (toURL) for destination. 931 * Example: /bin/cp $SCRATCH/(fromScratch) (toURL) 932 */ 933 public void setCopyCommand(String CopyCommand) { 934 this.CopyCommand = CopyCommand; 935 } 936 937 public String getCopyCommand() { 938 return CopyCommand; 939 } 940 941 private String recursiveCopyCommand = "(recursiveCopyCommand not set in config file file!)"; 942 943 /** 944 * sets the command for copying files back, when the user asks for <output fromScratch=x toURL="file:/path/"/> 945 * use (fromScratch) for source and (toURL) for destination. 946 * Example: /bin/cp -r $SCRATCH/(fromScratch) (toURL) 947 */ 948 public void setRecursiveCopyCommand(String recursiveCopyCommand) { 949 this.recursiveCopyCommand = recursiveCopyCommand; 950 } 951 952 public String getRecursiveCopyCommand() { 953 return recursiveCopyCommand; 954 } 955 956 /** 957 * replaces the (fromScratch) and the (toURL) in the copy command with the true path 958 */ 959 public String FillCopyCommand(String command, String fromScratch, String toURL, boolean isGrid) { 960 961 if (!isGrid) { 962 if (fromScratch.matches("^\\Q./\\E.*")) 963 fromScratch = fromScratch.replaceAll("(^\\Q.\\E)(.*)", "`pwd`$2"); 964 else 965 fromScratch = "$SCRATCH/".concat(fromScratch); 966 } 967 968 StringBuffer buffer = new StringBuffer(); 969 buffer.append(command); 970 while (buffer.indexOf("(fromScratch)") != -1) { 971 buffer.replace(buffer.indexOf("(fromScratch)"), (buffer.indexOf("(fromScratch)") + "(fromScratch)".length()), fromScratch); 972 } 973 974 while (buffer.indexOf("(toURL)") != -1) { 975 buffer.replace(buffer.indexOf("(toURL)"), (buffer.indexOf("(toURL)") + "(toURL)".length()), toURL); 976 } 977 978 979 while (buffer.indexOf("(copyBackGateKeeper)") != -1) { 980 //ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName(); 981 buffer.replace(buffer.indexOf("(copyBackGateKeeper)"), (buffer.indexOf("(copyBackGateKeeper)") + "(copyBackGateKeeper)".length()), ConfigToolkit.getToolkit().myLocalFileGateKeeper().getName()); 982 } 983 984 //This did not working, it would fail with "Illegal group reference" when fromScratch or toURL had a '$' in it. 985 //return command.replaceAll("\\(fromScratch\\)", fromScratch).replaceAll("\\(toURL\\)", toURL); 986 987 return buffer.toString(); 988 989 } 990 991 992 private String helperScripts = "[helper_script_path_not_set]"; 993 994 public String getHelperScripts() { 995 return helperScripts; 996 } 997 998 public void setHelperScripts(String helperScripts) { 999 this.helperScripts = helperScripts; 1000 } 1001 1002 private String initializationScript = ""; 1003 public String getInitializationScript(){return this.initializationScript; } 1004 public void setInitializationScript(String initializationScript){this.initializationScript = initializationScript; } 1005 1006 1007 public String importPakages(){ 1008 String importPakagesText = "\n#import packages \n"; 1009 1010 Sandbox sandbox = request.getSandbox(); 1011 1012 //Add warnings about special sandbox grid requirements here: 1013 if(! ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj())){ 1014 1015 if(sandbox.getClass().getName().matches(".*ZipSandbox$")){ 1016 if (! getEnvironmentVariables().containsKey("DATA")){ 1017 String Warning = "Warning: Scheduler has no configuration information for $DATA at site " + job.getAccessPoint().getSite().getSiteName() + ". If $DATA is not set in the environment of the worker node the sandbox will fail. For more information please contact your scheduler administrator."; 1018 System.out.println("\n\n" + Warning + "\n"); 1019 log.error(Warning); 1020 } 1021 } 1022 } 1023 1024 1025 1026 importPakagesText = importPakagesText.concat(sandbox.InitializationCommands()); 1027 1028 List packages = sandbox.getPackages(); 1029 1030 for(int i = 0; i != packages.size(); i++){ 1031 SandboxPackage sandboxPackage = (SandboxPackage) packages.get(i); 1032 1033 if(! sandbox.PackageExists(sandboxPackage)){ //This to see if the package dose not exist 1034 //make the package 1035 sandbox.MakePackage(sandboxPackage, request); 1036 } 1037 1038 importPakagesText = importPakagesText.concat(sandbox.installCommand(sandboxPackage, job)); 1039 } 1040 1041 return importPakagesText; 1042 } 1043 1044 1045 }