001 /*
002 * ZipSandbox.java
003 *
004 * Created on October 26, 2005, 12:54 PM
005 *
006 * This file is part of the STAR Scheduler.
007 * Copyright (c) 2003-2006 STAR Collaboration - Brookhaven National Laboratory
008 *
009 * STAR Scheduler is free software; you can redistribute it and/or modify
010 * it under the terms of the GNU General Public License as published by
011 * the Free Software Foundation; either version 2 of the License, or
012 * (at your option) any later version.
013 *
014 * STAR Scheduler is distributed in the hope that it will be useful,
015 * but WITHOUT ANY WARRANTY; without even the implied warranty of
016 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
017 * GNU General Public License for more details.
018 *
019 * You should have received a copy of the GNU General Public License
020 * along with STAR Scheduler; if not, write to the Free Software
021 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
022 */
023
024 package gov.bnl.star.offline.scheduler.util.sandbox;
025 import gov.bnl.star.offline.scheduler.util.sandbox.Sandbox;
026 import gov.bnl.star.offline.scheduler.util.CSHCommandLineTask;
027 import gov.bnl.star.offline.scheduler.util.FilesystemToolkit;
028 import gov.bnl.star.offline.scheduler.util.ConfigToolkit;
029
030 import gov.bnl.star.offline.scheduler.request.Request;
031
032 import gov.bnl.star.offline.scheduler.Job;
033 import java.net.MalformedURLException;
034
035
036 import java.net.URL;
037 import java.util.List;
038 import java.util.ArrayList;
039
040 import java.util.logging.Level;
041 import org.apache.log4j.Logger;
042
043
044 /**
045 * This sandbox variant will zip all files packaged by the user in the user's home directory. Then it will copy the zip to the users working directory and expand it.
046 * @author Levente Hajdu
047 */
048 public class ZipSandbox implements Sandbox{
049 static private Logger log = Logger.getLogger(ZipSandbox.class.getName());
050
051
052
053 /** Creates a new instance of ZipSandbox */
054 public ZipSandbox() {}
055
056 public String CopyCommand(SandboxPackage packageObj) { return ""; }
057
058 /**Zip the users requested files
059 * @param packageObj The package object holding inforamtion about the files to be packaged.
060 * @param request The request object of the job.
061 * @return True if the package was biult, else return false.
062 **/
063 public boolean MakePackage(SandboxPackage packageObj, Request request) {
064
065 int packageNumber = -1;
066 String dirName;
067 String buffer ="";
068
069 //Todo: if the package is not local the rep-path also has to be check for the package, think of addeding it to the file name
070 try{
071
072
073
074 packageObj.setPackageName(packageObj.getPackageName().trim()); //trim the package name
075
076 if(packageObj.getPackageName() != null){
077 if(FilesystemToolkit.checkIfFileExists( new URL("file:" + packageObj.getPackageName()) ,true)){
078 this.addSandboxedFiles((new URL(packageObj.getPackageName()).getFile()));
079 packageAlreadyExistWarning(packageObj.getPackageName());
080 return true;
081 }
082 else if(FilesystemToolkit.checkIfFileExists( new URL("file:" + packageObj.getPackageName() + ".zip" ) ,true)){
083 packageObj.setPackageName(packageObj.getPackageName() + ".zip");
084 this.addSandboxedFiles(packageObj.getPackageName());
085 packageAlreadyExistWarning(packageObj.getPackageName());
086 return true;
087 }
088 //The same as above but with uppercase ".ZIP"
089 else if(FilesystemToolkit.checkIfFileExists( new URL("file:" + packageObj.getPackageName() + ".ZIP" ) ,true)){
090 packageObj.setPackageName(packageObj.getPackageName() + ".ZIP");
091 this.addSandboxedFiles(packageObj.getPackageName());
092 packageAlreadyExistWarning(packageObj.getPackageName());
093 return true;
094 }
095 }
096
097 } catch (Exception e) {
098
099
100 buffer = "Could not find per-packed sandbox. This is normal if this is the first job being dispatched and you are having SUMS crate the pack the package for you.";
101 System.out.println(buffer);
102 log.error(buffer, e);
103 }
104
105 System.out.println("\nPacking Sandbox");
106
107
108 //If the package is not there and there is no files to biuld the package from, just give an error and exit
109 boolean packageNotValid = false;
110 if(packageObj.getFiles() == null) packageNotValid = true;
111 else if(packageObj.getFiles().size() == 0) packageNotValid = true;
112 if(packageNotValid){
113 System.out.println("\nZipsandbox could not find or make one of your packages because it contains zero files and does not already exist. \n\n");
114 System.exit(1);
115 }
116
117 //give the package dir a name if it dose not have not
118 if(packageObj.getPackageName() == null){
119 do{
120 packageNumber ++;
121 dirName = "sched" + request.getID() + "_" + packageNumber + ".package";
122 packageObj.setPackageName("sched" + request.getID() + "_" + packageNumber + ".package");
123 }while(FilesystemToolkit.checkIfDirExists( "./" + dirName , null, true));
124 }
125
126
127 //chech that the name is not already being used
128 else{
129 dirName = packageObj.getPackageName() + ".package";
130 if(FilesystemToolkit.checkIfDirExists( "./" + dirName , null, true)){
131 System.out.println("Folder " + "./" + dirName + " already exists.");
132 do{ //same loop as before
133 packageNumber ++;
134 dirName = "sched" + request.getID() + "_" + packageNumber + ".package";
135 }while(FilesystemToolkit.checkIfDirExists( "./" + dirName , null, true));
136 System.out.println("Renaming to " + dirName);
137 }
138 }
139
140 //Make the dir for the package
141 CSHCommandLineTask cwd = new CSHCommandLineTask("mkdir " + dirName, true);
142 System.out.print("makeing dir " + dirName);
143 cwd.run();
144 System.out.print("...done");
145 if(cwd.getExitStatus() != 0){
146 throw new RuntimeException("Error in creating directory \n\n" +
147 "mkdir " + dirName + "\n" +
148 cwd.getOutput() + "\n\n" +
149 "gov.bnl.star.offline.scheduler.util.sandbox.ZipSandbox.MakePackage(SandboxPackage packageObj, Request request)");
150 }
151
152
153 //copy all the file to the packing dir.
154 List files = new ArrayList();
155 for(int i = 0; i != packageObj.getFiles().size(); i++ ) files.add( packageObj.getFiles().get(i)); //copy all items from the packages file list the files array
156 for(int i = 0; i != packageObj.getDirectorys().size(); i++ ) files.add( packageObj.getDirectorys().get(i)); //copy all items from the packages directory list the files array
157
158 for(int i = 0; i != files.size(); i ++){
159 URL file = (URL) files.get(i);
160
161 if(! FilesystemToolkit.checkIfFileExists(file ,true) ){
162 System.out.println("\n\nWarning: The existence of the file or directory \" " + file.getFile() + "\" could not be validated. Please check that that the element exists. ");
163 }
164
165 // if( file.getFile().matches("^\\./.*/.*/.*$")){
166 if( file.getFile().matches("^\\./.*/.*$")){ //if the path relative
167
168
169 String relativePath = file.getFile().replaceAll("(^\\.)(/)(.*)(/.*$)", "$2$3");
170 //inInstallCommand = inInstallCommand + "/bin/mkdir -p " + installDir + relativePath + "\n";
171
172 buffer = "/bin/mkdir -p " + dirName + relativePath;
173 log.info("Executing ---> " + buffer);
174 cwd = new CSHCommandLineTask(buffer, true);
175 System.out.print("Executing-->" + buffer);
176 cwd.run();
177 System.out.println("...done");
178 if(cwd.getExitStatus() != 0) log.info("Command had a non-zero exit status \n" + buffer + "\n Command output \n" + cwd.getOutput());
179
180 String source = file.getFile();
181 source = source.endsWith("/") ? source + "*" : source; //remove the last slash if it has one
182 buffer = "/bin/cp -r -p " + source +" "+ dirName + relativePath;
183
184 //buffer = "/bin/cp -r -p " + file.getFile() +" "+ dirName + relativePath;
185 cwd = new CSHCommandLineTask(buffer, true);
186 log.info("Executing ---> " + buffer);
187 System.out.print("Executing ---> " + buffer);
188 cwd.run();
189 System.out.println("...done");
190 if(cwd.getExitStatus() != 0)
191 log.info("Command had a non-zero exit status \n" + buffer + "\n Command output \n" + cwd.getOutput());
192 //inInstallCommand = inInstallCommand + lncommand + " -s " + file.getFile() + " " + installDir + relativePath + "/\n";
193 }else{
194
195 //inInstallCommand = inInstallCommand + lncommand + " -s " + file.getFile() + " " + installDir + "\n";
196
197 buffer = "/bin/cp -r -p " + file.getFile() +" "+ dirName + "/";
198 cwd = new CSHCommandLineTask(buffer , true);
199 log.info("Executing ---> " + buffer);
200 System.out.print("Executing ---> " + buffer);
201 cwd.run();
202 System.out.println("...done");
203 if(cwd.getExitStatus() != 0)
204 log.info("Command had a non-zero exit status \n" + buffer + "\n Command output \n" + cwd.getOutput());
205 }
206 }
207
208 if(packageObj.getPackageName().matches("(?i).*\\.zip$")){ //don't add .zip on to the string if it is already there or else the name would be bla.zip.zip
209 //ZipUtility.Zipdir(dirName, packageObj.getPackageName());
210
211 //cwd = new CSHCommandLineTask("/usr/bin/zip ../" + packageObj.getPackageName() + " " + dirName + "/*", true);
212
213
214 //buffer = "cd " + dirName + "; find | /usr/bin/zip ../" + packageObj.getPackageName() + ".zip -@";
215 buffer = "cd " + dirName + "; find | /usr/bin/zip -y -v ../" + packageObj.getPackageName() + ".zip -@"; //cause a freez on some links
216
217 log.info("Executing ---> " + buffer);
218 cwd = new CSHCommandLineTask(buffer, true);
219 System.out.print("Executing ---> " + buffer);
220 cwd.run();
221 System.out.println("...done");
222 if(cwd.getExitStatus() != 0)
223 log.info("Command had a non-zero exit status \n" + buffer + "\n Command output \n" + cwd.getOutput());
224 this.addSandboxedFiles(packageObj.getPackageName()); //add this as a file that should be moved with the job
225 }
226 else{
227 //ZipUtility.Zipdir(dirName, packageObj.getPackageName() + ".zip" );
228 //cwd = new CSHCommandLineTask("/usr/bin/zip ../" + packageObj.getPackageName() + ".zip " + dirName + "/*", true);
229
230
231 //buffer = "cd " + dirName + "; find | /usr/bin/zip ../" + packageObj.getPackageName() + ".zip -@"; //cause a freez on some links
232 buffer = "cd " + dirName + "; find | /usr/bin/zip -y -v ../" + packageObj.getPackageName() + ".zip -@";
233
234 log.info("Executing ---> " + buffer);
235 //System.out.println("cd " + dirName + "; find | /usr/bin/zip ../" + packageObj.getPackageName() + ".zip -@");
236 cwd = new CSHCommandLineTask(buffer , true);
237 System.out.print("Executing ---> " + buffer);
238 cwd.run();
239 System.out.println("...done");
240 if(cwd.getExitStatus() != 0)
241 log.info("Command had a non-zero exit status \n" + buffer + "\n Command output \n" + cwd.getOutput());
242 packageObj.setPackageName(packageObj.getPackageName() + ".zip");
243 this.addSandboxedFiles(packageObj.getPackageName()); //add this as a file that should be moved with the job
244 }
245
246 System.out.println("\nZip Sandbox package "+ packageObj.getPackageName() +" created");
247
248 return true;
249 }
250
251 public void packageAlreadyExistWarning(String fileName){
252 System.out.println("\n\nWarning : The zip Sandbox package " + fileName + " already exist in this directory, it will not be recreated. If you have made changes to the files you are sandboxing please delete " + fileName + " and resubmit the request else the existing package will be used.\n");
253 return;
254 }
255
256 public boolean PackageExists(SandboxPackage packageObj) {
257
258
259 String packageURL = "file:" + packageObj.getRepository() + packageObj.getPackageName();
260
261 try{
262 if (FilesystemToolkit.checkIfFileExists(new URL(packageURL), true )){ return true; }
263 } catch (Exception e) {
264 System.out.println("Could not check if \""+ packageURL +"\" exists as there was a malformed URL exception.");
265 return false;
266 }
267
268 return false;
269 }
270
271 public void Sandbox() {
272 }
273
274
275
276 String findInstallDir(SandboxPackage packageObj){
277 String installDir;
278 if(packageObj.getInstalldir() == null){ //if nothing is stated use $SCRATCH
279 installDir = "$SCRATCH";
280 }
281 else if(packageObj.getInstalldir().compareTo("\\$.*") == 0){ // if the path is an inv. ver. just leave it
282 installDir = packageObj.getInstalldir();
283 }
284 else if(packageObj.getInstalldir().startsWith(System.getProperty("file.separator"))){ //if the path is absolute, just leave it
285 installDir = packageObj.getInstalldir();
286 }
287 else{
288 installDir = "$SCRATCH" + System.getProperty("file.separator") + packageObj.getInstalldir(); //if the path is relative, then it is relative to $SCRATCH
289 }
290 return installDir;
291 }
292
293
294 /**Chech program locations for unzip */
295 // public String findUnzipcommand(Job job){
296 // String unzipcommand = "/usr/bin/unzip";
297 // boolean foundProgram = false;
298 // if (job.getQueueObj().getBatchSystem().getSite().getProgramLocations() != null){
299 //
300 // if (job.getQueueObj().getBatchSystem().getSite().getProgramLocations().containsKey("unzip")){
301 // unzipcommand = (String) job.getQueueObj().getBatchSystem().getSite().getProgramLocations().get("unzip");
302 // foundProgram = true;
303 // }
304 // }
305 // if(! foundProgram){
306 // System.out.println("\n\nWarning the program locations table could not find a record for program \"unzip\" at site " + job.getQueueObj().getBatchSystem().getSite().getSiteName());
307 // }
308 // return unzipcommand;
309 // }
310
311
312 public String installCommand(SandboxPackage packageObj, Job job) {
313
314 if(ConfigToolkit.getToolkit().isLocalQueue(job.getQueueObj()))
315 return localInstallCommand(packageObj, job);
316 else
317 return gridInstallCommand(packageObj, job);
318
319 }
320
321
322 public String localInstallCommand(SandboxPackage packageObj, Job job) {
323
324 String installDir = findInstallDir(packageObj);
325 String unzipcommand = ConfigToolkit.findProgram("unzip", job);
326
327 //If the path to the unzip command is not found inside the sites program location table, we just have to hope the command is in the jobs path
328 return unzipcommand + " " + packageObj.getPackageName() + " -d " + installDir + "\n" ;
329 }
330
331
332 public String gridInstallCommand(SandboxPackage packageObj, Job job) {
333
334 // return "\nif (! -d $DATA) then" +"\n"+
335 // " echo 'Scheduler could not find $DATA\\n$DATA = '$DATA" +"\n"+
336 // " exit $STS" +"\n"+
337 // "endif" +
338 // "\n\n"+
339 // "setenv PACKAGE " + packageObj.getPackageName() +"\n"+
340 // "if (! -e $DATA/$REQUESTID/$PACKAGE) then" +"\n"+
341 // " /bin/mkdir $DATA/$REQUESTID" +"\n"+
342 // " /bin/cp $PACKAGE $DATA/$REQUESTID/$PACKAGE.$JOBID" +"\n"+
343 // " /bin/mv $DATA/$REQUESTID/$PACKAGE.$JOBID $DATA/$REQUESTID/$PACKAGE" +"\n"+
344 // "endif" +"\n"+
345 // "/bin/rm $PACKAGE" +"\n"+
346 // findUnzipcommand(job) + " -n $DATA/$REQUESTID/$PACKAGE -d $DATA/$REQUESTID/" +"\n"+
347 // "/bin/ln -s $DATA/$REQUESTID/* "+ findInstallDir(packageObj) + "/" +"\n";
348
349 //check that $APP is setup
350
351
352
353
354 return "\n"+
355 "if(! -d $DATA) then" +"\n"+
356 " echo 'Sandbox fault: $DATA ('$DATA') does not exist or does not allow user write access. Possible causes may include SUMS miss-configuration or remote site file system fault.'" +"\n"+
357 " exit" +"\n"+
358 "endif" +"\n"+
359 "setenv PACKAGE " + packageObj.getPackageName() +"\n"+
360 "if (! -e $DATA/STAR/$REQUESTID/$PACKAGE) then" +"\n"+
361 " if(! -d $DATA/STAR) then" +"\n"+
362 " /bin/mkdir -p $DATA/STAR" +"\n"+
363 " chmod 777 $DATA/STAR" + "\n"+
364 " endif" +"\n"+
365 " /bin/mkdir -p $DATA/STAR/$REQUESTID" +"\n"+
366 " /bin/cp $PACKAGE $DATA/STAR/$REQUESTID/$PACKAGE.$JOBID" +"\n"+
367 " /bin/mv $DATA/STAR/$REQUESTID/$PACKAGE.$JOBID $DATA/STAR/$REQUESTID/$PACKAGE" +"\n"+
368 "endif" +"\n"+
369 "/bin/rm $PACKAGE" +"\n"+
370 ConfigToolkit.findProgram("unzip", job) + " -n $DATA/STAR/$REQUESTID/$PACKAGE -d $DATA/STAR/$REQUESTID/" +"\n"+
371 "/bin/ln -s $DATA/STAR/$REQUESTID/* "+ findInstallDir(packageObj) + "/" +"\n";
372
373 }
374
375
376 List packages = new ArrayList();
377
378 public void setPackages(java.util.List packages) {
379 this.packages = packages;
380 }
381
382 /** Returns a List of sandboxPackage object that are part of the sandbox */
383 public List getPackages(){
384 return packages;
385 }
386
387 /** Used to add a package to the sand box */
388 public void addPackage(SandboxPackage sandboxPackage){
389 packages.add(sandboxPackage);
390 }
391
392
393
394 List SandboxedFiles = new ArrayList();
395 /* Gets the files that the sand box has packed. */
396 public List getSandboxedFiles(){ return SandboxedFiles; }
397 /* Sets the files that the sand box has packed. */
398 public void setSandboxedFiles(List SandboxedFiles){ this.SandboxedFiles = SandboxedFiles; }
399 /* Adds a file to the list of sandboxed files. */
400 public void addSandboxedFiles(String file){ SandboxedFiles.add(file); }
401
402 public String InitializationCommands() { return ""; }
403
404 }