001 /* 002 * BOSSResourceStrategy.java 003 * 004 * Created on April 1, 2003, 3:30 PM 005 * 006 * This file is part of the STAR Scheduler. 007 * Copyright (c) 2002-2003 STAR Collaboration - Brookhaven National Laboratory 008 * 009 * STAR Scheduler is free software; you can redistribute it and/or modify 010 * it under the terms of the GNU General Public License as published by 011 * the Free Software Foundation; either version 2 of the License, or 012 * (at your option) any later version. 013 * 014 * STAR Scheduler is distributed in the hope that it will be useful, 015 * but WITHOUT ANY WARRANTY; without even the implied warranty of 016 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 017 * GNU General Public License for more details. 018 * 019 * You should have received a copy of the GNU General Public License 020 * along with STAR Scheduler; if not, write to the Free Software 021 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 022 */ 023 package gov.bnl.star.offline.scheduler.Dispatchers.boss; 024 025 import gov.bnl.star.offline.scheduler.*; 026 import gov.bnl.star.offline.scheduler.catalog.PhysicalFile; 027 028 import java.util.Hashtable; 029 import java.util.Iterator; 030 import java.util.Set; 031 import java.util.logging.Logger; 032 import java.util.regex.Matcher; 033 import java.util.regex.Pattern; 034 035 036 /** Encapsulate the resource usage parameter (-R) for an BOSS farm. Essentially, 037 * its a placeholder for the prepareResourceUsageSwitch() which will calculate the 038 * appriate string. The administrator should be able to create different 039 * subclasses for each farm as needed. 040 * <p> 041 * This class provides a scheleton for a typical resource usage for files residing 042 * on different NFS servers. It was designed to fit STAR needs at two sites. In case 043 * it doesn't fit the needs, the class can be subclassed to define new behaviour. 044 * <p> 045 * The resource switch is prepared in this way: 046 * Takes the list of input files and iterates over them.<br> 047 * Checks if each file needs a resource (<CODE>resourceRequired()</CODE>).<br> 048 * To determine that, checks if the file is on NFS, and looks for the typical data 049 * vault prefix (<CODE>getDataVaultPrefix</CODE>). For example, "/star/dataxx/..."; 050 * it extract the vaultNumber and it substitute it in the resourceSyntax <br> 051 * Increases the count of the resource (<CODE>increaseResourceUsage</CODE>)<br> 052 * Prepares the switch (<CODE>prepareSwitch</CODE>) 053 * <p> 054 * The resource value for each resource will the be given by the following formula: <br> 055 * <code>min (base+nCount*fileFactor, max)</code> 056 * <p> 057 * The subclass need to implement the abstract methods, which defines things that 058 * always change between different farms. 059 * <p> 060 * TODO Further work should be done to be able to configure the ResourceStrategy 061 * depending on the farm on which the job will be dispatched. This should be 062 * included in the GRID dispatchers. 063 * @author Gabriele Carcassi 064 * @version July 24, 2003 065 */ 066 public class BOSSResourceStrategy { 067 static private Logger log = Logger.getLogger(BOSSResourceStrategy.class.getName()); 068 069 /** Base value for the resource value formula. */ 070 protected int base = 0; 071 072 /** Returns the base value for the resource value formula. */ 073 public int getBase() { 074 return base; 075 } 076 077 /** Changes the base value for the resource value formula. */ 078 public void setBase(int base) { 079 this.base = base; 080 } 081 082 /** FileFactor value for the resource value formula. */ 083 protected int fileFactor = 2; 084 085 /** Returns the fileFactor value for the resource value formula. */ 086 public int getFileFactor() { 087 return fileFactor; 088 } 089 090 /** Changes the fileFactor value for the resource value formula. */ 091 public void setFileFactor(int fileFactor) { 092 this.fileFactor = fileFactor; 093 } 094 095 /** Max parameter for the standard increaseResourceUsage implementation. */ 096 protected int max = 50; 097 098 /** Returns the max value for the resource value formula. */ 099 public int getMax() { 100 return max; 101 } 102 103 /** Changes the max value for the resource value formula. */ 104 public void setMax(int max) { 105 this.max = max; 106 } 107 108 private Hashtable resources; 109 110 private String dataVaultPrefix; 111 112 /** Returns the prefix with which a file on NFS identifies on which file server or 113 * data vault is residing. For example, "/star/data07/muDST/..." represents a file 114 * on data vault "07" (that is the data vault number or id), therefore the prefix is "/star/data". 115 * @return The prefix that identifies data vaults on NFS. 116 */ 117 public String getDataVaultPrefix() { 118 return dataVaultPrefix; 119 } 120 121 /** Changes the prefix of a NFS file. The resource identifies the data vault number 122 * by looking at a prefix in the file name. For example "/star/data07/minbias/..." 123 * would have "/star/data" as a prefix. 124 */ 125 public void setDataVaultPrefix(String dataVaultPrefix) { 126 this.dataVaultPrefix = dataVaultPrefix; 127 } 128 129 private String resourceNameSyntax; 130 131 /** Returns the syntax to be used to convert a data vault number to the BOSS resource. 132 */ 133 public String getResourceNameSyntax() { 134 return resourceNameSyntax; 135 } 136 137 /** Changes the syntax to be used to convert a data vault number to the BOSS resource. 138 * It can be any string, and it should contain the $vaultNumber variable, which will 139 * be substituted with the particular vault number. For example, to have something 140 * like "dv7io", you should have "dv$vaultNumberio". 141 */ 142 public void setResourceNameSyntax(String resourceNameSyntax) { 143 this.resourceNameSyntax = resourceNameSyntax; 144 } 145 146 private boolean trimVaultNumberLeadingZeros; 147 148 /** Returns true if the leading zeros of a vault number will be removed when 149 * creating the corresponding BOSS resource. 150 */ 151 public boolean isTrimVaultNumberLeadingZeros() { 152 return trimVaultNumberLeadingZeros; 153 } 154 155 /** Set to true if the leading zeros of a vault number will be removed when 156 * creating the corresponding BOSS resource. Basically, 07 will be changed to 7. 157 */ 158 public void setTrimVaultNumberLeadingZeros(boolean trimVaultNumberLeadingZeros) { 159 this.trimVaultNumberLeadingZeros = trimVaultNumberLeadingZeros; 160 } 161 162 /** Given the vault id, returns the BOSS resource name to be used in the -R 163 * parameter. 164 * @param vaultNumber the vault id, as identified in <CODE>getDataVaultPrefix</CODE> 165 * @return the BOSS resource name for the vault 166 */ 167 protected String prepareResourceName(String vaultNumber) { 168 if ((removeCharactersFromVaultNumber) && (!vaultNumber.matches("[a-zA-Z]+"))) { 169 Pattern pattern = Pattern.compile("[^a-zA-Z]+"); 170 Matcher matcher = pattern.matcher(vaultNumber); 171 StringBuffer buf = new StringBuffer(vaultNumber.length()); 172 StringBuffer vaultNumberBuf = new StringBuffer(vaultNumber); 173 while (matcher.find()) { 174 buf.append(vaultNumberBuf.subSequence(matcher.start(), matcher.end())); 175 } 176 vaultNumber = buf.toString(); 177 } 178 179 if (trimVaultNumberLeadingZeros) { 180 while (vaultNumber.startsWith("0") && (vaultNumber.length() != 1)) { 181 vaultNumber = vaultNumber.substring(1); 182 } 183 } 184 185 if (trimVaultNumberDecimal) { 186 int dec = vaultNumber.indexOf('.'); 187 if (dec != -1) vaultNumber = vaultNumber.substring(0, dec); 188 } 189 190 String name = resourceNameSyntax.replaceAll("\\$vaultNumber", vaultNumber); 191 192 return name; 193 } 194 195 private String switchSyntax; 196 197 /** Holds value of property trimVaultNumberDecimal. */ 198 private boolean trimVaultNumberDecimal; 199 200 /** Holds value of property removeCharactersFromVaultNumber. */ 201 private boolean removeCharactersFromVaultNumber; 202 203 /** Returns the syntax that will be used to prepare the final switch 204 */ 205 public String getSwitchSyntax() { 206 return switchSyntax; 207 } 208 209 /** Changes the syntax that will be used to prepare the final switch. 210 * You can use the $definedNames and the $nameEqualValueCommaSeparated 211 * variables in you switch. For example, the following switch: 212 * "\"select[$definedNames] rusage[$nameEqualValueCommaSeparated]\"" 213 * will produce something like: "select[defined(01)] rusage[01=50]" 214 */ 215 public void setSwitchSyntax(String switchSyntax) { 216 this.switchSyntax = switchSyntax; 217 } 218 219 /** Prepares the -R parameters to give to BOSS at the end of the resource 220 * distribution. It will use <CODE>definedNames()</CODE> and 221 * <CODE>nameEqualValueCommaSeparated()</CODE> to prepare it more easily. 222 * @return the -R parameter for the current job 223 */ 224 protected String prepareSwitch() { 225 if (determineResourceNotUsed()) { 226 return null; 227 } 228 229 String fullSwitch = switchSyntax.replaceAll("\\$definedNames", definedNames()); 230 fullSwitch = fullSwitch.replaceAll("\\$nameEqualValueCommaSeparated", nameEqualValueCommaSeparated()); 231 232 return fullSwitch; 233 } 234 235 /** Increase the resource usage as part of the standard implementation. The standard 236 * formula starts counting from <CODE>base</CODE>, adds <CODE>fileFactor</CODE> for 237 * each usage until <CODE>max</CODE> is reached. In pseudo-code: 238 * <CODE>if (nRes == 0) nRes = base; 239 * nRes += nRes; 240 * if (nRes > max) nRes = max; 241 * </CODE> 242 * <p> 243 * One can modify the three protected parameters (base, fileFactor, max) to tune 244 * the usage. 245 * @param resource the resource for which to increase the counting. 246 */ 247 protected void increaseResourceUsage(String resource) { 248 increaseResourceUptoLimit(resource, base, fileFactor, max); 249 } 250 251 /** Addes a new resource, and initializes the value to <CODE>base</CODE>. 252 * @param resName the name of the resource 253 * @param base the initial value for the resource 254 */ 255 protected void addResource(String resName, int base) { 256 if (!resources.containsKey(resName)) { 257 resources.put(resName, new Integer(base)); 258 } else { 259 throw new RuntimeException("Can't add resource " + resName + 260 " because it's already present."); 261 } 262 } 263 264 /** Increases the value associated to the given resource by a given amount. If the 265 * resource doesn't exists, first it will be created and initialized. 266 * @param resName the name of the resource 267 * @param base the initial value if the resource has to be created 268 * @param amount the amount to increase the resource by 269 * @return the new value associated to the resource 270 */ 271 protected int increaseResource(String resName, int base, int amount) { 272 if (!resources.containsKey(resName)) { 273 addResource(resName, base); 274 } 275 276 int returnValue = prepareResourceValue(resName) + amount; 277 Integer newValue = new Integer(returnValue); 278 279 resources.remove(resName); 280 resources.put(resName, newValue); 281 282 return returnValue; 283 } 284 285 /** Increases the value associated to the given resource by a given amount. If the 286 * resource doesn't exists, first it will be created and initialized. If the 287 * increased amount is over the limit, the value will be set to the limit. 288 * @return the new value associated to the resource 289 * @param limit upper limit to which the resource value will be set 290 * @param resName the name of the resource 291 * @param base the initial value if the resource has to be created 292 * @param amount the amount to increase the resource by 293 */ 294 protected int increaseResourceUptoLimit(String resName, int base, 295 int amount, int limit) { 296 if (!resources.containsKey(resName)) { 297 addResource(resName, base); 298 } 299 300 int returnValue = prepareResourceValue(resName) + amount; 301 302 if (returnValue > limit) { 303 return limit; 304 } 305 306 Integer newValue = new Integer(returnValue); 307 308 resources.remove(resName); 309 resources.put(resName, newValue); 310 311 return returnValue; 312 } 313 314 /** Returns the value currently associated to the resource. 315 * @param resName resource name 316 * @return resource value 317 */ 318 protected int prepareResourceValue(String resName) { 319 return ((Integer) resources.get(resName)).intValue(); 320 } 321 322 /** Returns the -R parameter to be used to dispatch the given job. 323 * @param job the job to be dispatched 324 * @return the -R parameter to use 325 */ 326 public String prepareResourceUsageSwitch(Job job) { 327 initResources(); 328 calculateBOSSResources(job); 329 330 return prepareSwitch(); 331 } 332 333 /** Initializes the resource table. */ 334 protected void initResources() { 335 resources = new Hashtable(); 336 } 337 338 /** Returns a string already formatted containing names and values comma separated 339 * (i.e. "res1=34,res2=12"). 340 * @return resource name/value pairs comma separated (i.e. "res1=34,res2=12") 341 */ 342 protected String nameEqualValueCommaSeparated() { 343 String rusage = ""; 344 Set resNames = resources.keySet(); 345 Iterator iter = resNames.iterator(); 346 boolean first = true; 347 348 while (iter.hasNext()) { 349 if (!first) { 350 rusage += ","; 351 } 352 353 Object key = iter.next(); 354 rusage += (key + "=" + resources.get(key)); 355 first = false; 356 } 357 358 return rusage; 359 } 360 361 /** A list of names, comma separated, of all the resources used. 362 * @return (i.e. "res1,res2") 363 */ 364 protected String definedNames() { 365 String defined = ""; 366 Set resNames = resources.keySet(); 367 Iterator iter = resNames.iterator(); 368 boolean first = true; 369 370 while (iter.hasNext()) { 371 if (!first) { 372 defined += " && "; 373 } 374 375 Object key = iter.next(); 376 defined += ("defined(" + key + ")"); 377 first = false; 378 } 379 380 return defined; 381 } 382 383 /** Filles the resource table with all the resources. It basically iterates over the 384 * input files and calls <CODE>increaseResourceUsage</CODE> for each file that map 385 * to a resource. 386 * @param job the job 387 */ 388 protected void calculateBOSSResources(Job job) { 389 log.fine("Calculating BOSS resources"); 390 391 for (int i = 0; i < job.getInput().size(); i++) { 392 PhysicalFile file = (PhysicalFile) job.getInput().get(i); 393 String res = resourceRequired(file); 394 395 if (res != null) { 396 increaseResourceUsage(res); 397 } 398 } 399 } 400 401 /** Determines which resource the job might need to access the file. 402 * @param file input file the job will be using 403 * @return the resource needed 404 */ 405 protected String resourceRequired(PhysicalFile file) { 406 if ("NFS".equals(file.getStorage())) { 407 log.finest("Found NFS file: " + file); 408 409 String path = file.getPath(); 410 411 if (path.startsWith(getDataVaultPrefix())) { 412 int bar = path.indexOf('/', getDataVaultPrefix().length()); 413 String serverNumber = path.substring(getDataVaultPrefix() 414 .length(), bar); 415 416 return prepareResourceName(serverNumber); 417 } 418 } 419 420 return null; 421 } 422 423 /** Determines whether any resources at all have been assigned. 424 * @return true if any resource is used. 425 */ 426 protected boolean determineResourceNotUsed() { 427 return resources.isEmpty(); 428 } 429 430 /** Getter for property trimVaultNumberDecimal. 431 * @return Value of property trimVaultNumberDecimal. 432 * 433 */ 434 public boolean isTrimVaultNumberDecimal() { 435 return this.trimVaultNumberDecimal; 436 } 437 438 /** Setter for property trimVaultNumberDecimal. 439 * @param trimVaultNumberDecimal New value of property trimVaultNumberDecimal. 440 * 441 */ 442 public void setTrimVaultNumberDecimal(boolean trimVaultNumberDecimal) { 443 this.trimVaultNumberDecimal = trimVaultNumberDecimal; 444 } 445 446 /** If true removes all the characters from the vault number. 447 * @return Value of property removeCharactersFromVaultNumber. 448 * 449 */ 450 public boolean isRemoveCharactersFromVaultNumber() { 451 return this.removeCharactersFromVaultNumber; 452 } 453 454 /** Set to true removes all the characters from the vault number 455 * @param removeCharactersFromVaultNumber New value of property removeCharactersFromVaultNumber. 456 * 457 */ 458 public void setRemoveCharactersFromVaultNumber(boolean removeCharactersFromVaultNumber) { 459 this.removeCharactersFromVaultNumber = removeCharactersFromVaultNumber; 460 } 461 462 }