001 /*
002 * PBSResourceStrategy.java
003 *
004 * Created on April 1, 2003, 3:30 PM
005 *
006 * This file is part of the STAR Scheduler.
007 * Copyright (c) 2002-2006 STAR Collaboration - Brookhaven National Laboratory
008 *
009 * STAR Scheduler is free software; you can redistribute it and/or modify
010 * it under the terms of the GNU General Public License as published by
011 * the Free Software Foundation; either version 2 of the License, or
012 * (at your option) any later version.
013 *
014 * STAR Scheduler is distributed in the hope that it will be useful,
015 * but WITHOUT ANY WARRANTY; without even the implied warranty of
016 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
017 * GNU General Public License for more details.
018 *
019 * You should have received a copy of the GNU General Public License
020 * along with STAR Scheduler; if not, write to the Free Software
021 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
022 */
023 package gov.bnl.star.offline.scheduler.Dispatchers.pbs;
024
025 import gov.bnl.star.offline.scheduler.*;
026 import gov.bnl.star.offline.scheduler.catalog.PhysicalFile;
027
028 import java.util.Hashtable;
029 import java.util.Iterator;
030 import java.util.Set;
031 import org.apache.log4j.Logger;
032 import java.util.regex.Matcher;
033 import java.util.regex.Pattern;
034
035
036 /**A dispatcher for the SGE batch system.
037 * @author Levente Hajdu
038 * @version July 24, 2003
039 */
040 public class PBSResourceStrategy {
041 static private Logger log = Logger.getLogger(PBSResourceStrategy.class.getName());
042
043 /** Base value for the resource value formula. */
044 protected int base = 0;
045
046 /** Returns the base value for the resource value formula. */
047 public int getBase() {
048 return base;
049 }
050
051 /** Changes the base value for the resource value formula. */
052 public void setBase(int base) {
053 this.base = base;
054 }
055
056 /** FileFactor value for the resource value formula. */
057 protected int fileFactor = 2;
058
059 /** Returns the fileFactor value for the resource value formula. */
060 public int getFileFactor() {
061 return fileFactor;
062 }
063
064 /** Changes the fileFactor value for the resource value formula. */
065 public void setFileFactor(int fileFactor) {
066 this.fileFactor = fileFactor;
067 }
068
069 /** Max parameter for the standard increaseResourceUsage implementation. */
070 protected int max = 50;
071
072 /** Returns the max value for the resource value formula. */
073 public int getMax() {
074 return max;
075 }
076
077 /** Changes the max value for the resource value formula. */
078 public void setMax(int max) {
079 this.max = max;
080 }
081
082 private Hashtable resources;
083
084 private String dataVaultPrefix;
085
086 /** Returns the prefix with which a file on NFS identifies on which file server or
087 * data vault is residing. For example, "/star/data07/muDST/..." represents a file
088 * on data vault "07" (that is the data vault number or id), therefore the prefix is "/star/data".
089 * @return The prefix that identifies data vaults on NFS.
090 */
091 public String getDataVaultPrefix() {
092 return dataVaultPrefix;
093 }
094
095 /** Changes the prefix of a NFS file. The resource identifies the data vault number
096 * by looking at a prefix in the file name. For example "/star/data07/minbias/..."
097 * would have "/star/data" as a prefix.
098 */
099 public void setDataVaultPrefix(String dataVaultPrefix) {
100 this.dataVaultPrefix = dataVaultPrefix;
101 }
102
103 private String resourceNameSyntax;
104
105 /** Returns the syntax to be used to convert a data vault number to the PBS resource.
106 */
107 public String getResourceNameSyntax() {
108 return resourceNameSyntax;
109 }
110
111 /** Changes the syntax to be used to convert a data vault number to the PBS resource.
112 * It can be any string, and it should contain the $vaultNumber variable, which will
113 * be substituted with the particular vault number. For example, to have something
114 * like "dv7io", you should have "dv$vaultNumberio".
115 */
116 public void setResourceNameSyntax(String resourceNameSyntax) {
117 this.resourceNameSyntax = resourceNameSyntax;
118 }
119
120 private boolean trimVaultNumberLeadingZeros;
121
122 /** Returns true if the leading zeros of a vault number will be removed when
123 * creating the corresponding PBS resource.
124 */
125 public boolean isTrimVaultNumberLeadingZeros() {
126 return trimVaultNumberLeadingZeros;
127 }
128
129 /** Set to true if the leading zeros of a vault number will be removed when
130 * creating the corresponding PBS resource. Basically, 07 will be changed to 7.
131 */
132 public void setTrimVaultNumberLeadingZeros(boolean trimVaultNumberLeadingZeros) {
133 this.trimVaultNumberLeadingZeros = trimVaultNumberLeadingZeros;
134 }
135
136 /** Given the vault id, returns the PBS resource name to be used in the -R
137 * parameter.
138 * @param vaultNumber the vault id, as identified in <CODE>getDataVaultPrefix</CODE>
139 * @return the PBS resource name for the vault
140 */
141 protected String prepareResourceName(String vaultNumber) {
142 if ((removeCharactersFromVaultNumber) && (!vaultNumber.matches("[a-zA-Z]+"))) {
143 Pattern pattern = Pattern.compile("[^a-zA-Z]+");
144 Matcher matcher = pattern.matcher(vaultNumber);
145 StringBuffer buf = new StringBuffer(vaultNumber.length());
146 StringBuffer vaultNumberBuf = new StringBuffer(vaultNumber);
147 while (matcher.find()) {
148 buf.append(vaultNumberBuf.subSequence(matcher.start(), matcher.end()));
149 }
150 vaultNumber = buf.toString();
151 }
152
153 if (trimVaultNumberLeadingZeros) {
154 while (vaultNumber.startsWith("0") && (vaultNumber.length() != 1)) {
155 vaultNumber = vaultNumber.substring(1);
156 }
157 }
158
159 if (trimVaultNumberDecimal) {
160 int dec = vaultNumber.indexOf('.');
161 if (dec != -1) vaultNumber = vaultNumber.substring(0, dec);
162 }
163
164 String name = resourceNameSyntax.replaceAll("\\$vaultNumber", vaultNumber);
165
166 return name;
167 }
168
169 private String switchSyntax;
170
171 /** Holds value of property trimVaultNumberDecimal. */
172 private boolean trimVaultNumberDecimal;
173
174 /** Holds value of property removeCharactersFromVaultNumber. */
175 private boolean removeCharactersFromVaultNumber;
176
177 /** Returns the syntax that will be used to prepare the final switch
178 */
179 public String getSwitchSyntax() {
180 return switchSyntax;
181 }
182
183 /** Changes the syntax that will be used to prepare the final switch.
184 * You can use the $definedNames and the $nameEqualValueCommaSeparated
185 * variables in you switch. For example, the following switch:
186 * "\"select[$definedNames] rusage[$nameEqualValueCommaSeparated]\""
187 * will produce something like: "select[defined(01)] rusage[01=50]"
188 */
189 public void setSwitchSyntax(String switchSyntax) {
190 this.switchSyntax = switchSyntax;
191 }
192
193 /** Prepares the -R parameters to give to PBS at the end of the resource
194 * distribution. It will use <CODE>definedNames()</CODE> and
195 * <CODE>nameEqualValueCommaSeparated()</CODE> to prepare it more easily.
196 * @return the -R parameter for the current job
197 */
198 protected String prepareSwitch() {
199 if (determineResourceNotUsed()) {
200 return null;
201 }
202
203 String fullSwitch = switchSyntax.replaceAll("\\$definedNames", definedNames());
204 fullSwitch = fullSwitch.replaceAll("\\$nameEqualValueCommaSeparated", nameEqualValueCommaSeparated());
205
206 return fullSwitch;
207 }
208
209 /** Increase the resource usage as part of the standard implementation. The standard
210 * formula starts counting from <CODE>base</CODE>, adds <CODE>fileFactor</CODE> for
211 * each usage until <CODE>max</CODE> is reached. In pseudo-code:
212 * <CODE>if (nRes == 0) nRes = base;
213 * nRes += nRes;
214 * if (nRes > max) nRes = max;
215 * </CODE>
216 * <p>
217 * One can modify the three protected parameters (base, fileFactor, max) to tune
218 * the usage.
219 * @param resource the resource for which to increase the counting.
220 */
221 protected void increaseResourceUsage(String resource) {
222 increaseResourceUptoLimit(resource, base, fileFactor, max);
223 }
224
225 /** Addes a new resource, and initializes the value to <CODE>base</CODE>.
226 * @param resName the name of the resource
227 * @param base the initial value for the resource
228 */
229 protected void addResource(String resName, int base) {
230 if (!resources.containsKey(resName)) {
231 resources.put(resName, new Integer(base));
232 } else {
233 throw new RuntimeException("Can't add resource " + resName +
234 " because it's already present.");
235 }
236 }
237
238 /** Increases the value associated to the given resource by a given amount. If the
239 * resource doesn't exists, first it will be created and initialized.
240 * @param resName the name of the resource
241 * @param base the initial value if the resource has to be created
242 * @param amount the amount to increase the resource by
243 * @return the new value associated to the resource
244 */
245 protected int increaseResource(String resName, int base, int amount) {
246 if (!resources.containsKey(resName)) {
247 addResource(resName, base);
248 }
249
250 int returnValue = prepareResourceValue(resName) + amount;
251 Integer newValue = new Integer(returnValue);
252
253 resources.remove(resName);
254 resources.put(resName, newValue);
255
256 return returnValue;
257 }
258
259 /** Increases the value associated to the given resource by a given amount. If the
260 * resource doesn't exists, first it will be created and initialized. If the
261 * increased amount is over the limit, the value will be set to the limit.
262 * @return the new value associated to the resource
263 * @param limit upper limit to which the resource value will be set
264 * @param resName the name of the resource
265 * @param base the initial value if the resource has to be created
266 * @param amount the amount to increase the resource by
267 */
268 protected int increaseResourceUptoLimit(String resName, int base,
269 int amount, int limit) {
270 if (!resources.containsKey(resName)) {
271 addResource(resName, base);
272 }
273
274 int returnValue = prepareResourceValue(resName) + amount;
275
276 if (returnValue > limit) {
277 return limit;
278 }
279
280 Integer newValue = new Integer(returnValue);
281
282 resources.remove(resName);
283 resources.put(resName, newValue);
284
285 return returnValue;
286 }
287
288 /** Returns the value currently associated to the resource.
289 * @param resName resource name
290 * @return resource value
291 */
292 protected int prepareResourceValue(String resName) {
293 return ((Integer) resources.get(resName)).intValue();
294 }
295
296 /** Returns the -R parameter to be used to dispatch the given job.
297 * @param job the job to be dispatched
298 * @return the -R parameter to use
299 */
300 public String prepareResourceUsageSwitch(Job job) {
301 initResources();
302 calculatePBSResources(job);
303
304 return prepareSwitch();
305 }
306
307 /** Initializes the resource table. */
308 protected void initResources() {
309 resources = new Hashtable();
310 }
311
312 /** Returns a string already formatted containing names and values comma separated
313 * (i.e. "res1=34,res2=12").
314 * @return resource name/value pairs comma separated (i.e. "res1=34,res2=12")
315 */
316 protected String nameEqualValueCommaSeparated() {
317 String rusage = "";
318 Set resNames = resources.keySet();
319 Iterator iter = resNames.iterator();
320 boolean first = true;
321
322 while (iter.hasNext()) {
323 if (!first) {
324 rusage += ",";
325 }
326
327 Object key = iter.next();
328 rusage += (key + "=" + resources.get(key));
329 first = false;
330 }
331
332 return rusage;
333 }
334
335 /** A list of names, comma separated, of all the resources used.
336 * @return (i.e. "res1,res2")
337 */
338 protected String definedNames() {
339 String defined = "";
340 Set resNames = resources.keySet();
341 Iterator iter = resNames.iterator();
342 boolean first = true;
343
344 while (iter.hasNext()) {
345 if (!first) {
346 defined += " && ";
347 }
348
349 Object key = iter.next();
350 defined += ("defined(" + key + ")");
351 first = false;
352 }
353
354 return defined;
355 }
356
357 /** Filles the resource table with all the resources. It basically iterates over the
358 * input files and calls <CODE>increaseResourceUsage</CODE> for each file that map
359 * to a resource.
360 * @param job the job
361 */
362 protected void calculatePBSResources(Job job) {
363 log.debug("Calculating PBS resources");
364
365 for (int i = 0; i < job.getInput().size(); i++) {
366 PhysicalFile file = (PhysicalFile) job.getInput().get(i);
367 String res = resourceRequired(file);
368
369 if (res != null) {
370 increaseResourceUsage(res);
371 }
372 }
373 }
374
375 /** Determines which resource the job might need to access the file.
376 * @param file input file the job will be using
377 * @return the resource needed
378 */
379 protected String resourceRequired(PhysicalFile file) {
380 if ("NFS".equals(file.getStorage())) {
381 log.debug("Found NFS file: " + file);
382
383 String path = file.getPath();
384
385 if (path.startsWith(getDataVaultPrefix())) {
386 int bar = path.indexOf('/', getDataVaultPrefix().length());
387 String serverNumber = path.substring(getDataVaultPrefix()
388 .length(), bar);
389
390 return prepareResourceName(serverNumber);
391 }
392 }
393
394 return null;
395 }
396
397 /** Determines whether any resources at all have been assigned.
398 * @return true if any resource is used.
399 */
400 protected boolean determineResourceNotUsed() {
401 return resources.isEmpty();
402 }
403
404 /** Getter for property trimVaultNumberDecimal.
405 * @return Value of property trimVaultNumberDecimal.
406 *
407 */
408 public boolean isTrimVaultNumberDecimal() {
409 return this.trimVaultNumberDecimal;
410 }
411
412 /** Setter for property trimVaultNumberDecimal.
413 * @param trimVaultNumberDecimal New value of property trimVaultNumberDecimal.
414 *
415 */
416 public void setTrimVaultNumberDecimal(boolean trimVaultNumberDecimal) {
417 this.trimVaultNumberDecimal = trimVaultNumberDecimal;
418 }
419
420 /** If true removes all the characters from the vault number.
421 * @return Value of property removeCharactersFromVaultNumber.
422 *
423 */
424 public boolean isRemoveCharactersFromVaultNumber() {
425 return this.removeCharactersFromVaultNumber;
426 }
427
428 /** Set to true removes all the characters from the vault number
429 * @param removeCharactersFromVaultNumber New value of property removeCharactersFromVaultNumber.
430 *
431 */
432 public void setRemoveCharactersFromVaultNumber(boolean removeCharactersFromVaultNumber) {
433 this.removeCharactersFromVaultNumber = removeCharactersFromVaultNumber;
434 }
435
436 }