001    /*
002     * BOSSResourceStrategy.java
003     *
004     * Created on April 1, 2003, 3:30 PM
005     *
006     * This file is part of the STAR Scheduler.
007     * Copyright (c) 2002-2003 STAR Collaboration - Brookhaven National Laboratory
008     *
009     * STAR Scheduler is free software; you can redistribute it and/or modify
010     * it under the terms of the GNU General Public License as published by
011     * the Free Software Foundation; either version 2 of the License, or
012     * (at your option) any later version.
013     *
014     * STAR Scheduler is distributed in the hope that it will be useful,
015     * but WITHOUT ANY WARRANTY; without even the implied warranty of
016     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
017     * GNU General Public License for more details.
018     *
019     * You should have received a copy of the GNU General Public License
020     * along with STAR Scheduler; if not, write to the Free Software
021     * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
022     */
023    package gov.bnl.star.offline.scheduler.Dispatchers.boss;
024    
025    import gov.bnl.star.offline.scheduler.*;
026    import gov.bnl.star.offline.scheduler.catalog.PhysicalFile;
027    
028    import java.util.Hashtable;
029    import java.util.Iterator;
030    import java.util.Set;
031    import java.util.logging.Logger;
032    import java.util.regex.Matcher;
033    import java.util.regex.Pattern;
034    
035    
036    /** Encapsulate the resource usage parameter (-R) for an BOSS farm. Essentially,
037     * its a placeholder for the prepareResourceUsageSwitch() which will calculate the
038     * appriate string. The administrator should be able to create different
039     * subclasses for each farm as needed.
040     * <p>
041     * This class provides a scheleton for a typical resource usage for files residing
042     * on different NFS servers. It was designed to fit STAR needs at two sites. In case
043     * it doesn't fit the needs, the class can be subclassed to define new behaviour.
044     * <p>
045     * The resource switch is prepared in this way:
046     * Takes the list of input files and iterates over them.<br>
047     * Checks if each file needs a resource (<CODE>resourceRequired()</CODE>).<br>
048     * To determine that, checks if the file is on NFS, and looks for the typical data
049     * vault prefix (<CODE>getDataVaultPrefix</CODE>). For example, "/star/dataxx/...";
050     * it extract the vaultNumber and it substitute it in the resourceSyntax <br>
051     * Increases the count of the resource (<CODE>increaseResourceUsage</CODE>)<br>
052     * Prepares the switch (<CODE>prepareSwitch</CODE>)
053     * <p>
054     * The resource value for each resource will the be given by the following formula: <br>
055     * <code>min (base+nCount*fileFactor, max)</code>
056     * <p>
057     * The subclass need to implement the abstract methods, which defines things that
058     * always change between different farms.
059     * <p>
060     * TODO Further work should be done to be able to configure the ResourceStrategy
061     * depending on the farm on which the job will be dispatched. This should be
062     * included in the GRID dispatchers.
063     * @author Gabriele Carcassi
064     * @version July 24, 2003
065     */
066    public class BOSSResourceStrategy {
067        static private Logger log = Logger.getLogger(BOSSResourceStrategy.class.getName());
068    
069        /** Base value for the resource value formula. */
070        protected int base = 0;
071        
072        /** Returns the base value for the resource value formula. */
073        public int getBase() {
074            return base;
075        }
076        
077        /** Changes the base value for the resource value formula. */
078        public void setBase(int base) {
079            this.base = base;
080        }
081    
082        /** FileFactor value for the resource value formula. */
083        protected int fileFactor = 2;
084        
085        /** Returns the fileFactor value for the resource value formula. */
086        public int getFileFactor() {
087            return fileFactor;
088        }
089        
090        /** Changes the fileFactor value for the resource value formula. */
091        public void setFileFactor(int fileFactor) {
092            this.fileFactor = fileFactor;
093        }
094    
095        /** Max parameter for the standard increaseResourceUsage implementation. */
096        protected int max = 50;
097        
098        /** Returns the max value for the resource value formula. */
099        public int getMax() {
100            return max;
101        }
102        
103        /** Changes the max value for the resource value formula. */
104        public void setMax(int max) {
105            this.max = max;
106        }
107        
108        private Hashtable resources;
109    
110        private String dataVaultPrefix;
111        
112        /** Returns the prefix with which a file on NFS identifies on which file server or
113         * data vault is residing. For example, "/star/data07/muDST/..." represents a file
114         * on data vault "07" (that is the data vault number or id), therefore the prefix is "/star/data".
115         * @return The prefix that identifies data vaults on NFS.
116         */
117        public String getDataVaultPrefix() {
118            return dataVaultPrefix;
119        }
120        
121        /** Changes the prefix of a NFS file. The resource identifies the data vault number
122         * by looking at a prefix in the file name. For example "/star/data07/minbias/..."
123         * would have "/star/data" as a prefix.
124         */
125        public void setDataVaultPrefix(String dataVaultPrefix) {
126            this.dataVaultPrefix = dataVaultPrefix;
127        }
128    
129        private String resourceNameSyntax;
130        
131        /** Returns the syntax to be used to convert a data vault number to the BOSS resource.
132         */
133        public String getResourceNameSyntax() {
134            return resourceNameSyntax;
135        }
136        
137        /** Changes the syntax to be used to convert a data vault number to the BOSS resource.
138         * It can be any string, and it should contain the $vaultNumber variable, which will
139         * be substituted with the particular vault number. For example, to have something
140         * like "dv7io", you should have "dv$vaultNumberio".
141         */
142        public void setResourceNameSyntax(String resourceNameSyntax) {
143            this.resourceNameSyntax = resourceNameSyntax;
144        }
145        
146        private boolean trimVaultNumberLeadingZeros;
147        
148        /** Returns true if the leading zeros of a vault number will be removed when
149         * creating the corresponding BOSS resource.
150         */
151        public boolean isTrimVaultNumberLeadingZeros() {
152            return trimVaultNumberLeadingZeros;
153        }
154        
155        /** Set to true if the leading zeros of a vault number will be removed when
156         * creating the corresponding BOSS resource. Basically, 07 will be changed to 7.
157         */
158        public void setTrimVaultNumberLeadingZeros(boolean trimVaultNumberLeadingZeros) {
159            this.trimVaultNumberLeadingZeros = trimVaultNumberLeadingZeros;
160        }
161        
162        /** Given the vault id, returns the BOSS resource name to be used in the -R
163         * parameter.
164         * @param vaultNumber the vault id, as identified in <CODE>getDataVaultPrefix</CODE>
165         * @return the BOSS resource name for the vault
166         */
167        protected String prepareResourceName(String vaultNumber) {
168            if ((removeCharactersFromVaultNumber) && (!vaultNumber.matches("[a-zA-Z]+"))) {
169                Pattern pattern = Pattern.compile("[^a-zA-Z]+");
170                Matcher matcher = pattern.matcher(vaultNumber);
171                StringBuffer buf = new StringBuffer(vaultNumber.length());
172                StringBuffer vaultNumberBuf = new StringBuffer(vaultNumber);
173                while (matcher.find()) {
174                    buf.append(vaultNumberBuf.subSequence(matcher.start(), matcher.end()));
175                }
176                vaultNumber = buf.toString();
177            }
178            
179            if (trimVaultNumberLeadingZeros) {
180                while (vaultNumber.startsWith("0") && (vaultNumber.length() != 1)) {
181                    vaultNumber = vaultNumber.substring(1);
182                }
183            }
184    
185            if (trimVaultNumberDecimal) {
186                int dec = vaultNumber.indexOf('.');
187                if (dec != -1) vaultNumber = vaultNumber.substring(0, dec);
188            }
189            
190            String name = resourceNameSyntax.replaceAll("\\$vaultNumber", vaultNumber);
191    
192            return name;
193        }
194        
195        private String switchSyntax;
196        
197        /** Holds value of property trimVaultNumberDecimal. */
198        private boolean trimVaultNumberDecimal;
199        
200        /** Holds value of property removeCharactersFromVaultNumber. */
201        private boolean removeCharactersFromVaultNumber;
202        
203        /** Returns the syntax that will be used to prepare the final switch
204         */
205        public String getSwitchSyntax() {
206            return switchSyntax;
207        }
208        
209        /** Changes the syntax that will be used to prepare the final switch.
210         * You can use the $definedNames and the $nameEqualValueCommaSeparated
211         * variables in you switch. For example, the following switch: 
212         * "\"select[$definedNames] rusage[$nameEqualValueCommaSeparated]\""
213         * will produce something like: "select[defined(01)] rusage[01=50]"
214         */
215        public void setSwitchSyntax(String switchSyntax) {
216            this.switchSyntax = switchSyntax;
217        }
218        
219        /** Prepares the -R parameters to give to BOSS at the end of the resource
220         * distribution. It will use <CODE>definedNames()</CODE> and
221         * <CODE>nameEqualValueCommaSeparated()</CODE> to prepare it more easily.
222         * @return the -R parameter for the current job
223         */
224        protected String prepareSwitch() {
225            if (determineResourceNotUsed()) {
226                return null;
227            }
228            
229            String fullSwitch = switchSyntax.replaceAll("\\$definedNames", definedNames());
230            fullSwitch = fullSwitch.replaceAll("\\$nameEqualValueCommaSeparated", nameEqualValueCommaSeparated());
231    
232            return fullSwitch;
233        }
234        
235        /** Increase the resource usage as part of the standard implementation. The standard
236         * formula starts counting from <CODE>base</CODE>, adds <CODE>fileFactor</CODE> for
237         * each usage until <CODE>max</CODE> is reached. In pseudo-code:
238         * <CODE>if (nRes == 0) nRes = base;
239         * nRes += nRes;
240         * if (nRes > max) nRes = max;
241         * </CODE>
242         * <p>
243         * One can modify the three protected parameters (base, fileFactor, max) to tune
244         * the usage.
245         * @param resource the resource for which to increase the counting.
246         */
247        protected void increaseResourceUsage(String resource) {
248            increaseResourceUptoLimit(resource, base, fileFactor, max);
249        }
250    
251        /** Addes a new resource, and initializes the value to <CODE>base</CODE>.
252         * @param resName the name of the resource
253         * @param base the initial value for the resource
254         */
255        protected void addResource(String resName, int base) {
256            if (!resources.containsKey(resName)) {
257                resources.put(resName, new Integer(base));
258            } else {
259                throw new RuntimeException("Can't add resource " + resName +
260                    " because it's already present.");
261            }
262        }
263    
264        /** Increases the value associated to the given resource by a given amount. If the
265         * resource doesn't exists, first it will be created and initialized.
266         * @param resName the name of the resource
267         * @param base the initial value if the resource has to be created
268         * @param amount the amount to increase the resource by
269         * @return the new value associated to the resource
270         */
271        protected int increaseResource(String resName, int base, int amount) {
272            if (!resources.containsKey(resName)) {
273                addResource(resName, base);
274            }
275    
276            int returnValue = prepareResourceValue(resName) + amount;
277            Integer newValue = new Integer(returnValue);
278    
279            resources.remove(resName);
280            resources.put(resName, newValue);
281    
282            return returnValue;
283        }
284    
285        /** Increases the value associated to the given resource by a given amount. If the
286         * resource doesn't exists, first it will be created and initialized. If the
287         * increased amount is over the limit, the value will be set to the limit.
288         * @return the new value associated to the resource
289         * @param limit upper limit to which the resource value will be set
290         * @param resName the name of the resource
291         * @param base the initial value if the resource has to be created
292         * @param amount the amount to increase the resource by
293         */
294        protected int increaseResourceUptoLimit(String resName, int base,
295            int amount, int limit) {
296            if (!resources.containsKey(resName)) {
297                addResource(resName, base);
298            }
299    
300            int returnValue = prepareResourceValue(resName) + amount;
301    
302            if (returnValue > limit) {
303                return limit;
304            }
305    
306            Integer newValue = new Integer(returnValue);
307    
308            resources.remove(resName);
309            resources.put(resName, newValue);
310    
311            return returnValue;
312        }
313    
314        /** Returns the value currently associated to the resource.
315         * @param resName resource name
316         * @return resource value
317         */
318        protected int prepareResourceValue(String resName) {
319            return ((Integer) resources.get(resName)).intValue();
320        }
321    
322        /** Returns the -R parameter to be used to dispatch the given job.
323         * @param job the job to be dispatched
324         * @return the -R parameter to use
325         */
326        public String prepareResourceUsageSwitch(Job job) {
327            initResources();
328            calculateBOSSResources(job);
329    
330            return prepareSwitch();
331        }
332    
333        /** Initializes the resource table. */
334        protected void initResources() {
335            resources = new Hashtable();
336        }
337    
338        /** Returns a string already formatted containing names and values comma separated
339         * (i.e. "res1=34,res2=12").
340         * @return resource name/value pairs comma separated (i.e. "res1=34,res2=12")
341         */
342        protected String nameEqualValueCommaSeparated() {
343            String rusage = "";
344            Set resNames = resources.keySet();
345            Iterator iter = resNames.iterator();
346            boolean first = true;
347    
348            while (iter.hasNext()) {
349                if (!first) {
350                    rusage += ",";
351                }
352    
353                Object key = iter.next();
354                rusage += (key + "=" + resources.get(key));
355                first = false;
356            }
357    
358            return rusage;
359        }
360    
361        /** A list of names, comma separated, of all the resources used.
362         * @return (i.e. "res1,res2")
363         */
364        protected String definedNames() {
365            String defined = "";
366            Set resNames = resources.keySet();
367            Iterator iter = resNames.iterator();
368            boolean first = true;
369    
370            while (iter.hasNext()) {
371                if (!first) {
372                    defined += " && ";
373                }
374    
375                Object key = iter.next();
376                defined += ("defined(" + key + ")");
377                first = false;
378            }
379    
380            return defined;
381        }
382    
383        /** Filles the resource table with all the resources. It basically iterates over the
384         * input files and calls <CODE>increaseResourceUsage</CODE> for each file that map
385         * to a resource.
386         * @param job the job
387         */
388        protected void calculateBOSSResources(Job job) {
389            log.fine("Calculating BOSS resources");
390    
391            for (int i = 0; i < job.getInput().size(); i++) {
392                PhysicalFile file = (PhysicalFile) job.getInput().get(i);
393                String res = resourceRequired(file);
394    
395                if (res != null) {
396                    increaseResourceUsage(res);
397                }
398            }
399        }
400    
401        /** Determines which resource the job might need to access the file.
402         * @param file input file the job will be using
403         * @return the resource needed
404         */
405        protected String resourceRequired(PhysicalFile file) {
406            if ("NFS".equals(file.getStorage())) {
407                log.finest("Found NFS file: " + file);
408    
409                String path = file.getPath();
410    
411                if (path.startsWith(getDataVaultPrefix())) {
412                    int bar = path.indexOf('/', getDataVaultPrefix().length());
413                    String serverNumber = path.substring(getDataVaultPrefix()
414                                                             .length(), bar);
415    
416                    return prepareResourceName(serverNumber);
417                }
418            }
419    
420            return null;
421        }
422    
423        /** Determines whether any resources at all have been assigned.
424         * @return true if any resource is used.
425         */
426        protected boolean determineResourceNotUsed() {
427            return resources.isEmpty();
428        }
429        
430        /** Getter for property trimVaultNumberDecimal.
431         * @return Value of property trimVaultNumberDecimal.
432         *
433         */
434        public boolean isTrimVaultNumberDecimal() {
435            return this.trimVaultNumberDecimal;
436        }
437        
438        /** Setter for property trimVaultNumberDecimal.
439         * @param trimVaultNumberDecimal New value of property trimVaultNumberDecimal.
440         *
441         */
442        public void setTrimVaultNumberDecimal(boolean trimVaultNumberDecimal) {
443            this.trimVaultNumberDecimal = trimVaultNumberDecimal;
444        }
445        
446        /** If true removes all the characters from the vault number.
447         * @return Value of property removeCharactersFromVaultNumber.
448         *
449         */
450        public boolean isRemoveCharactersFromVaultNumber() {
451            return this.removeCharactersFromVaultNumber;
452        }
453        
454        /** Set to true removes all the characters from the vault number
455         * @param removeCharactersFromVaultNumber New value of property removeCharactersFromVaultNumber.
456         *
457         */
458        public void setRemoveCharactersFromVaultNumber(boolean removeCharactersFromVaultNumber) {
459            this.removeCharactersFromVaultNumber = removeCharactersFromVaultNumber;
460        }
461        
462    }