001 /*
002 * MinMaxSingleCopySelector.java
003 *
004 * Created on April 4, 2003, 12:10 PM
005 *
006 * This file is part of the STAR Scheduler.
007 * Copyright (c) 2002-2006 STAR Collaboration - Brookhaven National Laboratory
008 *
009 * STAR Scheduler is free software; you can redistribute it and/or modify
010 * it under the terms of the GNU General Public License as published by
011 * the Free Software Foundation; either version 2 of the License, or
012 * (at your option) any later version.
013 *
014 * STAR Scheduler is distributed in the hope that it will be useful,
015 * but WITHOUT ANY WARRANTY; without even the implied warranty of
016 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
017 * GNU General Public License for more details.
018 *
019 * You should have received a copy of the GNU General Public License
020 * along with STAR Scheduler; if not, write to the Free Software
021 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
022 */
023 package gov.bnl.star.offline.scheduler.policy.copyselector;
024
025 import gov.bnl.star.offline.scheduler.CatalogQuery;
026 import gov.bnl.star.offline.scheduler.catalog.PhysicalFile;
027 import gov.bnl.star.offline.scheduler.catalog.QueryResult;
028 import gov.bnl.star.offline.scheduler.policy.FileAssignment;
029 import gov.bnl.star.offline.scheduler.policy.Location;
030
031 import java.util.Iterator;
032 import java.util.List;
033 import java.util.Set;
034 import java.util.Collection;
035 import java.util.logging.Logger;
036
037
038 /**
039 * Select the copies trying to comply with the minFilesPerProcess parameter.
040 * The selection is split in five staged.
041 * <p/>
042 * 1) Assigns all the files that are on one machine only. There is no chioce to
043 * be made there.
044 * <p/>
045 * 2) Checks if, due to the min limit, some groups can't be split into valid jobs.
046 * In that case, looks for files that have one copy on that machines, and assigns
047 * them
048 * <p/>
049 * 3) Assigns all the files that have multiple copies. If preferStorage is used,
050 * it chooses a file on the given storage. Local copies as chosen randomly.
051 * <p/>
052 * 4) Checks if, due to the min limits, some groups can't be split correctly.
053 * In that cases, uses files on NFS to fill the minimum requirements.
054 * <p/>
055 * 5) Assigns the rest of the files on NFS. First, it fills to the max all the
056 * processes already created. Then it assigns the rest to processes that run on
057 * NFS
058 *
059 * @author Gabreile Carcassi & Pavel Jakl
060 */
061 public class MinMaxSingleCopySelector implements CopySelector {
062 static private Logger log = Logger.getLogger(MinMaxSingleCopySelector.class.getName());
063
064 public int selectCopy(QueryResult list, CatalogQuery query,
065 FileAssignment assignment) {
066 // Files that have only one copy on local disk are forced
067 int nFiles = assignFilesWithSingleCopy(list, assignment);
068
069 // Make the jobs valid by distributing part of the files with multiple copies
070 // It can distribute also the one with a copy on NFS
071 nFiles += fillMinMultipleCopyFiles(list, assignment);
072
073 // Distribute the remaining local files
074 // It distributes the files with a copy on NFS depending on the preferStorage
075 nFiles += assignFilesWithMultipleCopy(list, query, assignment);
076
077 // Make the jobs valid by distributing the files on NFS
078 nFiles += fillMinNFSFiles(list, assignment);
079
080 // Distribute the remaining NFS files
081 nFiles += assignNFSFiles(list, assignment);
082
083 return nFiles;
084 }
085
086 private int assignFilesWithSingleCopy(QueryResult list,
087 FileAssignment assignment) {
088 //PJ: add all files which are not located in NFS or HPSS
089 List singleCopy = list.getSinglePhysicalNoNFSorHPSS();
090 // make sure that we can add something
091 if (singleCopy != null) {
092 assignment.addAllPhysical(singleCopy);
093 list.removeAllLogicalOfPhysical(singleCopy);
094 }
095 return singleCopy.size();
096 }
097
098 private int fillMinMultipleCopyFiles(QueryResult list,
099 FileAssignment assignment) {
100 int nFiles = 0;
101 List badLocations = assignment.getInvalidLocations();
102 if (badLocations != null) {
103 Iterator iter = badLocations.iterator();
104 while (iter.hasNext()) {
105 Location location = (Location) iter.next();
106 List files = list.getPhysicalOnLocation(location);
107 int nFilesToAssign = Math.min(assignment.needsMoreFiles(location),
108 files.size());
109 List filesToAssign = files.subList(0, nFilesToAssign);
110 // dont add null files
111 if (filesToAssign != null) {
112 assignment.addAllPhysical(filesToAssign);
113 list.removeAllLogicalOfPhysical(filesToAssign);
114 nFiles += nFilesToAssign;
115 }
116 }
117 }
118 return nFiles;
119 }
120
121 private int assignFilesWithMultipleCopy(QueryResult list, CatalogQuery query, FileAssignment assignment) {
122
123 int nFiles = 0;
124 List physicalFilesToAssign;
125
126 // getPreferStorage and call current method
127 if ("NFS".equals(query.getPreferStorage())) {
128 // get all copies on NFS or local (instance of one file is choosen randomly)
129 physicalFilesToAssign = list.getRandomPhysicalOnNFSOrLocal();
130 } else {
131 // get only files on local
132 physicalFilesToAssign = list.getRandomPhysicalLocal();
133 }
134
135 if (physicalFilesToAssign != null) {
136 Iterator iter = physicalFilesToAssign.iterator();
137 assignment.addAllPhysical(physicalFilesToAssign);
138 list.removeAllLogicalOfPhysical(physicalFilesToAssign);
139 nFiles+=physicalFilesToAssign.size();
140 }
141 return nFiles;
142 }
143
144 private int fillMinNFSFiles(QueryResult list, FileAssignment assignment) {
145 int nFiles = 0;
146 List badLocations = assignment.getInvalidLocations();
147
148 if (badLocations != null) {
149 Iterator iter = badLocations.iterator();
150 List files = list.getPhysicalOnLocation(Location.getNFS());
151 while (iter.hasNext()) {
152 Location location = (Location) iter.next();
153 int nFilesToAssign = Math.min(assignment.needsMoreFiles(location),
154 files.size());
155 List filesToAssign = files.subList(0, nFilesToAssign);
156 files = files.subList(nFilesToAssign, files.size());
157 if (filesToAssign != null) {
158 assignment.addAllPhysical(location, filesToAssign);
159 list.removeAllLogicalOfPhysical(filesToAssign);
160 nFiles += nFilesToAssign;
161 }
162 }
163 }
164 return nFiles;
165 }
166
167 private int assignNFSFiles(QueryResult list, FileAssignment assignment) {
168 int nFiles = 0;
169 Set locations = assignment.getLocations();
170 Iterator iter = locations.iterator();
171 List files = list.getPhysicalOnLocation(Location.getNFS());
172 if (locations != null) {
173 while (iter.hasNext()) {
174 Location location = (Location) iter.next();
175 int nFilesToAssign = Math.min(assignment.canAdd(location),
176 files.size());
177 List filesToAssign = files.subList(0, nFilesToAssign);
178 files = files.subList(nFilesToAssign, files.size());
179 if (filesToAssign != null) {
180 assignment.addAllPhysical(location, filesToAssign);
181 list.removeAllLogicalOfPhysical(filesToAssign);
182 nFiles += nFilesToAssign;
183 }
184 }
185 if (files != null) {
186 assignment.addAllPhysical(files);
187 list.removeAllLogicalOfPhysical(files);
188 }
189 }
190 return files.size();
191 }
192
193 }