001 /*
002 * STARCatalog.java
003 *
004 * Created on June 27, 2006, 7:33 PM
005 *
006 * This file is part of the STAR Scheduler.
007 * Copyright (c) 2002-2006 STAR Collaboration - Brookhaven National Laboratory
008 *
009 * STAR Scheduler is free software; you can redistribute it and/or modify
010 * it under the terms of the GNU General Public License as published by
011 * the Free Software Foundation; either version 2 of the License, or
012 * (at your option) any later version.
013 *
014 * STAR Scheduler is distributed in the hope that it will be useful,
015 * but WITHOUT ANY WARRANTY; without even the implied warranty of
016 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
017 * GNU General Public License for more details.
018 *
019 * You should have received a copy of the GNU General Public License
020 * along with STAR Scheduler; if not, write to the Free Software
021 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
022 */
023
024 package gov.bnl.star.offline.scheduler.dataset.catalog;
025
026 import gov.bnl.star.offline.scheduler.request.Request;
027 import java.io.BufferedReader;
028 import java.io.File;
029 import java.io.FileNotFoundException;
030 import java.io.FileOutputStream;
031 import java.io.InputStreamReader;
032 import java.io.PrintStream;
033 import java.io.Reader;
034
035 import java.net.URL;
036 import java.util.*;
037 import java.util.logging.Level;
038 import org.apache.log4j.Logger;
039
040 import gov.bnl.star.offline.scheduler.util.CSHCommandLineTask; //for testing
041 import gov.bnl.star.offline.scheduler.CatalogQuery;
042 import gov.bnl.star.offline.scheduler.dataset.Dataset;
043 import gov.bnl.star.offline.scheduler.dataset.catalog.PrintDots;
044
045 import java.io.IOException;
046
047 /** The hard implementation of the SUMS catalog interface for the S.T.A.R. experiment.
048 *
049 * @author lbhajdu
050 */
051 public class STARCatalog {
052
053 static private Logger log = Logger.getLogger(STARCatalog.class.getName());
054 private static Dataset dataset = null;
055 private static Request request;
056 private static PrintStream datasetFileStream;
057
058 PrintDots printDots;
059
060 /** Creates a new instance of STARCatalog */
061 public STARCatalog() {
062
063
064 }
065
066
067
068 public void fillDataset(Dataset dataset, Request request){
069
070 this.dataset = dataset;
071 this.request = request;
072
073 try {//Open the file we will be writting to
074 datasetFileStream = new PrintStream(new FileOutputStream(new File(dataset.getDatasetName()),true));
075 } catch (FileNotFoundException ex) {
076 ex.printStackTrace();
077 }
078
079 int nQueries = 0;
080
081 for (int n = 0; n < request.getInputList().size(); n++) {
082 if (request.getInputList().get(n) instanceof CatalogQuery) {
083 nQueries++;
084 }
085 else if (request.getInputList().get(n) instanceof URL){
086 URLToEntryWriter((URL) request.getInputList().get(n));
087 }
088 }
089
090 CatalogQuery[] queries = new CatalogQuery[nQueries];
091
092 for (int n = 0; n < queries.length; n++) {
093 //queries[n] = (CatalogQuery) request.getInputList().get(n);
094 getDataSet((CatalogQuery) request.getInputList().get(n));
095 }
096
097 // Close to stream now that we have filled the dataset file
098 datasetFileStream.flush();
099 datasetFileStream.close();
100
101 }
102
103
104
105
106 /*
107 public Dataset getDataSet(CatalogQuery[] querys, String name){
108
109
110 this.dataset = new Dataset(name);
111
112 for(int i = 0; i < querys.length ; i++){
113 getDataSet(querys[i], name);
114 }
115
116 return dataset;
117
118 }
119 */
120
121 CatalogQuery currentQuery;
122
123 private void getDataSet(CatalogQuery query){
124
125 currentQuery = query;
126
127 if(query.getNFiles() == null){
128 throw new RuntimeException("The query query.getNFiles() file limit was null");
129 }
130 else if(query.getNFiles().intValue() == -1){ //If the user asks for alll the files
131 executeQuery(query.getQuery(), null, null);
132 }
133 else if(query.getNFiles().intValue() > 0){ //If the user set nfiles to a real value
134 executeQuery(query.getQuery(), null, query.getNFiles());
135 }
136 else{
137 throw new RuntimeException("Couldn't execute query the query.getNFiles() = " + query.getNFiles() + " is not valid value.");
138 }
139
140 /*
141 }
142 else{
143 System.out.println("The limit was null");
144 executeQuery(query.getQuery(), null, null);
145 }
146 */
147
148 }
149
150
151
152
153
154 private void executeQuery(String query, Integer start, Integer limit) {
155
156 String inputOrder = "";
157 List inputOrderList = request.getInputOrder();
158 if(inputOrderList != null){
159 for(int i = 0; i != inputOrderList.size(); i++){
160 String item = (String) inputOrderList.get(i);
161 boolean isAlreadyOnList = item.equals("fdid")||item.equals("storage")||item.equals("site")||item.equals("node")||item.equals("path")||item.equals("filename")||item.equals("events");
162 if(!isAlreadyOnList) inputOrder = inputOrder + "," + item;
163 }
164 }
165
166
167
168 String commandLine = "get_file_list.pl -keys fdid,storage,site,node,path,filename,events" + inputOrder + " -cond " + query + "";
169
170
171 if (start != null) {
172 commandLine += (" -start " + start);
173 }
174
175 if (limit != null) {
176
177 commandLine += (" -limit " + limit);
178 } else {
179 commandLine += (" -limit 0");
180 }
181
182 executeCommand(commandLine);
183
184 }
185
186
187
188 private void executeCommand(String commandLine) {
189
190 log.info("Executing : " + commandLine);
191 System.out.print("Executing : " + commandLine);
192 int linesRecovered = 0; // Entries Recored
193
194 PrintDots printDots = new PrintDots();
195 printDots.start();
196 printDots.resumeDots();
197
198 try {
199
200 String line;
201 Process p = Runtime.getRuntime().exec(commandLine);
202
203 BufferedReader input = new BufferedReader(new InputStreamReader(p.getInputStream()));
204 while ((line = input.readLine()) != null) {
205
206 if(line.matches("[0-9]*::[a-zA-Z]*::[a-zA-Z]*::[a-zA-Z0-9:.]*::[^:]*::[^:/]*::[0-9]*.*")){
207 //Write everyting out to the data set
208 datasetFileStream.println(line);
209 linesRecovered ++;
210 dataset.incrementSize();
211 }else{
212 printDots.pauseDots();
213 System.out.println("Line does not match expected format, dropping line >>>> " + line);
214 printDots.resumeDots();
215 }
216
217
218 }
219 input.close();
220 printDots.pauseDots();
221
222 if( p.waitFor() != 0){
223 log.fatal("There was an error recovering the dataset (catalog error) :\n" + commandLine);
224 throw new RuntimeException("There was an error recovering the dataset (catalog error) :\n" + commandLine);
225 }
226
227 } catch (Exception err) {
228 err.printStackTrace();
229 }
230
231 printDots.exitPrinting();
232 printDots.stop();
233 System.out.println("...." + linesRecovered +" Entries Recovered ");
234
235
236 currentQuery.setFilesReturned(linesRecovered);
237
238 }
239
240
241
242
243
244 /**
245 *Convert a file url to an entry in the database
246 *
247 *basic format example:
248 *
249 * {FLN}::{STORAGE_SERVISE}::{SITE}::{HOST}::{PATH}::{FILE_NAME}::{NUMBER_OF_EVENTS}{other}
250 * 9404397::NFS::BNL::localhost::/star/data41/reco/productionMinBias/ReversedFullField/P05ic/2004/023::st_physics_adc_5023001_raw_1050014.MuDst.root::155
251 *
252 */
253 private void URLToEntryWriter(URL url){
254
255
256 //String baseEntry = "{FLN}::{STORAGE_SERVISE}::{SITE}::{HOST}::{PATH}::{FILE_NAME}::{NUMBER_OF_EVENTS}{OTHER}";
257 String baseEntry = "::{STORAGE_SERVISE}::::{HOST}::{PATH}::{FILE_NAME}::{NUMBER_OF_EVENTS}{OTHER}";
258
259
260 String host = url.getHost(); //if this is an ip address to format may come out funny
261 int port = url.getPort();
262 if(host != null){ //fill in the host block
263 host = host.trim();
264 if(! host.equals("")){ //if there is some kind of string for the host fill it in
265 if(port != -1) host = host + ":" + String.valueOf(port);
266 baseEntry = baseEntry.replaceFirst("\\{HOST\\}",host);
267 baseEntry = baseEntry.replaceFirst("\\{STORAGE_SERVISE\\}","local");
268 } else{
269 baseEntry = baseEntry.replaceFirst("\\{HOST\\}",""); //if the host is "" leave it blank
270 baseEntry = baseEntry.replaceFirst("\\{STORAGE_SERVISE\\}","NFS");
271 }
272 } else{
273 baseEntry = baseEntry.replaceFirst("\\{HOST\\}",""); //if the host is null leave it blank
274 baseEntry = baseEntry.replaceFirst("\\{STORAGE_SERVISE\\}","NFS");
275 }
276
277
278 String path_filename = url.getFile();
279 String path = "";
280 String filename = "";
281 String events = "";
282 if(path_filename.matches(".*/[^/]*") ){ //if it is not just a file name, but also a path then it needs to be split
283 path = path_filename.replaceAll("(.*)/[^/]*","$1");
284 filename = path_filename.replaceAll(".*/([^/]*)","$1");
285 }
286 else{ //if it has no '/' in it hen it is just a file file name without a path
287 filename = path_filename;
288 }
289
290 if(filename.matches("^.* [0-9]*$")){ //if it end with the number of events
291 events = filename.replaceAll(".* ([0-9]*) *$","$1");
292 filename = filename.replaceAll("(.*) [0-9]* *$","$1");
293 //System.out.println("name has events -->" + filename + "------->" + events); //checks the split in case there is ever a problem
294
295 }
296
297
298 baseEntry = baseEntry.replaceAll("\\{PATH\\}", path);
299 baseEntry = baseEntry.replaceAll("\\{FILE_NAME\\}", filename);
300 baseEntry = baseEntry.replaceAll("\\{NUMBER_OF_EVENTS\\}", events);
301
302
303 if(request.getOutputList() != null){
304 if(request.getOutputList().size() != 0 ) baseEntry = baseEntry.replaceFirst("\\{OTHER\\}", "::");
305 else baseEntry = baseEntry.replaceFirst("\\{OTHER\\}", "");
306 } else baseEntry = baseEntry.replaceFirst("\\{OTHER\\}", "");
307
308
309 datasetFileStream.println(baseEntry);
310
311
312 }
313
314
315
316
317
318
319 /*
320
321 System.out.println("bebug step 4");
322
323
324
325 try {
326
327
328 //This code caused more problems then it has solved. The ,<,>,|| don"t work when it is used.
329 //This code replaces the code commented out below. The changes where needed because the code below does not allow the user to use " " in there code.
330 //CSHCommandLineTask task = new CSHCommandLineTask(commandLine, true, 0);
331 //task.execute(); //run the get_file_list.pl command
332 //List result = parseOutput(task.getOutputReader(), attributeNames); //get the output back
333
334
335 System.out.println("Execute query: " + commandLine );
336
337 Process proc = Runtime.getRuntime().exec("touch blax.tmp");
338 proc.waitFor();
339 proc = Runtime.getRuntime().exec(commandLine);
340
341 // List result = parseOutput(new InputStreamReader(proc.getInputStream()), attributeNames);
342 proc.waitFor();
343
344 } catch (Exception e) {
345 throw new RuntimeException("Couldn't execute query: " + commandLine + "\n" + e.getMessage());
346 }
347
348 */
349
350
351
352
353
354
355 /*
356 public void resolveQuires(){
357 try {
358
359 //how to get catalog Q
360 Process proc = Runtime.getRuntime().exec("bla >> file");
361
362 } catch (IOException ex) {
363 ex.printStackTrace();
364 }
365
366
367 }
368 */
369
370
371
372
373
374
375
376 //public static List bla = new ArrayList();
377
378 /** Parses the output of a get_file_list.pl command.
379 * @param in the input reader with the command output
380 * @return a list of PhysicalFiles
381 */
382 /*
383 protected static List parseOutput(Reader in, List attributeNames) {
384 try {
385 List files = new ArrayList();
386 BufferedReader reader = new BufferedReader(in);
387 String line;
388
389 while ((line = reader.readLine()) != null) {
390
391
392 List tokens = parseOutputLine(line);
393 PhysicalFile newFile = new PhysicalFile((String) tokens.get(0), (String) tokens.get(1),(String) tokens.get(2), (String) tokens.get(3), (String) tokens.get(4), null);
394 for (int nAtt = 5; nAtt < tokens.size(); nAtt++) {
395 int nName = nAtt - 5;
396 String name = (String) attributeNames.get(nName);
397 String token = (String) tokens.get(nAtt);
398 // Hack to prevent garbage values: 0 for nEvents really means that
399 // the value is not set
400 if ("events".equals(name)) {
401 if (!"0".equals(token)) {
402 newFile.setAttribute(name, new Integer(token));
403 }
404 } else {
405 newFile.setAttribute(name, new Integer(token));
406 }
407 }
408 files.add(newFile);
409 }
410
411 return files;
412 } catch (Exception e) {
413 log.error("Couldn't parse output of STAR catalog query", e);
414
415 return new ArrayList();
416 }
417 }
418
419 private static List parseOutputLine(String line) {
420 log.debug("Parsing : " + line);
421
422 List tokens = new ArrayList();
423 StringTokenizer tokenizer = new StringTokenizer(line, ":");
424 int nToken = 0;
425 int parsedLength = 0;
426
427 while (tokenizer.hasMoreTokens()) {
428 String token = tokenizer.nextToken();
429
430 // If more spaces is detected, than a token was empty, and a position
431 // should be skipped.
432 if (nToken != 0) {
433 int nWhiteSpaces = line.indexOf(token, parsedLength) -
434 parsedLength;
435 nToken += ((nWhiteSpaces / 2) - 1);
436 parsedLength += nWhiteSpaces;
437 }
438
439 // Saves the value of the token in the correct position
440 tokens.add(token);
441 parsedLength += token.length();
442 nToken++;
443 }
444
445 return tokens;
446 }
447 */
448 /*
449 private List executeQuery(String query, Integer start, Integer limit) {
450 if (CatalogManager.getAttrList() != null) {
451 return executeQuery(query, start, limit, CatalogManager.getAttrList());
452 } else {
453 return executeQuery(query, start, limit, new ArrayList());
454 }
455 }*/
456
457
458 /*
459 public void executeQuery(String query) {
460 executeQuery(query, null, null);
461 }
462
463 public Iterator executeQuery(String query, int nExpectedResults) {
464 return new CatalogIterator(query, nExpectedResults);
465 }
466
467 public Iterator executeQuery(String query, int start, int limit) {
468 return executeQuery(query, new Integer(start), new Integer(limit))
469 .iterator();
470 }
471
472 private class CatalogIterator implements Iterator {
473 private String query;
474 private List buffer;
475 private int nCurrentInBuffer;
476 private int nCurrentStart;
477 private int bufferSize;
478
479 public CatalogIterator(String query, int bufferSize) {
480 this.query = query;
481 this.bufferSize = bufferSize;
482 }
483
484 private void fetchData() {
485 buffer = executeQuery(query, new Integer(nCurrentStart),
486 new Integer(bufferSize));
487 nCurrentInBuffer = 0;
488 nCurrentStart += bufferSize;
489 }
490
491 public boolean hasNext() {
492 if (buffer == null) {
493 fetchData();
494
495 boolean test = hasNext();
496
497 return test;
498 }
499
500 if (buffer.size() == 0) {
501 return false;
502 }
503
504 if (nCurrentInBuffer < buffer.size()) {
505 return true;
506 }
507
508 fetchData();
509
510 return hasNext();
511 }
512
513 public Object next() {
514 if (hasNext()) {
515 Object temp = buffer.get(nCurrentInBuffer);
516 nCurrentInBuffer++;
517
518 return temp;
519 }
520
521 throw new NoSuchElementException(
522 "Catalog query has no more results.");
523 }
524
525 public void remove() {
526 throw new UnsupportedOperationException(
527 "Catalog query result don't support remove");
528 }
529 }
530
531 */
532
533
534
535
536
537
538 }