// scan has been done, create FmrcInv
private FmrcInv makeFmrcInv(Formatter debug) throws IOException {
try {
Map<CalendarDate, FmrInv> fmrMap =
new HashMap<CalendarDate, FmrInv>(); // all files are grouped by run date in an FmrInv
List<FmrInv> fmrList = new ArrayList<FmrInv>(); // an fmrc is a collection of fmr
// get the inventory, sorted by path
for (MFile f : manager.getFiles()) {
if (logger.isDebugEnabled()) logger.debug("Fmrc: " + config.spec + ": file=" + f.getPath());
GridDatasetInv inv = null;
try {
inv =
GridDatasetInv.open(
manager, f, config.innerNcml); // inventory is discovered for each GDS
} catch (IOException ioe) {
logger.warn("Error opening " + f.getPath() + "(skipped)", ioe);
continue; // skip
}
CalendarDate runDate = inv.getRunDate();
if (debug != null)
debug.format(" opened %s rundate = %s%n", f.getPath(), inv.getRunDateString());
// add to fmr for that rundate
FmrInv fmr = fmrMap.get(runDate);
if (fmr == null) {
fmr = new FmrInv(runDate);
fmrMap.put(runDate, fmr);
fmrList.add(fmr);
}
fmr.addDataset(inv, debug);
}
if (debug != null) debug.format("%n");
// finish the FmrInv
Collections.sort(fmrList);
for (FmrInv fmr : fmrList) {
fmr.finish();
if (logger.isDebugEnabled())
logger.debug(
"Fmrc: spec="
+ config.spec
+ ": fmr rundate="
+ fmr.getRunDate()
+ " nfiles= "
+ fmr.getFiles().size());
}
return new FmrcInv(
"fmrc:" + manager.getCollectionName(), fmrList, config.fmrcConfig.regularize);
} catch (Throwable t) {
logger.error("makeFmrcInv", t);
throw new RuntimeException(t);
}
}
@Test
public void testStreamFilterInDirPartition() throws IOException {
// this dataset 0-6 hour forecasts x 124 runtimes (4x31)
// there are 2 groups, likely miscoded, the smaller group are 0 hour, duplicates, possibly
// miscoded
FeatureCollectionConfig config =
new FeatureCollectionConfig(
"cfrsAnalysis_46",
"test/testCfrsAnalysisOnly",
FeatureCollectionType.GRIB2,
TestDir.cdmUnitTestDir + "gribCollections/cfsr/.*grb2",
null,
null,
null,
"directory",
null);
Formatter errlog = new Formatter();
CollectionSpecParser specp = new CollectionSpecParser(config.spec, errlog);
Path rootPath = Paths.get(specp.getRootDir());
try (DirectoryCollection dcm =
new DirectoryCollection(config.collectionName, rootPath, true, config.olderThan, logger)) {
dcm.putAuxInfo(FeatureCollectionConfig.AUX_CONFIG, config);
if (specp.getFilter() != null)
dcm.setStreamFilter(new StreamFilter(specp.getFilter(), specp.getFilterOnName()));
int count = 0;
for (MFile mfile : dcm.getFilesSorted()) {
System.out.printf("%s%n", mfile);
assert mfile.getName().equals("pwat.gdas.199612.grb2");
count++;
}
assert count == 1;
}
}
@Override
public int compareTo(MFile o) {
return getPath().compareTo(o.getPath());
}
