/**
* Accept datasets whose last modified date is at least the last modified limit of
* milliseconds in the past.
*
* @param dataset the dataset to filter
* @return true if the datasets last modified date is at least lastModifiedLimitInMillis in
* the past.
*/
public boolean accept(MFile dataset) {
long lastModDate = dataset.getLastModified();
if (lastModDate > 0) {
long now = System.currentTimeMillis();
if (now - lastModDate > lastModifiedLimitInMillis) return true;
}
return false;
}
// read all records in all files,
// divide into groups based on GDS hash
// each group has an arraylist of all records that belong to it.
// for each group, run rectlizer to derive the coordinates and variables
public List<Group> makeAggregatedGroups(
ArrayList<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();
f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
int total = 0;
int fileno = 0;
for (CollectionManager dcm : collections) {
// dcm.scanIfNeeded(); // LOOK ??
f.format(" dcm= %s%n", dcm);
Map<Integer, Integer> gdsConvert = (Map<Integer, Integer>) dcm.getAuxInfo("gdsHash");
for (MFile mfile : dcm.getFiles()) {
// f.format("%3d: %s%n", fileno, mfile.getPath());
filenames.add(mfile.getPath());
Grib2Index index = new Grib2Index();
try {
if (!index.readIndex(
mfile.getPath(),
mfile.getLastModified(),
force)) { // heres where the index date is checked against the data file
index.makeIndex(mfile.getPath(), f);
f.format(
" Index written: %s == %d records %n",
mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
} else if (debug) {
f.format(
" Index read: %s == %d records %n",
mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
}
} catch (IOException ioe) {
f.format(
"GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n skipping %s%n",
ioe.getMessage(), mfile.getPath() + Grib2Index.IDX_EXT);
continue;
}
for (Grib2Record gr : index.getRecords()) {
gr.setFile(fileno); // each record tracks which file it belongs to
int gdsHash =
gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
if (gdsConvert != null
&& gdsConvert.get(gdsHash)
!= null) { // allow external config to muck with gdsHash. Why? because of error in
// encoding
gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching
}
Group g = gdsMap.get(gdsHash);
if (g == null) {
g = new Group(gr.getGDSsection(), gdsHash);
gdsMap.put(gdsHash, g);
}
g.records.add(gr);
total++;
}
fileno++;
}
}
f.format(" total grib records= %d%n", total);
Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter();
List<Group> result = new ArrayList<Group>(gdsMap.values());
for (Group g : result) {
g.rect = new Grib2Rectilyser(g.records, g.gdsHash);
f.format(" GDS hash %d == ", g.gdsHash);
g.rect.make(f, c);
}
f.format(
" Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);
return result;
}