private boolean filterTinv(
Grib2Record gr, FeatureCollectionConfig.GribIntvFilter intvFilter, Formatter f) {
int[] intv = tables.getForecastTimeIntervalOffset(gr);
if (intv == null) return false;
int haveLength = intv[1] - intv[0];
// HACK
if (haveLength == 0 && intvFilter.isZeroExcluded()) { // discard 0,0
if ((intv[0] == 0) && (intv[1] == 0)) {
f.format(" FILTER INTV [0, 0] %s%n", gr);
return true;
}
return false;
} else if (intvFilter.hasMap()) {
int discipline = gr.getIs().getDiscipline();
int category = gr.getPDS().getParameterCategory();
int number = gr.getPDS().getParameterNumber();
int id = (discipline << 16) + (category << 8) + number;
Integer needLength = intvFilter.getLengthById(id);
if (needLength != null && needLength != haveLength) {
f.format(" FILTER INTV [%d != %d] %s%n", haveLength, needLength, gr);
return true;
}
}
return false;
}
/**
* tricky bit of business. recapture the entire record based on drs position. for validation.
*
* @param raf from this RandomAccessFile
* @param drsPos Grib2SectionDataRepresentation starts here
*/
public static Grib2Record findRecordByDrspos(RandomAccessFile raf, long drsPos)
throws IOException {
Grib2Record result = null;
Grib2RecordScanner scanner = new Grib2RecordScanner(raf);
long pos = Math.max(0, drsPos - 10000); // go back 10000 bytes
raf.seek(pos);
while (scanner.hasNext()) { // find GRIB header
result = scanner.next();
if (result.getDataRepresentationSection().getStartingPosition() == drsPos) return result;
if (raf.getFilePointer() > drsPos) break;
}
return null;
}
// return true if got another repeat out of this record
// side effect is that the new record is in repeatRecord
private boolean nextRepeating() throws IOException {
raf.seek(repeatPos);
// octets 1-4 (Length of GDS)
int length = GribNumbers.int4(raf);
int section = raf.read();
raf.seek(repeatPos);
if (section == 2) {
repeatRecord.setLus(new Grib2SectionLocalUse(raf));
repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else if (section == 3) {
repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else if (section == 4) {
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else {
if (debugRepeat) System.out.printf(" REPEAT Terminate %d%n", section);
repeatPos = -1;
repeatRecord = null;
repeatBms = null;
return false;
}
// look for repeating bms
Grib2SectionBitMap bms = repeatRecord.getBitmapSection();
if (bms.getBitMapIndicator() == 254) {
// replace BMS with last good one
if (repeatBms == null) throw new IllegalStateException("No bms in repeating section");
repeatRecord.setBms(repeatBms, true);
// debug
if (debugRepeat) System.out.printf("replaced bms %d%n", section);
repeatRecord.repeat += 1000;
} else if (bms.getBitMapIndicator() == 0) {
// track last good bms
repeatBms = repeatRecord.getBitmapSection();
}
// keep only unique gds
if ((section == 2) || (section == 3)) {
// look for duplicate gds
Grib2SectionGridDefinition gds = repeatRecord.getGDSsection();
long crc = gds.calcCRC();
Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
if (gdsCached != null) repeatRecord.setGdss(gdsCached);
else gdsMap.put(crc, gds);
}
// check to see if we are at the end
long pos = raf.getFilePointer();
long ending = repeatRecord.getIs().getEndPos();
if (pos + 34 < ending) { // give it 30 bytes of slop
if (debugRepeat) System.out.printf(" REPEAT AGAIN %d != %d%n", pos + 4, ending);
repeatPos = pos;
return true;
}
if (debug)
System.out.printf(
" REPEAT read until %d grib ending at %d header ='%s'%n",
raf.getFilePointer(), ending, StringUtil2.cleanup(header));
// check that end section is correct
raf.seek(ending - 4);
for (int i = 0; i < 4; i++) {
if (raf.read() != 55) {
log.warn(
" REPEAT Missing End of GRIB message at pos="
+ ending
+ " header= "
+ StringUtil2.cleanup(header)
+ " for="
+ raf.getLocation());
break;
}
}
lastPos = raf.getFilePointer();
if (debugRepeat) System.out.printf(" REPEAT DONE%n");
repeatPos = -1; // no more repeats in this record
return true;
}
private void createIndex(
File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
throws IOException {
Grib2Record first = null; // take global metadata from here
if (indexFile.exists()) indexFile.delete(); // replace it
f.format(" createIndex for %s%n", indexFile.getPath());
RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
for (Group g : groups) {
g.fileSet = new HashSet<Integer>();
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
countRecords += vb.recordMap.length;
}
}
long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
f.format(
" write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
countBytes, countRecords, bytesPerRecord);
if (first == null) {
logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
}
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(gc.getName());
for (String fn : filenames) indexBuilder.addFiles(fn);
for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));
/* int count = 0;
for (DatasetCollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
count++;
} */
// what about just storing first ??
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
Grib2Pds pds = first.getPDS();
indexBuilder.setGenProcessType(pds.getGenProcessType());
indexBuilder.setGenProcessId(pds.getGenProcessId());
indexBuilder.setBackProcessId(pds.getBackProcessId());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format(" write GribCollectionIndex= %d bytes%n", b.length);
} finally {
f.format(" file size = %d bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
}
// read all records in all files,
// divide into groups based on GDS hash
// each group has an arraylist of all records that belong to it.
// for each group, run rectlizer to derive the coordinates and variables
public List<Group> makeAggregatedGroups(
ArrayList<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();
f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
int total = 0;
int fileno = 0;
for (CollectionManager dcm : collections) {
// dcm.scanIfNeeded(); // LOOK ??
f.format(" dcm= %s%n", dcm);
Map<Integer, Integer> gdsConvert = (Map<Integer, Integer>) dcm.getAuxInfo("gdsHash");
for (MFile mfile : dcm.getFiles()) {
// f.format("%3d: %s%n", fileno, mfile.getPath());
filenames.add(mfile.getPath());
Grib2Index index = new Grib2Index();
try {
if (!index.readIndex(
mfile.getPath(),
mfile.getLastModified(),
force)) { // heres where the index date is checked against the data file
index.makeIndex(mfile.getPath(), f);
f.format(
" Index written: %s == %d records %n",
mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
} else if (debug) {
f.format(
" Index read: %s == %d records %n",
mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
}
} catch (IOException ioe) {
f.format(
"GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n skipping %s%n",
ioe.getMessage(), mfile.getPath() + Grib2Index.IDX_EXT);
continue;
}
for (Grib2Record gr : index.getRecords()) {
gr.setFile(fileno); // each record tracks which file it belongs to
int gdsHash =
gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
if (gdsConvert != null
&& gdsConvert.get(gdsHash)
!= null) { // allow external config to muck with gdsHash. Why? because of error in
// encoding
gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching
}
Group g = gdsMap.get(gdsHash);
if (g == null) {
g = new Group(gr.getGDSsection(), gdsHash);
gdsMap.put(gdsHash, g);
}
g.records.add(gr);
total++;
}
fileno++;
}
}
f.format(" total grib records= %d%n", total);
Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter();
List<Group> result = new ArrayList<Group>(gdsMap.values());
for (Group g : result) {
g.rect = new Grib2Rectilyser(g.records, g.gdsHash);
f.format(" GDS hash %d == ", g.gdsHash);
g.rect.make(f, c);
}
f.format(
" Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);
return result;
}
/**
* Constructor.
*
* @param firstRecord in the Grib file
*/
public Grib2Lookup(Grib2Record firstRecord) {
this.firstPDS = firstRecord.getPDS();
this.firstID = firstRecord.getId();
this.firstDRS = firstRecord.getDRS();
}
// read all records in all files,
// divide into groups based on GDS hash
// each group has an arraylist of all records that belong to it.
// for each group, run rectlizer to derive the coordinates and variables
public List<Group> makeAggregatedGroups(
List<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();
boolean intvMerge = mergeIntvDefault;
f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
int total = 0;
int fileno = 0;
for (CollectionManager dcm : collections) {
f.format(" dcm= %s%n", dcm);
FeatureCollectionConfig.GribConfig config =
(FeatureCollectionConfig.GribConfig)
dcm.getAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG);
Map<Integer, Integer> gdsConvert = (config != null) ? config.gdsHash : null;
FeatureCollectionConfig.GribIntvFilter intvMap = (config != null) ? config.intvFilter : null;
intvMerge =
(config == null) || (config.intvMerge == null) ? mergeIntvDefault : config.intvMerge;
for (MFile mfile : dcm.getFiles()) {
// f.format("%3d: %s%n", fileno, mfile.getPath());
filenames.add(mfile.getPath());
Grib2Index index = null;
try {
index =
(Grib2Index)
GribIndex.readOrCreateIndexFromSingleFile(
false, !isSingleFile, mfile, config, force, f);
} catch (IOException ioe) {
logger.warn(
"GribCollectionBuilder {}: reading/Creating gbx9 index failed err={}",
gc.getName(),
ioe.getMessage());
f.format(
"GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n skipping %s%n",
ioe.getMessage(), mfile.getPath() + GribIndex.IDX_EXT);
continue;
}
for (Grib2Record gr : index.getRecords()) {
if (this.tables == null) {
Grib2SectionIdentification ids =
gr.getId(); // so all records must use the same table (!)
this.tables =
Grib2Customizer.factory(
ids.getCenter_id(),
ids.getSubcenter_id(),
ids.getMaster_table_version(),
ids.getLocal_table_version());
if (config != null)
tables.setTimeUnitConverter(
config
.getTimeUnitConverter()); // LOOK doesnt really work with multiple collections
}
if (intvMap != null && filterTinv(gr, intvMap, f)) continue; // skip
gr.setFile(fileno); // each record tracks which file it belongs to
int gdsHash =
gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
if (gdsConvert != null
&& gdsConvert.get(gdsHash)
!= null) // allow external config to muck with gdsHash. Why? because of error in
// encoding
gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching
Group g = gdsMap.get(gdsHash);
if (g == null) {
g = new Group(gr.getGDSsection(), gdsHash);
gdsMap.put(gdsHash, g);
}
g.records.add(gr);
total++;
}
fileno++;
}
}
f.format(" total grib records= %d%n", total);
Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter(); // debugging
List<Group> result = new ArrayList<Group>(gdsMap.values());
for (Group g : result) {
g.rect = new Grib2Rectilyser(tables, g.records, g.gdsHash, intvMerge);
f.format(" GDS hash %d == ", g.gdsHash);
g.rect.make(f, c, filenames);
}
f.format(
" Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);
return result;
}
