// read all records in all files, // divide into groups based on GDS hash // each group has an arraylist of all records that belong to it. // for each group, run rectlizer to derive the coordinates and variables public List<Group> makeAggregatedGroups( ArrayList<String> filenames, CollectionManager.Force force, Formatter f) throws IOException { Map<Integer, Group> gdsMap = new HashMap<Integer, Group>(); f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName()); int total = 0; int fileno = 0; for (CollectionManager dcm : collections) { // dcm.scanIfNeeded(); // LOOK ?? f.format(" dcm= %s%n", dcm); Map<Integer, Integer> gdsConvert = (Map<Integer, Integer>) dcm.getAuxInfo("gdsHash"); for (MFile mfile : dcm.getFiles()) { // f.format("%3d: %s%n", fileno, mfile.getPath()); filenames.add(mfile.getPath()); Grib2Index index = new Grib2Index(); try { if (!index.readIndex( mfile.getPath(), mfile.getLastModified(), force)) { // heres where the index date is checked against the data file index.makeIndex(mfile.getPath(), f); f.format( " Index written: %s == %d records %n", mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size()); } else if (debug) { f.format( " Index read: %s == %d records %n", mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size()); } } catch (IOException ioe) { f.format( "GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n skipping %s%n", ioe.getMessage(), mfile.getPath() + Grib2Index.IDX_EXT); continue; } for (Grib2Record gr : index.getRecords()) { gr.setFile(fileno); // each record tracks which file it belongs to int gdsHash = gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records if (gdsConvert != null && gdsConvert.get(gdsHash) != null) { // allow external config to muck with gdsHash. Why? because of error in // encoding gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching } Group g = gdsMap.get(gdsHash); if (g == null) { g = new Group(gr.getGDSsection(), gdsHash); gdsMap.put(gdsHash, g); } g.records.add(gr); total++; } fileno++; } } f.format(" total grib records= %d%n", total); Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter(); List<Group> result = new ArrayList<Group>(gdsMap.values()); for (Group g : result) { g.rect = new Grib2Rectilyser(g.records, g.gdsHash); f.format(" GDS hash %d == ", g.gdsHash); g.rect.make(f, c); } f.format( " Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n", c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records); return result; }
// return true if got another repeat out of this record // side effect is that the new record is in repeatRecord private boolean nextRepeating() throws IOException { raf.seek(repeatPos); // octets 1-4 (Length of GDS) int length = GribNumbers.int4(raf); int section = raf.read(); raf.seek(repeatPos); if (section == 2) { repeatRecord.setLus(new Grib2SectionLocalUse(raf)); repeatRecord.setGdss(new Grib2SectionGridDefinition(raf)); repeatRecord.setPdss(new Grib2SectionProductDefinition(raf)); repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf)); repeatRecord.setBms(new Grib2SectionBitMap(raf), false); repeatRecord.setDataSection(new Grib2SectionData(raf)); repeatRecord.repeat = section; } else if (section == 3) { repeatRecord.setGdss(new Grib2SectionGridDefinition(raf)); repeatRecord.setPdss(new Grib2SectionProductDefinition(raf)); repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf)); repeatRecord.setBms(new Grib2SectionBitMap(raf), false); repeatRecord.setDataSection(new Grib2SectionData(raf)); repeatRecord.repeat = section; } else if (section == 4) { repeatRecord.setPdss(new Grib2SectionProductDefinition(raf)); repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf)); repeatRecord.setBms(new Grib2SectionBitMap(raf), false); repeatRecord.setDataSection(new Grib2SectionData(raf)); repeatRecord.repeat = section; } else { if (debugRepeat) System.out.printf(" REPEAT Terminate %d%n", section); repeatPos = -1; repeatRecord = null; repeatBms = null; return false; } // look for repeating bms Grib2SectionBitMap bms = repeatRecord.getBitmapSection(); if (bms.getBitMapIndicator() == 254) { // replace BMS with last good one if (repeatBms == null) throw new IllegalStateException("No bms in repeating section"); repeatRecord.setBms(repeatBms, true); // debug if (debugRepeat) System.out.printf("replaced bms %d%n", section); repeatRecord.repeat += 1000; } else if (bms.getBitMapIndicator() == 0) { // track last good bms repeatBms = repeatRecord.getBitmapSection(); } // keep only unique gds if ((section == 2) || (section == 3)) { // look for duplicate gds Grib2SectionGridDefinition gds = repeatRecord.getGDSsection(); long crc = gds.calcCRC(); Grib2SectionGridDefinition gdsCached = gdsMap.get(crc); if (gdsCached != null) repeatRecord.setGdss(gdsCached); else gdsMap.put(crc, gds); } // check to see if we are at the end long pos = raf.getFilePointer(); long ending = repeatRecord.getIs().getEndPos(); if (pos + 34 < ending) { // give it 30 bytes of slop if (debugRepeat) System.out.printf(" REPEAT AGAIN %d != %d%n", pos + 4, ending); repeatPos = pos; return true; } if (debug) System.out.printf( " REPEAT read until %d grib ending at %d header ='%s'%n", raf.getFilePointer(), ending, StringUtil2.cleanup(header)); // check that end section is correct raf.seek(ending - 4); for (int i = 0; i < 4; i++) { if (raf.read() != 55) { log.warn( " REPEAT Missing End of GRIB message at pos=" + ending + " header= " + StringUtil2.cleanup(header) + " for=" + raf.getLocation()); break; } } lastPos = raf.getFilePointer(); if (debugRepeat) System.out.printf(" REPEAT DONE%n"); repeatPos = -1; // no more repeats in this record return true; }
// read all records in all files, // divide into groups based on GDS hash // each group has an arraylist of all records that belong to it. // for each group, run rectlizer to derive the coordinates and variables public List<Group> makeAggregatedGroups( List<String> filenames, CollectionManager.Force force, Formatter f) throws IOException { Map<Integer, Group> gdsMap = new HashMap<Integer, Group>(); boolean intvMerge = mergeIntvDefault; f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName()); int total = 0; int fileno = 0; for (CollectionManager dcm : collections) { f.format(" dcm= %s%n", dcm); FeatureCollectionConfig.GribConfig config = (FeatureCollectionConfig.GribConfig) dcm.getAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG); Map<Integer, Integer> gdsConvert = (config != null) ? config.gdsHash : null; FeatureCollectionConfig.GribIntvFilter intvMap = (config != null) ? config.intvFilter : null; intvMerge = (config == null) || (config.intvMerge == null) ? mergeIntvDefault : config.intvMerge; for (MFile mfile : dcm.getFiles()) { // f.format("%3d: %s%n", fileno, mfile.getPath()); filenames.add(mfile.getPath()); Grib2Index index = null; try { index = (Grib2Index) GribIndex.readOrCreateIndexFromSingleFile( false, !isSingleFile, mfile, config, force, f); } catch (IOException ioe) { logger.warn( "GribCollectionBuilder {}: reading/Creating gbx9 index failed err={}", gc.getName(), ioe.getMessage()); f.format( "GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n skipping %s%n", ioe.getMessage(), mfile.getPath() + GribIndex.IDX_EXT); continue; } for (Grib2Record gr : index.getRecords()) { if (this.tables == null) { Grib2SectionIdentification ids = gr.getId(); // so all records must use the same table (!) this.tables = Grib2Customizer.factory( ids.getCenter_id(), ids.getSubcenter_id(), ids.getMaster_table_version(), ids.getLocal_table_version()); if (config != null) tables.setTimeUnitConverter( config .getTimeUnitConverter()); // LOOK doesnt really work with multiple collections } if (intvMap != null && filterTinv(gr, intvMap, f)) continue; // skip gr.setFile(fileno); // each record tracks which file it belongs to int gdsHash = gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records if (gdsConvert != null && gdsConvert.get(gdsHash) != null) // allow external config to muck with gdsHash. Why? because of error in // encoding gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching Group g = gdsMap.get(gdsHash); if (g == null) { g = new Group(gr.getGDSsection(), gdsHash); gdsMap.put(gdsHash, g); } g.records.add(gr); total++; } fileno++; } } f.format(" total grib records= %d%n", total); Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter(); // debugging List<Group> result = new ArrayList<Group>(gdsMap.values()); for (Group g : result) { g.rect = new Grib2Rectilyser(tables, g.records, g.gdsHash, intvMerge); f.format(" GDS hash %d == ", g.gdsHash); g.rect.make(f, c, filenames); } f.format( " Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n", c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records); return result; }