public boolean isValidFile(RandomAccessFile raf) {
int data_msecs = 0;
short data_julian_date = 0;
try {
raf.order(RandomAccessFile.LITTLE_ENDIAN);
raf.seek(0);
raf.skipBytes(14);
short message_type = raf.readShort();
if (message_type != 1) return false;
raf.skipBytes(12);
// data header
byte[] b4 = raf.readBytes(4);
data_msecs = bytesToInt(b4, true);
byte[] b2 = raf.readBytes(2);
data_julian_date = (short) bytesToShort(b2, true);
java.util.Date dd = Cinrad2Record.getDate(data_julian_date, data_msecs);
Calendar cal = new GregorianCalendar(new SimpleTimeZone(0, "GMT"));
cal.clear();
cal.setTime(dd);
int year = cal.get(Calendar.YEAR);
cal.setTime(new Date());
int cyear = cal.get(Calendar.YEAR);
if (year < 1990 || year > cyear) return false;
return true;
} catch (IOException ioe) {
return false;
}
}
public static void main(String[] args) throws IOException {
RandomAccessFile raf =
new RandomAccessFile("Q:/cdmUnitTest/formats/grib2/sfc_d01_20080430_1200_f00000.grb2", "r");
Grib2RecordScanner scan = new Grib2RecordScanner(raf);
while (scan.hasNext()) scan.next();
raf.close();
}
/**
* _more_
*
* @param raf _more_
* @param stop _more_
* @return _more_
* @throws IOException _more_
*/
private boolean seekHeader(RandomAccessFile raf, long stop) throws IOException {
// seek header
StringBuffer hdr = new StringBuffer();
int match = 0;
while (raf.getFilePointer() < stop) {
// code must be "G" "R" "I" "B"
char c = (char) raf.read();
hdr.append((char) c);
if (c == 'G') {
match = 1;
} else if ((c == 'R') && (match == 1)) {
match = 2;
} else if ((c == 'I') && (match == 2)) {
match = 3;
} else if ((c == 'B') && (match == 3)) {
match = 4;
Matcher m = productID.matcher(hdr.toString());
if (m.find()) {
header = m.group(1);
} else {
// header = hdr.toString();
header = "GRIB1";
}
// System.out.println( "header =" + header.toString() );
return true;
} else {
match = 0; /* Needed to protect against "GaRaIaB" case. */
}
}
return false;
} // end seekHeader
@Test
public void testDBZ() throws IOException {
TaskConfiguration taskConfiguration = new TaskConfiguration();
taskConfiguration.builder(dbzFile, 160);
System.out.println(dbzFile.getFilePointer());
System.out.println(taskConfiguration);
assertTrue(taskConfiguration.getCutNumber() == 8);
for (int i = 0; i < taskConfiguration.getCutNumber(); i++) {
CutConfiguration cutConfiguration = new CutConfiguration();
cutConfiguration.builder(dbzFile, -1);
System.out.println(cutConfiguration);
System.out.println(dbzFile.getFilePointer());
}
ProductHeader productHeader = new ProductHeader();
productHeader.builder(dbzFile, -1);
System.out.println(productHeader);
assertTrue(productHeader.getProductNumber() == 1);
ProductDependentParameter productDependentParameter =
new ProductDependentParameter(
CinradXUtils.getProductType(productHeader.getProductNumber()), dbzFile.readBytes(64));
assertTrue(productDependentParameter.getProductType().name().equalsIgnoreCase("PPI"));
// productDependentParameter.buildProductParameter(dbzFile.readBytes(64));
System.out.println(productDependentParameter.getProductType());
System.out.println(productDependentParameter);
}
/**
* Sync the file
*
* @return true if needed to sync
* @throws IOException problem synching the file
*/
public boolean sync() throws IOException {
// printStack("***************************** sync ************************", 100);
// System.out.printf("check sync on %s (%s) %n", raf.getLocation(),
// Calendar.getInstance().getTime());
if (gemreader.getInitFileSize() < raf.length()) {
long start = System.currentTimeMillis();
Trace.msg(
"GEMPAKStationIOSP.sync: file "
+ raf.getLocation()
+ " is bigger: "
+ raf.length()
+ " > "
+ gemreader.getInitFileSize());
Trace.call1("GEMPAKStationIOSP.sync: reader.init");
gemreader.init(raf, true);
Trace.call2("GEMPAKStationIOSP.sync: reader.init");
Trace.call1("GEMPAKStationIOSP.sync: buildNCFile");
// reconstruct the ncfile objects
buildNCFile();
Trace.call2("GEMPAKSIOSP.sync: buildNCFile");
// System.out.printf("sync on %s took %d msecs%n", raf.getLocation(),
// (System.currentTimeMillis()-start));
return true;
}
return false;
}
/**
* Read the grid
*
* @param gr the grid record
* @return the data
*/
public float[] readGrid(McIDASGridRecord gr) {
float[] data = null;
try {
int te = (gr.getOffsetToHeader() + 64) * 4;
int rows = gr.getRows();
int cols = gr.getColumns();
rf.seek(te);
float scale = (float) gr.getParamScale();
data = new float[rows * cols];
rf.order(needToSwap ? rf.LITTLE_ENDIAN : rf.BIG_ENDIAN);
int n = 0;
// store such that 0,0 is in lower left corner...
for (int nc = 0; nc < cols; nc++) {
for (int nr = 0; nr < rows; nr++) {
int temp = rf.readInt(); // check for missing value
data[(rows - nr - 1) * cols + nc] =
(temp == McIDASUtil.MCMISSING) ? Float.NaN : ((float) temp) / scale;
}
}
rf.order(rf.BIG_ENDIAN);
} catch (Exception esc) {
System.out.println(esc);
}
return data;
}
@After
public void tearDown() throws Exception {
dbzFile.flush();
dbzFile.close();
cappiFile.flush();
cappiFile.close();
}
public Grib2RecordScanner(RandomAccessFile raf) throws IOException {
startPos = 0;
this.raf = raf;
raf.seek(startPos);
raf.order(RandomAccessFile.BIG_ENDIAN);
lastPos = startPos;
if (debugRepeat) System.out.printf(" Grib2RecordScanner %s%n", raf.getLocation());
}
/**
* Grib edition number 1, 2 or 0 not a Grib file.
*
* @return int 0 not a Grib file, 1 Grib1, 2 Grib2
* @throws NotSupportedException
* @throws IOException
*/
public final int getEdition() throws IOException, NotSupportedException {
int check = 0; // Not a valid Grib file
long length = (raf.length() < 4000L) ? raf.length() : 4000L;
if (!seekHeader(raf, length)) {
return 0; // not valid Grib file
}
// Read Section 0 Indicator Section to get Edition number
Grib1IndicatorSection is = new Grib1IndicatorSection(raf); // section 0
return is.getGribEdition();
} // end getEdition
// LOOK what about extending an index ??
public boolean makeIndex(String filename, RandomAccessFile dataRaf, Formatter f)
throws IOException {
File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
FileOutputStream fout = new FileOutputStream(idxFile);
RandomAccessFile raf = null;
try {
//// header message
fout.write(MAGIC_START.getBytes("UTF-8"));
NcStream.writeVInt(fout, version);
Map<Long, Integer> gdsMap = new HashMap<Long, Integer>();
gdsList = new ArrayList<Grib1SectionGridDefinition>();
records = new ArrayList<Grib1Record>(200);
Grib1IndexProto.Grib1Index.Builder rootBuilder = Grib1IndexProto.Grib1Index.newBuilder();
rootBuilder.setFilename(filename);
if (dataRaf == null) {
raf = new RandomAccessFile(filename, "r");
dataRaf = raf;
}
Grib1RecordScanner scan = new Grib1RecordScanner(dataRaf);
while (scan.hasNext()) {
Grib1Record r = scan.next();
if (r == null) break; // done
records.add(r);
Grib1SectionGridDefinition gds = r.getGDSsection();
Integer index = gdsMap.get(gds.calcCRC());
if (gds.getPredefinedGridDefinition()
>= 0) // skip predefined gds - they dont have raw bytes
index = 0;
else if (index == null) {
gdsList.add(gds);
index = gdsList.size() - 1;
gdsMap.put(gds.calcCRC(), index);
rootBuilder.addGdsList(makeGdsProto(gds));
}
rootBuilder.addRecords(makeRecordProto(r, index));
}
ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Index index = rootBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(fout, b.length); // message size
fout.write(b); // message - all in one gulp
f.format(" made gbx9 index for %s size=%d%n", filename, b.length);
return true;
} finally {
fout.close();
if (raf != null) raf.close();
}
}
/**
* tricky bit of business. recapture the entire record based on drs position. for validation.
*
* @param raf from this RandomAccessFile
* @param drsPos Grib2SectionDataRepresentation starts here
*/
public static Grib2Record findRecordByDrspos(RandomAccessFile raf, long drsPos)
throws IOException {
Grib2Record result = null;
Grib2RecordScanner scanner = new Grib2RecordScanner(raf);
long pos = Math.max(0, drsPos - 10000); // go back 10000 bytes
raf.seek(pos);
while (scanner.hasNext()) { // find GRIB header
result = scanner.next();
if (result.getDataRepresentationSection().getStartingPosition() == drsPos) return result;
if (raf.getFilePointer() > drsPos) break;
}
return null;
}
/**
* Read an integer
*
* @param word word in file (0 based) to read
* @return int read
* @throws IOException problem reading file
*/
public int readInt(int word) throws IOException {
if (rf == null) {
throw new IOException("no file to read from");
}
rf.seek(word * 4);
// set the order
if (needToSwap) {
rf.order(RandomAccessFile.LITTLE_ENDIAN); // swap
} else {
rf.order(RandomAccessFile.BIG_ENDIAN);
}
int idata = rf.readInt();
rf.order(RandomAccessFile.BIG_ENDIAN);
return idata;
}
/** Read StationName strings */
public java.util.Map<String, String> readStnNames(ucar.unidata.io.RandomAccessFile raf) {
java.util.Map<String, String> hdrNames = new java.util.HashMap<String, String>();
try {
raf.seek(6288);
String stnName = raf.readString(16); // System.out.println(" stnName="+stnName.trim());
raf.seek(6306);
String stnName_util = raf.readString(16);
hdrNames.put("StationName", stnName.trim());
hdrNames.put("StationName_SetupUtility", stnName_util.trim());
} catch (Exception e) {
System.out.println(e.toString());
e.printStackTrace();
}
return hdrNames;
}
/**
* Open the service provider for reading.
*
* @param raf file to read from
* @param ncfile netCDF file we are writing to (memory)
* @param cancelTask task for cancelling
* @throws IOException problem reading file
*/
public void open(RandomAccessFile raf, NetcdfFile ncfile, CancelTask cancelTask)
throws IOException {
super.open(raf, ncfile, cancelTask);
if (areaReader == null) areaReader = new AreaReader();
try {
areaReader.init(raf.getLocation(), ncfile);
} catch (Throwable e) {
close(); // try not to leak files
throw new IOException(e);
} finally {
raf.close(); // avoid leaks
}
}
public static boolean isValidFile(RandomAccessFile raf) {
try {
raf.seek(0);
while (raf.getFilePointer() < maxScan) {
boolean found = raf.searchForward(matcher, maxScan); // look in first 16K
if (!found) return false;
raf.skipBytes(7); // will be positioned on byte 0 of indicator section
int edition = raf.read(); // read at byte 8
if (edition == 2) return true;
}
} catch (IOException e) {
return false;
}
return false;
}
/**
* Initialize the file, read in all the metadata (ala DM_OPEN)
*
* @param fullCheck for a full check reading grids
* @param raf RandomAccessFile to read.
* @throws IOException problem reading file
*/
public final void init(RandomAccessFile raf, boolean fullCheck) throws IOException {
rf = raf;
raf.order(RandomAccessFile.BIG_ENDIAN);
boolean ok = init(fullCheck);
if (!ok) {
throw new IOException("Unable to open McIDAS Grid file: " + errorMessage);
}
}
/** Check if this is a valid SIGMET-IRIS file for this IOServiceProvider. */
public boolean isValidFile(ucar.unidata.io.RandomAccessFile raf) {
try {
raf.order(RandomAccessFile.LITTLE_ENDIAN);
// The first struct in the file is the product_hdr, which will have the
// standard structure_header, followed by other embedded structures.
// Each of these structures also have a structure header. To validate
// the file we check for a product_hdr (by looking for type 27 in the
// structure_header), then a product_configuration structure (by looking
// for type 26 in its structure_header), then checking that that
// the product_configuration does indicate a type of RAW data (type 15)
raf.seek(0);
short[] data = new short[13];
raf.readShort(data, 0, 13);
return (data[0] == (short) 27 && data[6] == (short) 26 && data[12] == (short) 15);
} catch (IOException ioe) {
System.out.println("In isValidFile(): " + ioe.toString());
return false;
}
}
public boolean isValidFileOld(RandomAccessFile raf) {
try {
String loc = raf.getLocation();
int posFirst = loc.lastIndexOf('/') + 1;
if (posFirst < 0) posFirst = 0;
String stationId = loc.substring(posFirst, posFirst + 4);
NexradStationDB.init();
NexradStationDB.Station station = NexradStationDB.get("K" + stationId);
if (station != null) return true;
else return false;
} catch (IOException ioe) {
return false;
}
}
/*
MAGIC_START
version
sizeRecords
VariableRecords (sizeRecords bytes)
sizeIndex
GribCollectionIndex (sizeIndex bytes)
*/
private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
File file = tp.getIndexFile();
if (file.exists()) {
if (!file.delete()) logger.error("Cant delete " + file.getPath());
}
RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
raf.writeLong(0); // no record section
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(tp.getName());
GribCollection canonGc = canon.makeGribCollection(f);
for (GribCollection.GroupHcs g : canonGc.getGroups())
indexBuilder.addGroups(writeGroupProto(g));
indexBuilder.setCenter(canonGc.getCenter());
indexBuilder.setSubcenter(canonGc.getSubcenter());
indexBuilder.setMaster(canonGc.getMaster());
indexBuilder.setLocal(canonGc.getLocal());
for (TimePartition.Partition p : tp.getPartitions()) {
indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
}
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);
} finally {
f.format("file size = %d bytes%n", raf.length());
raf.close();
}
return true;
}
public boolean hasNext() throws IOException {
if (lastPos >= raf.length()) return false;
if (repeatPos > 0) {
if (nextRepeating()) // this has created a new repeatRecord
return true;
} else {
repeatRecord = null;
repeatBms = null;
// fall through to new record
}
boolean more;
long stop = 0;
while (true) { // scan until we get a GRIB-2 or more == false
raf.seek(lastPos);
more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
if (!more) break;
stop = raf.getFilePointer();
// see if its GRIB-2
raf.skipBytes(7);
int edition = raf.read();
if (edition == 2) break;
lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
}
if (more) {
int sizeHeader =
(int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
// if (sizeHeader > 30) sizeHeader = 30;
header = new byte[sizeHeader];
startPos = stop - sizeHeader;
raf.seek(startPos);
raf.read(header);
}
if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
return more;
}
public float[] getData(RandomAccessFile raf, byte[] bitmap, Grib2Drs gdrs) throws IOException {
this.bitmap = bitmap;
if (bitmap != null) { // is bitmap ok ?
if (bitmap.length * 8 < totalNPoints) { // gdsNumberPoints == nx * ny ??
System.out.printf(
"Bitmap section length = %d != grid length %d (%d,%d)",
bitmap.length, totalNPoints, nx, totalNPoints / nx);
throw new IllegalStateException("Bitmap section length!= grid length %");
}
}
raf.seek(startPos + 5); // skip past first 5 bytes in data section, now ready to read
float[] data;
switch (dataTemplate) {
case 0:
data = getData0(raf, (Grib2Drs.Type0) gdrs);
break;
case 2:
data = getData2(raf, (Grib2Drs.Type2) gdrs);
break;
case 3:
data = getData3(raf, (Grib2Drs.Type3) gdrs);
break;
case 40:
data = getData40(raf, (Grib2Drs.Type40) gdrs);
break;
default:
throw new UnsupportedOperationException("Unsupported DRS type = " + dataTemplate);
}
// int scanMode = gds.getGds().getScanMode();
// int nx = gds.getGds().getNx(); // needs some smarts for different type Grids
scanningModeCheck(data, scanMode, nx);
return data;
}
/**
* Read some global data from SIGMET file. The SIGMET file consists of records with fixed
* length=6144 bytes.
*/
public static java.util.Map<String, Number> readRecordsHdr(ucar.unidata.io.RandomAccessFile raf) {
java.util.Map<String, Number> recHdr1 = new java.util.HashMap<String, Number>();
try {
int nparams = 0;
// -- Read from <product_end> of the 1st record -- 12+320+120
// -- Calculate Nyquist velocity --------------------
raf.seek(452);
int prf = raf.readInt();
raf.seek(480);
int wave = raf.readInt();
float vNyq = calcNyquist(prf, wave);
recHdr1.put("vNyq", vNyq);
// -- Read from the 2nd record----------- 6144+12(strucr_hdr)+168(from ingest_config)
raf.seek(6324);
int radar_lat = raf.readInt();
int radar_lon = raf.readInt(); // 6328
short ground_height = raf.readShort(); // 6332
short radar_height = raf.readShort(); // 6334
raf.skipBytes(4);
short num_rays = raf.readShort(); // 6340
raf.skipBytes(2);
int radar_alt = raf.readInt(); // 6344
raf.seek(6648);
int time_beg = raf.readInt();
raf.seek(6652);
int time_end = raf.readInt();
raf.seek(6772);
int data_mask = raf.readInt();
for (int j = 0; j < 32; j++) {
nparams += ((data_mask >> j) & (0x1));
}
raf.seek(6912);
short multiprf = raf.readShort();
raf.seek(7408);
int range_first = raf.readInt(); // cm 7408
int range_last = raf.readInt(); // cm 7412
raf.skipBytes(2);
short bins = raf.readShort(); // 7418
if (bins % 2 != 0) bins = (short) (bins + 1);
raf.skipBytes(4);
int step = raf.readInt(); // cm 7424
raf.seek(7574);
short number_sweeps = raf.readShort(); // 7574
raf.seek(12312);
int base_time = raf.readInt(); // <ingest_data_header> 3d rec
raf.skipBytes(2);
short year = raf.readShort();
short month = raf.readShort();
short day = raf.readShort();
recHdr1.put("radar_lat", calcAngle(radar_lat));
recHdr1.put("radar_lon", calcAngle(radar_lon));
recHdr1.put("range_first", range_first);
recHdr1.put("range_last", range_last);
recHdr1.put("ground_height", ground_height);
recHdr1.put("radar_height", radar_height);
recHdr1.put("radar_alt", radar_alt);
recHdr1.put("step", step);
recHdr1.put("bins", bins); // System.out.println(" bins="+bins);
recHdr1.put("num_rays", num_rays); // System.out.println(" rays="+num_rays);
recHdr1.put("nparams", nparams); // System.out.println(" nparams="+nparams);
recHdr1.put("multiprf", multiprf);
recHdr1.put(
"number_sweeps",
number_sweeps); // System.out.println("IN HDR: number_sweeps="+number_sweeps);
recHdr1.put("year", year);
recHdr1.put("month", month);
recHdr1.put("day", day);
recHdr1.put("base_time", base_time);
} catch (Exception e) {
System.out.println(e.toString());
e.printStackTrace();
}
return recHdr1;
}
// check if compressed file seems ok
public static long testValid(String ufilename) throws IOException {
boolean lookForHeader = false;
// gotta make it
RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
byte[] b = new byte[8];
raf.read(b);
String test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println("--Good header= " + test);
raf.seek(24);
} else {
System.out.println("--No header ");
lookForHeader = true;
raf.seek(0);
}
boolean eof = false;
int numCompBytes;
try {
while (!eof) {
if (lookForHeader) {
raf.read(b);
test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println(" found header= " + test);
raf.skipBytes(16);
lookForHeader = false;
} else {
raf.skipBytes(-8);
}
}
try {
numCompBytes = raf.readInt();
if (numCompBytes == -1) {
System.out.println("\n--done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
System.out.println("\n--got EOFException ");
break; // assume this is ok
}
System.out.print(" " + numCompBytes + ",");
if (numCompBytes < 0) {
System.out.println("\n--last block " + numCompBytes);
numCompBytes = -numCompBytes;
if (!lookForHeader) eof = true;
}
raf.skipBytes(numCompBytes);
}
} catch (EOFException e) {
e.printStackTrace();
}
return raf.getFilePointer();
}
/**
* Close this IOSP
*
* @throws IOException problem closing file
*/
public void close() throws IOException {
raf.close();
}
/**
* scans a Grib file to gather information that could be used to create an index or dump the
* metadata contents.
*
* @param parm
* @throws NotSupportedException
* @throws IOException if raf does not contain a valid GRIB record
*/
public final boolean scan(int[] parm) throws NotSupportedException, IOException {
long start = System.currentTimeMillis();
// stores the number of times a particular GDS is used
HashMap gdsCounter = new HashMap();
Grib1ProductDefinitionSection pds = null;
Grib1GridDefinitionSection gds = null;
long gdsOffset = 0;
DataOutputStream dos = new DataOutputStream(System.out);
// System.out.println("file position =" + raf.getFilePointer());
long SOR = raf.getFilePointer();
while (raf.getFilePointer() < raf.length()) {
SOR = raf.getFilePointer();
if (seekHeader(raf, raf.length())) {
// Read Section 0 Indicator Section
Grib1IndicatorSection is = new Grib1IndicatorSection(raf);
// System.out.println( "Grib record length=" + is.getGribLength());
// EOR (EndOfRecord) calculated so skipping data sections is faster
long EOR = raf.getFilePointer() + is.getGribLength() - is.getLength();
// long SOR = raf.getFilePointer() - is.getLength();
// skip Grib 2 records in a Grib 1 file
if (is.getGribEdition() == 2) {
// System.out.println( "Error Grib 2 record in Grib1 file" ) ;
raf.seek(EOR);
continue;
}
if (parm[0] == -1) { // extract only 1st record
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
break;
}
long dataOffset = 0;
try { // catch all exceptions and seek to EOR
// Read Section 1 Product Definition Section PDS
pds = new Grib1ProductDefinitionSection(raf);
if (pds.getLengthErr()) {
raf.seek(EOR);
continue;
}
Grib1PDSVariables pdsv = pds.getPdsVars();
if (getParameter && parm[0] == pdsv.getParameterNumber()) {
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
} else if (parm[0] <= pdsv.getForecastTime() && parm[1] >= pdsv.getForecastTime()) {
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
}
raf.seek(EOR);
continue;
} catch (Exception e) {
// .println( "Caught Exception scannning record" );
e.printStackTrace();
raf.seek(EOR);
continue;
}
} // end if seekHeader
// System.out.println( "raf.getFilePointer()=" + raf.getFilePointer());
// System.out.println( "raf.length()=" + raf.length() );
} // end while raf.getFilePointer() < raf.length()
// System.out.println("GribInput: processed in " +
// (System.currentTimeMillis()- start) + " milliseconds");
dos.close();
return true;
} // end scan
public boolean readIndex(RandomAccessFile raf) {
gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
try {
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
//// header message
if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
logger.error("GribCollection {}: invalid index", gc.getName());
return false;
}
int v = raf.readInt();
if (v != getVersion()) {
logger.warn(
"GribCollection {}: index found version={}, want version= {} on file {}",
new Object[] {gc.getName(), v, version, raf.getLocation()});
return false;
}
long skip = raf.readLong();
raf.skipBytes(skip);
int size = NcStream.readVInt(raf);
if ((size < 0) || (size > 100 * 1000 * 1000)) {
logger.warn("GribCollection {}: invalid index ", gc.getName());
return false;
}
byte[] m = new byte[size];
raf.readFully(m);
GribCollectionProto.GribCollectionIndex proto =
GribCollectionProto.GribCollectionIndex.parseFrom(m);
gc.center = proto.getCenter();
gc.subcenter = proto.getSubcenter();
gc.master = proto.getMaster();
gc.local = proto.getLocal();
gc.genProcessType = proto.getGenProcessType();
gc.genProcessId = proto.getGenProcessId();
gc.backProcessId = proto.getBackProcessId();
gc.local = proto.getLocal();
// gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);
gc.filenames = new ArrayList<String>(proto.getFilesCount());
for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));
// error condition on a GribCollection Index
if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
return false;
}
gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
for (int i = 0; i < proto.getGroupsCount(); i++)
gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
Collections.sort(gc.groups);
gc.params = new ArrayList<Parameter>(proto.getParamsCount());
for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));
if (!readPartitions(proto)) {
logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
return false;
}
return true;
} catch (Throwable t) {
logger.error("Error reading index " + raf.getLocation(), t);
return false;
}
}
private void createIndex(
File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
throws IOException {
Grib2Record first = null; // take global metadata from here
if (indexFile.exists()) indexFile.delete(); // replace it
f.format(" createIndex for %s%n", indexFile.getPath());
RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
for (Group g : groups) {
g.fileSet = new HashSet<Integer>();
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
countRecords += vb.recordMap.length;
}
}
long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
f.format(
" write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
countBytes, countRecords, bytesPerRecord);
if (first == null) {
logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
}
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(gc.getName());
for (String fn : filenames) indexBuilder.addFiles(fn);
for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));
/* int count = 0;
for (DatasetCollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
count++;
} */
// what about just storing first ??
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
Grib2Pds pds = first.getPDS();
indexBuilder.setGenProcessType(pds.getGenProcessType());
indexBuilder.setGenProcessId(pds.getGenProcessId());
indexBuilder.setBackProcessId(pds.getBackProcessId());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format(" write GribCollectionIndex= %d bytes%n", b.length);
} finally {
f.format(" file size = %d bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
}
// Grid point data - JPEG 2000 code stream format
public float[] getData40(RandomAccessFile raf, Grib2Drs.Type40 gdrs) throws IOException {
// 6-xx jpeg2000 data block to decode
// dataPoints are number of points encoded, it could be less than the
// totalNPoints in the grid record if bitMap is used, otherwise equal
// int dataPoints = drs.getDataPoints();
int nb = gdrs.numberOfBits;
int D = gdrs.decimalScaleFactor;
float DD = (float) java.lang.Math.pow((double) 10, (double) D);
float R = gdrs.referenceValue;
int E = gdrs.binaryScaleFactor;
float EE = (float) java.lang.Math.pow((double) 2.0, (double) E);
Grib2JpegDecoder g2j = null;
try {
if (nb != 0) { // there's data to decode
String[] argv = new String[6];
argv[0] = "-rate";
argv[1] = Integer.toString(nb);
argv[2] = "-verbose";
argv[3] = "off";
argv[4] = "-debug";
argv[5] = "on";
// argv[ 2 ] = "-nocolorspace" ;
// argv[ 3 ] = "-Rno_roi" ;
// argv[ 4 ] = "-cdstr_info" ;
// argv[ 5 ] = "-verbose" ;
g2j = new Grib2JpegDecoder(argv);
// how jpeg2000.jar use to decode, used raf
// g2j.decode(raf, length - 5);
// jpeg-1.0.jar added method to have the data read first
byte[] buf = new byte[dataLength - 5];
raf.read(buf);
g2j.decode(buf);
gdrs.hasSignedProblem = g2j.hasSignedProblem();
}
float[] data = new float[totalNPoints];
if (nb == 0) { // no data to decoded, set to reference or MissingValue
for (int i = 0; i < dataNPoints; i++) data[i] = R;
} else if (bitmap == null) {
if (g2j.data.length != dataNPoints) {
data = null;
return data;
}
for (int i = 0; i < dataNPoints; i++) {
// Y * 10^D = R + (X1 + X2) * 2^E ; // regulation 92.9.4
// Y = (R + ( 0 + X2) * EE)/DD ;
data[i] = (R + g2j.data[i] * EE) / DD;
}
} else { // use bitmap
for (int i = 0, j = 0; i < totalNPoints; i++) {
if ((bitmap[i / 8] & GribNumbers.bitmask[i % 8]) != 0) {
data[i] = (R + g2j.data[j++] * EE) / DD;
} else {
data[i] = staticMissingValue; // LOOK ??
}
}
}
return data;
} catch (NullPointerException npe) {
log.error(
"Grib2DataSection.jpeg2000Unpacking: bit rate too small nb ="
+ nb
+ " for file"
+ raf.getLocation());
float[] data = new float[dataNPoints];
for (int i = 0; i < dataNPoints; i++) {
data[i] = staticMissingValue; // LOOK ??
}
return data;
}
}
public Grib2Record next() throws IOException {
if (repeatRecord != null) { // serve current repeatRecord if it exists
return new Grib2Record(repeatRecord);
}
Grib2SectionIndicator is = null;
try {
is = new Grib2SectionIndicator(raf);
Grib2SectionIdentification ids = new Grib2SectionIdentification(raf);
Grib2SectionLocalUse lus = new Grib2SectionLocalUse(raf);
Grib2SectionGridDefinition gds = new Grib2SectionGridDefinition(raf);
Grib2SectionProductDefinition pds = new Grib2SectionProductDefinition(raf);
Grib2SectionDataRepresentation drs = new Grib2SectionDataRepresentation(raf);
Grib2SectionBitMap bms = new Grib2SectionBitMap(raf);
Grib2SectionData dataSection = new Grib2SectionData(raf);
if (dataSection.getMsgLength() > is.getMessageLength()) { // presumably corrupt
raf.seek(drs.getStartingPosition()); // go back to before the dataSection
throw new IllegalStateException("Illegal Grib2SectionData Message Length");
}
// look for duplicate gds
long crc = gds.calcCRC();
Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
if (gdsCached != null) gds = gdsCached;
else gdsMap.put(crc, gds);
// look for duplicate pds
/* crc = pds.calcCRC();
Grib2SectionProductDefinition pdsCached = pdsMap.get(crc);
if (pdsCached != null)
pds = pdsCached;
else
pdsMap.put(crc, pds); */
// check to see if we have a repeating record
long pos = raf.getFilePointer();
long ending = is.getEndPos();
if (pos + 34 < ending) { // give it 30 bytes of slop
if (debugRepeat) System.out.printf(" REPEAT AT %d != %d%n", pos + 4, ending);
repeatPos = pos;
repeatRecord =
new Grib2Record(
header,
is,
ids,
lus,
gds,
pds,
drs,
bms,
dataSection,
false); // this assumes immutable sections
// track bms in case its a repeat
if (bms.getBitMapIndicator() == 0) repeatBms = bms;
return new Grib2Record(
repeatRecord); // GribRecord isnt immutable; still may not be necessary
}
if (debug)
System.out.printf(
" read until %d grib ending at %d header ='%s'%n",
raf.getFilePointer(), ending, StringUtil2.cleanup(header));
// check that end section is correct
raf.seek(ending - 4);
for (int i = 0; i < 4; i++) {
if (raf.read() != 55) {
log.warn(
"Missing End of GRIB message at pos="
+ ending
+ " header= "
+ StringUtil2.cleanup(header)
+ " for="
+ raf.getLocation());
break;
}
}
lastPos = raf.getFilePointer();
return new Grib2Record(header, is, ids, lus, gds, pds, drs, bms, dataSection, false);
} catch (Throwable t) {
long pos = (is == null) ? -1 : is.getStartPos();
log.warn("Bad GRIB2 record in file {}, skipping pos={}", raf.getLocation(), pos);
lastPos = raf.getFilePointer();
if (hasNext()) // skip forward
return next();
}
return null; // last record was incomplete
}
// return true if got another repeat out of this record
// side effect is that the new record is in repeatRecord
private boolean nextRepeating() throws IOException {
raf.seek(repeatPos);
// octets 1-4 (Length of GDS)
int length = GribNumbers.int4(raf);
int section = raf.read();
raf.seek(repeatPos);
if (section == 2) {
repeatRecord.setLus(new Grib2SectionLocalUse(raf));
repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else if (section == 3) {
repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else if (section == 4) {
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else {
if (debugRepeat) System.out.printf(" REPEAT Terminate %d%n", section);
repeatPos = -1;
repeatRecord = null;
repeatBms = null;
return false;
}
// look for repeating bms
Grib2SectionBitMap bms = repeatRecord.getBitmapSection();
if (bms.getBitMapIndicator() == 254) {
// replace BMS with last good one
if (repeatBms == null) throw new IllegalStateException("No bms in repeating section");
repeatRecord.setBms(repeatBms, true);
// debug
if (debugRepeat) System.out.printf("replaced bms %d%n", section);
repeatRecord.repeat += 1000;
} else if (bms.getBitMapIndicator() == 0) {
// track last good bms
repeatBms = repeatRecord.getBitmapSection();
}
// keep only unique gds
if ((section == 2) || (section == 3)) {
// look for duplicate gds
Grib2SectionGridDefinition gds = repeatRecord.getGDSsection();
long crc = gds.calcCRC();
Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
if (gdsCached != null) repeatRecord.setGdss(gdsCached);
else gdsMap.put(crc, gds);
}
// check to see if we are at the end
long pos = raf.getFilePointer();
long ending = repeatRecord.getIs().getEndPos();
if (pos + 34 < ending) { // give it 30 bytes of slop
if (debugRepeat) System.out.printf(" REPEAT AGAIN %d != %d%n", pos + 4, ending);
repeatPos = pos;
return true;
}
if (debug)
System.out.printf(
" REPEAT read until %d grib ending at %d header ='%s'%n",
raf.getFilePointer(), ending, StringUtil2.cleanup(header));
// check that end section is correct
raf.seek(ending - 4);
for (int i = 0; i < 4; i++) {
if (raf.read() != 55) {
log.warn(
" REPEAT Missing End of GRIB message at pos="
+ ending
+ " header= "
+ StringUtil2.cleanup(header)
+ " for="
+ raf.getLocation());
break;
}
}
lastPos = raf.getFilePointer();
if (debugRepeat) System.out.printf(" REPEAT DONE%n");
repeatPos = -1; // no more repeats in this record
return true;
}