@Test
public void testDBZ() throws IOException {
TaskConfiguration taskConfiguration = new TaskConfiguration();
taskConfiguration.builder(dbzFile, 160);
System.out.println(dbzFile.getFilePointer());
System.out.println(taskConfiguration);
assertTrue(taskConfiguration.getCutNumber() == 8);
for (int i = 0; i < taskConfiguration.getCutNumber(); i++) {
CutConfiguration cutConfiguration = new CutConfiguration();
cutConfiguration.builder(dbzFile, -1);
System.out.println(cutConfiguration);
System.out.println(dbzFile.getFilePointer());
}
ProductHeader productHeader = new ProductHeader();
productHeader.builder(dbzFile, -1);
System.out.println(productHeader);
assertTrue(productHeader.getProductNumber() == 1);
ProductDependentParameter productDependentParameter =
new ProductDependentParameter(
CinradXUtils.getProductType(productHeader.getProductNumber()), dbzFile.readBytes(64));
assertTrue(productDependentParameter.getProductType().name().equalsIgnoreCase("PPI"));
// productDependentParameter.buildProductParameter(dbzFile.readBytes(64));
System.out.println(productDependentParameter.getProductType());
System.out.println(productDependentParameter);
}
/**
* _more_
*
* @param raf _more_
* @param stop _more_
* @return _more_
* @throws IOException _more_
*/
private boolean seekHeader(RandomAccessFile raf, long stop) throws IOException {
// seek header
StringBuffer hdr = new StringBuffer();
int match = 0;
while (raf.getFilePointer() < stop) {
// code must be "G" "R" "I" "B"
char c = (char) raf.read();
hdr.append((char) c);
if (c == 'G') {
match = 1;
} else if ((c == 'R') && (match == 1)) {
match = 2;
} else if ((c == 'I') && (match == 2)) {
match = 3;
} else if ((c == 'B') && (match == 3)) {
match = 4;
Matcher m = productID.matcher(hdr.toString());
if (m.find()) {
header = m.group(1);
} else {
// header = hdr.toString();
header = "GRIB1";
}
// System.out.println( "header =" + header.toString() );
return true;
} else {
match = 0; /* Needed to protect against "GaRaIaB" case. */
}
}
return false;
} // end seekHeader
public boolean hasNext() throws IOException {
if (lastPos >= raf.length()) return false;
if (repeatPos > 0) {
if (nextRepeating()) // this has created a new repeatRecord
return true;
} else {
repeatRecord = null;
repeatBms = null;
// fall through to new record
}
boolean more;
long stop = 0;
while (true) { // scan until we get a GRIB-2 or more == false
raf.seek(lastPos);
more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
if (!more) break;
stop = raf.getFilePointer();
// see if its GRIB-2
raf.skipBytes(7);
int edition = raf.read();
if (edition == 2) break;
lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
}
if (more) {
int sizeHeader =
(int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
// if (sizeHeader > 30) sizeHeader = 30;
header = new byte[sizeHeader];
startPos = stop - sizeHeader;
raf.seek(startPos);
raf.read(header);
}
if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
return more;
}
/**
* tricky bit of business. recapture the entire record based on drs position. for validation.
*
* @param raf from this RandomAccessFile
* @param drsPos Grib2SectionDataRepresentation starts here
*/
public static Grib2Record findRecordByDrspos(RandomAccessFile raf, long drsPos)
throws IOException {
Grib2Record result = null;
Grib2RecordScanner scanner = new Grib2RecordScanner(raf);
long pos = Math.max(0, drsPos - 10000); // go back 10000 bytes
raf.seek(pos);
while (scanner.hasNext()) { // find GRIB header
result = scanner.next();
if (result.getDataRepresentationSection().getStartingPosition() == drsPos) return result;
if (raf.getFilePointer() > drsPos) break;
}
return null;
}
public static boolean isValidFile(RandomAccessFile raf) {
try {
raf.seek(0);
while (raf.getFilePointer() < maxScan) {
boolean found = raf.searchForward(matcher, maxScan); // look in first 16K
if (!found) return false;
raf.skipBytes(7); // will be positioned on byte 0 of indicator section
int edition = raf.read(); // read at byte 8
if (edition == 2) return true;
}
} catch (IOException e) {
return false;
}
return false;
}
/**
* scans a Grib file to gather information that could be used to create an index or dump the
* metadata contents.
*
* @param parm
* @throws NotSupportedException
* @throws IOException if raf does not contain a valid GRIB record
*/
public final boolean scan(int[] parm) throws NotSupportedException, IOException {
long start = System.currentTimeMillis();
// stores the number of times a particular GDS is used
HashMap gdsCounter = new HashMap();
Grib1ProductDefinitionSection pds = null;
Grib1GridDefinitionSection gds = null;
long gdsOffset = 0;
DataOutputStream dos = new DataOutputStream(System.out);
// System.out.println("file position =" + raf.getFilePointer());
long SOR = raf.getFilePointer();
while (raf.getFilePointer() < raf.length()) {
SOR = raf.getFilePointer();
if (seekHeader(raf, raf.length())) {
// Read Section 0 Indicator Section
Grib1IndicatorSection is = new Grib1IndicatorSection(raf);
// System.out.println( "Grib record length=" + is.getGribLength());
// EOR (EndOfRecord) calculated so skipping data sections is faster
long EOR = raf.getFilePointer() + is.getGribLength() - is.getLength();
// long SOR = raf.getFilePointer() - is.getLength();
// skip Grib 2 records in a Grib 1 file
if (is.getGribEdition() == 2) {
// System.out.println( "Error Grib 2 record in Grib1 file" ) ;
raf.seek(EOR);
continue;
}
if (parm[0] == -1) { // extract only 1st record
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
break;
}
long dataOffset = 0;
try { // catch all exceptions and seek to EOR
// Read Section 1 Product Definition Section PDS
pds = new Grib1ProductDefinitionSection(raf);
if (pds.getLengthErr()) {
raf.seek(EOR);
continue;
}
Grib1PDSVariables pdsv = pds.getPdsVars();
if (getParameter && parm[0] == pdsv.getParameterNumber()) {
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
} else if (parm[0] <= pdsv.getForecastTime() && parm[1] >= pdsv.getForecastTime()) {
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
}
raf.seek(EOR);
continue;
} catch (Exception e) {
// .println( "Caught Exception scannning record" );
e.printStackTrace();
raf.seek(EOR);
continue;
}
} // end if seekHeader
// System.out.println( "raf.getFilePointer()=" + raf.getFilePointer());
// System.out.println( "raf.length()=" + raf.length() );
} // end while raf.getFilePointer() < raf.length()
// System.out.println("GribInput: processed in " +
// (System.currentTimeMillis()- start) + " milliseconds");
dos.close();
return true;
} // end scan
// return true if got another repeat out of this record
// side effect is that the new record is in repeatRecord
private boolean nextRepeating() throws IOException {
raf.seek(repeatPos);
// octets 1-4 (Length of GDS)
int length = GribNumbers.int4(raf);
int section = raf.read();
raf.seek(repeatPos);
if (section == 2) {
repeatRecord.setLus(new Grib2SectionLocalUse(raf));
repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else if (section == 3) {
repeatRecord.setGdss(new Grib2SectionGridDefinition(raf));
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else if (section == 4) {
repeatRecord.setPdss(new Grib2SectionProductDefinition(raf));
repeatRecord.setDrs(new Grib2SectionDataRepresentation(raf));
repeatRecord.setBms(new Grib2SectionBitMap(raf), false);
repeatRecord.setDataSection(new Grib2SectionData(raf));
repeatRecord.repeat = section;
} else {
if (debugRepeat) System.out.printf(" REPEAT Terminate %d%n", section);
repeatPos = -1;
repeatRecord = null;
repeatBms = null;
return false;
}
// look for repeating bms
Grib2SectionBitMap bms = repeatRecord.getBitmapSection();
if (bms.getBitMapIndicator() == 254) {
// replace BMS with last good one
if (repeatBms == null) throw new IllegalStateException("No bms in repeating section");
repeatRecord.setBms(repeatBms, true);
// debug
if (debugRepeat) System.out.printf("replaced bms %d%n", section);
repeatRecord.repeat += 1000;
} else if (bms.getBitMapIndicator() == 0) {
// track last good bms
repeatBms = repeatRecord.getBitmapSection();
}
// keep only unique gds
if ((section == 2) || (section == 3)) {
// look for duplicate gds
Grib2SectionGridDefinition gds = repeatRecord.getGDSsection();
long crc = gds.calcCRC();
Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
if (gdsCached != null) repeatRecord.setGdss(gdsCached);
else gdsMap.put(crc, gds);
}
// check to see if we are at the end
long pos = raf.getFilePointer();
long ending = repeatRecord.getIs().getEndPos();
if (pos + 34 < ending) { // give it 30 bytes of slop
if (debugRepeat) System.out.printf(" REPEAT AGAIN %d != %d%n", pos + 4, ending);
repeatPos = pos;
return true;
}
if (debug)
System.out.printf(
" REPEAT read until %d grib ending at %d header ='%s'%n",
raf.getFilePointer(), ending, StringUtil2.cleanup(header));
// check that end section is correct
raf.seek(ending - 4);
for (int i = 0; i < 4; i++) {
if (raf.read() != 55) {
log.warn(
" REPEAT Missing End of GRIB message at pos="
+ ending
+ " header= "
+ StringUtil2.cleanup(header)
+ " for="
+ raf.getLocation());
break;
}
}
lastPos = raf.getFilePointer();
if (debugRepeat) System.out.printf(" REPEAT DONE%n");
repeatPos = -1; // no more repeats in this record
return true;
}
public Grib2Record next() throws IOException {
if (repeatRecord != null) { // serve current repeatRecord if it exists
return new Grib2Record(repeatRecord);
}
Grib2SectionIndicator is = null;
try {
is = new Grib2SectionIndicator(raf);
Grib2SectionIdentification ids = new Grib2SectionIdentification(raf);
Grib2SectionLocalUse lus = new Grib2SectionLocalUse(raf);
Grib2SectionGridDefinition gds = new Grib2SectionGridDefinition(raf);
Grib2SectionProductDefinition pds = new Grib2SectionProductDefinition(raf);
Grib2SectionDataRepresentation drs = new Grib2SectionDataRepresentation(raf);
Grib2SectionBitMap bms = new Grib2SectionBitMap(raf);
Grib2SectionData dataSection = new Grib2SectionData(raf);
if (dataSection.getMsgLength() > is.getMessageLength()) { // presumably corrupt
raf.seek(drs.getStartingPosition()); // go back to before the dataSection
throw new IllegalStateException("Illegal Grib2SectionData Message Length");
}
// look for duplicate gds
long crc = gds.calcCRC();
Grib2SectionGridDefinition gdsCached = gdsMap.get(crc);
if (gdsCached != null) gds = gdsCached;
else gdsMap.put(crc, gds);
// look for duplicate pds
/* crc = pds.calcCRC();
Grib2SectionProductDefinition pdsCached = pdsMap.get(crc);
if (pdsCached != null)
pds = pdsCached;
else
pdsMap.put(crc, pds); */
// check to see if we have a repeating record
long pos = raf.getFilePointer();
long ending = is.getEndPos();
if (pos + 34 < ending) { // give it 30 bytes of slop
if (debugRepeat) System.out.printf(" REPEAT AT %d != %d%n", pos + 4, ending);
repeatPos = pos;
repeatRecord =
new Grib2Record(
header,
is,
ids,
lus,
gds,
pds,
drs,
bms,
dataSection,
false); // this assumes immutable sections
// track bms in case its a repeat
if (bms.getBitMapIndicator() == 0) repeatBms = bms;
return new Grib2Record(
repeatRecord); // GribRecord isnt immutable; still may not be necessary
}
if (debug)
System.out.printf(
" read until %d grib ending at %d header ='%s'%n",
raf.getFilePointer(), ending, StringUtil2.cleanup(header));
// check that end section is correct
raf.seek(ending - 4);
for (int i = 0; i < 4; i++) {
if (raf.read() != 55) {
log.warn(
"Missing End of GRIB message at pos="
+ ending
+ " header= "
+ StringUtil2.cleanup(header)
+ " for="
+ raf.getLocation());
break;
}
}
lastPos = raf.getFilePointer();
return new Grib2Record(header, is, ids, lus, gds, pds, drs, bms, dataSection, false);
} catch (Throwable t) {
long pos = (is == null) ? -1 : is.getStartPos();
log.warn("Bad GRIB2 record in file {}, skipping pos={}", raf.getLocation(), pos);
lastPos = raf.getFilePointer();
if (hasNext()) // skip forward
return next();
}
return null; // last record was incomplete
}
private void createIndex(
File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
throws IOException {
Grib2Record first = null; // take global metadata from here
if (indexFile.exists()) indexFile.delete(); // replace it
f.format(" createIndex for %s%n", indexFile.getPath());
RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
for (Group g : groups) {
g.fileSet = new HashSet<Integer>();
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
countRecords += vb.recordMap.length;
}
}
long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
f.format(
" write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
countBytes, countRecords, bytesPerRecord);
if (first == null) {
logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
}
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(gc.getName());
for (String fn : filenames) indexBuilder.addFiles(fn);
for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));
/* int count = 0;
for (DatasetCollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
count++;
} */
// what about just storing first ??
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
Grib2Pds pds = first.getPDS();
indexBuilder.setGenProcessType(pds.getGenProcessType());
indexBuilder.setGenProcessId(pds.getGenProcessId());
indexBuilder.setBackProcessId(pds.getBackProcessId());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format(" write GribCollectionIndex= %d bytes%n", b.length);
} finally {
f.format(" file size = %d bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
}
// check if compressed file seems ok
public static long testValid(String ufilename) throws IOException {
boolean lookForHeader = false;
// gotta make it
RandomAccessFile raf = new RandomAccessFile(ufilename, "r");
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
byte[] b = new byte[8];
raf.read(b);
String test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println("--Good header= " + test);
raf.seek(24);
} else {
System.out.println("--No header ");
lookForHeader = true;
raf.seek(0);
}
boolean eof = false;
int numCompBytes;
try {
while (!eof) {
if (lookForHeader) {
raf.read(b);
test = new String(b);
if (test.equals(Level2VolumeScan.ARCHIVE2) || test.equals(Level2VolumeScan.AR2V0001)) {
System.out.println(" found header= " + test);
raf.skipBytes(16);
lookForHeader = false;
} else {
raf.skipBytes(-8);
}
}
try {
numCompBytes = raf.readInt();
if (numCompBytes == -1) {
System.out.println("\n--done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
System.out.println("\n--got EOFException ");
break; // assume this is ok
}
System.out.print(" " + numCompBytes + ",");
if (numCompBytes < 0) {
System.out.println("\n--last block " + numCompBytes);
numCompBytes = -numCompBytes;
if (!lookForHeader) eof = true;
}
raf.skipBytes(numCompBytes);
}
} catch (EOFException e) {
e.printStackTrace();
}
return raf.getFilePointer();
}
/**
* Write equivilent uncompressed version of the file.
*
* @param inputRaf file to uncompress
* @param ufilename write to this file
* @return raf of uncompressed file
* @throws IOException on read error
*/
private RandomAccessFile uncompress(RandomAccessFile inputRaf, String ufilename)
throws IOException {
RandomAccessFile outputRaf = new RandomAccessFile(ufilename, "rw");
FileLock lock = null;
while (true) { // loop waiting for the lock
try {
lock = outputRaf.getRandomAccessFile().getChannel().lock(0, 1, false);
break;
} catch (OverlappingFileLockException oe) { // not sure why lock() doesnt block
try {
Thread.sleep(100); // msecs
} catch (InterruptedException e1) {
}
}
}
try {
inputRaf.seek(0);
byte[] header = new byte[Level2Record.FILE_HEADER_SIZE];
inputRaf.read(header);
outputRaf.write(header);
boolean eof = false;
int numCompBytes;
byte[] ubuff = new byte[40000];
byte[] obuff = new byte[40000];
try {
CBZip2InputStream cbzip2 = new CBZip2InputStream();
while (!eof) {
try {
numCompBytes = inputRaf.readInt();
if (numCompBytes == -1) {
if (log.isDebugEnabled()) log.debug(" done: numCompBytes=-1 ");
break;
}
} catch (EOFException ee) {
log.warn(" got EOFException ");
break; // assume this is ok
}
if (log.isDebugEnabled()) {
log.debug(
"reading compressed bytes "
+ numCompBytes
+ " input starts at "
+ inputRaf.getFilePointer()
+ "; output starts at "
+ outputRaf.getFilePointer());
}
/*
* For some stupid reason, the last block seems to
* have the number of bytes negated. So, we just
* assume that any negative number (other than -1)
* is the last block and go on our merry little way.
*/
if (numCompBytes < 0) {
if (log.isDebugEnabled()) log.debug("last block?" + numCompBytes);
numCompBytes = -numCompBytes;
eof = true;
}
byte[] buf = new byte[numCompBytes];
inputRaf.readFully(buf);
ByteArrayInputStream bis = new ByteArrayInputStream(buf, 2, numCompBytes - 2);
// CBZip2InputStream cbzip2 = new CBZip2InputStream(bis);
cbzip2.setStream(bis);
int total = 0;
int nread;
/*
while ((nread = cbzip2.read(ubuff)) != -1) {
dout2.write(ubuff, 0, nread);
total += nread;
}
*/
try {
while ((nread = cbzip2.read(ubuff)) != -1) {
if (total + nread > obuff.length) {
byte[] temp = obuff;
obuff = new byte[temp.length * 2];
System.arraycopy(temp, 0, obuff, 0, temp.length);
}
System.arraycopy(ubuff, 0, obuff, total, nread);
total += nread;
}
if (obuff.length >= 0) outputRaf.write(obuff, 0, total);
} catch (BZip2ReadException ioe) {
log.warn("Nexrad2IOSP.uncompress ", ioe);
}
float nrecords = (float) (total / 2432.0);
if (log.isDebugEnabled())
log.debug(
" unpacked "
+ total
+ " num bytes "
+ nrecords
+ " records; ouput ends at "
+ outputRaf.getFilePointer());
}
} catch (Exception e) {
if (outputRaf != null) outputRaf.close();
outputRaf = null;
// dont leave bad files around
File ufile = new File(ufilename);
if (ufile.exists()) {
if (!ufile.delete())
log.warn("failed to delete uncompressed file (IOException)" + ufilename);
}
if (e instanceof IOException) throw (IOException) e;
else throw new RuntimeException(e);
}
} finally {
if (null != outputRaf) outputRaf.flush();
if (lock != null) lock.release();
}
return outputRaf;
}