/**
* Sync the file
*
* @return true if needed to sync
* @throws IOException problem synching the file
*/
public boolean sync() throws IOException {
// printStack("***************************** sync ************************", 100);
// System.out.printf("check sync on %s (%s) %n", raf.getLocation(),
// Calendar.getInstance().getTime());
if (gemreader.getInitFileSize() < raf.length()) {
long start = System.currentTimeMillis();
Trace.msg(
"GEMPAKStationIOSP.sync: file "
+ raf.getLocation()
+ " is bigger: "
+ raf.length()
+ " > "
+ gemreader.getInitFileSize());
Trace.call1("GEMPAKStationIOSP.sync: reader.init");
gemreader.init(raf, true);
Trace.call2("GEMPAKStationIOSP.sync: reader.init");
Trace.call1("GEMPAKStationIOSP.sync: buildNCFile");
// reconstruct the ncfile objects
buildNCFile();
Trace.call2("GEMPAKSIOSP.sync: buildNCFile");
// System.out.printf("sync on %s took %d msecs%n", raf.getLocation(),
// (System.currentTimeMillis()-start));
return true;
}
return false;
}
/**
* Grib edition number 1, 2 or 0 not a Grib file.
*
* @return int 0 not a Grib file, 1 Grib1, 2 Grib2
* @throws NotSupportedException
* @throws IOException
*/
public final int getEdition() throws IOException, NotSupportedException {
int check = 0; // Not a valid Grib file
long length = (raf.length() < 4000L) ? raf.length() : 4000L;
if (!seekHeader(raf, length)) {
return 0; // not valid Grib file
}
// Read Section 0 Indicator Section to get Edition number
Grib1IndicatorSection is = new Grib1IndicatorSection(raf); // section 0
return is.getGribEdition();
} // end getEdition
/*
MAGIC_START
version
sizeRecords
VariableRecords (sizeRecords bytes)
sizeIndex
GribCollectionIndex (sizeIndex bytes)
*/
private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
File file = tp.getIndexFile();
if (file.exists()) {
if (!file.delete()) logger.error("Cant delete " + file.getPath());
}
RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
raf.writeLong(0); // no record section
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(tp.getName());
GribCollection canonGc = canon.makeGribCollection(f);
for (GribCollection.GroupHcs g : canonGc.getGroups())
indexBuilder.addGroups(writeGroupProto(g));
indexBuilder.setCenter(canonGc.getCenter());
indexBuilder.setSubcenter(canonGc.getSubcenter());
indexBuilder.setMaster(canonGc.getMaster());
indexBuilder.setLocal(canonGc.getLocal());
for (TimePartition.Partition p : tp.getPartitions()) {
indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
}
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);
} finally {
f.format("file size = %d bytes%n", raf.length());
raf.close();
}
return true;
}
public boolean hasNext() throws IOException {
if (lastPos >= raf.length()) return false;
if (repeatPos > 0) {
if (nextRepeating()) // this has created a new repeatRecord
return true;
} else {
repeatRecord = null;
repeatBms = null;
// fall through to new record
}
boolean more;
long stop = 0;
while (true) { // scan until we get a GRIB-2 or more == false
raf.seek(lastPos);
more = raf.searchForward(matcher, -1); // will scan to end for a 'GRIB' string
if (!more) break;
stop = raf.getFilePointer();
// see if its GRIB-2
raf.skipBytes(7);
int edition = raf.read();
if (edition == 2) break;
lastPos = raf.getFilePointer(); // not edition 2 ! just skip it !!
}
if (more) {
int sizeHeader =
(int) (stop - lastPos); // wmo headers are embedded between records in some idd streams
// if (sizeHeader > 30) sizeHeader = 30;
header = new byte[sizeHeader];
startPos = stop - sizeHeader;
raf.seek(startPos);
raf.read(header);
}
if (debug) System.out.println(" more " + more + " at " + startPos + " lastPos " + lastPos);
return more;
}
/**
* scans a Grib file to gather information that could be used to create an index or dump the
* metadata contents.
*
* @param parm
* @throws NotSupportedException
* @throws IOException if raf does not contain a valid GRIB record
*/
public final boolean scan(int[] parm) throws NotSupportedException, IOException {
long start = System.currentTimeMillis();
// stores the number of times a particular GDS is used
HashMap gdsCounter = new HashMap();
Grib1ProductDefinitionSection pds = null;
Grib1GridDefinitionSection gds = null;
long gdsOffset = 0;
DataOutputStream dos = new DataOutputStream(System.out);
// System.out.println("file position =" + raf.getFilePointer());
long SOR = raf.getFilePointer();
while (raf.getFilePointer() < raf.length()) {
SOR = raf.getFilePointer();
if (seekHeader(raf, raf.length())) {
// Read Section 0 Indicator Section
Grib1IndicatorSection is = new Grib1IndicatorSection(raf);
// System.out.println( "Grib record length=" + is.getGribLength());
// EOR (EndOfRecord) calculated so skipping data sections is faster
long EOR = raf.getFilePointer() + is.getGribLength() - is.getLength();
// long SOR = raf.getFilePointer() - is.getLength();
// skip Grib 2 records in a Grib 1 file
if (is.getGribEdition() == 2) {
// System.out.println( "Error Grib 2 record in Grib1 file" ) ;
raf.seek(EOR);
continue;
}
if (parm[0] == -1) { // extract only 1st record
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
break;
}
long dataOffset = 0;
try { // catch all exceptions and seek to EOR
// Read Section 1 Product Definition Section PDS
pds = new Grib1ProductDefinitionSection(raf);
if (pds.getLengthErr()) {
raf.seek(EOR);
continue;
}
Grib1PDSVariables pdsv = pds.getPdsVars();
if (getParameter && parm[0] == pdsv.getParameterNumber()) {
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
} else if (parm[0] <= pdsv.getForecastTime() && parm[1] >= pdsv.getForecastTime()) {
raf.seek(SOR);
byte[] oneRecord = new byte[(int) is.getGribLength()];
raf.read(oneRecord);
dos.write(oneRecord, 0, oneRecord.length);
dos.flush();
}
raf.seek(EOR);
continue;
} catch (Exception e) {
// .println( "Caught Exception scannning record" );
e.printStackTrace();
raf.seek(EOR);
continue;
}
} // end if seekHeader
// System.out.println( "raf.getFilePointer()=" + raf.getFilePointer());
// System.out.println( "raf.length()=" + raf.length() );
} // end while raf.getFilePointer() < raf.length()
// System.out.println("GribInput: processed in " +
// (System.currentTimeMillis()- start) + " milliseconds");
dos.close();
return true;
} // end scan
private void createIndex(
File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
throws IOException {
Grib2Record first = null; // take global metadata from here
if (indexFile.exists()) indexFile.delete(); // replace it
f.format(" createIndex for %s%n", indexFile.getPath());
RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
for (Group g : groups) {
g.fileSet = new HashSet<Integer>();
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
countRecords += vb.recordMap.length;
}
}
long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
f.format(
" write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
countBytes, countRecords, bytesPerRecord);
if (first == null) {
logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
}
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(gc.getName());
for (String fn : filenames) indexBuilder.addFiles(fn);
for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));
/* int count = 0;
for (DatasetCollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
count++;
} */
// what about just storing first ??
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
Grib2Pds pds = first.getPDS();
indexBuilder.setGenProcessType(pds.getGenProcessType());
indexBuilder.setGenProcessId(pds.getGenProcessId());
indexBuilder.setBackProcessId(pds.getBackProcessId());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format(" write GribCollectionIndex= %d bytes%n", b.length);
} finally {
f.format(" file size = %d bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
}
