// LOOK what about extending an index ??
public boolean makeIndex(String filename, RandomAccessFile dataRaf, Formatter f)
throws IOException {
File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
FileOutputStream fout = new FileOutputStream(idxFile);
RandomAccessFile raf = null;
try {
//// header message
fout.write(MAGIC_START.getBytes("UTF-8"));
NcStream.writeVInt(fout, version);
Map<Long, Integer> gdsMap = new HashMap<Long, Integer>();
gdsList = new ArrayList<Grib1SectionGridDefinition>();
records = new ArrayList<Grib1Record>(200);
Grib1IndexProto.Grib1Index.Builder rootBuilder = Grib1IndexProto.Grib1Index.newBuilder();
rootBuilder.setFilename(filename);
if (dataRaf == null) {
raf = new RandomAccessFile(filename, "r");
dataRaf = raf;
}
Grib1RecordScanner scan = new Grib1RecordScanner(dataRaf);
while (scan.hasNext()) {
Grib1Record r = scan.next();
if (r == null) break; // done
records.add(r);
Grib1SectionGridDefinition gds = r.getGDSsection();
Integer index = gdsMap.get(gds.calcCRC());
if (gds.getPredefinedGridDefinition()
>= 0) // skip predefined gds - they dont have raw bytes
index = 0;
else if (index == null) {
gdsList.add(gds);
index = gdsList.size() - 1;
gdsMap.put(gds.calcCRC(), index);
rootBuilder.addGdsList(makeGdsProto(gds));
}
rootBuilder.addRecords(makeRecordProto(r, index));
}
ucar.nc2.grib.grib1.Grib1IndexProto.Grib1Index index = rootBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(fout, b.length); // message size
fout.write(b); // message - all in one gulp
f.format(" made gbx9 index for %s size=%d%n", filename, b.length);
return true;
} finally {
fout.close();
if (raf != null) raf.close();
}
}
private boolean needsUpdate(long collectionLastModified, Formatter f) throws IOException {
CollectionManager.ChangeChecker cc = Grib1Index.getChangeChecker();
for (CollectionManager dcm :
tpc.makePartitions()) { // LOOK not really right, since we dont know if these files are the
// same as in the index
File idxFile = GribCollection.getIndexFile(dcm);
if (!idxFile.exists()) return true;
if (collectionLastModified < idxFile.lastModified()) return true;
for (MFile mfile : dcm.getFiles()) {
if (cc.hasChangedSince(mfile, idxFile.lastModified())) return true;
}
}
return false;
}
// read or create index
private void readOrCreateIndex(CollectionManager.Force ff, Formatter f) throws IOException {
// force new index or test for new index needed
boolean force =
((ff == CollectionManager.Force.always)
|| (ff == CollectionManager.Force.test && needsUpdate()));
// otherwise, we're good as long as the index file exists
File idx = gc.getIndexFile();
if (force || !idx.exists() || !readIndex(idx.getPath())) {
logger.info("GribCollection {}: createIndex {}", gc.getName(), idx.getPath());
createIndex(idx, ff, f); // write out index
gc.rafLocation = idx.getPath();
gc.setRaf(new RandomAccessFile(idx.getPath(), "r"));
readIndex(gc.getRaf()); // read back in index
}
}
public boolean readIndex(String filename, long gribLastModified, CollectionManager.Force force)
throws IOException {
File idxFile = GribCollection.getIndexFile(filename + GBX9_IDX);
if (!idxFile.exists()) return false;
long idxModified = idxFile.lastModified();
if ((force != CollectionManager.Force.nocheck) && (idxModified < gribLastModified))
return false;
// force new index if file was updated
FileInputStream fin =
new FileInputStream(idxFile); // LOOK need DiskCache for non-writeable directories
try {
//// check header is ok
if (!NcStream.readAndTest(fin, MAGIC_START.getBytes())) {
log.info("Bad magic number of grib index, on file" + idxFile);
return false;
}
int v = NcStream.readVInt(fin);
if (v != version) {
if ((v == 0) || (v > version))
throw new IOException(
"Grib1Index found version " + v + ", want version " + version + " on " + filename);
if (log.isDebugEnabled())
log.debug(
"Grib1Index found version " + v + ", want version " + version + " on " + filename);
return false;
}
int size = NcStream.readVInt(fin);
if (size <= 0 || size > 100 * 1000 * 1000) { // try to catch garbage
log.warn("Grib1Index bad size = {} for {} ", size, filename);
return false;
}
byte[] m = new byte[size];
NcStream.readFully(fin, m);
Grib1IndexProto.Grib1Index proto = Grib1IndexProto.Grib1Index.parseFrom(m);
String fname = proto.getFilename();
if (debug) System.out.printf("%s for %s%n", fname, filename);
gdsList = new ArrayList<Grib1SectionGridDefinition>(proto.getGdsListCount());
for (Grib1IndexProto.Grib1GdsSection pgds : proto.getGdsListList()) {
Grib1SectionGridDefinition gds = readGds(pgds);
gdsList.add(gds);
}
if (debug) System.out.printf(" read %d gds%n", gdsList.size());
records = new ArrayList<Grib1Record>(proto.getRecordsCount());
for (Grib1IndexProto.Grib1Record precord : proto.getRecordsList()) {
records.add(readRecord(precord));
}
if (debug) System.out.printf(" read %d records%n", records.size());
} catch (java.lang.NegativeArraySizeException e) {
log.error("GribIndex failed on " + filename, e);
return false;
} catch (IOException e) {
log.error("GribIndex failed on " + filename, e);
return false;
} finally {
fin.close();
}
return true;
}
public boolean needsUpdate() {
File idx = gc.getIndexFile();
return !idx.exists() || needsUpdate(idx.lastModified());
}