/*
MAGIC_START
version
sizeRecords
VariableRecords (sizeRecords bytes)
sizeIndex
GribCollectionIndex (sizeIndex bytes)
*/
private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
File file = tp.getIndexFile();
if (file.exists()) {
if (!file.delete()) logger.error("Cant delete " + file.getPath());
}
RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
raf.writeLong(0); // no record section
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(tp.getName());
GribCollection canonGc = canon.makeGribCollection(f);
for (GribCollection.GroupHcs g : canonGc.getGroups())
indexBuilder.addGroups(writeGroupProto(g));
indexBuilder.setCenter(canonGc.getCenter());
indexBuilder.setSubcenter(canonGc.getSubcenter());
indexBuilder.setMaster(canonGc.getMaster());
indexBuilder.setLocal(canonGc.getLocal());
for (TimePartition.Partition p : tp.getPartitions()) {
indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
}
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);
} finally {
f.format("file size = %d bytes%n", raf.length());
raf.close();
}
return true;
}
private void createIndex(
File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
throws IOException {
Grib2Record first = null; // take global metadata from here
if (indexFile.exists()) indexFile.delete(); // replace it
f.format(" createIndex for %s%n", indexFile.getPath());
RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
for (Group g : groups) {
g.fileSet = new HashSet<Integer>();
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
countRecords += vb.recordMap.length;
}
}
long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
f.format(
" write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
countBytes, countRecords, bytesPerRecord);
if (first == null) {
logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
}
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(gc.getName());
for (String fn : filenames) indexBuilder.addFiles(fn);
for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));
/* int count = 0;
for (DatasetCollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
count++;
} */
// what about just storing first ??
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
Grib2Pds pds = first.getPDS();
indexBuilder.setGenProcessType(pds.getGenProcessType());
indexBuilder.setGenProcessId(pds.getGenProcessId());
indexBuilder.setBackProcessId(pds.getBackProcessId());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format(" write GribCollectionIndex= %d bytes%n", b.length);
} finally {
f.format(" file size = %d bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
}