@Override
protected boolean readPartitions(GribCollectionProto.GribCollectionIndex proto) {
for (int i = 0; i < proto.getPartitionsCount(); i++) {
GribCollectionProto.Partition pp = proto.getPartitions(i);
tp.addPartition(pp.getName(), pp.getFilename());
}
return proto.getPartitionsCount() > 0;
}
/*
MAGIC_START
version
sizeRecords
VariableRecords (sizeRecords bytes)
sizeIndex
GribCollectionIndex (sizeIndex bytes)
*/
private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
File file = tp.getIndexFile();
if (file.exists()) {
if (!file.delete()) logger.error("Cant delete " + file.getPath());
}
RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
raf.writeLong(0); // no record section
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(tp.getName());
GribCollection canonGc = canon.makeGribCollection(f);
for (GribCollection.GroupHcs g : canonGc.getGroups())
indexBuilder.addGroups(writeGroupProto(g));
indexBuilder.setCenter(canonGc.getCenter());
indexBuilder.setSubcenter(canonGc.getSubcenter());
indexBuilder.setMaster(canonGc.getMaster());
indexBuilder.setLocal(canonGc.getLocal());
for (TimePartition.Partition p : tp.getPartitions()) {
indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
}
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);
} finally {
f.format("file size = %d bytes%n", raf.length());
raf.close();
}
return true;
}
private void createIndex(
File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
throws IOException {
Grib2Record first = null; // take global metadata from here
if (indexFile.exists()) indexFile.delete(); // replace it
f.format(" createIndex for %s%n", indexFile.getPath());
RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
raf.order(RandomAccessFile.BIG_ENDIAN);
try {
//// header message
raf.write(MAGIC_START.getBytes("UTF-8"));
raf.writeInt(version);
long lenPos = raf.getFilePointer();
raf.writeLong(0); // save space to write the length of the record section
long countBytes = 0;
int countRecords = 0;
for (Group g : groups) {
g.fileSet = new HashSet<Integer>();
for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
if (first == null) first = vb.first;
GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
byte[] b = vr.toByteArray();
vb.pos = raf.getFilePointer();
vb.length = b.length;
raf.write(b);
countBytes += b.length;
countRecords += vb.recordMap.length;
}
}
long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
f.format(
" write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
countBytes, countRecords, bytesPerRecord);
if (first == null) {
logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
}
long pos = raf.getFilePointer();
raf.seek(lenPos);
raf.writeLong(countBytes);
raf.seek(pos); // back to the output.
GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
GribCollectionProto.GribCollectionIndex.newBuilder();
indexBuilder.setName(gc.getName());
for (String fn : filenames) indexBuilder.addFiles(fn);
for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));
/* int count = 0;
for (DatasetCollectionManager dcm : collections) {
indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
count++;
} */
// what about just storing first ??
Grib2SectionIdentification ids = first.getId();
indexBuilder.setCenter(ids.getCenter_id());
indexBuilder.setSubcenter(ids.getSubcenter_id());
indexBuilder.setMaster(ids.getMaster_table_version());
indexBuilder.setLocal(ids.getLocal_table_version());
Grib2Pds pds = first.getPDS();
indexBuilder.setGenProcessType(pds.getGenProcessType());
indexBuilder.setGenProcessId(pds.getGenProcessId());
indexBuilder.setBackProcessId(pds.getBackProcessId());
GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
byte[] b = index.toByteArray();
NcStream.writeVInt(raf, b.length); // message size
raf.write(b); // message - all in one gulp
f.format(" write GribCollectionIndex= %d bytes%n", b.length);
} finally {
f.format(" file size = %d bytes%n", raf.length());
raf.close();
if (raf != null) raf.close();
}
}
public boolean readIndex(RandomAccessFile raf) {
gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection
try {
raf.order(RandomAccessFile.BIG_ENDIAN);
raf.seek(0);
//// header message
if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) {
logger.error("GribCollection {}: invalid index", gc.getName());
return false;
}
int v = raf.readInt();
if (v != getVersion()) {
logger.warn(
"GribCollection {}: index found version={}, want version= {} on file {}",
new Object[] {gc.getName(), v, version, raf.getLocation()});
return false;
}
long skip = raf.readLong();
raf.skipBytes(skip);
int size = NcStream.readVInt(raf);
if ((size < 0) || (size > 100 * 1000 * 1000)) {
logger.warn("GribCollection {}: invalid index ", gc.getName());
return false;
}
byte[] m = new byte[size];
raf.readFully(m);
GribCollectionProto.GribCollectionIndex proto =
GribCollectionProto.GribCollectionIndex.parseFrom(m);
gc.center = proto.getCenter();
gc.subcenter = proto.getSubcenter();
gc.master = proto.getMaster();
gc.local = proto.getLocal();
gc.genProcessType = proto.getGenProcessType();
gc.genProcessId = proto.getGenProcessId();
gc.backProcessId = proto.getBackProcessId();
gc.local = proto.getLocal();
// gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local);
gc.filenames = new ArrayList<String>(proto.getFilesCount());
for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i));
// error condition on a GribCollection Index
if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) {
logger.warn("GribCollection {}: has no files, force recreate ", gc.getName());
return false;
}
gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount());
for (int i = 0; i < proto.getGroupsCount(); i++)
gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup()));
Collections.sort(gc.groups);
gc.params = new ArrayList<Parameter>(proto.getParamsCount());
for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i)));
if (!readPartitions(proto)) {
logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName());
return false;
}
return true;
} catch (Throwable t) {
logger.error("Error reading index " + raf.getLocation(), t);
return false;
}
}