@Override protected boolean readPartitions(GribCollectionProto.GribCollectionIndex proto) { for (int i = 0; i < proto.getPartitionsCount(); i++) { GribCollectionProto.Partition pp = proto.getPartitions(i); tp.addPartition(pp.getName(), pp.getFilename()); } return proto.getPartitionsCount() > 0; }
/* MAGIC_START version sizeRecords VariableRecords (sizeRecords bytes) sizeIndex GribCollectionIndex (sizeIndex bytes) */ private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException { File file = tp.getIndexFile(); if (file.exists()) { if (!file.delete()) logger.error("Cant delete " + file.getPath()); } RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw"); raf.order(RandomAccessFile.BIG_ENDIAN); try { //// header message raf.write(MAGIC_START.getBytes("UTF-8")); raf.writeInt(version); raf.writeLong(0); // no record section GribCollectionProto.GribCollectionIndex.Builder indexBuilder = GribCollectionProto.GribCollectionIndex.newBuilder(); indexBuilder.setName(tp.getName()); GribCollection canonGc = canon.makeGribCollection(f); for (GribCollection.GroupHcs g : canonGc.getGroups()) indexBuilder.addGroups(writeGroupProto(g)); indexBuilder.setCenter(canonGc.getCenter()); indexBuilder.setSubcenter(canonGc.getSubcenter()); indexBuilder.setMaster(canonGc.getMaster()); indexBuilder.setLocal(canonGc.getLocal()); for (TimePartition.Partition p : tp.getPartitions()) { indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p)); } GribCollectionProto.GribCollectionIndex index = indexBuilder.build(); byte[] b = index.toByteArray(); NcStream.writeVInt(raf, b.length); // message size raf.write(b); // message - all in one gulp f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length); } finally { f.format("file size = %d bytes%n", raf.length()); raf.close(); } return true; }
private void createIndex( File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f) throws IOException { Grib2Record first = null; // take global metadata from here if (indexFile.exists()) indexFile.delete(); // replace it f.format(" createIndex for %s%n", indexFile.getPath()); RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw"); raf.order(RandomAccessFile.BIG_ENDIAN); try { //// header message raf.write(MAGIC_START.getBytes("UTF-8")); raf.writeInt(version); long lenPos = raf.getFilePointer(); raf.writeLong(0); // save space to write the length of the record section long countBytes = 0; int countRecords = 0; for (Group g : groups) { g.fileSet = new HashSet<Integer>(); for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) { if (first == null) first = vb.first; GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet); byte[] b = vr.toByteArray(); vb.pos = raf.getFilePointer(); vb.length = b.length; raf.write(b); countBytes += b.length; countRecords += vb.recordMap.length; } } long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords); f.format( " write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n", countBytes, countRecords, bytesPerRecord); if (first == null) { logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString()); throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files"); } long pos = raf.getFilePointer(); raf.seek(lenPos); raf.writeLong(countBytes); raf.seek(pos); // back to the output. GribCollectionProto.GribCollectionIndex.Builder indexBuilder = GribCollectionProto.GribCollectionIndex.newBuilder(); indexBuilder.setName(gc.getName()); for (String fn : filenames) indexBuilder.addFiles(fn); for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g)); /* int count = 0; for (DatasetCollectionManager dcm : collections) { indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.()))); count++; } */ // what about just storing first ?? Grib2SectionIdentification ids = first.getId(); indexBuilder.setCenter(ids.getCenter_id()); indexBuilder.setSubcenter(ids.getSubcenter_id()); indexBuilder.setMaster(ids.getMaster_table_version()); indexBuilder.setLocal(ids.getLocal_table_version()); Grib2Pds pds = first.getPDS(); indexBuilder.setGenProcessType(pds.getGenProcessType()); indexBuilder.setGenProcessId(pds.getGenProcessId()); indexBuilder.setBackProcessId(pds.getBackProcessId()); GribCollectionProto.GribCollectionIndex index = indexBuilder.build(); byte[] b = index.toByteArray(); NcStream.writeVInt(raf, b.length); // message size raf.write(b); // message - all in one gulp f.format(" write GribCollectionIndex= %d bytes%n", b.length); } finally { f.format(" file size = %d bytes%n", raf.length()); raf.close(); if (raf != null) raf.close(); } }
public boolean readIndex(RandomAccessFile raf) { gc.setRaf(raf); // LOOK leaving the raf open in the GribCollection try { raf.order(RandomAccessFile.BIG_ENDIAN); raf.seek(0); //// header message if (!NcStream.readAndTest(raf, MAGIC_START.getBytes())) { logger.error("GribCollection {}: invalid index", gc.getName()); return false; } int v = raf.readInt(); if (v != getVersion()) { logger.warn( "GribCollection {}: index found version={}, want version= {} on file {}", new Object[] {gc.getName(), v, version, raf.getLocation()}); return false; } long skip = raf.readLong(); raf.skipBytes(skip); int size = NcStream.readVInt(raf); if ((size < 0) || (size > 100 * 1000 * 1000)) { logger.warn("GribCollection {}: invalid index ", gc.getName()); return false; } byte[] m = new byte[size]; raf.readFully(m); GribCollectionProto.GribCollectionIndex proto = GribCollectionProto.GribCollectionIndex.parseFrom(m); gc.center = proto.getCenter(); gc.subcenter = proto.getSubcenter(); gc.master = proto.getMaster(); gc.local = proto.getLocal(); gc.genProcessType = proto.getGenProcessType(); gc.genProcessId = proto.getGenProcessId(); gc.backProcessId = proto.getBackProcessId(); gc.local = proto.getLocal(); // gc.tables = Grib2Tables.factory(gc.center, gc.subcenter, gc.master, gc.local); gc.filenames = new ArrayList<String>(proto.getFilesCount()); for (int i = 0; i < proto.getFilesCount(); i++) gc.filenames.add(proto.getFiles(i)); // error condition on a GribCollection Index if ((proto.getFilesCount() == 0) && !(this instanceof TimePartitionBuilder)) { logger.warn("GribCollection {}: has no files, force recreate ", gc.getName()); return false; } gc.groups = new ArrayList<GribCollection.GroupHcs>(proto.getGroupsCount()); for (int i = 0; i < proto.getGroupsCount(); i++) gc.groups.add(readGroup(proto.getGroups(i), gc.makeGroup())); Collections.sort(gc.groups); gc.params = new ArrayList<Parameter>(proto.getParamsCount()); for (int i = 0; i < proto.getParamsCount(); i++) gc.params.add(readParam(proto.getParams(i))); if (!readPartitions(proto)) { logger.warn("TimePartition {}: has no partitions, force recreate ", gc.getName()); return false; } return true; } catch (Throwable t) { logger.error("Error reading index " + raf.getLocation(), t); return false; } }