private boolean createPartitionedIndex(Formatter f) throws IOException { long start = System.currentTimeMillis(); // create partitions based on TimePartitionCollections object for (CollectionManager dcm : tpc.makePartitions()) { tp.addPartition(dcm); } List<TimePartition.Partition> bad = new ArrayList<TimePartition.Partition>(); for (TimePartition.Partition dc : tp.getPartitions()) { try { dc.makeGribCollection(f); // ensure collection has been read successfully if (trace) f.format(" Open partition %s%n", dc.getDcm().getCollectionName()); } catch (Throwable t) { logger.error(" Failed to open partition " + dc.getName(), t); f.format(" FAIL on partition %s (remove) %n", dc.getDcm().getCollectionName()); bad.add(dc); // LOOK may be a file leak ? } } // remove ones that failed for (TimePartition.Partition p : bad) tp.removePartition(p); // choose the "canonical" partition, aka prototype int n = tp.getPartitions().size(); if (n == 0) { logger.error(" Nothing in this partition = " + tp.getName()); f.format(" FAIL Partition empty collection = %s%n", tp.getName()); return false; } int idx = tpc.getProtoIndex(n); TimePartition.Partition canon = tp.getPartitions().get(idx); f.format(" Using canonical partition %s%n", canon.getDcm().getCollectionName()); // check consistency across vert and ens coords if (!checkPartitions(canon, f)) { logger.error( " Partition check failed, index not written on {} message = {}", tp.getName(), f.toString()); f.format(" FAIL Partition check collection = %s%n", tp.getName()); return false; } // make the time coordinates, place results into canon createPartitionedTimeCoordinates(canon, f); // ready to write the index file writeIndex(canon, f); // close open gc's tp.cleanup(); long took = System.currentTimeMillis() - start; f.format(" CreatePartitionedIndex took %d msecs%n", took); return true; }
private void createIndex( File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f) throws IOException { Grib2Record first = null; // take global metadata from here if (indexFile.exists()) indexFile.delete(); // replace it f.format(" createIndex for %s%n", indexFile.getPath()); RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw"); raf.order(RandomAccessFile.BIG_ENDIAN); try { //// header message raf.write(MAGIC_START.getBytes("UTF-8")); raf.writeInt(version); long lenPos = raf.getFilePointer(); raf.writeLong(0); // save space to write the length of the record section long countBytes = 0; int countRecords = 0; for (Group g : groups) { g.fileSet = new HashSet<Integer>(); for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) { if (first == null) first = vb.first; GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet); byte[] b = vr.toByteArray(); vb.pos = raf.getFilePointer(); vb.length = b.length; raf.write(b); countBytes += b.length; countRecords += vb.recordMap.length; } } long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords); f.format( " write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n", countBytes, countRecords, bytesPerRecord); if (first == null) { logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString()); throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files"); } long pos = raf.getFilePointer(); raf.seek(lenPos); raf.writeLong(countBytes); raf.seek(pos); // back to the output. GribCollectionProto.GribCollectionIndex.Builder indexBuilder = GribCollectionProto.GribCollectionIndex.newBuilder(); indexBuilder.setName(gc.getName()); for (String fn : filenames) indexBuilder.addFiles(fn); for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g)); /* int count = 0; for (DatasetCollectionManager dcm : collections) { indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.()))); count++; } */ // what about just storing first ?? Grib2SectionIdentification ids = first.getId(); indexBuilder.setCenter(ids.getCenter_id()); indexBuilder.setSubcenter(ids.getSubcenter_id()); indexBuilder.setMaster(ids.getMaster_table_version()); indexBuilder.setLocal(ids.getLocal_table_version()); Grib2Pds pds = first.getPDS(); indexBuilder.setGenProcessType(pds.getGenProcessType()); indexBuilder.setGenProcessId(pds.getGenProcessId()); indexBuilder.setBackProcessId(pds.getBackProcessId()); GribCollectionProto.GribCollectionIndex index = indexBuilder.build(); byte[] b = index.toByteArray(); NcStream.writeVInt(raf, b.length); // message size raf.write(b); // message - all in one gulp f.format(" write GribCollectionIndex= %d bytes%n", b.length); } finally { f.format(" file size = %d bytes%n", raf.length()); raf.close(); if (raf != null) raf.close(); } }