private GribCollectionProto.Partition writePartitionProto(String name, TimePartition.Partition p) throws IOException { GribCollectionProto.Partition.Builder b = GribCollectionProto.Partition.newBuilder(); b.setFilename(p.getIndexFilename()); b.setName(name); return b.build(); }
/* MAGIC_START version sizeRecords VariableRecords (sizeRecords bytes) sizeIndex GribCollectionIndex (sizeIndex bytes) */ private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException { File file = tp.getIndexFile(); if (file.exists()) { if (!file.delete()) logger.error("Cant delete " + file.getPath()); } RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw"); raf.order(RandomAccessFile.BIG_ENDIAN); try { //// header message raf.write(MAGIC_START.getBytes("UTF-8")); raf.writeInt(version); raf.writeLong(0); // no record section GribCollectionProto.GribCollectionIndex.Builder indexBuilder = GribCollectionProto.GribCollectionIndex.newBuilder(); indexBuilder.setName(tp.getName()); GribCollection canonGc = canon.makeGribCollection(f); for (GribCollection.GroupHcs g : canonGc.getGroups()) indexBuilder.addGroups(writeGroupProto(g)); indexBuilder.setCenter(canonGc.getCenter()); indexBuilder.setSubcenter(canonGc.getSubcenter()); indexBuilder.setMaster(canonGc.getMaster()); indexBuilder.setLocal(canonGc.getLocal()); for (TimePartition.Partition p : tp.getPartitions()) { indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p)); } GribCollectionProto.GribCollectionIndex index = indexBuilder.build(); byte[] b = index.toByteArray(); NcStream.writeVInt(raf, b.length); // message size raf.write(b); // message - all in one gulp f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length); } finally { f.format("file size = %d bytes%n", raf.length()); raf.close(); } return true; }
private boolean createPartitionedIndex(Formatter f) throws IOException { long start = System.currentTimeMillis(); // create partitions based on TimePartitionCollections object for (CollectionManager dcm : tpc.makePartitions()) { tp.addPartition(dcm); } List<TimePartition.Partition> bad = new ArrayList<TimePartition.Partition>(); for (TimePartition.Partition dc : tp.getPartitions()) { try { dc.makeGribCollection(f); // ensure collection has been read successfully if (trace) f.format(" Open partition %s%n", dc.getDcm().getCollectionName()); } catch (Throwable t) { logger.error(" Failed to open partition " + dc.getName(), t); f.format(" FAIL on partition %s (remove) %n", dc.getDcm().getCollectionName()); bad.add(dc); // LOOK may be a file leak ? } } // remove ones that failed for (TimePartition.Partition p : bad) tp.removePartition(p); // choose the "canonical" partition, aka prototype int n = tp.getPartitions().size(); if (n == 0) { logger.error(" Nothing in this partition = " + tp.getName()); f.format(" FAIL Partition empty collection = %s%n", tp.getName()); return false; } int idx = tpc.getProtoIndex(n); TimePartition.Partition canon = tp.getPartitions().get(idx); f.format(" Using canonical partition %s%n", canon.getDcm().getCollectionName()); // check consistency across vert and ens coords if (!checkPartitions(canon, f)) { logger.error( " Partition check failed, index not written on {} message = {}", tp.getName(), f.toString()); f.format(" FAIL Partition check collection = %s%n", tp.getName()); return false; } // make the time coordinates, place results into canon createPartitionedTimeCoordinates(canon, f); // ready to write the index file writeIndex(canon, f); // close open gc's tp.cleanup(); long took = System.currentTimeMillis() - start; f.format(" CreatePartitionedIndex took %d msecs%n", took); return true; }
private boolean createPartitionedTimeCoordinates(TimePartition.Partition canon, Formatter f) throws IOException { List<TimePartition.Partition> partitions = tp.getPartitions(); boolean ok = true; // for each group in canonical Partition for (GribCollection.GroupHcs firstGroup : canon.makeGribCollection(f).getGroups()) { String gname = firstGroup.getId(); if (trace) f.format(" Check Group %s%n", gname); // get list of corresponding groups from all the time partition, so we dont have to keep // looking it up List<PartGroup> pgList = new ArrayList<PartGroup>(partitions.size()); for (TimePartition.Partition dc : partitions) { GribCollection.GroupHcs gg = dc.makeGribCollection(f).findGroupById(gname); if (gg == null) logger.error(" Cant find group {} in partition {}", gname, dc.getName()); else pgList.add(new PartGroup(gg, dc)); } // unique time coordinate unions List<TimeCoordUnion> unionList = new ArrayList<TimeCoordUnion>(); // for each variable in canonical Partition for (GribCollection.VariableIndex viCanon : firstGroup.varIndex) { if (trace) f.format(" Check variable %s%n", viCanon); TimeCoord tcCanon = viCanon.getTimeCoord(); List<TimeCoord> tcPartitions = new ArrayList<TimeCoord>(pgList.size()); // for each partition, get the time index for (PartGroup pg : pgList) { // get corresponding variable GribCollection.VariableIndex vi2 = pg.group.findVariableByHash(viCanon.cdmHash); if (vi2 == null) { // apparently not in the file f.format( " WARN Cant find variable %s in partition %s / %s%n", viCanon, pg.tpp.getName(), pg.group.getId()); tcPartitions.add(null); } else { if (vi2.timeIdx < 0 || vi2.timeIdx >= pg.group.timeCoords.size()) { logger.error(" timeIdx out of range var= {} on partition {}", vi2, pg.tpp.getName()); tcPartitions.add(null); } else { TimeCoord tc2 = vi2.getTimeCoord(); if (tc2.isInterval() != tcCanon.isInterval()) { logger.error( " timeIdx wrong interval type var= {} on partition {}", vi2, pg.tpp.getName()); tcPartitions.add(null); } else { tcPartitions.add(tc2); } } } } // union of time coordinates TimeCoordUnion union = new TimeCoordUnion(tcCanon.getCode(), tcPartitions, tcCanon); // store result in the first group viCanon.partTimeCoordIdx = TimeCoordUnion.findUnique(unionList, union); // this merges identical TimeCoordUnion } /* turn TimeIndex into TimeCoord for (int tidx = 0; tidx <unionList.size(); tidx++) { TimeCoordUnion union = unionList.get(tidx); f.format(" %s %d: timeIndexList=", firstGroup.hcs.getName(), tidx); for (int idx : union.) f.format("%d,",idx); f.format("%n"); } */ // store results in first group firstGroup.timeCoordPartitions = unionList; } return ok; }
// consistency check on variables : compare each variable to corresponding one in proto // also set the groupno and partno for each partition private boolean checkPartitions(TimePartition.Partition canon, Formatter f) throws IOException { List<TimePartition.Partition> partitions = tp.getPartitions(); int npart = partitions.size(); boolean ok = true; // for each group in canonical Partition GribCollection canonGc = canon.makeGribCollection(f); for (GribCollection.GroupHcs firstGroup : canonGc.getGroups()) { String gname = firstGroup.getId(); if (trace) f.format(" Check Group %s%n", gname); // hash proto variables for quick lookup Map<Integer, GribCollection.VariableIndex> check = new HashMap<Integer, GribCollection.VariableIndex>(firstGroup.varIndex.size()); List<GribCollection.VariableIndex> varIndexP = new ArrayList<GribCollection.VariableIndex>(firstGroup.varIndex.size()); for (GribCollection.VariableIndex vi : firstGroup.varIndex) { TimePartition.VariableIndexPartitioned vip = tp.makeVariableIndexPartitioned(vi, npart); varIndexP.add(vip); check.put(vi.cdmHash, vip); // replace with its evil twin } firstGroup.varIndex = varIndexP; // replace with its evil twin // for each partition for (int partno = 0; partno < npart; partno++) { TimePartition.Partition tpp = partitions.get(partno); if (trace) f.format(" Check Partition %s%n", tpp.getName()); // get corresponding group GribCollection gc = tpp.makeGribCollection(f); int groupIdx = gc.findGroupIdxById(firstGroup.getId()); if (groupIdx < 0) { f.format(" Cant find group %s in partition %s%n", gname, tpp.getName()); ok = false; continue; } GribCollection.GroupHcs group = gc.getGroup(groupIdx); // for each variable in partition group for (int varIdx = 0; varIdx < group.varIndex.size(); varIdx++) { GribCollection.VariableIndex vi2 = group.varIndex.get(varIdx); if (trace) f.format(" Check variable %s%n", vi2); int flag = 0; GribCollection.VariableIndex vi1 = check.get(vi2.cdmHash); // compare with proto variable if (vi1 == null) { f.format( " WARN Cant find variable %s from %s in proto - ignoring that variable%n", vi2, tpp.getName()); continue; // we can tolerate this } // compare vert coordinates VertCoord vc1 = vi1.getVertCoord(); VertCoord vc2 = vi2.getVertCoord(); if ((vc1 == null) != (vc2 == null)) { f.format( " ERR Vert coordinates existence on variable %s in %s doesnt match%n", vi2, tpp.getName()); ok = false; } else if ((vc1 != null) && !vc1.equalsData(vc2)) { f.format( " WARN Vert coordinates values on variable %s in %s dont match%n", vi2, tpp.getName()); f.format(" canon vc = %s%n", vc1); f.format(" this vc = %s%n", vc2); flag |= TimePartition.VERT_COORDS_DIFFER; } // compare ens coordinates EnsCoord ec1 = vi1.getEnsCoord(); EnsCoord ec2 = vi2.getEnsCoord(); if ((ec1 == null) != (ec2 == null)) { f.format( " ERR Ensemble coordinates existence on variable %s in %s doesnt match%n", vi2, tpp.getName()); ok = false; } else if ((ec1 != null) && !ec1.equalsData(ec2)) { f.format( " WARN Ensemble coordinates values on variable %s in %s dont match%n", vi2, tpp.getName()); f.format(" canon ec = %s%n", ec1); f.format(" this ec = %s%n", ec2); flag |= TimePartition.ENS_COORDS_DIFFER; } ((TimePartition.VariableIndexPartitioned) vi1) .setPartitionIndex(partno, groupIdx, varIdx, flag); } // loop over variable } // loop over partition } // loop over group if (ok) f.format(" Partition check: vert, ens coords OK%n"); return ok; }