private boolean filterTinv(
      Grib2Record gr, FeatureCollectionConfig.GribIntvFilter intvFilter, Formatter f) {
    int[] intv = tables.getForecastTimeIntervalOffset(gr);
    if (intv == null) return false;
    int haveLength = intv[1] - intv[0];

    // HACK
    if (haveLength == 0 && intvFilter.isZeroExcluded()) { // discard 0,0
      if ((intv[0] == 0) && (intv[1] == 0)) {
        f.format(" FILTER INTV [0, 0] %s%n", gr);
        return true;
      }
      return false;

    } else if (intvFilter.hasMap()) {
      int discipline = gr.getIs().getDiscipline();
      int category = gr.getPDS().getParameterCategory();
      int number = gr.getPDS().getParameterNumber();
      int id = (discipline << 16) + (category << 8) + number;
      Integer needLength = intvFilter.getLengthById(id);

      if (needLength != null && needLength != haveLength) {
        f.format(" FILTER INTV [%d != %d] %s%n", haveLength, needLength, gr);
        return true;
      }
    }
    return false;
  }
  /*
  MAGIC_START
  version
  sizeRecords
  VariableRecords (sizeRecords bytes)
  sizeIndex
  GribCollectionIndex (sizeIndex bytes)
  */
  private boolean writeIndex(TimePartition.Partition canon, Formatter f) throws IOException {
    File file = tp.getIndexFile();
    if (file.exists()) {
      if (!file.delete()) logger.error("Cant delete " + file.getPath());
    }

    RandomAccessFile raf = new RandomAccessFile(file.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      raf.writeLong(0); // no record section

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(tp.getName());

      GribCollection canonGc = canon.makeGribCollection(f);
      for (GribCollection.GroupHcs g : canonGc.getGroups())
        indexBuilder.addGroups(writeGroupProto(g));

      indexBuilder.setCenter(canonGc.getCenter());
      indexBuilder.setSubcenter(canonGc.getSubcenter());
      indexBuilder.setMaster(canonGc.getMaster());
      indexBuilder.setLocal(canonGc.getLocal());

      for (TimePartition.Partition p : tp.getPartitions()) {
        indexBuilder.addPartitions(writePartitionProto(p.getName(), (TimePartition.Partition) p));
      }

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("GribCollectionTimePartitionedIndex= %d bytes%n", b.length);

    } finally {
      f.format("file size =  %d bytes%n", raf.length());
      raf.close();
    }

    return true;
  }
  private boolean createPartitionedIndex(Formatter f) throws IOException {
    long start = System.currentTimeMillis();

    // create partitions based on TimePartitionCollections object
    for (CollectionManager dcm : tpc.makePartitions()) {
      tp.addPartition(dcm);
    }

    List<TimePartition.Partition> bad = new ArrayList<TimePartition.Partition>();
    for (TimePartition.Partition dc : tp.getPartitions()) {
      try {
        dc.makeGribCollection(f); // ensure collection has been read successfully
        if (trace) f.format(" Open partition %s%n", dc.getDcm().getCollectionName());
      } catch (Throwable t) {
        logger.error(" Failed to open partition " + dc.getName(), t);
        f.format(" FAIL on partition %s (remove) %n", dc.getDcm().getCollectionName());
        bad.add(dc); // LOOK may be a file leak ?
      }
    }

    // remove ones that failed
    for (TimePartition.Partition p : bad) tp.removePartition(p);

    // choose the "canonical" partition, aka prototype
    int n = tp.getPartitions().size();
    if (n == 0) {
      logger.error(" Nothing in this partition = " + tp.getName());
      f.format(" FAIL Partition empty collection = %s%n", tp.getName());
      return false;
    }
    int idx = tpc.getProtoIndex(n);
    TimePartition.Partition canon = tp.getPartitions().get(idx);
    f.format(" Using canonical partition %s%n", canon.getDcm().getCollectionName());

    // check consistency across vert and ens coords
    if (!checkPartitions(canon, f)) {
      logger.error(
          " Partition check failed, index not written on {} message = {}",
          tp.getName(),
          f.toString());
      f.format(" FAIL Partition check collection = %s%n", tp.getName());
      return false;
    }

    // make the time coordinates, place results into canon
    createPartitionedTimeCoordinates(canon, f);

    // ready to write the index file
    writeIndex(canon, f);

    // close open gc's
    tp.cleanup();

    long took = System.currentTimeMillis() - start;
    f.format(" CreatePartitionedIndex took %d msecs%n", took);
    return true;
  }
  private boolean createIndex(File indexFile, CollectionManager.Force ff, Formatter f)
      throws IOException {
    long start = System.currentTimeMillis();

    ArrayList<String> filenames = new ArrayList<String>();
    List<Group> groups = makeAggregatedGroups(filenames, ff, f);
    createIndex(indexFile, groups, filenames, f);

    long took = System.currentTimeMillis() - start;
    f.format("That took %d msecs%n", took);
    return true;
  }
  // single file
  private Grib2CollectionBuilder(File file, Formatter f) throws IOException {
    try {
      // String spec = StringUtil2.substitute(file.getPath(), "\\", "/");
      CollectionManager dcm = new DatasetCollectionSingleFile(file);
      this.collections.add(dcm);
      this.gc = new Grib2Collection(file.getName(), new File(dcm.getRoot()));

    } catch (Exception e) {
      ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
      e.printStackTrace(new PrintStream(bos));
      f.format("%s", bos.toString());
      throw new IOException(e);
    }
  }
  // single file
  private Grib2CollectionBuilder(MFile file, FeatureCollectionConfig.GribConfig config, Formatter f)
      throws IOException {
    this.isSingleFile = true;
    try {
      // String spec = StringUtil2.substitute(file.getPath(), "\\", "/");
      CollectionManager dcm = new CollectionManagerSingleFile(file);
      if (config != null) dcm.putAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG, config);
      this.collections.add(dcm);
      this.gc = new Grib2Collection(file.getName(), new File(dcm.getRoot()), config);

    } catch (Exception e) {
      ByteArrayOutputStream bos = new ByteArrayOutputStream(10000);
      e.printStackTrace(new PrintStream(bos));
      f.format("%s", bos.toString());
      throw new IOException(e);
    }
  }
  private boolean createPartitionedTimeCoordinates(TimePartition.Partition canon, Formatter f)
      throws IOException {
    List<TimePartition.Partition> partitions = tp.getPartitions();
    boolean ok = true;

    // for each group in canonical Partition
    for (GribCollection.GroupHcs firstGroup : canon.makeGribCollection(f).getGroups()) {
      String gname = firstGroup.getId();
      if (trace) f.format(" Check Group %s%n", gname);

      // get list of corresponding groups from all the time partition, so we dont have to keep
      // looking it up
      List<PartGroup> pgList = new ArrayList<PartGroup>(partitions.size());
      for (TimePartition.Partition dc : partitions) {
        GribCollection.GroupHcs gg = dc.makeGribCollection(f).findGroupById(gname);
        if (gg == null) logger.error(" Cant find group {} in partition {}", gname, dc.getName());
        else pgList.add(new PartGroup(gg, dc));
      }

      // unique time coordinate unions
      List<TimeCoordUnion> unionList = new ArrayList<TimeCoordUnion>();

      // for each variable in canonical Partition
      for (GribCollection.VariableIndex viCanon : firstGroup.varIndex) {
        if (trace) f.format(" Check variable %s%n", viCanon);
        TimeCoord tcCanon = viCanon.getTimeCoord();

        List<TimeCoord> tcPartitions = new ArrayList<TimeCoord>(pgList.size());

        // for each partition, get the time index
        for (PartGroup pg : pgList) {
          // get corresponding variable
          GribCollection.VariableIndex vi2 = pg.group.findVariableByHash(viCanon.cdmHash);
          if (vi2 == null) { // apparently not in the file
            f.format(
                "   WARN Cant find variable %s in partition %s / %s%n",
                viCanon, pg.tpp.getName(), pg.group.getId());
            tcPartitions.add(null);
          } else {
            if (vi2.timeIdx < 0 || vi2.timeIdx >= pg.group.timeCoords.size()) {
              logger.error(" timeIdx out of range var= {} on partition {}", vi2, pg.tpp.getName());
              tcPartitions.add(null);
            } else {
              TimeCoord tc2 = vi2.getTimeCoord();
              if (tc2.isInterval() != tcCanon.isInterval()) {
                logger.error(
                    " timeIdx wrong interval type var= {} on partition {}", vi2, pg.tpp.getName());
                tcPartitions.add(null);
              } else {
                tcPartitions.add(tc2);
              }
            }
          }
        }

        // union of time coordinates
        TimeCoordUnion union = new TimeCoordUnion(tcCanon.getCode(), tcPartitions, tcCanon);

        // store result in the first group
        viCanon.partTimeCoordIdx =
            TimeCoordUnion.findUnique(unionList, union); // this merges identical TimeCoordUnion
      }

      /* turn TimeIndex into TimeCoord
      for (int tidx = 0; tidx <unionList.size(); tidx++) {
        TimeCoordUnion union = unionList.get(tidx);
        f.format(" %s %d: timeIndexList=", firstGroup.hcs.getName(), tidx);
        for (int idx : union.) f.format("%d,",idx);
        f.format("%n");
      } */

      // store results in first group
      firstGroup.timeCoordPartitions = unionList;
    }

    return ok;
  }
  // consistency check on variables : compare each variable to corresponding one in proto
  // also set the groupno and partno for each partition
  private boolean checkPartitions(TimePartition.Partition canon, Formatter f) throws IOException {
    List<TimePartition.Partition> partitions = tp.getPartitions();
    int npart = partitions.size();
    boolean ok = true;

    // for each group in canonical Partition
    GribCollection canonGc = canon.makeGribCollection(f);
    for (GribCollection.GroupHcs firstGroup : canonGc.getGroups()) {
      String gname = firstGroup.getId();
      if (trace) f.format(" Check Group %s%n", gname);

      // hash proto variables for quick lookup
      Map<Integer, GribCollection.VariableIndex> check =
          new HashMap<Integer, GribCollection.VariableIndex>(firstGroup.varIndex.size());
      List<GribCollection.VariableIndex> varIndexP =
          new ArrayList<GribCollection.VariableIndex>(firstGroup.varIndex.size());
      for (GribCollection.VariableIndex vi : firstGroup.varIndex) {
        TimePartition.VariableIndexPartitioned vip = tp.makeVariableIndexPartitioned(vi, npart);
        varIndexP.add(vip);
        check.put(vi.cdmHash, vip); // replace with its evil twin
      }
      firstGroup.varIndex = varIndexP; // replace with its evil twin

      // for each partition
      for (int partno = 0; partno < npart; partno++) {
        TimePartition.Partition tpp = partitions.get(partno);
        if (trace) f.format(" Check Partition %s%n", tpp.getName());

        // get corresponding group
        GribCollection gc = tpp.makeGribCollection(f);
        int groupIdx = gc.findGroupIdxById(firstGroup.getId());
        if (groupIdx < 0) {
          f.format(" Cant find group %s in partition %s%n", gname, tpp.getName());
          ok = false;
          continue;
        }
        GribCollection.GroupHcs group = gc.getGroup(groupIdx);

        // for each variable in partition group
        for (int varIdx = 0; varIdx < group.varIndex.size(); varIdx++) {
          GribCollection.VariableIndex vi2 = group.varIndex.get(varIdx);
          if (trace) f.format(" Check variable %s%n", vi2);
          int flag = 0;

          GribCollection.VariableIndex vi1 = check.get(vi2.cdmHash); // compare with proto variable
          if (vi1 == null) {
            f.format(
                "   WARN Cant find variable %s from %s in proto - ignoring that variable%n",
                vi2, tpp.getName());
            continue; // we can tolerate this
          }

          // compare vert coordinates
          VertCoord vc1 = vi1.getVertCoord();
          VertCoord vc2 = vi2.getVertCoord();
          if ((vc1 == null) != (vc2 == null)) {
            f.format(
                "   ERR Vert coordinates existence on variable %s in %s doesnt match%n",
                vi2, tpp.getName());
            ok = false;
          } else if ((vc1 != null) && !vc1.equalsData(vc2)) {
            f.format(
                "   WARN Vert coordinates values on variable %s in %s dont match%n",
                vi2, tpp.getName());
            f.format("    canon vc = %s%n", vc1);
            f.format("    this vc = %s%n", vc2);
            flag |= TimePartition.VERT_COORDS_DIFFER;
          }

          // compare ens coordinates
          EnsCoord ec1 = vi1.getEnsCoord();
          EnsCoord ec2 = vi2.getEnsCoord();
          if ((ec1 == null) != (ec2 == null)) {
            f.format(
                "   ERR Ensemble coordinates existence on variable %s in %s doesnt match%n",
                vi2, tpp.getName());
            ok = false;
          } else if ((ec1 != null) && !ec1.equalsData(ec2)) {
            f.format(
                "   WARN Ensemble coordinates values on variable %s in %s dont match%n",
                vi2, tpp.getName());
            f.format("    canon ec = %s%n", ec1);
            f.format("    this ec = %s%n", ec2);
            flag |= TimePartition.ENS_COORDS_DIFFER;
          }

          ((TimePartition.VariableIndexPartitioned) vi1)
              .setPartitionIndex(partno, groupIdx, varIdx, flag);
        } // loop over variable
      } // loop over partition
    } // loop over group

    if (ok) f.format("  Partition check: vert, ens coords OK%n");
    return ok;
  }
  private void createIndex(
      File indexFile, List<Group> groups, ArrayList<String> filenames, Formatter f)
      throws IOException {
    Grib2Record first = null; // take global metadata from here

    if (indexFile.exists()) indexFile.delete(); // replace it
    f.format(" createIndex for %s%n", indexFile.getPath());

    RandomAccessFile raf = new RandomAccessFile(indexFile.getPath(), "rw");
    raf.order(RandomAccessFile.BIG_ENDIAN);
    try {
      //// header message
      raf.write(MAGIC_START.getBytes("UTF-8"));
      raf.writeInt(version);
      long lenPos = raf.getFilePointer();
      raf.writeLong(0); // save space to write the length of the record section
      long countBytes = 0;
      int countRecords = 0;
      for (Group g : groups) {
        g.fileSet = new HashSet<Integer>();
        for (Grib2Rectilyser.VariableBag vb : g.rect.getGribvars()) {
          if (first == null) first = vb.first;
          GribCollectionProto.VariableRecords vr = writeRecordsProto(vb, g.fileSet);
          byte[] b = vr.toByteArray();
          vb.pos = raf.getFilePointer();
          vb.length = b.length;
          raf.write(b);
          countBytes += b.length;
          countRecords += vb.recordMap.length;
        }
      }
      long bytesPerRecord = countBytes / ((countRecords == 0) ? 1 : countRecords);
      f.format(
          "  write RecordMaps: bytes = %d record = %d bytesPerRecord=%d%n",
          countBytes, countRecords, bytesPerRecord);

      if (first == null) {
        logger.error("GribCollection {}: has no files\n{}", gc.getName(), f.toString());
        throw new IllegalArgumentException("GribCollection " + gc.getName() + " has no files");
      }

      long pos = raf.getFilePointer();
      raf.seek(lenPos);
      raf.writeLong(countBytes);
      raf.seek(pos); // back to the output.

      GribCollectionProto.GribCollectionIndex.Builder indexBuilder =
          GribCollectionProto.GribCollectionIndex.newBuilder();
      indexBuilder.setName(gc.getName());

      for (String fn : filenames) indexBuilder.addFiles(fn);

      for (Group g : groups) indexBuilder.addGroups(writeGroupProto(g));

      /* int count = 0;
      for (DatasetCollectionManager dcm : collections) {
        indexBuilder.addParams(makeParamProto(new Parameter("spec" + count, dcm.())));
        count++;
      } */

      // what about just storing first ??
      Grib2SectionIdentification ids = first.getId();
      indexBuilder.setCenter(ids.getCenter_id());
      indexBuilder.setSubcenter(ids.getSubcenter_id());
      indexBuilder.setMaster(ids.getMaster_table_version());
      indexBuilder.setLocal(ids.getLocal_table_version());

      Grib2Pds pds = first.getPDS();
      indexBuilder.setGenProcessType(pds.getGenProcessType());
      indexBuilder.setGenProcessId(pds.getGenProcessId());
      indexBuilder.setBackProcessId(pds.getBackProcessId());

      GribCollectionProto.GribCollectionIndex index = indexBuilder.build();
      byte[] b = index.toByteArray();
      NcStream.writeVInt(raf, b.length); // message size
      raf.write(b); // message  - all in one gulp
      f.format("  write GribCollectionIndex= %d bytes%n", b.length);

    } finally {
      f.format("  file size =  %d bytes%n", raf.length());
      raf.close();
      if (raf != null) raf.close();
    }
  }
  // read all records in all files,
  // divide into groups based on GDS hash
  // each group has an arraylist of all records that belong to it.
  // for each group, run rectlizer to derive the coordinates and variables
  public List<Group> makeAggregatedGroups(
      ArrayList<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
    Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();

    f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
    int total = 0;
    int fileno = 0;
    for (CollectionManager dcm : collections) {
      // dcm.scanIfNeeded(); // LOOK ??
      f.format(" dcm= %s%n", dcm);
      Map<Integer, Integer> gdsConvert = (Map<Integer, Integer>) dcm.getAuxInfo("gdsHash");

      for (MFile mfile : dcm.getFiles()) {
        // f.format("%3d: %s%n", fileno, mfile.getPath());
        filenames.add(mfile.getPath());

        Grib2Index index = new Grib2Index();
        try {
          if (!index.readIndex(
              mfile.getPath(),
              mfile.getLastModified(),
              force)) { // heres where the index date is checked against the data file
            index.makeIndex(mfile.getPath(), f);
            f.format(
                "  Index written: %s == %d records %n",
                mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
          } else if (debug) {
            f.format(
                "  Index read: %s == %d records %n",
                mfile.getName() + Grib2Index.IDX_EXT, index.getRecords().size());
          }
        } catch (IOException ioe) {
          f.format(
              "GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n  skipping %s%n",
              ioe.getMessage(), mfile.getPath() + Grib2Index.IDX_EXT);
          continue;
        }

        for (Grib2Record gr : index.getRecords()) {
          gr.setFile(fileno); // each record tracks which file it belongs to
          int gdsHash =
              gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
          if (gdsConvert != null
              && gdsConvert.get(gdsHash)
                  != null) { // allow external config to muck with gdsHash. Why? because of error in
            // encoding
            gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching
          }

          Group g = gdsMap.get(gdsHash);
          if (g == null) {
            g = new Group(gr.getGDSsection(), gdsHash);
            gdsMap.put(gdsHash, g);
          }
          g.records.add(gr);
          total++;
        }
        fileno++;
      }
    }
    f.format(" total grib records= %d%n", total);

    Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter();
    List<Group> result = new ArrayList<Group>(gdsMap.values());
    for (Group g : result) {
      g.rect = new Grib2Rectilyser(g.records, g.gdsHash);
      f.format(" GDS hash %d == ", g.gdsHash);
      g.rect.make(f, c);
    }
    f.format(
        " Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
        c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);

    return result;
  }
  // read all records in all files,
  // divide into groups based on GDS hash
  // each group has an arraylist of all records that belong to it.
  // for each group, run rectlizer to derive the coordinates and variables
  public List<Group> makeAggregatedGroups(
      List<String> filenames, CollectionManager.Force force, Formatter f) throws IOException {
    Map<Integer, Group> gdsMap = new HashMap<Integer, Group>();
    boolean intvMerge = mergeIntvDefault;

    f.format("GribCollection %s: makeAggregatedGroups%n", gc.getName());
    int total = 0;
    int fileno = 0;

    for (CollectionManager dcm : collections) {
      f.format(" dcm= %s%n", dcm);
      FeatureCollectionConfig.GribConfig config =
          (FeatureCollectionConfig.GribConfig)
              dcm.getAuxInfo(FeatureCollectionConfig.AUX_GRIB_CONFIG);
      Map<Integer, Integer> gdsConvert = (config != null) ? config.gdsHash : null;
      FeatureCollectionConfig.GribIntvFilter intvMap = (config != null) ? config.intvFilter : null;
      intvMerge =
          (config == null) || (config.intvMerge == null) ? mergeIntvDefault : config.intvMerge;

      for (MFile mfile : dcm.getFiles()) {
        // f.format("%3d: %s%n", fileno, mfile.getPath());
        filenames.add(mfile.getPath());

        Grib2Index index = null;
        try {
          index =
              (Grib2Index)
                  GribIndex.readOrCreateIndexFromSingleFile(
                      false, !isSingleFile, mfile, config, force, f);

        } catch (IOException ioe) {
          logger.warn(
              "GribCollectionBuilder {}: reading/Creating gbx9 index failed err={}",
              gc.getName(),
              ioe.getMessage());
          f.format(
              "GribCollectionBuilder: reading/Creating gbx9 index failed err=%s%n  skipping %s%n",
              ioe.getMessage(), mfile.getPath() + GribIndex.IDX_EXT);
          continue;
        }

        for (Grib2Record gr : index.getRecords()) {
          if (this.tables == null) {
            Grib2SectionIdentification ids =
                gr.getId(); // so all records must use the same table (!)
            this.tables =
                Grib2Customizer.factory(
                    ids.getCenter_id(),
                    ids.getSubcenter_id(),
                    ids.getMaster_table_version(),
                    ids.getLocal_table_version());
            if (config != null)
              tables.setTimeUnitConverter(
                  config
                      .getTimeUnitConverter()); // LOOK doesnt really work with multiple collections
          }
          if (intvMap != null && filterTinv(gr, intvMap, f)) continue; // skip

          gr.setFile(fileno); // each record tracks which file it belongs to
          int gdsHash =
              gr.getGDSsection().getGDS().hashCode(); // use GDS hash code to group records
          if (gdsConvert != null
              && gdsConvert.get(gdsHash)
                  != null) // allow external config to muck with gdsHash. Why? because of error in
            // encoding
            gdsHash = (Integer) gdsConvert.get(gdsHash); // and we need exact hash matching

          Group g = gdsMap.get(gdsHash);
          if (g == null) {
            g = new Group(gr.getGDSsection(), gdsHash);
            gdsMap.put(gdsHash, g);
          }
          g.records.add(gr);
          total++;
        }
        fileno++;
      }
    }
    f.format(" total grib records= %d%n", total);

    Grib2Rectilyser.Counter c = new Grib2Rectilyser.Counter(); // debugging
    List<Group> result = new ArrayList<Group>(gdsMap.values());
    for (Group g : result) {
      g.rect = new Grib2Rectilyser(tables, g.records, g.gdsHash, intvMerge);
      f.format(" GDS hash %d == ", g.gdsHash);
      g.rect.make(f, c, filenames);
    }
    f.format(
        " Rectilyser: nvars=%d records unique=%d total=%d dups=%d (%f) %n",
        c.vars, c.recordsUnique, c.records, c.dups, ((float) c.dups) / c.records);

    return result;
  }