/**
   * Partitions the data represented by dims into groups of records where records are whole
   * subarrays with size 1 on the zero-th dimension. This may not work for all formats, revisit this
   * later TODO
   *
   * @param dims List of Dimension objects representing the dimensions of the input data that we are
   *     generating partitions for
   * @param varName Name of the variable we are generating partitions for
   * @param fileName name of the file that contains the variable we are generating partitions for
   * @param partMode the partitioning mode being used to generate the partitions
   * @param startOffset the logical offset in the input data to start creating partitions at
   * @param conf Configuration object for this execution the given MR program
   * @return an array of ArraySpec objects that represent the partitions this function generated
   */
  protected ArraySpec[] recordBasedPartition(
      int[] dims,
      String varName,
      String fileName,
      PartMode partMode,
      int[] startOffset,
      Configuration conf)
      throws IOException {

    int ndims = dims.length;
    int recDimLen = dims[0];

    int[] recordShape = new int[ndims];
    int[] recordCorner = new int[ndims];

    ArrayList<ArraySpec> records = new ArrayList<ArraySpec>(recDimLen);

    for (int i = 0; i < ndims; i++) {
      recordShape[i] = dims[i];
      recordCorner[i] = 0;
    }

    recordShape[0] = 1;

    if (Utils.queryDependantEnabled(conf)) {
      LOG.info("Query Dependant enabled");
      recordShape[0] = Utils.getExtractionShape(conf, recordShape.length)[0];
    } else {
      LOG.info("Query Dependant NOT enabled");
    }

    ArraySpec tempSpec = null;
    for (int i = 0; i < recDimLen; i += recordShape[0]) {
      recordCorner[0] = i;
      // FIXME: this is clunky
      try {
        // if this is optC and the record is not valid, do not add it,
        if (Utils.noScanEnabled(conf)) {
          if (Utils.isValid(recordCorner, conf)) {
            tempSpec = new ArraySpec(recordCorner, recordShape, varName, fileName);
            tempSpec.setLogicalStartOffset(startOffset);
            records.add(tempSpec);
            // records.add(new ArraySpec(recordCorner, recordShape, varName, fileName));
          }
        } else { // else wise do add it
          tempSpec = new ArraySpec(recordCorner, recordShape, varName, fileName);
          tempSpec.setLogicalStartOffset(startOffset);
          records.add(tempSpec);
          // records.add(new ArraySpec(recordCorner, recordShape, varName, fileName));
        }
      } catch (Exception e) {
        throw new IOException(e);
      }
    }

    ArraySpec[] returnArray = new ArraySpec[records.size()];
    returnArray = records.toArray(returnArray);

    return returnArray;
  }
  /**
   * The partitioning scheme creates partitions distributes the data to be read (approximately)
   * evenly over all the blocks in the file. This is a very naive approach and should not be used
   * other than as a point of refernece.
   *
   * @param dims represent the logical input space
   * @param varName name of the variable that we're creating partitions for
   * @param blockSize the size, in bytes, of the blocks used to store the file that contains the
   *     data being partitioned
   * @param numBlocks the number of blocks to use for generating the per-partition size
   * @param fileLen the length of the file, in bytes
   * @param dataTypeSize the size, in bytes, of a single cell for the given data type stored in the
   *     file for which partitions are being generated
   * @param fileName name of the file that partitions are being generated for
   * @param startOffset the logical position in the file to begin generating partitions from
   * @param conf Configuration object for this current MR program
   * @return an array of ArraySpecs that is the same length as dims, where each ArraySpec
   *     corresponds to a partition
   */
  protected ArraySpec[] proportionalPartition(
      int[] dims,
      String varName,
      long blockSize,
      long numBlocks,
      long fileLen,
      int dataTypeSize,
      String fileName,
      int[] startOffset,
      Configuration conf)
      throws IOException {
    int ndims = dims.length;
    ArrayList<ArraySpec> records = new ArrayList<ArraySpec>();

    // this next bit is fairly hard-coded and specific to our tests.
    // it represents a naive split that a human might come up with

    // sort out the step size
    int[] stepShape = calcStepShape(dims, blockSize, numBlocks, fileLen, dataTypeSize, conf);

    int[] tempCorner = new int[ndims];
    int[] tempStep = new int[ndims];

    // initialize the temporary step shape to be the first step
    for (int i = 0; i < ndims; i++) {
      tempStep[i] = stepShape[i];
      tempCorner[i] = 0;
    }

    LOG.info("Calculated stepshape: " + Utils.arrayToString(stepShape));

    ArraySpec tempSpec = new ArraySpec();
    int stepSize = tempStep[0];

    // create the actual splits
    while (tempCorner[0] < dims[0]) {
      try {
        if (Utils.noScanEnabled(conf)) {
          if (Utils.isValid(tempCorner, conf)) {
            tempSpec = new ArraySpec(tempCorner, tempStep, varName, fileName);
            tempSpec.setLogicalStartOffset(startOffset);
            records.add(tempSpec);
          }
        } else {
          tempSpec = new ArraySpec(tempCorner, tempStep, varName, fileName);
          tempSpec.setLogicalStartOffset(startOffset);
          records.add(tempSpec);
        }
      } catch (Exception e) {
        throw new IOException(e);
      }

      // update the corner
      tempCorner[0] += stepSize;

      // use MIN here to make sure we don't over run the constraining space
      stepSize = Math.min(tempStep[0], dims[0] - tempCorner[0]);

      // update the shape of the next write appropriately
      tempStep[0] = stepSize;
    }

    ArraySpec[] returnArray = new ArraySpec[records.size()];
    returnArray = records.toArray(returnArray);

    return returnArray;
  }