private void createDataVariables(List<VariableSimpleIF> dataVars) throws IOException {

    /* height variable
    Variable heightVar = ncfile.addStringVariable(altName, recordDims, 20);
    ncfile.addVariableAttribute(heightVar, new Attribute("long_name", "height of observation"));
    ncfile.addVariableAttribute(heightVar, new Attribute("units", altUnits));  */

    Variable v = ncfile.addVariable(parentProfileIndex, DataType.INT, recordDimName);
    ncfile.addVariableAttribute(v, new Attribute("long_name", "index of parent profile"));

    v = ncfile.addVariable(nextObsName, DataType.INT, recordDimName);
    ncfile.addVariableAttribute(
        v, new Attribute("long_name", "record number of next obs in linked list for this profile"));

    // find all dimensions needed by the data variables
    for (VariableSimpleIF var : dataVars) {
      List<Dimension> dims = var.getDimensions();
      dimSet.addAll(dims);
    }

    // add them
    for (Dimension d : dimSet) {
      if (!d.isUnlimited())
        ncfile.addDimension(d.getName(), d.getLength(), d.isShared(), false, d.isVariableLength());
    }

    // add the data variables all using the record dimension
    for (VariableSimpleIF oldVar : dataVars) {
      List<Dimension> dims = oldVar.getDimensions();
      StringBuffer dimNames = new StringBuffer(recordDimName);
      for (Dimension d : dims) {
        if (!d.isUnlimited()) dimNames.append(" ").append(d.getName());
      }
      Variable newVar =
          ncfile.addVariable(oldVar.getName(), oldVar.getDataType(), dimNames.toString());

      List<Attribute> atts = oldVar.getAttributes();
      for (Attribute att : atts) {
        ncfile.addVariableAttribute(newVar, att);
      }
    }
  }
Пример #2
0
 public AbstractCubeTable getQueriedTable(String alias) {
   if (cube != null && cube.getName().equalsIgnoreCase(qb.getTabNameForAlias((alias)))) {
     return (AbstractCubeTable) cube;
   }
   for (Dimension dim : dimensions) {
     if (dim.getName().equalsIgnoreCase(qb.getTabNameForAlias(alias))) {
       return dim;
     }
   }
   return null;
 }
Пример #3
0
 String getQBFromString(CandidateFact fact, Map<Dimension, CandidateDim> dimsToQuery)
     throws LensException {
   String fromString;
   if (getJoinAST() == null) {
     if (cube != null) {
       if (dimensions.size() > 0) {
         throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
       }
       fromString = fact.getStorageString(getAliasForTableName(cube.getName()));
     } else {
       if (dimensions.size() != 1) {
         throw new LensException(LensCubeErrorCode.NO_JOIN_CONDITION_AVAILABLE.getLensErrorInfo());
       }
       Dimension dim = dimensions.iterator().next();
       fromString = dimsToQuery.get(dim).getStorageString(getAliasForTableName(dim.getName()));
     }
   } else {
     StringBuilder builder = new StringBuilder();
     getQLString(qb.getQbJoinTree(), builder, fact, dimsToQuery);
     fromString = builder.toString();
   }
   return fromString;
 }
  /**
   * Find all de-normalized columns, if these columns are not directly available in candidate
   * tables, query will be replaced with the corresponding table reference
   */
  @Override
  public void rewriteContext(CubeQueryContext cubeql) throws LensException {
    DenormalizationContext denormCtx = cubeql.getDeNormCtx();
    if (denormCtx == null) {
      // Adds all the reference dimensions as eligible for denorm fields
      denormCtx = new DenormalizationContext(cubeql);
      cubeql.setDeNormCtx(denormCtx);
      // add ref columns in cube
      addRefColsQueried(cubeql, cubeql, denormCtx);
      // add ref columns from expressions
      for (Set<ExpressionContext> ecSet : cubeql.getExprCtx().getAllExprsQueried().values()) {
        for (ExpressionContext ec : ecSet) {
          for (ExprSpecContext esc : ec.getAllExprs()) {
            addRefColsQueried(cubeql, esc, denormCtx);
          }
        }
      }
    } else if (!denormCtx.tableToRefCols.isEmpty()) {
      // In the second iteration of denorm resolver
      // candidate tables which require denorm fields and the refernces are no
      // more valid will be pruned
      if (cubeql.getCube() != null && !cubeql.getCandidateFacts().isEmpty()) {
        for (Iterator<CandidateFact> i = cubeql.getCandidateFacts().iterator(); i.hasNext(); ) {
          CandidateFact cfact = i.next();
          if (denormCtx.tableToRefCols.containsKey(cfact.getName())) {
            for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cfact.getName())) {
              if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) {
                log.info(
                    "Not considering fact table:{} as column {} is not available",
                    cfact,
                    refcol.col);
                cubeql.addFactPruningMsgs(
                    cfact.fact, CandidateTablePruneCause.columnNotFound(refcol.col.getName()));
                i.remove();
              }
            }
          }
        }
        if (cubeql.getCandidateFacts().size() == 0) {
          throw new LensException(
              LensCubeErrorCode.NO_FACT_HAS_COLUMN.getLensErrorInfo(),
              cubeql.getColumnsQueried(cubeql.getCube().getName()).toString());
        }
        cubeql.pruneCandidateFactSet(CandidateTablePruneCode.COLUMN_NOT_FOUND);
      }
      if (cubeql.getDimensions() != null && !cubeql.getDimensions().isEmpty()) {
        for (Dimension dim : cubeql.getDimensions()) {
          for (Iterator<CandidateDim> i = cubeql.getCandidateDimTables().get(dim).iterator();
              i.hasNext(); ) {
            CandidateDim cdim = i.next();
            if (denormCtx.tableToRefCols.containsKey(cdim.getName())) {
              for (ReferencedQueriedColumn refcol : denormCtx.tableToRefCols.get(cdim.getName())) {
                if (denormCtx.getReferencedCols().get(refcol.col.getName()).isEmpty()) {
                  log.info(
                      "Not considering dim table:{} as column {} is not available",
                      cdim,
                      refcol.col);
                  cubeql.addDimPruningMsgs(
                      dim,
                      cdim.dimtable,
                      CandidateTablePruneCause.columnNotFound(refcol.col.getName()));
                  i.remove();
                }
              }
            }
          }

          if (cubeql.getCandidateDimTables().get(dim).size() == 0) {
            throw new LensException(
                LensCubeErrorCode.NO_DIM_HAS_COLUMN.getLensErrorInfo(),
                dim.toString(),
                cubeql.getColumnsQueried(dim.getName()).toString());
          }
        }
      }
    }
  }
Пример #5
0
  /**
   * This reads an arbitrary data slice, returning the data in canonical order (rt-e-t-z-y-x). If
   * any dimension does not exist, ignore it.
   *
   * @param rt if < 0, get all of runtime dim; if valid index, fix slice to that value.
   * @param e if < 0, get all of ensemble dim; if valid index, fix slice to that value.
   * @param t if < 0, get all of time dim; if valid index, fix slice to that value.
   * @param z if < 0, get all of z dim; if valid index, fix slice to that value.
   * @param y if < 0, get all of y dim; if valid index, fix slice to that value.
   * @param x if < 0, get all of x dim; if valid index, fix slice to that value.
   * @return data[rt,e,t,z,y,x], eliminating missing or fixed dimension.
   */
  public Array readDataSlice(int rt, int e, int t, int z, int y, int x) throws java.io.IOException {

    int rank = vs.getRank();
    int[] start = new int[rank];
    int[] shape = new int[rank];
    for (int i = 0; i < rank; i++) {
      start[i] = 0;
      shape[i] = 1;
    }
    Dimension xdim = getXDimension();
    Dimension ydim = getYDimension();
    Dimension zdim = getZDimension();
    Dimension tdim = getTimeDimension();
    Dimension edim = getEnsembleDimension();
    Dimension rtdim = getRunTimeDimension();

    // construct the shape of the data volume to be read
    if (rtdim != null) {
      if ((rt >= 0) && (rt < rtdim.getLength())) start[rtDimOrgIndex] = rt; // fix rt
      else {
        shape[rtDimOrgIndex] = rtdim.getLength(); // all of rt
      }
    }

    if (edim != null) {
      if ((e >= 0) && (e < edim.getLength())) start[eDimOrgIndex] = e; // fix e
      else {
        shape[eDimOrgIndex] = edim.getLength(); // all of e
      }
    }

    if (tdim != null) {
      if ((t >= 0) && (t < tdim.getLength())) start[tDimOrgIndex] = t; // fix t
      else {
        shape[tDimOrgIndex] = tdim.getLength(); // all of t
      }
    }

    if (zdim != null) {
      if ((z >= 0) && (z < zdim.getLength())) start[zDimOrgIndex] = z; // fix z
      else {
        shape[zDimOrgIndex] = zdim.getLength(); // all of z
      }
    }

    if (ydim != null) {
      if ((y >= 0) && (y < ydim.getLength())) start[yDimOrgIndex] = y; // fix y
      else {
        shape[yDimOrgIndex] = ydim.getLength(); // all of y
      }
    }

    if (xdim != null) {
      if ((x >= 0) && (x < xdim.getLength())) // all of x
      start[xDimOrgIndex] = x; // fix x
      else {
        shape[xDimOrgIndex] = xdim.getLength(); // all of x
      }
    }

    if (debugArrayShape) {
      System.out.println("read shape from org variable = ");
      for (int i = 0; i < rank; i++)
        System.out.println(
            "   start = "
                + start[i]
                + " shape = "
                + shape[i]
                + " name = "
                + vs.getDimension(i).getName());
    }

    // read it
    Array dataVolume;
    try {
      dataVolume = vs.read(start, shape);
    } catch (Exception ex) {
      log.error(
          "GeoGrid.getdataSlice() on dataset " + getFullName() + " " + dataset.getLocation(), ex);
      throw new java.io.IOException(ex.getMessage());
    }

    // LOOK: the real problem is the lack of named dimensions in the Array object
    // figure out correct permutation for canonical ordering for permute
    List<Dimension> oldDims = new ArrayList<Dimension>(vs.getDimensions());
    int[] permuteIndex = new int[dataVolume.getRank()];
    int count = 0;
    if (oldDims.contains(rtdim)) permuteIndex[count++] = oldDims.indexOf(rtdim);
    if (oldDims.contains(edim)) permuteIndex[count++] = oldDims.indexOf(edim);
    if (oldDims.contains(tdim)) permuteIndex[count++] = oldDims.indexOf(tdim);
    if (oldDims.contains(zdim)) permuteIndex[count++] = oldDims.indexOf(zdim);
    if (oldDims.contains(ydim)) permuteIndex[count++] = oldDims.indexOf(ydim);
    if (oldDims.contains(xdim)) permuteIndex[count] = oldDims.indexOf(xdim);

    if (debugArrayShape) {
      System.out.println("oldDims = ");
      for (Dimension oldDim : oldDims) System.out.println("   oldDim = " + oldDim.getName());
      System.out.println("permute dims = ");
      for (int aPermuteIndex : permuteIndex)
        System.out.println("   oldDim index = " + aPermuteIndex);
    }

    // check to see if we need to permute
    boolean needPermute = false;
    for (int i = 0; i < permuteIndex.length; i++) {
      if (i != permuteIndex[i]) needPermute = true;
    }

    // permute to the order rt,e,t,z,y,x
    if (needPermute) dataVolume = dataVolume.permute(permuteIndex);

    // eliminate fixed dimensions, but not all dimensions of length 1.
    count = 0;
    if (rtdim != null) {
      if (rt >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (edim != null) {
      if (e >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (tdim != null) {
      if (t >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (zdim != null) {
      if (z >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (ydim != null) {
      if (y >= 0) dataVolume = dataVolume.reduce(count);
      else count++;
    }
    if (xdim != null) {
      if (x >= 0) dataVolume = dataVolume.reduce(count);
    }

    return dataVolume;
  }
Пример #6
0
  public Variable makeVariable(
      NetcdfFile ncfile,
      int datatype,
      String shortName,
      String longName,
      String abbrev,
      List groups)
      throws IOException {
    int nscans = groups.size();

    if (nscans == 0) {
      throw new IllegalStateException("No data for " + shortName);
    }

    // get representative record
    List firstGroup = (List) groups.get(0);
    Cinrad2Record firstRecord = (Cinrad2Record) firstGroup.get(0);
    int ngates = firstRecord.getGateCount(datatype);

    String scanDimName = "scan" + abbrev;
    String gateDimName = "gate" + abbrev;
    Dimension scanDim = new Dimension(scanDimName, nscans);
    Dimension gateDim = new Dimension(gateDimName, ngates);
    ncfile.addDimension(null, scanDim);
    ncfile.addDimension(null, gateDim);

    ArrayList dims = new ArrayList();
    dims.add(scanDim);
    dims.add(radialDim);
    dims.add(gateDim);

    Variable v = new Variable(ncfile, null, null, shortName);
    v.setDataType(DataType.BYTE);
    v.setDimensions(dims);
    ncfile.addVariable(null, v);

    v.addAttribute(new Attribute(CDM.UNITS, Cinrad2Record.getDatatypeUnits(datatype)));
    v.addAttribute(new Attribute(CDM.LONG_NAME, longName));

    byte[] b = new byte[2];
    b[0] = Cinrad2Record.MISSING_DATA;
    b[1] = Cinrad2Record.BELOW_THRESHOLD;
    Array missingArray = Array.factory(DataType.BYTE.getPrimitiveClassType(), new int[] {2}, b);

    v.addAttribute(new Attribute(CDM.MISSING_VALUE, missingArray));
    v.addAttribute(
        new Attribute("signal_below_threshold", new Byte(Cinrad2Record.BELOW_THRESHOLD)));
    v.addAttribute(
        new Attribute(CDM.SCALE_FACTOR, new Float(Cinrad2Record.getDatatypeScaleFactor(datatype))));
    v.addAttribute(
        new Attribute(CDM.ADD_OFFSET, new Float(Cinrad2Record.getDatatypeAddOffset(datatype))));
    v.addAttribute(new Attribute(CDM.UNSIGNED, "true"));

    ArrayList dim2 = new ArrayList();
    dim2.add(scanDim);
    dim2.add(radialDim);

    // add time coordinate variable
    String timeCoordName = "time" + abbrev;
    Variable timeVar = new Variable(ncfile, null, null, timeCoordName);
    timeVar.setDataType(DataType.INT);
    timeVar.setDimensions(dim2);
    ncfile.addVariable(null, timeVar);

    // int julianDays = volScan.getTitleJulianDays();
    // Date d = Cinrad2Record.getDate( julianDays, 0);
    // Date d = Cinrad2Record.getDate(volScan.getTitleJulianDays(), volScan.getTitleMsecs());
    Date d = volScan.getStartDate();
    String units = "msecs since " + formatter.toDateTimeStringISO(d);

    timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "time since base date"));
    timeVar.addAttribute(new Attribute(CDM.UNITS, units));
    timeVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Integer(MISSING_INT)));
    timeVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.Time.toString()));

    // add elevation coordinate variable
    String elevCoordName = "elevation" + abbrev;
    Variable elevVar = new Variable(ncfile, null, null, elevCoordName);
    elevVar.setDataType(DataType.FLOAT);
    elevVar.setDimensions(dim2);
    ncfile.addVariable(null, elevVar);

    elevVar.addAttribute(new Attribute(CDM.UNITS, "degrees"));
    elevVar.addAttribute(
        new Attribute(
            CDM.LONG_NAME,
            "elevation angle in degres: 0 = parallel to pedestal base, 90 = perpendicular"));
    elevVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Float(MISSING_FLOAT)));
    elevVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialElevation.toString()));

    // add azimuth coordinate variable
    String aziCoordName = "azimuth" + abbrev;
    Variable aziVar = new Variable(ncfile, null, null, aziCoordName);
    aziVar.setDataType(DataType.FLOAT);
    aziVar.setDimensions(dim2);
    ncfile.addVariable(null, aziVar);

    aziVar.addAttribute(new Attribute(CDM.UNITS, "degrees"));
    aziVar.addAttribute(
        new Attribute(CDM.LONG_NAME, "azimuth angle in degrees: 0 = true north, 90 = east"));
    aziVar.addAttribute(new Attribute(CDM.MISSING_VALUE, new Float(MISSING_FLOAT)));
    aziVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialAzimuth.toString()));

    // add gate coordinate variable
    String gateCoordName = "distance" + abbrev;
    Variable gateVar = new Variable(ncfile, null, null, gateCoordName);
    gateVar.setDataType(DataType.FLOAT);
    gateVar.setDimensions(gateDimName);
    Array data =
        Array.makeArray(
            DataType.FLOAT,
            ngates,
            (double) firstRecord.getGateStart(datatype),
            (double) firstRecord.getGateSize(datatype));
    gateVar.setCachedData(data, false);
    ncfile.addVariable(null, gateVar);
    radarRadius = firstRecord.getGateStart(datatype) + ngates * firstRecord.getGateSize(datatype);

    gateVar.addAttribute(new Attribute(CDM.UNITS, "m"));
    gateVar.addAttribute(new Attribute(CDM.LONG_NAME, "radial distance to start of gate"));
    gateVar.addAttribute(new Attribute(_Coordinate.AxisType, AxisType.RadialDistance.toString()));

    // add number of radials variable
    String nradialsName = "numRadials" + abbrev;
    Variable nradialsVar = new Variable(ncfile, null, null, nradialsName);
    nradialsVar.setDataType(DataType.INT);
    nradialsVar.setDimensions(scanDim.getName());
    nradialsVar.addAttribute(new Attribute(CDM.LONG_NAME, "number of valid radials in this scan"));
    ncfile.addVariable(null, nradialsVar);

    // add number of gates variable
    String ngateName = "numGates" + abbrev;
    Variable ngateVar = new Variable(ncfile, null, null, ngateName);
    ngateVar.setDataType(DataType.INT);
    ngateVar.setDimensions(scanDim.getName());
    ngateVar.addAttribute(new Attribute(CDM.LONG_NAME, "number of valid gates in this scan"));
    ncfile.addVariable(null, ngateVar);

    makeCoordinateDataWithMissing(
        datatype, timeVar, elevVar, aziVar, nradialsVar, ngateVar, groups);

    // back to the data variable
    String coordinates =
        timeCoordName + " " + elevCoordName + " " + aziCoordName + " " + gateCoordName;
    v.addAttribute(new Attribute(_Coordinate.Axes, coordinates));

    // make the record map
    int nradials = radialDim.getLength();
    Cinrad2Record[][] map = new Cinrad2Record[nscans][nradials];
    for (int i = 0; i < groups.size(); i++) {
      Cinrad2Record[] mapScan = map[i];
      List group = (List) groups.get(i);
      for (int j = 0; j < group.size(); j++) {
        Cinrad2Record r = (Cinrad2Record) group.get(j);
        int radial = r.radial_num - 1;
        mapScan[radial] = r;
      }
    }

    Vgroup vg = new Vgroup(datatype, map);
    v.setSPobject(vg);

    return v;
  }