Exemple #1
0
  public boolean compareVariables(NetcdfFile org, NetcdfFile copy) {
    f.format("Original = %s%n", org.getLocation());
    f.format("CompareTo= %s%n", copy.getLocation());
    boolean ok = true;

    for (Variable orgV : org.getVariables()) {
      if (orgV.isCoordinateVariable()) continue;
      Variable copyVar = copy.findVariable(orgV.getShortName());
      if (copyVar == null) {
        f.format(" MISSING '%s' in 2nd file%n", orgV.getFullName());
        ok = false;
      } else {
        List<Dimension> dims1 = orgV.getDimensions();
        List<Dimension> dims2 = copyVar.getDimensions();
        if (!compare(dims1, dims2)) {
          f.format(" %s != %s%n", orgV.getNameAndDimensions(), copyVar.getNameAndDimensions());
        } else {
          // f.format("   ok %s%n", orgV.getName());
        }
      }
    }

    f.format("%n");
    for (Variable orgV : copy.getVariables()) {
      if (orgV.isCoordinateVariable()) continue;
      Variable copyVar = org.findVariable(orgV.getShortName());
      if (copyVar == null) {
        f.format(" MISSING '%s' in 1st file%n", orgV.getFullName());
        ok = false;
      }
    }

    return ok;
  }
Exemple #2
0
  private void copySome(NetcdfFileWriteable ncfile, Variable oldVar, int nelems)
      throws IOException {
    String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());

    int[] shape = oldVar.getShape();
    int[] origin = new int[oldVar.getRank()];
    int size = shape[0];

    for (int i = 0; i < size; i += nelems) {
      origin[0] = i;
      int left = size - i;
      shape[0] = Math.min(nelems, left);

      Array data;
      try {
        data = oldVar.read(origin, shape);
        if (oldVar.getDataType() == DataType.STRING) {
          data = convertToChar(ncfile.findVariable(newName), data);
        }
        if (data.getSize() > 0) { // zero when record dimension = 0
          ncfile.write(newName, origin, data);
          if (debug) System.out.println("write " + data.getSize() + " bytes");
        }

      } catch (InvalidRangeException e) {
        e.printStackTrace();
        throw new IOException(e.getMessage());
      }
    }
  }
  public void testAggCoordVar2(NetcdfFile ncfile) {

    Variable time = ncfile.findVariable("time");
    assert null != time;
    assert time.getShortName().equals("time");
    assert time.getRank() == 1 : time.getRank();
    assert time.getShape()[0] == 3;
    assert time.getDataType() == DataType.INT;

    assert time.getDimension(0) == ncfile.findDimension("time");

    try {
      Array data = time.read();

      assert (data instanceof ArrayInt);
      IndexIterator dataI = data.getIndexIterator();
      assert dataI.getIntNext() == 0 : dataI.getIntCurrent();
      assert dataI.getIntNext() == 1 : dataI.getIntCurrent();
      assert dataI.getIntNext() == 2 : dataI.getIntCurrent();

    } catch (IOException io) {
      io.printStackTrace();
      assert false;
    }
  }
  public void testAggCoordVar(NetcdfFile ncfile) {

    Variable time = ncfile.findVariable("time");
    assert null != time;
    assert time.getShortName().equals("time");
    assert time.getRank() == 1 : time.getRank();
    assert time.getShape()[0] == 3;
    assert time.getDataType() == DataType.INT;

    assert time.getDimension(0) == ncfile.findDimension("time");

    try {
      Array data = time.read();

      assert (data instanceof ArrayInt.D1) : data.getClass().getName();
      ArrayInt.D1 dataI = (ArrayInt.D1) data;
      assert dataI.get(0) == 0;
      assert dataI.get(1) == 10;
      assert dataI.get(2) == 99;

    } catch (IOException io) {
      io.printStackTrace();
      assert false;
    }
  }
Exemple #5
0
  private boolean compareGroups(Group org, Group copy) {
    if (showCompare) f.format("compare Group %s to %s %n", org.getName(), copy.getName());
    boolean ok = true;

    if (!org.getName().equals(copy.getName())) {
      f.format(" ** names are different %s != %s %n", org.getName(), copy.getName());
      ok = false;
    }

    // dimensions
    ok &= checkAll(org.getDimensions(), copy.getDimensions(), null);

    // attributes
    ok &= checkAll(org.getAttributes(), copy.getAttributes(), null);

    // variables
    // cant use object equality, just match on short name
    for (Variable orgV : org.getVariables()) {
      Variable copyVar = copy.findVariable(orgV.getShortName());
      if (copyVar == null) {
        f.format(" ** cant find variable %s in 2nd file%n", orgV.getFullName());
        ok = false;
      } else {
        ok &= compareVariables(orgV, copyVar, compareData, true);
      }
    }

    for (Variable copyV : copy.getVariables()) {
      Variable orgV = org.findVariable(copyV.getShortName());
      if (orgV == null) {
        f.format(" ** cant find variable %s in 1st file%n", copyV.getFullName());
        ok = false;
      }
    }

    // nested groups
    List groups = new ArrayList();
    ok &= checkAll(org.getGroups(), copy.getGroups(), groups);
    for (int i = 0; i < groups.size(); i += 2) {
      Group orgGroup = (Group) groups.get(i);
      Group ncmlGroup = (Group) groups.get(i + 1);
      ok &= compareGroups(orgGroup, ncmlGroup);
    }

    return ok;
  }
Exemple #6
0
  private void copyAll(NetcdfFileWriteable ncfile, Variable oldVar) throws IOException {
    String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());

    Array data = oldVar.read();
    try {
      if (oldVar.getDataType() == DataType.STRING) {
        data = convertToChar(ncfile.findVariable(newName), data);
      }
      if (data.getSize() > 0) // zero when record dimension = 0
      ncfile.write(newName, data);

    } catch (InvalidRangeException e) {
      e.printStackTrace();
      throw new IOException(e.getMessage() + " for Variable " + oldVar.getFullName());
    }
  }
  public void testAggCoordVarScan(NetcdfFile ncfile) throws IOException {
    Variable time = ncfile.findVariable("time");
    assert null != time;
    assert time.getShortName().equals("time");
    assert time.getRank() == 1 : time.getRank();
    assert time.getShape()[0] == 3;
    assert time.getDataType() == DataType.INT : time.getDataType();

    assert time.getDimension(0) == ncfile.findDimension("time");

    int count = 0;
    Array data = time.read();
    assert (data instanceof ArrayInt);
    while (data.hasNext()) {
      int val = data.nextInt();
      assert val == count * 10 : val + "!=" + count * 10;
      count++;
    }
  }
  public void testAggCoordVar3(NetcdfFile ncfile) throws IOException {
    Variable time = ncfile.findVariable("time");
    assert null != time;
    assert time.getShortName().equals("time");
    assert time.getRank() == 1 : time.getRank();
    assert time.getShape()[0] == 3;
    assert time.getDataType() == DataType.DOUBLE : time.getDataType();

    assert time.getDimension(0) == ncfile.findDimension("time");

    Array data = time.read();

    assert (data instanceof ArrayDouble);
    IndexIterator dataI = data.getIndexIterator();
    double val = dataI.getDoubleNext();
    assert Misc.closeEnough(val, 0.0) : val;
    assert Misc.closeEnough(dataI.getDoubleNext(), 10.0) : dataI.getDoubleCurrent();
    assert Misc.closeEnough(dataI.getDoubleNext(), 99.0) : dataI.getDoubleCurrent();
  }
  public void testAggCoordVarNoCoordsDir(NetcdfFile ncfile) throws IOException {
    Variable time = ncfile.findVariable("time");
    assert null != time;
    assert time.getShortName().equals("time");
    assert time.getRank() == 1 : time.getRank();
    assert time.getShape()[0] == 3;
    assert time.getDataType() == DataType.STRING : time.getDataType();

    assert time.getDimension(0) == ncfile.findDimension("time");

    Array data = time.read();

    assert (data instanceof ArrayObject);
    IndexIterator dataI = data.getIndexIterator();
    String coordName = (String) dataI.getObjectNext();
    assert coordName.equals("time0Dir.nc") : coordName;
    coordName = (String) dataI.getObjectNext();
    assert coordName.equals("time1Dir.nc") : coordName;
    coordName = (String) dataI.getObjectNext();
    assert coordName.equals("time2Dir.nc") : coordName;
  }
  public void testReadData(NetcdfFile ncfile, String name) throws IOException {

    Variable v = ncfile.findVariable(name);
    assert null != v;
    assert v.getShortName().equals(name);
    assert v.getRank() == 3;
    assert v.getSize() == 36 : v.getSize();
    assert v.getShape()[0] == 3;
    assert v.getShape()[1] == 3;
    assert v.getShape()[2] == 4;
    assert v.getDataType() == DataType.DOUBLE;

    assert !v.isCoordinateVariable();

    assert v.getDimension(0) == ncfile.findDimension("time");
    assert v.getDimension(1) == ncfile.findDimension("lat");
    assert v.getDimension(2) == ncfile.findDimension("lon");

    Array data = v.read();
    assert data.getRank() == 3;
    assert data.getSize() == 36;
    assert data.getShape()[0] == 3;
    assert data.getShape()[1] == 3;
    assert data.getShape()[2] == 4;
    assert data.getElementType() == double.class;

    int[] shape = data.getShape();
    Index tIndex = data.getIndex();
    for (int i = 0; i < shape[0]; i++)
      for (int j = 0; j < shape[1]; j++)
        for (int k = 0; k < shape[2]; k++) {
          double val = data.getDouble(tIndex.set(i, j, k));
          // System.out.println(" "+val);
          assert TestUtils.close(val, 100 * i + 10 * j + k) : val;
        }
  }
  public void testCoordVar(NetcdfFile ncfile) {

    Variable lat = ncfile.findVariable("lat");
    assert null != lat;
    assert lat.getShortName().equals("lat");
    assert lat.getRank() == 1;
    assert lat.getSize() == 3;
    assert lat.getShape()[0] == 3;
    assert lat.getDataType() == DataType.FLOAT;

    assert !lat.isUnlimited();
    assert lat.getDimension(0).equals(ncfile.findDimension("lat"));

    Attribute att = lat.findAttribute("units");
    assert null != att;
    assert !att.isArray();
    assert att.isString();
    assert att.getDataType() == DataType.STRING;
    assert att.getStringValue().equals("degrees_north");
    assert att.getNumericValue() == null;
    assert att.getNumericValue(3) == null;

    try {
      Array data = lat.read();
      assert data.getRank() == 1;
      assert data.getSize() == 3;
      assert data.getShape()[0] == 3;
      assert data.getElementType() == float.class;

      IndexIterator dataI = data.getIndexIterator();
      assert TestUtils.close(dataI.getDoubleNext(), 41.0);
      assert TestUtils.close(dataI.getDoubleNext(), 40.0);
      assert TestUtils.close(dataI.getDoubleNext(), 39.0);
    } catch (IOException io) {
    }
  }
  public static void main(String args[]) throws Exception {
    long start = System.currentTimeMillis();
    Map<String, ucar.unidata.geoloc.Station> staHash =
        new HashMap<String, ucar.unidata.geoloc.Station>();

    String location = "R:/testdata/sounding/netcdf/Upperair_20070401_0000.nc";
    NetcdfDataset ncfile = NetcdfDataset.openDataset(location);
    ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);

    // look through record varibles, for those that have "manLevel" dimension
    // make a StructureData object for those
    StructureMembers sm = new StructureMembers("manLevel");
    Dimension manDim = ncfile.findDimension("manLevel");
    Structure record = (Structure) ncfile.findVariable("record");
    List<Variable> allList = record.getVariables();
    List<VariableSimpleIF> varList = new ArrayList<VariableSimpleIF>();
    for (Variable v : allList) {
      if ((v.getRank() == 1) && v.getDimension(0).equals(manDim)) {
        // public VariableDS(NetcdfDataset ds, Group group, Structure parentStructure, String
        // shortName, DataType dataType,
        // String dims, String units, String desc) {
        varList.add(
            new VariableDS(
                ncfile,
                null,
                null,
                v.getShortName(),
                v.getDataType(),
                "",
                v.getUnitsString(),
                v.getDescription()));
        // (String name, String desc, String units, DataType dtype, int []shape)
        sm.addMember(
            v.getShortName(),
            v.getDescription(),
            v.getUnitsString(),
            v.getDataType(),
            new int[0]); // scalar
      }
    }

    ArrayStructureMA manAS = new ArrayStructureMA(sm, new int[] {manDim.getLength()});

    // need the date units
    Variable time = ncfile.findVariable("synTime");
    String timeUnits = ncfile.findAttValueIgnoreCase(time, "units", null);
    timeUnits = StringUtil.remove(timeUnits, '('); // crappy fsl'ism
    timeUnits = StringUtil.remove(timeUnits, ')');
    DateUnit timeUnit = new DateUnit(timeUnits);

    // extract stations
    int nrecs = 0;
    StructureDataIterator iter = record.getStructureIterator();
    while (iter.hasNext()) {
      StructureData sdata = iter.next();
      String name = sdata.getScalarString("staName");
      ucar.unidata.geoloc.Station s = staHash.get(name);
      if (s == null) {
        float lat = sdata.convertScalarFloat("staLat");
        float lon = sdata.convertScalarFloat("staLon");
        float elev = sdata.convertScalarFloat("staElev");
        s = new StationImpl(name, "", lat, lon, elev);
        staHash.put(name, s);
      }
      nrecs++;
    }
    List<ucar.unidata.geoloc.Station> stnList =
        Arrays.asList(staHash.values().toArray(new ucar.unidata.geoloc.Station[staHash.size()]));
    Collections.sort(stnList);

    // create the writer
    WriterProfileObsDataset writer =
        new WriterProfileObsDataset(location + ".out", "rewrite " + location);
    writer.writeHeader(stnList, varList, nrecs, "prMan");

    // extract records
    iter = record.getStructureIterator();
    while (iter.hasNext()) {
      StructureData sdata = iter.next();
      String name = sdata.getScalarString("staName");
      double timeValue = sdata.convertScalarDouble("synTime");
      Date date = timeUnit.makeDate(timeValue);

      // transfer to the ArrayStructure
      List<String> names = sm.getMemberNames();
      for (String mname : names) {
        manAS.setMemberArray(mname, sdata.getArray(mname));
      }

      // each level is weritten as a seperate structure
      int numMand = sdata.getScalarInt("numMand");
      if (numMand >= manDim.getLength()) continue;

      for (int i = 0; i < numMand; i++) {
        StructureData useData = manAS.getStructureData(i);
        writer.writeRecord(name, date, useData);
      }
    }

    writer.finish();

    long took = System.currentTimeMillis() - start;
    System.out.println("That took = " + took);
  }
Exemple #13
0
  Write2ncRect(NetcdfFile bufr, String fileOutName, boolean fill)
      throws IOException, InvalidRangeException {

    NetcdfFileWriteable ncfile = NetcdfFileWriteable.createNew(fileOutName, fill);
    if (debug) {
      System.out.println("FileWriter write " + bufr.getLocation() + " to " + fileOutName);
    }

    // global attributes
    List<Attribute> glist = bufr.getGlobalAttributes();
    for (Attribute att : glist) {
      String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
      Attribute useAtt;
      if (att.isArray()) useAtt = ncfile.addGlobalAttribute(useName, att.getValues());
      else if (att.isString()) useAtt = ncfile.addGlobalAttribute(useName, att.getStringValue());
      else useAtt = ncfile.addGlobalAttribute(useName, att.getNumericValue());
      if (debug) System.out.println("add gatt= " + useAtt);
    }

    // global dimensions
    Dimension recordDim = null;
    Map<String, Dimension> dimHash = new HashMap<String, Dimension>();
    for (Dimension oldD : bufr.getDimensions()) {
      String useName = N3iosp.makeValidNetcdfObjectName(oldD.getName());
      boolean isRecord = useName.equals("record");
      Dimension newD = ncfile.addDimension(useName, oldD.getLength(), true, false, false);
      dimHash.put(newD.getName(), newD);
      if (isRecord) recordDim = newD;
      if (debug) System.out.println("add dim= " + newD);
    }

    // Variables
    Structure recordStruct = (Structure) bufr.findVariable(BufrIosp.obsRecord);
    for (Variable oldVar : recordStruct.getVariables()) {
      if (oldVar.getDataType() == DataType.STRUCTURE) continue;

      String varName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
      DataType newType = oldVar.getDataType();

      List<Dimension> newDims = new ArrayList<Dimension>();
      newDims.add(recordDim);
      for (Dimension dim : oldVar.getDimensions()) {
        newDims.add(ncfile.addDimension(oldVar.getShortName() + "_strlen", dim.getLength()));
      }

      Variable newVar = ncfile.addVariable(varName, newType, newDims);
      if (debug) System.out.println("add var= " + newVar);

      // attributes
      List<Attribute> attList = oldVar.getAttributes();
      for (Attribute att : attList) {
        String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
        if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
        else if (att.isString())
          ncfile.addVariableAttribute(varName, useName, att.getStringValue());
        else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
      }
    }

    // int max_seq = countSeq(recordStruct);
    // Dimension seqD = ncfile.addDimension("level", max_seq);

    for (Variable v : recordStruct.getVariables()) {
      if (v.getDataType() != DataType.STRUCTURE) continue;
      String structName = N3iosp.makeValidNetcdfObjectName(v.getShortName());
      int shape[] = v.getShape();

      Dimension structDim = ncfile.addDimension(structName, shape[0]);

      Structure struct = (Structure) v;
      for (Variable seqVar : struct.getVariables()) {
        String varName = N3iosp.makeValidNetcdfObjectName(seqVar.getShortName() + "-" + structName);
        DataType newType = seqVar.getDataType();

        List<Dimension> newDims = new ArrayList<Dimension>();
        newDims.add(recordDim);
        newDims.add(structDim);
        for (Dimension dim : seqVar.getDimensions()) {
          newDims.add(ncfile.addDimension(seqVar.getShortName() + "_strlen", dim.getLength()));
        }

        Variable newVar = ncfile.addVariable(varName, newType, newDims);
        if (debug) System.out.println("add var= " + newVar);

        // attributes
        List<Attribute> attList = seqVar.getAttributes();
        for (Attribute att : attList) {
          String useName = N3iosp.makeValidNetcdfObjectName(att.getName());
          if (att.isArray()) ncfile.addVariableAttribute(varName, useName, att.getValues());
          else if (att.isString())
            ncfile.addVariableAttribute(varName, useName, att.getStringValue());
          else ncfile.addVariableAttribute(varName, useName, att.getNumericValue());
        }
      }
    }

    // create the file
    ncfile.create();
    if (debug) System.out.println("File Out= " + ncfile.toString());

    // boolean ok = (Boolean) ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);

    double total = copyVarData(ncfile, recordStruct);
    ncfile.flush();
    System.out.println("FileWriter done total bytes = " + total);
    ncfile.close();
  }
 /**
  * Constructor
  *
  * @param v : the netcdf Variable
  */
 NcSDUInt16(Variable v) {
   super(NcDDS.escapeName(v.getShortName()));
   this.ncVar = v;
 }
Exemple #15
0
 /**
  * Constructor
  *
  * @param ncVar : the netcdf Variable
  */
 NcSDByte(Variable ncVar) {
   super((ncVar.getShortName()));
   this.ncVar = ncVar;
 }