/* Structure {
      int LAT[0];
      ...
      int LAT[149];
  } IMAGE_LAT_ARRAY(3600);
     type = Layout(8);  type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
  */
  @Test
  public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
    try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {

      Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
      assert (null != dset);
      assert (dset.getDataType() == DataType.STRUCTURE);
      assert (dset.getRank() == 1);
      assert (dset.getSize() == 3600);

      Dimension d = dset.getDimension(0);
      assert (d.getLength() == 3600);

      Structure s = (Structure) dset;

      // read last one - chunked
      StructureData sd = s.readStructure(3599);
      assert sd.getScalarInt("LAT[0]") == 70862722;
      assert sd.getScalarInt("LAT[149]") == 85302263;

      // read one at a time
      for (int i = 3590; i < d.getLength(); i++) {
        s.readStructure(i);
        System.out.println(" read structure " + i);
      }
    }
    System.out.println("*** testReadIASI ok");
  }
  /*
   Structure {
    int a_name;
    byte b_name(3);
    byte c_name(3);
    short d_name(3);
    int e_name(3);
    long f_name(3);
    int g_name(3);
    short h_name(3);
    int i_name(3);
    long j_name(3);
    float k_name(3);
    double l_name(3);
  } CompoundNative(15);
      type = Layout(8);  type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
  */
  @Test
  public void testReadH5StructureArrayMembers() throws java.io.IOException {
    try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {

      Variable dset = ncfile.findVariable("CompoundNative");
      assert (null != dset);
      assert (dset.getDataType() == DataType.STRUCTURE);
      assert (dset.getRank() == 1);
      assert (dset.getSize() == 15);

      Dimension d = dset.getDimension(0);
      assert (d.getLength() == 15);

      Structure s = (Structure) dset;

      // read all with the iterator
      StructureDataIterator iter = s.getStructureIterator();
      while (iter.hasNext()) {
        StructureData sd = iter.next();

        for (StructureMembers.Member m : sd.getMembers()) {
          Array data = sd.getArray(m);
          NCdumpW.printArray(data, m.getName(), out, null);
        }
      }
    }
    System.out.println("*** testReadH5StructureArrayMembers ok");
  }
  /**
   * Constructor.
   *
   * @param ncfile the netccdf file
   * @param typedDataVariables list of data variables; all record variables will be added to this
   *     list, except . You can remove extra
   * @param obsTimeVName observation time variable name (required)
   * @param nomTimeVName nominal time variable name (may be null)
   * @throws IllegalArgumentException if ncfile has no unlimited dimension and recDimName is null.
   */
  public RecordDatasetHelper(
      NetcdfDataset ncfile,
      String obsTimeVName,
      String nomTimeVName,
      List<VariableSimpleIF> typedDataVariables,
      String recDimName,
      Formatter errBuffer) {
    this.ncfile = ncfile;
    this.obsTimeVName = obsTimeVName;
    this.nomTimeVName = nomTimeVName;
    this.errs = errBuffer;

    // check if we already have a structure vs if we have to add it.

    if (this.ncfile.hasUnlimitedDimension()) {
      this.ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);
      this.recordVar = (StructureDS) this.ncfile.getRootGroup().findVariable("record");
      this.obsDim = ncfile.getUnlimitedDimension();

    } else {
      if (recDimName == null)
        throw new IllegalArgumentException(
            "File <"
                + this.ncfile.getLocation()
                + "> has no unlimited dimension, specify psuedo record dimension with observationDimension global attribute.");
      this.obsDim = this.ncfile.getRootGroup().findDimension(recDimName);
      this.recordVar = new StructurePseudoDS(this.ncfile, null, "record", null, obsDim);
    }

    // create member variables
    List<Variable> recordMembers = ncfile.getVariables();
    for (Variable v : recordMembers) {
      if (v == recordVar) continue;
      if (v.isScalar()) continue;
      if (v.getDimension(0) == this.obsDim) typedDataVariables.add(v);
    }

    // need the time units
    Variable timeVar = ncfile.findVariable(obsTimeVName);
    String timeUnitString =
        ncfile.findAttValueIgnoreCase(timeVar, CDM.UNITS, "seconds since 1970-01-01");
    try {
      timeUnit = new DateUnit(timeUnitString);
    } catch (Exception e) {
      if (null != errs) errs.format("Error on string = %s == %s%n", timeUnitString, e.getMessage());
      try {
        timeUnit = new DateUnit("seconds since 1970-01-01");
      } catch (Exception e1) {
        // cant happen
      }
    }
  }
Exemple #4
0
  private int[] getWeights(Variable v) {
    int rank = v.getRank();
    int[] w = new int[rank];

    for (int n = 0; n < rank; n++) {
      Dimension dim = v.getDimension(n);
      String dimName = dim.getName();
      if (dimName.equals("time")) w[n] = 1000;
      if (dimName.equals("z")) w[n] = 100;
      if (dimName.equals("y")) w[n] = 10;
      if (dimName.equals("x")) w[n] = 1;
    }

    return w;
  }
  @Test
  public void testH5StructureDS() throws java.io.IOException {
    int a_name = 0;
    String[] b_name =
        new String[] {
          "A fight is a contract that takes two people to honor.",
          "A combative stance means that you've accepted the contract.",
          "In which case, you deserve what you get.",
          "  --  Professor Cheng Man-ch'ing"
        };
    String c_name = "Hello!";

    // H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
    try (NetcdfDataset ncfile =
        NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {

      Variable dset = ncfile.findVariable("CompoundComplex");
      assert (null != dset);
      assert (dset.getDataType() == DataType.STRUCTURE);
      assert (dset.getRank() == 1);
      assert (dset.getSize() == 6);

      Dimension d = dset.getDimension(0);
      assert (d.getLength() == 6);

      Structure s = (Structure) dset;

      // read all with the iterator
      StructureDataIterator iter = s.getStructureIterator();
      while (iter.hasNext()) {
        StructureData sd = iter.next();
        assert sd.getScalarInt("a_name") == a_name;
        a_name++;
        assert sd.getScalarString("c_name").equals(c_name);
        String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
        assert results.length == b_name.length;
        int count = 0;
        for (String r : results) assert r.equals(b_name[count++]);

        for (StructureMembers.Member m : sd.getMembers()) {
          Array data = sd.getArray(m);
          NCdumpW.printArray(data, m.getName(), out, null);
        }
      }
    }
    System.out.println("*** testH5StructureDS ok");
  }
  /*  Structure {
    char EntryName(64);
    char Definition(1024);
    char Unit(1024);
    char Scale Factor(1024);
  } TIME_DESCR(60);
     type = Layout(8);  type= 2 (chunked) storageSize = (1,3136) dataSize=0 dataAddress=684294
  */
  @Test
  public void testReadManyAtATime() throws java.io.IOException, InvalidRangeException {
    try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {

      Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/TIME_DESCR");
      assert (null != dset);
      assert (dset.getDataType() == DataType.STRUCTURE);
      assert (dset.getRank() == 1);
      assert (dset.getSize() == 60);

      Dimension d = dset.getDimension(0);
      assert (d.getLength() == 60);

      ArrayStructure data = (ArrayStructure) dset.read();
      StructureMembers.Member m = data.getStructureMembers().findMember("EntryName");
      assert m != null;
      for (int i = 0; i < dset.getSize(); i++) {
        String r = data.getScalarString(i, m);
        if (i % 2 == 0) assert r.equals("TIME[" + i / 2 + "]-days") : r + " at " + i;
        else assert r.equals("TIME[" + i / 2 + "]-milliseconds") : r + " at " + i;
      }
    }
    System.out.println("*** testReadManyAtATime ok");
  }
  public static void main(String args[]) throws Exception {
    long start = System.currentTimeMillis();
    Map<String, ucar.unidata.geoloc.Station> staHash =
        new HashMap<String, ucar.unidata.geoloc.Station>();

    String location = "R:/testdata/sounding/netcdf/Upperair_20070401_0000.nc";
    NetcdfDataset ncfile = NetcdfDataset.openDataset(location);
    ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);

    // look through record varibles, for those that have "manLevel" dimension
    // make a StructureData object for those
    StructureMembers sm = new StructureMembers("manLevel");
    Dimension manDim = ncfile.findDimension("manLevel");
    Structure record = (Structure) ncfile.findVariable("record");
    List<Variable> allList = record.getVariables();
    List<VariableSimpleIF> varList = new ArrayList<VariableSimpleIF>();
    for (Variable v : allList) {
      if ((v.getRank() == 1) && v.getDimension(0).equals(manDim)) {
        // public VariableDS(NetcdfDataset ds, Group group, Structure parentStructure, String
        // shortName, DataType dataType,
        // String dims, String units, String desc) {
        varList.add(
            new VariableDS(
                ncfile,
                null,
                null,
                v.getShortName(),
                v.getDataType(),
                "",
                v.getUnitsString(),
                v.getDescription()));
        // (String name, String desc, String units, DataType dtype, int []shape)
        sm.addMember(
            v.getShortName(),
            v.getDescription(),
            v.getUnitsString(),
            v.getDataType(),
            new int[0]); // scalar
      }
    }

    ArrayStructureMA manAS = new ArrayStructureMA(sm, new int[] {manDim.getLength()});

    // need the date units
    Variable time = ncfile.findVariable("synTime");
    String timeUnits = ncfile.findAttValueIgnoreCase(time, "units", null);
    timeUnits = StringUtil.remove(timeUnits, '('); // crappy fsl'ism
    timeUnits = StringUtil.remove(timeUnits, ')');
    DateUnit timeUnit = new DateUnit(timeUnits);

    // extract stations
    int nrecs = 0;
    StructureDataIterator iter = record.getStructureIterator();
    while (iter.hasNext()) {
      StructureData sdata = iter.next();
      String name = sdata.getScalarString("staName");
      ucar.unidata.geoloc.Station s = staHash.get(name);
      if (s == null) {
        float lat = sdata.convertScalarFloat("staLat");
        float lon = sdata.convertScalarFloat("staLon");
        float elev = sdata.convertScalarFloat("staElev");
        s = new StationImpl(name, "", lat, lon, elev);
        staHash.put(name, s);
      }
      nrecs++;
    }
    List<ucar.unidata.geoloc.Station> stnList =
        Arrays.asList(staHash.values().toArray(new ucar.unidata.geoloc.Station[staHash.size()]));
    Collections.sort(stnList);

    // create the writer
    WriterProfileObsDataset writer =
        new WriterProfileObsDataset(location + ".out", "rewrite " + location);
    writer.writeHeader(stnList, varList, nrecs, "prMan");

    // extract records
    iter = record.getStructureIterator();
    while (iter.hasNext()) {
      StructureData sdata = iter.next();
      String name = sdata.getScalarString("staName");
      double timeValue = sdata.convertScalarDouble("synTime");
      Date date = timeUnit.makeDate(timeValue);

      // transfer to the ArrayStructure
      List<String> names = sm.getMemberNames();
      for (String mname : names) {
        manAS.setMemberArray(mname, sdata.getArray(mname));
      }

      // each level is weritten as a seperate structure
      int numMand = sdata.getScalarInt("numMand");
      if (numMand >= manDim.getLength()) continue;

      for (int i = 0; i < numMand; i++) {
        StructureData useData = manAS.getStructureData(i);
        writer.writeRecord(name, date, useData);
      }
    }

    writer.finish();

    long took = System.currentTimeMillis() - start;
    System.out.println("That took = " + took);
  }