/* Structure {
      int LAT[0];
      ...
      int LAT[149];
  } IMAGE_LAT_ARRAY(3600);
     type = Layout(8);  type= 2 (chunked) storageSize = (1,600) dataSize=0 dataAddress=2548046
  */
  @Test
  public void testReadOneAtATime() throws java.io.IOException, InvalidRangeException {
    try (NetcdfFile ncfile = TestH5.openH5("IASI/IASI.h5")) {

      Variable dset = ncfile.findVariable("U-MARF/EPS/IASI_xxx_1C/DATA/IMAGE_LAT_ARRAY");
      assert (null != dset);
      assert (dset.getDataType() == DataType.STRUCTURE);
      assert (dset.getRank() == 1);
      assert (dset.getSize() == 3600);

      Dimension d = dset.getDimension(0);
      assert (d.getLength() == 3600);

      Structure s = (Structure) dset;

      // read last one - chunked
      StructureData sd = s.readStructure(3599);
      assert sd.getScalarInt("LAT[0]") == 70862722;
      assert sd.getScalarInt("LAT[149]") == 85302263;

      // read one at a time
      for (int i = 3590; i < d.getLength(); i++) {
        s.readStructure(i);
        System.out.println(" read structure " + i);
      }
    }
    System.out.println("*** testReadIASI ok");
  }
Пример #2
0
  private int countSeq(Structure recordStruct) throws IOException {
    int total = 0;
    int count = 0;
    int max = 0;

    StructureDataIterator iter = recordStruct.getStructureIterator();
    try {
      while (iter.hasNext()) {

        StructureData sdata = iter.next();
        ArraySequence seq1 = sdata.getArraySequence("seq1");
        int n = seq1.getStructureDataCount();
        total += n;
        count++;
        max = Math.max(max, n);
      }
    } finally {
      iter.finish();
    }
    double avg = total / count;
    int wasted = count * max - total;
    double wp = (double) wasted / (count * max);
    System.out.println(" Max = " + max + " avg = " + avg + " wasted = " + wasted + " %= " + wp);
    return max;
  }
  /*
   Structure {
    int a_name;
    byte b_name(3);
    byte c_name(3);
    short d_name(3);
    int e_name(3);
    long f_name(3);
    int g_name(3);
    short h_name(3);
    int i_name(3);
    long j_name(3);
    float k_name(3);
    double l_name(3);
  } CompoundNative(15);
      type = Layout(8);  type= 1 (contiguous) storageSize = (15,144) dataSize=0 dataAddress=2048
  */
  @Test
  public void testReadH5StructureArrayMembers() throws java.io.IOException {
    try (NetcdfFile ncfile = TestH5.openH5("complex/compound_native.h5")) {

      Variable dset = ncfile.findVariable("CompoundNative");
      assert (null != dset);
      assert (dset.getDataType() == DataType.STRUCTURE);
      assert (dset.getRank() == 1);
      assert (dset.getSize() == 15);

      Dimension d = dset.getDimension(0);
      assert (d.getLength() == 15);

      Structure s = (Structure) dset;

      // read all with the iterator
      StructureDataIterator iter = s.getStructureIterator();
      while (iter.hasNext()) {
        StructureData sd = iter.next();

        for (StructureMembers.Member m : sd.getMembers()) {
          Array data = sd.getArray(m);
          NCdumpW.printArray(data, m.getName(), out, null);
        }
      }
    }
    System.out.println("*** testReadH5StructureArrayMembers ok");
  }
  public void writeRecord(String stnName, Date obsDate, StructureData sdata) throws IOException {
    StationTracker stnTracker = stationMap.get(stnName);
    ProfileTracker proTracker = stnTracker.profileMap.get(obsDate);

    if (proTracker == null) {
      proTracker = new ProfileTracker(profileIndex);
      stnTracker.profileMap.put(obsDate, proTracker);

      stnTracker.link.add(profileIndex);
      stnTracker.lastChild = profileIndex;
      stnTracker.numChildren++;

      try {
        originTime[0] = profileIndex; // 2d index
        timeArray.set(0, dateFormatter.toDateTimeStringISO(obsDate));
        parentArray.set(0, stnTracker.parent_index);
        ncfile.writeStringData(timeName, originTime, timeArray);
        ncfile.write(parentStationIndex, originTime, parentArray);
      } catch (InvalidRangeException e) {
        e.printStackTrace();
        throw new IllegalStateException(e);
      }

      profileIndex++;
    }

    // needs to be wrapped as an ArrayStructure, even though we are only writing one at a time.
    ArrayStructureW sArray = new ArrayStructureW(sdata.getStructureMembers(), new int[] {1});
    sArray.setStructureData(sdata, 0);

    // track the min and max date
    if ((minDate == null) || minDate.after(obsDate)) minDate = obsDate;
    if ((maxDate == null) || maxDate.before(obsDate)) maxDate = obsDate;

    timeArray.set(0, dateFormatter.toDateTimeStringISO(obsDate));
    parentArray.set(0, proTracker.parent_index);

    proTracker.link.add(recno);
    proTracker.lastChild = recno; // not currently used
    proTracker.numChildren++;

    // write the recno record
    origin[0] = recno; // 1d index
    originTime[0] = recno; // 2d index
    try {
      ncfile.write("record", origin, sArray);
      ncfile.write(parentProfileIndex, originTime, parentArray);
    } catch (InvalidRangeException e) {
      e.printStackTrace();
      throw new IllegalStateException(e);
    }

    recno++;
  }
Пример #5
0
  public boolean compareStructureData(
      StructureData sdata1, StructureData sdata2, double tol, boolean justOne) {
    boolean ok = true;

    StructureMembers sm1 = sdata1.getStructureMembers();
    StructureMembers sm2 = sdata2.getStructureMembers();
    if (sm1.getMembers().size() != sm2.getMembers().size()) {
      f.format(" size %d !== %d%n", sm1.getMembers().size(), sm2.getMembers().size());
      ok = false;
    }

    for (StructureMembers.Member m1 : sm1.getMembers()) {
      if (m1.getName().equals("time")) continue;
      StructureMembers.Member m2 = sm2.findMember(m1.getName());
      Array data1 = sdata1.getArray(m1);
      Array data2 = sdata2.getArray(m2);
      ok &= compareData(m1.getName(), data1, data2, tol, justOne);
    }

    return ok;
  }
    StationObs(Station s, StructureData data) {
      this.station = s;
      this.data = data;

      int cyd = data.getScalarInt("DAY");
      int year = cyd / 1000;
      int doy = cyd % 1000;
      int hms = data.getScalarInt("TIME");
      int hour = hms / 10000;
      hms %= 10000;
      int min = hms / 100;
      int sec = hms % 100;

      calendar.clear();
      calendar.set(GregorianCalendar.YEAR, year);
      calendar.set(GregorianCalendar.DAY_OF_YEAR, doy);
      calendar.set(GregorianCalendar.HOUR_OF_DAY, hour);
      calendar.set(GregorianCalendar.MINUTE, min);
      calendar.set(GregorianCalendar.SECOND, sec);
      // Date check = calendar.getTime();
      // System.out.println(" check="+DateUnit.getStandardDateString(check));
      obsTime = calendar.getTimeInMillis() / 1000.0;
      nomTime = obsTime; // LOOK get real nominal time
    }
  protected void makeFeatureVariables(StructureData featureData, boolean isExtended)
      throws IOException {

    // LOOK why not unlimited here ?
    Dimension profileDim = writer.addDimension(null, profileDimName, nfeatures);
    // Dimension profileDim = isExtendedModel ?  writer.addUnlimitedDimension(profileDimName) :
    // writer.addDimension(null, profileDimName, nprofiles);

    // add the profile Variables using the profile dimension
    List<VariableSimpleIF> profileVars = new ArrayList<>();
    profileVars.add(
        VariableSimpleImpl.makeScalar(latName, "profile latitude", CDM.LAT_UNITS, DataType.DOUBLE));
    profileVars.add(
        VariableSimpleImpl.makeScalar(
            lonName, "profile longitude", CDM.LON_UNITS, DataType.DOUBLE));
    profileVars.add(
        VariableSimpleImpl.makeString(profileIdName, "profile identifier", null, id_strlen)
            .add(new Attribute(CF.CF_ROLE, CF.PROFILE_ID))); // profileId:cf_role = "profile_id";

    profileVars.add(
        VariableSimpleImpl.makeScalar(
                numberOfObsName, "number of obs for this profile", null, DataType.INT)
            .add(
                new Attribute(
                    CF.SAMPLE_DIMENSION, recordDimName))); // rowSize:sample_dimension = "obs"

    profileVars.add(
        VariableSimpleImpl.makeScalar(
            profileTimeName,
            "nominal time of profile",
            timeUnit.getUnitsString(),
            DataType.DOUBLE));

    for (StructureMembers.Member m : featureData.getMembers()) {
      VariableSimpleIF dv = getDataVar(m.getName());
      if (dv != null) profileVars.add(dv);
    }

    if (isExtended) {
      profileStruct =
          (Structure)
              writer.addVariable(null, profileStructName, DataType.STRUCTURE, profileDimName);
      addCoordinatesExtended(profileStruct, profileVars);
    } else {
      addCoordinatesClassic(profileDim, profileVars, featureVarMap);
    }
  }
  @Test
  public void testH5StructureDS() throws java.io.IOException {
    int a_name = 0;
    String[] b_name =
        new String[] {
          "A fight is a contract that takes two people to honor.",
          "A combative stance means that you've accepted the contract.",
          "In which case, you deserve what you get.",
          "  --  Professor Cheng Man-ch'ing"
        };
    String c_name = "Hello!";

    // H5header.setDebugFlags(new ucar.nc2.util.DebugFlagsImpl("H5header/header"));
    try (NetcdfDataset ncfile =
        NetcdfDataset.openDataset(TestH5.testDir + "complex/compound_complex.h5")) {

      Variable dset = ncfile.findVariable("CompoundComplex");
      assert (null != dset);
      assert (dset.getDataType() == DataType.STRUCTURE);
      assert (dset.getRank() == 1);
      assert (dset.getSize() == 6);

      Dimension d = dset.getDimension(0);
      assert (d.getLength() == 6);

      Structure s = (Structure) dset;

      // read all with the iterator
      StructureDataIterator iter = s.getStructureIterator();
      while (iter.hasNext()) {
        StructureData sd = iter.next();
        assert sd.getScalarInt("a_name") == a_name;
        a_name++;
        assert sd.getScalarString("c_name").equals(c_name);
        String[] results = sd.getJavaArrayString(sd.findMember("b_name"));
        assert results.length == b_name.length;
        int count = 0;
        for (String r : results) assert r.equals(b_name[count++]);

        for (StructureMembers.Member m : sd.getMembers()) {
          Array data = sd.getArray(m);
          NCdumpW.printArray(data, m.getName(), out, null);
        }
      }
    }
    System.out.println("*** testH5StructureDS ok");
  }
  public static void main(String args[]) throws Exception {
    long start = System.currentTimeMillis();
    Map<String, ucar.unidata.geoloc.Station> staHash =
        new HashMap<String, ucar.unidata.geoloc.Station>();

    String location = "R:/testdata/sounding/netcdf/Upperair_20070401_0000.nc";
    NetcdfDataset ncfile = NetcdfDataset.openDataset(location);
    ncfile.sendIospMessage(NetcdfFile.IOSP_MESSAGE_ADD_RECORD_STRUCTURE);

    // look through record varibles, for those that have "manLevel" dimension
    // make a StructureData object for those
    StructureMembers sm = new StructureMembers("manLevel");
    Dimension manDim = ncfile.findDimension("manLevel");
    Structure record = (Structure) ncfile.findVariable("record");
    List<Variable> allList = record.getVariables();
    List<VariableSimpleIF> varList = new ArrayList<VariableSimpleIF>();
    for (Variable v : allList) {
      if ((v.getRank() == 1) && v.getDimension(0).equals(manDim)) {
        // public VariableDS(NetcdfDataset ds, Group group, Structure parentStructure, String
        // shortName, DataType dataType,
        // String dims, String units, String desc) {
        varList.add(
            new VariableDS(
                ncfile,
                null,
                null,
                v.getShortName(),
                v.getDataType(),
                "",
                v.getUnitsString(),
                v.getDescription()));
        // (String name, String desc, String units, DataType dtype, int []shape)
        sm.addMember(
            v.getShortName(),
            v.getDescription(),
            v.getUnitsString(),
            v.getDataType(),
            new int[0]); // scalar
      }
    }

    ArrayStructureMA manAS = new ArrayStructureMA(sm, new int[] {manDim.getLength()});

    // need the date units
    Variable time = ncfile.findVariable("synTime");
    String timeUnits = ncfile.findAttValueIgnoreCase(time, "units", null);
    timeUnits = StringUtil.remove(timeUnits, '('); // crappy fsl'ism
    timeUnits = StringUtil.remove(timeUnits, ')');
    DateUnit timeUnit = new DateUnit(timeUnits);

    // extract stations
    int nrecs = 0;
    StructureDataIterator iter = record.getStructureIterator();
    while (iter.hasNext()) {
      StructureData sdata = iter.next();
      String name = sdata.getScalarString("staName");
      ucar.unidata.geoloc.Station s = staHash.get(name);
      if (s == null) {
        float lat = sdata.convertScalarFloat("staLat");
        float lon = sdata.convertScalarFloat("staLon");
        float elev = sdata.convertScalarFloat("staElev");
        s = new StationImpl(name, "", lat, lon, elev);
        staHash.put(name, s);
      }
      nrecs++;
    }
    List<ucar.unidata.geoloc.Station> stnList =
        Arrays.asList(staHash.values().toArray(new ucar.unidata.geoloc.Station[staHash.size()]));
    Collections.sort(stnList);

    // create the writer
    WriterProfileObsDataset writer =
        new WriterProfileObsDataset(location + ".out", "rewrite " + location);
    writer.writeHeader(stnList, varList, nrecs, "prMan");

    // extract records
    iter = record.getStructureIterator();
    while (iter.hasNext()) {
      StructureData sdata = iter.next();
      String name = sdata.getScalarString("staName");
      double timeValue = sdata.convertScalarDouble("synTime");
      Date date = timeUnit.makeDate(timeValue);

      // transfer to the ArrayStructure
      List<String> names = sm.getMemberNames();
      for (String mname : names) {
        manAS.setMemberArray(mname, sdata.getArray(mname));
      }

      // each level is weritten as a seperate structure
      int numMand = sdata.getScalarInt("numMand");
      if (numMand >= manDim.getLength()) continue;

      for (int i = 0; i < numMand; i++) {
        StructureData useData = manAS.getStructureData(i);
        writer.writeRecord(name, date, useData);
      }
    }

    writer.finish();

    long took = System.currentTimeMillis() - start;
    System.out.println("That took = " + took);
  }
Пример #10
0
  private double copyVarData(NetcdfFileWriteable ncfile, Structure recordStruct)
      throws IOException, InvalidRangeException {
    int nrecs = (int) recordStruct.getSize();
    int sdataSize = recordStruct.getElementSize();

    double total = 0;
    double totalRecordBytes = 0;
    for (int count = 0; count < nrecs; count++) {

      StructureData recordData = recordStruct.readStructure(count);
      for (StructureMembers.Member m : recordData.getMembers()) {

        if (m.getDataType() == DataType.STRUCTURE) {
          int countLevel = 0;
          ArrayStructure seq1 = recordData.getArrayStructure(m);
          StructureDataIterator iter = seq1.getStructureDataIterator();
          try {
            while (iter.hasNext()) {
              StructureData seqData = iter.next();
              for (StructureMembers.Member seqm : seqData.getMembers()) {
                Array data = seqData.getArray(seqm);
                int[] shape = data.getShape();
                int[] newShape = new int[data.getRank() + 2];
                newShape[0] = 1;
                newShape[1] = 1;
                for (int i = 0; i < data.getRank(); i++) newShape[i + 1] = shape[i];

                int[] origin = new int[data.getRank() + 2];
                origin[0] = count;
                origin[1] = countLevel;

                String mname = seqm.getName() + "-" + m.getName();
                if (debug && (count == 0) && (countLevel == 0))
                  System.out.println("write to = " + mname);
                ncfile.write(mname, origin, data.reshape(newShape));
              }
              countLevel++;
            }
          } finally {
            iter.finish();
          }
        } else {

          Array data = recordData.getArray(m);
          int[] shape = data.getShape();
          int[] newShape = new int[data.getRank() + 1];
          newShape[0] = 1;
          for (int i = 0; i < data.getRank(); i++) newShape[i + 1] = shape[i];

          int[] origin = new int[data.getRank() + 1];
          origin[0] = count;

          if (debug && (count == 0)) System.out.println("write to = " + m.getName());
          ncfile.write(m.getName(), origin, data.reshape(newShape));
        }
      }
      totalRecordBytes += sdataSize;
    }

    total += totalRecordBytes;
    totalRecordBytes /= 1000 * 1000;
    if (debug)
      System.out.println(
          "write record var; total = " + totalRecordBytes + " Mbytes # recs=" + nrecs);

    return total;
  }
 private MyEarthLocation(StructureData sdata) {
   this.latitude = sdata.convertScalarDouble(SingleTrajectory.this.latVar.getShortName());
   this.longitude = sdata.convertScalarDouble(SingleTrajectory.this.lonVar.getShortName());
   this.elevation = sdata.convertScalarDouble(SingleTrajectory.this.elevVar.getShortName());
   if (elevVarUnitsConversionFactor != 1.0) this.elevation *= elevVarUnitsConversionFactor;
 }
 private MyPointObsDatatype(int point, StructureData sdata) {
   this.point = point;
   this.sdata = sdata;
   this.time = sdata.convertScalarDouble(SingleTrajectory.this.timeVar.getShortName());
   this.earthLoc = new MyEarthLocation(sdata);
 }
Пример #13
0
 public void serialize(DataOutputStream sink, StructureData sdata, StructureMembers.Member m)
     throws IOException {
   setValue(sdata.getScalarShort(m));
   externalize(sink);
 }