Example #1
0
  /**
   * Open a ucar.nc2.ft.PointFeatureCollection, write out in CF point format.
   *
   * @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..)
   * @param fileOut write to this netcdf-3 file
   * @param inMemory if true, read file into memory for efficiency
   * @return true on success
   * @throws IOException on read/write error
   */
  public static boolean rewritePointFeatureDataset(String fileIn, String fileOut, boolean inMemory)
      throws IOException {
    System.out.println(
        "Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory);

    long start = System.currentTimeMillis();

    // do it in memory for speed
    NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn);
    NetcdfDataset ncd = new NetcdfDataset(ncfile);

    Formatter errlog = new Formatter();
    FeatureDataset fd = FeatureDatasetFactoryManager.wrap(FeatureType.ANY_POINT, ncd, null, errlog);
    if (fd == null) return false;

    if (fd instanceof FeatureDatasetPoint) {
      writePointFeatureCollection((FeatureDatasetPoint) fd, fileOut);
      fd.close();
      long took = System.currentTimeMillis() - start;
      System.out.println(" that took " + (took - start) + " msecs");
      return true;
    }

    return false;
  }
 private NetcdfDataset getBufrMessageAsDataset(Message m) throws IOException {
   byte[] mbytes = scan.getMessageBytes(m);
   NetcdfFile ncfile = null;
   try {
     ncfile = NetcdfFile.openInMemory("test", mbytes, "ucar.nc2.iosp.bufr.BufrIosp");
   } catch (Exception e) {
     throw new IOException(e);
   }
   return new NetcdfDataset(ncfile);
 }
Example #3
0
  @Test
  public void testReadnetCDFinSequnceFileFormat() throws Exception {

    String path = this.getClass().getResource("/ncar.nc").getPath();
    Util.writeToSequenceFile("file://" + path, hadoopMaster + "/tmp/ncar.seq", new DefaultCodec());
    Map<Text, byte[]> netcdfsequnce = Util.readSequenceFile(hadoopMaster + "/tmp/ncar.seq");
    for (Map.Entry<Text, byte[]> entry : netcdfsequnce.entrySet()) {
      NetcdfFile ncFile = NetcdfFile.openInMemory(entry.getKey().toString(), entry.getValue());
      assertEquals(ncFile.getDimensions().size(), 5);
    }
  }
Example #4
0
 @Test
 /**
  * TODO: python API:
  * http://stackoverflow.com/questions/16654251/can-h5py-load-a-file-from-a-byte-array-in-memory
  *
  * @throws Exception
  */
 public void testNASAModisHDFAccess() throws Exception {
   File file =
       new File(
           this.getClass()
               .getResource("/MYD13Q1.A2014121.h23v04.005.2014138045119.hdf")
               .getPath());
   byte[] netcdfinbyte = FileUtils.readFileToByteArray(file);
   NetcdfFile netCDFfile = NetcdfFile.openInMemory("inmemory.hdf", netcdfinbyte);
   // TODO Processing hdf files
 }
Example #5
0
  @Test
  /**
   * TODO: python API:
   * http://stackoverflow.com/questions/16654251/can-h5py-load-a-file-from-a-byte-array-in-memory
   *
   * @throws Exception
   */
  public void testNetCDFInterfaceToACcessH5() throws Exception {
    H5File h5 =
        hdf5_getters.hdf5_open_readonly(
            this.getClass().getResource("/TRAXLZU12903D05F94.h5").getPath());
    double h5_temp = hdf5_getters.get_tempo(h5);

    File file = new File(this.getClass().getResource("/TRAXLZU12903D05F94.h5").getPath());
    byte[] netcdfinbyte = FileUtils.readFileToByteArray(file);

    NetcdfFile netCDFfile = NetcdfFile.openInMemory("inmemory.h5", netcdfinbyte);
    Variable var = (Variable) netCDFfile.findVariable("/analysis/songs.tempo");
    Array content = var.read(); // 1D array
    double netcdf_tempo = content.getDouble(0); // 1 column only
    assertEquals(h5_temp, netcdf_tempo, 0.001);
  }
Example #6
0
  /**
   * Open a ucar.nc2.dt.PointObsDataset, write out in CF point format.
   *
   * @param fileIn open through TypedDatasetFactory.open(FeatureType.POINT, ..)
   * @param fileOut write to this netcdf-3 file
   * @param inMemory if true, read file into memory for efficiency
   * @return true on success
   * @throws IOException on read/write error
   */
  public static boolean rewritePointObsDataset(String fileIn, String fileOut, boolean inMemory)
      throws IOException {
    System.out.println(
        "Rewrite2 .nc files from " + fileIn + " to " + fileOut + " inMemory= " + inMemory);

    long start = System.currentTimeMillis();

    // do it in memory for speed
    NetcdfFile ncfile = inMemory ? NetcdfFile.openInMemory(fileIn) : NetcdfFile.open(fileIn);
    NetcdfDataset ncd = new NetcdfDataset(ncfile);

    StringBuilder errlog = new StringBuilder();
    PointObsDataset pobsDataset =
        (PointObsDataset) TypedDatasetFactory.open(FeatureType.POINT, ncd, null, errlog);
    if (pobsDataset == null) return false;

    writePointObsDataset(pobsDataset, fileOut);
    pobsDataset.close();

    long took = System.currentTimeMillis() - start;
    System.out.println(" that took " + (took - start) + " msecs");
    return true;
  }
Example #7
0
  @Test
  /**
   * Find min/max/average precipitation for a randomly-positioned but fixed-size region from a nc
   * file output: filename,origin,size key: value:min, max, average
   *
   * @throws Exception
   */
  public void testProcessingNASANexDataInNetCDF() throws Exception {
    final int SIZE = 100;
    File file = new File(this.getClass().getResource("/ncar.nc").getPath());
    byte[] netcdfinbyte = FileUtils.readFileToByteArray(file);
    // use any dummy filename for file in memory
    NetcdfFile netCDFfile = NetcdfFile.openInMemory("inmemory.nc", netcdfinbyte);

    Variable time = netCDFfile.findVariable("time");
    ArrayDouble.D1 days = (ArrayDouble.D1) time.read();
    Variable lat = netCDFfile.findVariable("lat");
    if (lat == null) {
      logger.error("Cannot find Variable latitude(lat)");
      return;
    }
    ArrayFloat.D1 absolutelat = (ArrayFloat.D1) lat.read();
    Variable lon = netCDFfile.findVariable("lon");
    if (lon == null) {
      logger.error("Cannot find Variable longitude(lon)");
      return;
    }
    ArrayFloat.D1 absolutelon = (ArrayFloat.D1) lon.read();
    Variable pres = netCDFfile.findVariable("pr");
    if (pres == null) {
      logger.error("Cannot find Variable precipitation(pr)");
      return;
    }

    Random rand = new Random();
    int orig_lat = rand.nextInt((int) lat.getSize());
    orig_lat = Math.min(orig_lat, (int) (lat.getSize() - SIZE));
    int orig_lon = rand.nextInt((int) lon.getSize());
    orig_lon = Math.min(orig_lon, (int) (lon.getSize() - SIZE));

    int[] origin = new int[] {0, orig_lat, orig_lon};
    int[] size = new int[] {1, SIZE, SIZE};
    ArrayFloat.D3 data3D = (ArrayFloat.D3) pres.read(origin, size);
    double max = Double.NEGATIVE_INFINITY;
    double min = Double.POSITIVE_INFINITY;
    double sum = 0;
    for (int j = 0; j < SIZE; j++) {
      for (int k = 0; k < SIZE; k++) {
        double current = data3D.get(0, j, k);
        max = (current > max ? current : max);
        min = (current < min ? current : min);
        sum += current;
      }
    }
    logger.info(
        days
            + ","
            + absolutelat.get(orig_lat)
            + ","
            + absolutelon.get(orig_lon)
            + ","
            + SIZE
            + ":"
            + min
            + ","
            + max
            + ","
            + sum / (SIZE * SIZE));
  }