void initTransientFire(boolean useseverity, String dir) {
String filename = dir + "fire.nc";
NetcdfFile ncfile = null;
File spfile = new File(filename);
if (spfile.exists()) {
try {
ncfile = NetcdfFile.open(filename);
Variable fyearV = ncfile.findVariable("FIRE");
trfireyearA = fyearV.read();
Variable fseasonV = ncfile.findVariable("SEASON");
trfireseasonA = fseasonV.read();
if (useseverity) {
Variable fsevV = ncfile.findVariable("SEVERITY");
trseverityA = fsevV.read();
}
} catch (IOException ioe) {
System.out.println(ioe.getMessage());
} finally {
if (ncfile != null) {
try {
ncfile.close();
} catch (IOException ioee) {
System.out.println(ioee.getMessage());
}
}
}
} else { // file not exist
System.out.println("Input file: " + filename + " NOT existed");
System.exit(-1);
}
};
Array readFully() throws IOException {
if (origin == null) {
return binVariable.read();
} else {
try {
return binVariable.read(origin, shape).reduce();
} catch (InvalidRangeException e) {
throw new IOException(e.getMessage());
}
}
}
private void compareVariableData(
Variable var1, Variable var2, boolean showCompare, boolean justOne) throws IOException {
Array data1 = var1.read();
Array data2 = var2.read();
if (showCompare)
f.format(
" compareArrays %s unlimited=%s size=%d%n",
var1.getNameAndDimensions(), var1.isUnlimited(), data1.getSize());
compareData(var1.getFullName(), data1, data2, justOne);
if (showCompare) f.format(" ok%n");
}
Array read(int firstIndex, int length) throws IOException {
try {
if (origin == null) {
return binVariable.read(new int[] {firstIndex}, new int[] {length});
} else {
int[] originFull = origin.clone();
int[] shapeFull = shape.clone();
originFull[binDimIndex] = firstIndex;
shapeFull[binDimIndex] = length;
return binVariable.read(originFull, shapeFull).reduce();
}
} catch (InvalidRangeException e) {
throw new IOException(e.getMessage());
}
}
/** Read the value (parameters are ignored). */
public boolean read(String datasetName, Object specialO) throws IOException {
ArrayDouble.D0 a = (ArrayDouble.D0) ncVar.read();
setValue(a.get());
setRead(true);
return (false);
}
public void addInputParamMetadata(Product product) throws ProductIOException {
Variable inputParams = ncFile.findVariable("Input_Parameters");
if (inputParams != null) {
final MetadataElement inputParamsMeta = new MetadataElement("Input_Parameters");
Array array;
try {
array = inputParams.read();
} catch (IOException e) {
throw new ProductIOException(e.getMessage());
}
String[] lines = array.toString().split("\n");
for (String line : lines) {
String[] parts = line.split("=");
if (parts.length == 2) {
final String name = parts[0].trim();
final String value = parts[1].trim();
final ProductData data = ProductData.createInstance(ProductData.TYPE_ASCII, value);
final MetadataAttribute attribute = new MetadataAttribute(name, data, true);
inputParamsMeta.addAttribute(attribute);
}
}
final MetadataElement metadataRoot = product.getMetadataRoot();
metadataRoot.addElement(inputParamsMeta);
}
}
// @todo Make sure units are meters
public Array getElevation(Range range) throws IOException, InvalidRangeException {
List section = new ArrayList(1);
section.add(range);
Array a = elevVar.read(section);
if (elevVarUnitsConversionFactor == 1.0) return (a);
for (IndexIterator it = a.getIndexIterator(); it.hasNext(); ) {
if (elevVar.getDataType() == DataType.DOUBLE) {
double val = it.getDoubleNext();
it.setDoubleCurrent(val * elevVarUnitsConversionFactor);
} else if (elevVar.getDataType() == DataType.FLOAT) {
float val = it.getFloatNext();
it.setFloatCurrent((float) (val * elevVarUnitsConversionFactor));
} else if (elevVar.getDataType() == DataType.INT) {
int val = it.getIntNext();
it.setIntCurrent((int) (val * elevVarUnitsConversionFactor));
} else if (elevVar.getDataType() == DataType.LONG) {
long val = it.getLongNext();
it.setLongCurrent((long) (val * elevVarUnitsConversionFactor));
} else {
throw new IllegalStateException(
"Elevation variable type <"
+ elevVar.getDataType().toString()
+ "> not double, float, int, or long.");
}
}
return (a);
}
private void copySome(NetcdfFileWriteable ncfile, Variable oldVar, int nelems)
throws IOException {
String newName = N3iosp.makeValidNetcdfObjectName(oldVar.getShortName());
int[] shape = oldVar.getShape();
int[] origin = new int[oldVar.getRank()];
int size = shape[0];
for (int i = 0; i < size; i += nelems) {
origin[0] = i;
int left = size - i;
shape[0] = Math.min(nelems, left);
Array data;
try {
data = oldVar.read(origin, shape);
if (oldVar.getDataType() == DataType.STRING) {
data = convertToChar(ncfile.findVariable(newName), data);
}
if (data.getSize() > 0) { // zero when record dimension = 0
ncfile.write(newName, origin, data);
if (debug) System.out.println("write " + data.getSize() + " bytes");
}
} catch (InvalidRangeException e) {
e.printStackTrace();
throw new IOException(e.getMessage());
}
}
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt.D1) : data.getClass().getName();
ArrayInt.D1 dataI = (ArrayInt.D1) data;
assert dataI.get(0) == 0;
assert dataI.get(1) == 10;
assert dataI.get(2) == 99;
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testAggCoordVar2(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1 : time.getRank();
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.INT;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert (data instanceof ArrayInt);
IndexIterator dataI = data.getIndexIterator();
assert dataI.getIntNext() == 0 : dataI.getIntCurrent();
assert dataI.getIntNext() == 1 : dataI.getIntCurrent();
assert dataI.getIntNext() == 2 : dataI.getIntCurrent();
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
public void testAggCoordVar(NetcdfFile ncfile) {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1;
assert time.getSize() == 3;
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE;
assert time.getDimension(0) == ncfile.findDimension("time");
try {
Array data = time.read();
assert data.getRank() == 1;
assert data.getSize() == 3;
assert data.getShape()[0] == 3;
assert data.getElementType() == double.class;
int count = 0;
IndexIterator dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count]);
count++;
}
} catch (IOException io) {
io.printStackTrace();
assert false;
}
}
private static String getStructMetadata(Group eosGroup) throws IOException {
StringBuilder sbuff = null;
String structMetadata = null;
int n = 0;
while (true) {
Variable structMetadataVar = eosGroup.findVariable("StructMetadata." + n);
if (structMetadataVar == null) break;
if ((structMetadata != null) && (sbuff == null)) { // more than 1 StructMetadata
sbuff = new StringBuilder(64000);
sbuff.append(structMetadata);
}
// read and parse the ODL
Array A = structMetadataVar.read();
if (A instanceof ArrayChar.D1) {
ArrayChar ca = (ArrayChar) A;
structMetadata = ca.getString(); // common case only StructMetadata.0, avoid extra copy
} else if (A instanceof ArrayObject.D0) {
ArrayObject ao = (ArrayObject) A;
structMetadata = (String) ao.getObject(0);
} else {
log.error("Unsupported array type {} for StructMetadata", A.getElementType());
}
if (sbuff != null) sbuff.append(structMetadata);
n++;
}
return (sbuff != null) ? sbuff.toString() : structMetadata;
}
private void handleMetadataGroup(Group group, MetadataElement metadataElement)
throws ProductIOException {
List<Variable> variables = group.getVariables();
for (Variable variable : variables) {
final String name = variable.getShortName();
final int dataType = getProductDataType(variable);
Array array;
try {
array = variable.read();
} catch (IOException e) {
throw new ProductIOException(e.getMessage());
}
final ProductData data = ProductData.createInstance(dataType, array.getStorage());
final MetadataAttribute attribute = new MetadataAttribute("data", data, true);
final MetadataElement sdsElement = new MetadataElement(name);
sdsElement.addAttribute(attribute);
metadataElement.addElement(sdsElement);
final List<Attribute> list = variable.getAttributes();
for (Attribute hdfAttribute : list) {
final String attribName = hdfAttribute.getShortName();
if ("units".equals(attribName)) {
attribute.setUnit(hdfAttribute.getStringValue());
} else if ("long_name".equals(attribName)) {
attribute.setDescription(hdfAttribute.getStringValue());
} else {
addAttributeToElement(sdsElement, hdfAttribute);
}
}
}
}
private float testReadScalar(Variable v) throws IOException {
if (show) System.out.printf(" read %s%n", v.getNameAndDimensions());
assert (null != v);
Array a = v.read();
assert (null != a);
IndexIterator ii = a.getIndexIterator();
return ii.getFloatNext();
}
@Ignore("files not available")
@Test
public void testCacheTiming() throws IOException, InvalidRangeException {
String filename = "file:testCacheTiming.xml";
System.out.printf("%s%n", filename);
String cacheDirName = TestDir.temporaryLocalDataDir + "testAggExistingCache/";
System.out.printf("cacheDir=%s%n", cacheDirName);
File cacheDir = new File(cacheDirName);
FileUtils.deleteDirectory(cacheDir); // from commons-io
assert !cacheDir.exists();
DiskCache2 cache = new DiskCache2(cacheDirName, false, 0, 0);
cache.setAlwaysUseCache(true);
Assert.assertEquals(cache.getRootDirectory(), cacheDirName);
assert new File(cache.getRootDirectory()).exists();
Aggregation.setPersistenceCache(cache);
AggregationExisting.countCacheUse = 0;
long start = System.currentTimeMillis();
try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml2), filename, null)) {
System.out.printf("%nTestNcmlAggExisting.open %s%n", filename);
Variable time = ncfile.findVariable("time");
System.out.printf(" Variable %s%n", time.getNameAndDimensions());
time.read();
}
System.out.printf(" countCacheUse = %d%n", AggregationExisting.countCacheUse);
long took = System.currentTimeMillis() - start;
System.out.printf(" first took %d msecs%n", took);
AggregationExisting.countCacheUse = 0;
start = System.currentTimeMillis();
try (NetcdfFile ncfile = NcMLReader.readNcML(new StringReader(ncml2), filename, null)) {
System.out.printf("%nTestNcmlAggExisting.open %s%n", filename);
Variable time = ncfile.findVariable("time");
System.out.printf(" Variable %s%n", time.getNameAndDimensions());
time.read();
}
System.out.printf(" countCacheUse = %d%n", AggregationExisting.countCacheUse);
took = System.currentTimeMillis() - start;
System.out.printf(" second took %d msecs%n", took);
}
/* (non-Javadoc)
* @see loci.formats.NetCDFService#getArray(java.lang.String, int[], int[])
*/
public Object getArray(String path, int[] origin, int[] shape) throws ServiceException {
String groupName = getDirectory(path);
String variableName = getName(path);
Group group = getGroup(groupName);
Variable variable = group.findVariable(variableName);
try {
if (origin != null && shape != null) {
return variable.read(origin, shape).reduce().copyToNDJavaArray();
}
return variable.read().copyToNDJavaArray();
} catch (InvalidRangeException e) {
throw new ServiceException(e);
} catch (IOException e) {
throw new ServiceException(e);
}
}
void CheckInt16(Variable v) throws IOException {
// int16
// assert(null != (v = dodsfile.findVariable("types.integers.i16")));
// assert v.getName().equals("types.integers.i16");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.SHORT;
CheckInt16Value(v.read());
}
private void testReadData(Variable v) throws IOException {
if (show) System.out.printf(" read %s%n", v.getNameAndDimensions());
assert (null != v);
assert (null != v.getDimension(0));
Array a = v.read();
assert (null != a);
assert (v.getSize() == a.getSize());
}
void CheckS(Variable v) throws IOException {
// string
// assert(null != (v = dodsfile.findVariable("types.strings.s")));
// assert v.getName().equals("types.strings.s");
assert v.getRank() == 0;
assert v.getDataType() == DataType.STRING : v.getDataType();
CheckSValue(v.read());
}
public synchronized void readBandData(
Band destBand,
int sourceOffsetX,
int sourceOffsetY,
int sourceWidth,
int sourceHeight,
int sourceStepX,
int sourceStepY,
ProductData destBuffer,
ProgressMonitor pm)
throws IOException, InvalidRangeException {
if (mustFlipY) {
sourceOffsetY = destBand.getSceneRasterHeight() - (sourceOffsetY + sourceHeight);
}
if (mustFlipX) {
sourceOffsetX = destBand.getSceneRasterWidth() - (sourceOffsetX + sourceWidth);
}
sourceOffsetY += leadLineSkip;
start[0] = sourceOffsetY;
start[1] = sourceOffsetX;
stride[0] = sourceStepY;
stride[1] = sourceStepX;
count[0] = sourceHeight;
count[1] = sourceWidth;
Object buffer = destBuffer.getElems();
Variable variable = variableMap.get(destBand);
pm.beginTask("Reading band '" + variable.getShortName() + "'...", sourceHeight);
try {
Section section = new Section(start, count, stride);
Array array;
int[] newshape = {sourceHeight, sourceWidth};
array = variable.read(section);
if (array.getRank() == 3) {
array = array.reshapeNoCopy(newshape);
}
Object storage;
if (mustFlipX && !mustFlipY) {
storage = array.flip(1).copyTo1DJavaArray();
} else if (!mustFlipX && mustFlipY) {
storage = array.flip(0).copyTo1DJavaArray();
} else if (mustFlipX && mustFlipY) {
storage = array.flip(0).flip(1).copyTo1DJavaArray();
} else {
storage = array.copyTo1DJavaArray();
}
arraycopy(storage, 0, buffer, 0, destBuffer.getNumElems());
} finally {
pm.done();
}
}
void CheckLong32(Variable v) throws IOException {
// uint32
// assert(null != (v = dodsfile.findVariable("types.integers.ui32")));
// assert v.getName().equals("types.integers.ui32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.LONG : v.getDataType();
CheckLongValue(v.read());
}
void CheckF(Variable v) throws IOException {
// float
// assert(null != (v = dodsfile.findVariable("types.floats.f32")));
// assert v.getName().equals("types.floats.f32");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.FLOAT : v.getDataType();
CheckFValue(v.read());
}
/**
* Read data from a top level Variable of FLOAT data type and return a memory resident Array.
*
* @param index LayoutRegular object
* @param v2 Variable has FLOAT data type.
* @return Array of data which will be read from Variable through this call.
*/
public Array readFloatData(LayoutRegular index, Variable v2) throws IOException {
float[] var = (float[]) (v2.read().get1DJavaArray(v2.getDataType().getPrimitiveClassType()));
float[] data = new float[(int) index.getTotalNelems()];
while (index.hasNext()) {
Layout.Chunk chunk = index.next();
System.arraycopy(
var, (int) chunk.getSrcPos() / 4, data, (int) chunk.getDestElem(), chunk.getNelems());
}
return Array.factory(data);
}
void CheckD(Variable v) throws IOException {
// double
// assert(null != (v = dodsfile.findVariable("types.floats.f64")));
// assert v.getName().equals("types.floats.f64");
assert v.getRank() == 0;
assert v.getSize() == 1;
assert v.getDataType() == DataType.DOUBLE : v.getDataType();
CheckDValue(v.read());
}
public void testStride(String stride) throws IOException, InvalidRangeException {
Variable time = ncfile.findVariable("time");
ArrayInt all = (ArrayInt) time.read();
ArrayInt correct = (ArrayInt) all.section(new Section(stride).getRanges());
System.out.printf("correct(%s) %s", stride, NCdumpW.toString(correct));
ArrayInt data = (ArrayInt) time.read(stride);
System.out.printf("data(%s) %s%n", stride, NCdumpW.toString(data));
Index ci = correct.getIndex();
Index di = data.getIndex();
for (int i = 0; i < data.getSize(); i++)
assert (data.getInt(di.set(i)) == correct.getInt(ci.set(i)))
: stride
+ " index "
+ i
+ " = "
+ data.getInt(di.set(i))
+ " != "
+ correct.getInt(ci.set(i));
}
public void testStride() throws IOException, InvalidRangeException {
System.out.println("ncfile opened = " + location + "\n" + ncfile);
Variable time = ncfile.findVariable("time");
ArrayInt all = (ArrayInt) time.read();
for (int i = 0; i < all.getSize(); i++) assert (all.getInt(i) == i + 1);
testStride("0:13:3");
for (int i = 1; i < 12; i++) testStride("0:13:" + i);
}
// do not call directly
@Override
public Array reallyRead(Variable client, Section section, CancelTask cancelTask)
throws IOException, InvalidRangeException {
// see if its really a full read
if ((null == section) || section.computeSize() == getSize())
return reallyRead(client, cancelTask);
if (orgVar == null) return getMissingDataArray(section.getShape());
return orgVar.read(section);
}
public ProductData readData(Variable variable) throws ProductIOException {
final int dataType = getProductDataType(variable);
Array array;
try {
array = variable.read();
} catch (IOException e) {
throw new ProductIOException(e.getMessage());
}
return ProductData.createInstance(dataType, array.getStorage());
}
public void testAggCoordVarSubsetDefeatLocalCache(NetcdfFile ncfile)
throws InvalidRangeException, IOException {
Variable time = ncfile.findVariable("time");
assert null != time;
assert time.getShortName().equals("time");
assert time.getRank() == 1;
assert time.getSize() == 3;
assert time.getShape()[0] == 3;
assert time.getDataType() == DataType.DOUBLE;
assert time.getDimension(0) == ncfile.findDimension("time");
time.setCachedData(null, false);
Array data = time.read("1:2");
assert data.getRank() == 1;
assert data.getSize() == 2;
assert data.getShape()[0] == 2;
assert data.getElementType() == double.class;
int count = 0;
IndexIterator dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count + 1]);
count++;
}
time.setCachedData(null, false);
data = time.read("0:2:2");
assert data.getRank() == 1;
assert data.getSize() == 2;
assert data.getShape()[0] == 2;
assert data.getElementType() == double.class;
count = 0;
dataI = data.getIndexIterator();
while (dataI.hasNext()) {
assert Misc.closeEnough(dataI.getDoubleNext(), result[count * 2]);
count++;
}
}
private RowInfo[] createRowInfos() throws IOException {
final ISINGrid grid = this.grid;
final RowInfo[] binLines = new RowInfo[sceneHeight];
final Variable idxVariable =
ncFile.getRootGroup().findGroup("Level-3_Binned_Data").findVariable("BinList");
final Structure idxStructure = (Structure) idxVariable;
final Variable idx = idxStructure.findVariable("bin_num");
final int[] idxValues;
synchronized (ncFile) {
idxValues = (int[]) idx.read().getStorage();
}
if (bins == null) {
bins = idxValues; // (int[]) idxVariable.read().copyTo1DJavaArray();
}
final Point gridPoint = new Point();
int lastBinIndex = -1;
int lastRowIndex = -1;
int lineOffset = 0;
int lineLength = 0;
for (int i = 0; i < idxValues.length; i++) {
final int binIndex = idxValues[i];
if (binIndex < lastBinIndex) {
throw new IOException(
"Unrecognized level-3 format. Bins numbers expected to appear in ascending order.");
}
lastBinIndex = binIndex;
grid.getGridPoint(binIndex, gridPoint);
final int rowIndex = gridPoint.y;
if (rowIndex != lastRowIndex) {
if (lineLength > 0) {
binLines[lastRowIndex] = new RowInfo(lineOffset, lineLength);
}
lineOffset = i;
lineLength = 0;
}
lineLength++;
lastRowIndex = rowIndex;
}
if (lineLength > 0) {
binLines[lastRowIndex] = new RowInfo(lineOffset, lineLength);
}
return binLines;
}