/**
* Read data from encoded values and run len into regular data array
*
* @param ddata is encoded data values
* @return the data array of row data
*/
public byte[] readOneRowData(byte[] ddata, int rLen, int xt)
throws IOException, InvalidRangeException {
int run;
byte[] bdata = new byte[xt];
int nbin = 0;
int total = 0;
for (run = 0; run < rLen; run++) {
int drun = DataType.unsignedByteToShort(ddata[run]) >> 4;
byte dcode1 = (byte) (DataType.unsignedByteToShort(ddata[run]) & 0Xf);
for (int i = 0; i < drun; i++) {
bdata[nbin++] = dcode1;
total++;
}
}
if (total < xt) {
for (run = total; run < xt; run++) {
bdata[run] = 0;
}
}
return bdata;
}
long computeFieldSize(BaseType bt, boolean isAscii) throws Exception {
long fieldsize = 0;
// Figure out what this field is (e.g. primitive or not)
// Somewhat convoluted.
if (bt instanceof DConstructor) {
// simple struct, seq, or grid => recurse
fieldsize = computeSize((DConstructor) bt, isAscii);
} else if (bt instanceof DArray) {
SDArray da = (SDArray) bt;
// Separate structure arrays from primitive arrays
if (da.getContainerVar() instanceof DPrimitive) {
fieldsize = computeArraySize(da);
} else if (da.getContainerVar() instanceof DStructure) {
fieldsize = computeSize((DStructure) da.getContainerVar(), isAscii); // recurse
} else { // Some kind of problem
throw new NoSuchTypeException("Computesize: unexpected type for " + bt.getLongName());
}
} else if (bt instanceof DPrimitive) {
DPrimitive dp = (DPrimitive) bt;
if (dp instanceof DString) {
String v = ((DString) dp).getValue();
fieldsize = (v == null ? 0 : v.length());
} else {
DataType dtype = DODSNetcdfFile.convertToNCType(bt);
fieldsize = dtype.getSize();
}
} else { // Some kind of problem
throw new NoSuchTypeException("Computesize: unknown type for " + bt.getLongName());
}
return fieldsize;
}
/**
* Create a scalar numeric-valued Attribute.
*
* @param name name of Attribute
* @param val value of Attribute
*/
public Attribute(String name, Number val) {
this.name = name;
int[] shape = new int[1];
shape[0] = 1;
DataType dt = DataType.getType(val.getClass());
Array vala = Array.factory(dt.getPrimitiveClassType(), shape);
Index ima = vala.getIndex();
vala.setDouble(ima.set0(0), val.doubleValue());
setValues(vala);
}
/**
* Get a 1DArray for the type and length
*
* @param type DataType
* @param len length
* @return the array
*/
private Array get1DArray(DataType type, int len) {
Array varArray = null;
if (type.equals(DataType.FLOAT)) {
varArray = new ArrayFloat.D1(len);
} else if (type.equals(DataType.DOUBLE)) {
varArray = new ArrayDouble.D1(len);
} else if (type.equals(DataType.INT)) {
varArray = new ArrayInt.D1(len);
}
return varArray;
}
protected Array convertEnums(Array values) {
DataType dt = DataType.getType(values.getElementType());
if (!dt.isNumeric()) System.out.println("HEY");
Array result = Array.factory(DataType.STRING, values.getShape());
IndexIterator ii = result.getIndexIterator();
values.resetLocalIterator();
while (values.hasNext()) {
String sval = lookupEnumString(values.nextInt());
ii.setObjectNext(sval);
}
return result;
}
long computeArraySize(SDArray da) throws Exception {
assert (da.getContainerVar() instanceof DPrimitive);
BaseType base = da.getPrimitiveVector().getTemplate();
DataType dtype = DODSNetcdfFile.convertToNCType(base);
int elemSize = dtype.getSize();
int n = da.numDimensions();
List<Range> ranges = new ArrayList<Range>(n);
long size = 0;
for (int i = 0; i < n; i++) {
ranges.add(new Range(da.getStart(i), da.getStop(i), da.getStride(i)));
Section s = new Section(ranges);
size += s.computeSize() * elemSize;
}
return size;
}
int getInt(byte[] b, int offset, int num) {
int base = 1;
int i;
int word = 0;
int bv[] = new int[num];
for (i = 0; i < num; i++) {
bv[i] = DataType.unsignedByteToShort(b[offset + i]);
}
if (bv[0] > 127) {
bv[0] -= 128;
base = -1;
}
/*
* Calculate the integer value of the byte sequence
*/
for (i = num - 1; i >= 0; i--) {
word += base * bv[i];
base *= 256;
}
return word;
}
/**
* set the values from an Array
*
* @param arr value of Attribute
*/
protected void setValues(Array arr) {
if (arr == null) {
dataType = DataType.STRING;
return;
}
if (DataType.getType(arr.getElementType()) == null)
throw new IllegalArgumentException("Cant set Attribute with type " + arr.getElementType());
if (arr.getElementType() == char.class) { // turn CHAR into STRING
ArrayChar carr = (ArrayChar) arr;
if (carr.getRank() == 1) { // common case
svalue = carr.getString();
this.nelems = 1;
this.dataType = DataType.STRING;
return;
}
// otherwise its an array of Strings
arr = carr.make1DStringArray();
}
// this should be a utility somewhere
if (arr.getElementType() == ByteBuffer.class) { // turn OPAQUE into BYTE
int totalLen = 0;
arr.resetLocalIterator();
while (arr.hasNext()) {
ByteBuffer bb = (ByteBuffer) arr.next();
totalLen += bb.limit();
}
byte[] ba = new byte[totalLen];
int pos = 0;
arr.resetLocalIterator();
while (arr.hasNext()) {
ByteBuffer bb = (ByteBuffer) arr.next();
System.arraycopy(bb.array(), 0, ba, pos, bb.limit());
pos += bb.limit();
}
arr = Array.factory(DataType.BYTE, new int[] {totalLen}, ba);
}
if (arr.getRank() > 1) arr = arr.reshape(new int[] {(int) arr.getSize()}); // make sure 1D
this.values = arr;
this.nelems = (int) arr.getSize();
this.dataType = DataType.getType(arr.getElementType());
hashCode = 0;
}
private void convertScaleOffsetUnsignedInt(IndexIterator iterIn, IndexIterator iterOut) {
boolean checkMissing = useNaNs && hasMissing();
while (iterIn.hasNext()) {
int valb = iterIn.getIntNext();
double val = scale * DataType.unsignedIntToLong(valb) + offset;
iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
}
}
public double convertScaleOffsetMissing(int vali) {
if (!hasScaleOffset) return useNaNs && isMissing((double) vali) ? Double.NaN : (double) vali;
double convertedValue;
if (isUnsigned) convertedValue = scale * DataType.unsignedIntToLong(vali) + offset;
else convertedValue = scale * vali + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
public double convertScaleOffsetMissing(byte valb) {
if (!hasScaleOffset) return useNaNs && isMissing((double) valb) ? Double.NaN : (double) valb;
double convertedValue;
if (isUnsigned) convertedValue = scale * DataType.unsignedByteToShort(valb) + offset;
else convertedValue = scale * valb + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
public double convertScaleOffsetMissing(short vals) {
if (!hasScaleOffset) return useNaNs && isMissing((double) vals) ? Double.NaN : (double) vals;
double convertedValue;
if (isUnsigned) convertedValue = scale * DataType.unsignedShortToInt(vals) + offset;
else convertedValue = scale * vals + offset;
return useNaNs && isMissing(convertedValue) ? Double.NaN : convertedValue;
}
/** Instances which have same content are equal. */
@Override
public boolean equals(Object o) {
if (this == o) return true;
if ((o == null) || !(o instanceof Attribute)) return false;
final Attribute att = (Attribute) o;
if (!name.equals(att.name)) return false;
if (nelems != att.nelems) return false;
if (!dataType.equals(att.dataType)) return false;
if (svalue != null) return svalue.equals(att.getStringValue());
if (values != null) {
for (int i = 0; i < getLength(); i++) {
int r1 = isString() ? getStringValue(i).hashCode() : getNumericValue(i).hashCode();
int r2 =
att.isString() ? att.getStringValue(i).hashCode() : att.getNumericValue(i).hashCode();
if (r1 != r2) return false;
}
}
return true;
}
/**
* Convert Data with scale and offset. Also translate missing data to NaNs if useNaNs = true.
*
* @param in data to convert
* @return converted data.
*/
private Array convertScaleOffset(Array in) {
if (!hasScaleOffset) return in;
if (debugRead) System.out.println("convertScaleOffset ");
Array out = Array.factory(convertedDataType.getPrimitiveClassType(), in.getShape());
IndexIterator iterIn = in.getIndexIterator();
IndexIterator iterOut = out.getIndexIterator();
if (isUnsigned && in.getElementType() == byte.class)
convertScaleOffsetUnsignedByte(iterIn, iterOut);
else if (isUnsigned && in.getElementType() == short.class)
convertScaleOffsetUnsignedShort(iterIn, iterOut);
else if (isUnsigned && in.getElementType() == int.class)
convertScaleOffsetUnsignedInt(iterIn, iterOut);
else {
boolean checkMissing = useNaNs && hasMissing();
while (iterIn.hasNext()) {
double val = scale * iterIn.getDoubleNext() + offset;
iterOut.setDoubleNext(checkMissing && isMissing_(val) ? Double.NaN : val);
}
}
return out;
}
private boolean compareData(String name, Array data1, Array data2, double tol, boolean justOne) {
boolean ok = true;
if (data1.getSize() != data2.getSize()) {
f.format(" DIFF %s: size %d !== %d%n", name, data1.getSize(), data2.getSize());
ok = false;
}
if (data1.getElementType() != data2.getElementType()) {
f.format(
" DIFF %s: element type %s !== %s%n",
name, data1.getElementType(), data2.getElementType());
ok = false;
}
if (!ok) return false;
DataType dt = DataType.getType(data1.getElementType());
IndexIterator iter1 = data1.getIndexIterator();
IndexIterator iter2 = data2.getIndexIterator();
if (dt == DataType.DOUBLE) {
while (iter1.hasNext() && iter2.hasNext()) {
double v1 = iter1.getDoubleNext();
double v2 = iter2.getDoubleNext();
if (!Double.isNaN(v1) || !Double.isNaN(v2))
if (!closeEnough(v1, v2, tol)) {
f.format(
" DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n",
name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
ok = false;
if (justOne) break;
}
}
} else if (dt == DataType.FLOAT) {
while (iter1.hasNext() && iter2.hasNext()) {
float v1 = iter1.getFloatNext();
float v2 = iter2.getFloatNext();
if (!Float.isNaN(v1) || !Float.isNaN(v2))
if (!closeEnough(v1, v2, (float) tol)) {
f.format(
" DIFF %s: %f != %f count=%s diff = %f pdiff = %f %n",
name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
ok = false;
if (justOne) break;
}
}
} else if (dt == DataType.INT) {
while (iter1.hasNext() && iter2.hasNext()) {
int v1 = iter1.getIntNext();
int v2 = iter2.getIntNext();
if (v1 != v2) {
f.format(
" DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n",
name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
ok = false;
if (justOne) break;
}
}
} else if (dt == DataType.SHORT) {
while (iter1.hasNext() && iter2.hasNext()) {
short v1 = iter1.getShortNext();
short v2 = iter2.getShortNext();
if (v1 != v2) {
f.format(
" DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n",
name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
ok = false;
if (justOne) break;
}
}
} else if (dt == DataType.BYTE) {
while (iter1.hasNext() && iter2.hasNext()) {
byte v1 = iter1.getByteNext();
byte v2 = iter2.getByteNext();
if (v1 != v2) {
f.format(
" DIFF %s: %d != %d count=%s diff = %f pdiff = %f %n",
name, v1, v2, iter1, diff(v1, v2), pdiff(v1, v2));
ok = false;
if (justOne) break;
}
}
} else if (dt == DataType.STRUCTURE) {
while (iter1.hasNext() && iter2.hasNext()) {
compareStructureData(
(StructureData) iter1.next(), (StructureData) iter2.next(), tol, justOne);
}
}
return ok;
}
// all the work is here, so can be called recursively
public Object readOneScanData(ByteBuffer bos, NOWRadheader.Vinfo vinfo, String vName)
throws IOException, InvalidRangeException {
int doff = (int) vinfo.hoff;
int npixel = vinfo.yt * vinfo.xt;
byte[] rdata = null;
byte[] ldata = new byte[vinfo.xt];
byte[] pdata = new byte[npixel];
byte[] b2 = new byte[2];
bos.position(doff);
// begining of image data
if ((DataType.unsignedByteToShort(bos.get()) != 0xF0) || (bos.get() != 0x0C)) {
return null;
}
int ecode;
int color;
int datapos;
int offset = 0;
int roffset = 0;
boolean newline = true;
int linenum = 0;
while (true) {
// line number
if (newline) {
bos.get(b2);
linenum = (DataType.unsignedByteToShort(b2[1]) << 8) + DataType.unsignedByteToShort(b2[0]);
// System.out.println("Line Number = " + linenum);
}
// int linenum = bytesToInt(b2[0], b2[1], true);
// System.out.println("Line Number = " + linenum);
// if(linenum == 1225)
// System.out.println(" HHHHH");
short b = DataType.unsignedByteToShort(bos.get());
color = b & 0xF;
ecode = b >> 4;
datapos = bos.position();
int datarun;
if (ecode == 0xF) {
byte bb1 = bos.get(datapos - 2);
byte bb2 = bos.get(datapos);
if ((color == 0x0) && (bb1 == 0x00) && (bb2 == 0x00)) {
datapos += 1;
}
bos.position(datapos);
datarun = 0;
} else if (ecode == 0xE) {
byte b0 = bos.get(datapos);
datarun = DataType.unsignedByteToShort(b0) + 1;
datapos += 1;
bos.position(datapos);
} else if (ecode == 0xD) {
b2[0] = bos.get(datapos);
b2[1] = bos.get(datapos + 1);
datarun =
(DataType.unsignedByteToShort(b2[1]) << 8) + DataType.unsignedByteToShort(b2[0]) + 1;
datapos += 2;
bos.position(datapos);
} else {
datarun = ecode + 1;
}
// move the unpacked data in the data line
rdata = new byte[datarun];
for (int i = 0; i < datarun; i++) {
rdata[i] = (byte) color;
}
System.arraycopy(rdata, 0, ldata, roffset, datarun);
roffset = roffset + datarun;
// System.out.println("run ecode = " + ecode + " and data run " + datarun + " and totalrun " +
// roffset);
// check to see if the beginning of the next line or at the end of the file
short c0 = DataType.unsignedByteToShort(bos.get());
if (c0 == 0x00) {
short c1 = DataType.unsignedByteToShort(bos.get());
short c2 = DataType.unsignedByteToShort(bos.get());
// System.out.println("c1 and c2 " + c1 + " " + c2);
if ((c0 == 0x00) && (c1 == 0xF0) && (c2 == 0x0C)) {
// beginning of next line
// System.out.println("linenum " + linenum + " and this line total " + roffset);
// if (roffset != 3661) {
// System.out.println("ERROR missing data, this line total only " + roffset);
// }
System.arraycopy(ldata, 0, pdata, offset, roffset);
offset = offset + vinfo.xt;
roffset = 0;
newline = true;
ldata = new byte[vinfo.xt];
} else if ((c1 == 0xF0) && (c2 == 0x02)) {
// end of the file
break;
} else {
datapos = bos.position() - 3;
bos.position(datapos);
newline = false;
}
} else {
newline = false;
datapos = bos.position();
bos.position(datapos - 1);
}
}
return pdata;
}
/**
* Make a station variable
*
* @param varname variable name
* @param firstDim station dimension
* @return corresponding variable
*/
protected Variable makeStationVariable(String varname, Dimension firstDim) {
String longName = varname;
String unit = null;
DataType type = DataType.CHAR;
List<Dimension> dims = new ArrayList<Dimension>();
List<Attribute> attrs = new ArrayList<Attribute>();
if (firstDim != null) {
dims.add(firstDim);
}
if (varname.equals(GempakStation.STID)) {
longName = "Station identifier";
dims.add(DIM_LEN8);
} else if (varname.equals(GempakStation.STNM)) {
longName = "WMO station id";
type = DataType.INT;
} else if (varname.equals(GempakStation.SLAT)) {
longName = "latitude";
unit = "degrees_north";
type = DataType.FLOAT;
attrs.add(new Attribute("standard_name", "latitude"));
} else if (varname.equals(GempakStation.SLON)) {
longName = "longitude";
unit = "degrees_east";
type = DataType.FLOAT;
attrs.add(new Attribute("standard_name", "longitude"));
} else if (varname.equals(GempakStation.SELV)) {
longName = "altitude";
unit = "meter";
type = DataType.FLOAT;
attrs.add(new Attribute(CF.POSITIVE, CF.POSITIVE_UP));
attrs.add(new Attribute("standard_name", "station_altitude"));
} else if (varname.equals(GempakStation.STAT)) {
longName = "state or province";
dims.add(DIM_LEN2);
} else if (varname.equals(GempakStation.COUN)) {
longName = "country code";
dims.add(DIM_LEN2);
} else if (varname.equals(GempakStation.STD2)) {
longName = "Extended station id";
dims.add(DIM_LEN4);
} else if (varname.equals(GempakStation.SPRI)) {
longName = "Station priority";
type = DataType.INT;
} else if (varname.equals(GempakStation.SWFO)) {
longName = "WFO code";
dims.add(DIM_LEN4);
} else if (varname.equals(GempakStation.WFO2)) {
longName = "Second WFO code";
dims.add(DIM_LEN4);
}
Variable v = new Variable(ncfile, null, null, varname);
v.setDataType(type);
v.addAttribute(new Attribute("long_name", longName));
if (unit != null) {
v.addAttribute(new Attribute(CDM.UNITS, unit));
}
if (type.equals(DataType.FLOAT)) {
v.addAttribute(new Attribute("missing_value", RMISS));
} else if (type.equals(DataType.INT)) {
v.addAttribute(new Attribute("missing_value", IMISS));
}
if (!attrs.isEmpty()) {
for (Attribute attr : attrs) {
v.addAttribute(attr);
}
}
if (!dims.isEmpty()) {
v.setDimensions(dims);
} else {
v.setDimensions((String) null);
}
return v;
}
private int populateFile2(
NetcdfFileWriteable ncFile,
String varName,
DataType dataType,
int[] dimensions,
int valueCounter,
Random generator)
throws IOException, InvalidRangeException {
long singleWriteSize = maxMemory / dataType.getSize();
int highestNonWriteDim = determineHighestNonWriteDimension(dimensions, singleWriteSize);
int[] singleStep = createWriteStep(dimensions, singleWriteSize, highestNonWriteDim);
int numStepWrites = determineNumberOfStepWrites(singleStep, singleWriteSize);
System.out.println(
"SingleWriteSize: "
+ singleWriteSize
+ " datatype size: "
+ dataType.getSize()
+ " singleStep: "
+ arrayToString(singleStep)
+ " numberStepsPerWrite: "
+ numStepWrites);
int[] allOnes = new int[dimensions.length];
int[] allZeros = new int[dimensions.length];
for (int i = 0; i < allOnes.length; i++) {
allOnes[i] = 1;
allZeros[i] = 0;
}
Index origin = new Index(dimensions, allOnes);
long totalSize = origin.getSize();
long writtenSoFar = 0;
boolean done = false;
while (!done) {
for (int i = 0; i < numStepWrites; i++) {
// this is a crack at an optimization
if ((highestNonWriteDim >= 0)
&& ((origin.getCurrentCounter()[highestNonWriteDim] + numStepWrites)
<= dimensions[highestNonWriteDim])) {
singleStep[highestNonWriteDim] = numStepWrites;
System.out.println("JUST OPTIMIZED. New write step: " + arrayToString(singleStep));
// keep 'i' right
i += numStepWrites - 1;
} else {
singleStep[highestNonWriteDim] = 1;
}
Array array = new ArrayInt(singleStep);
IndexIterator iter = array.getIndexIterator();
while (iter.hasNext()) {
iter.getIntNext();
// uncomment the following line for a random distribution
// iter.setIntCurrent((int) (Math.abs(generator.nextGaussian()) * 40) );
// uncomment this line for an incrementing value
iter.setIntCurrent(valueCounter);
valueCounter++;
// book keeping
writtenSoFar++;
// origin.incr();
}
System.out.println(
"Writing to file: "
+ ncFile.getLocation()
+ ". var_name: "
+ varName
+ " origin: "
+ arrayToString(origin.getCurrentCounter())
+ " writeSize: "
+ array.getSize()
+ " write shape: "
+ arrayToString(singleStep));
// write to the actual file
ncFile.write(varName, origin.getCurrentCounter(), array);
// update origin accordingly
for (int j = 0; j < Index.computeSize(singleStep); j++) {
// writtenSoFar++;
origin.incr();
}
System.out.println(
"\tcurrentIndex: "
+ arrayToString(origin.getCurrentCounter())
+ " currentElement: "
+ origin.currentElement()
+ " totalsize: "
+ (totalSize)
+ " writtenSoFar: "
+ writtenSoFar);
if (writtenSoFar >= totalSize) {
done = true;
break;
}
}
}
return valueCounter;
}