private int getElementSize(int offset, int ordinal) {
if (ordinal == numElements - 1) {
return sizeInBytes - offset;
} else {
return Math.abs(getElementOffset(ordinal + 1)) - offset;
}
}
/**
* Write this UnsafeRow's underlying bytes to the given OutputStream.
*
* @param out the stream to write to.
* @param writeBuffer a byte array for buffering chunks of off-heap data while writing to the
* output stream. If this row is backed by an on-heap byte array, then this buffer will not be
* used and may be null.
*/
public void writeToStream(OutputStream out, byte[] writeBuffer) throws IOException {
if (baseObject instanceof byte[]) {
int offsetInByteArray = (int) (Platform.BYTE_ARRAY_OFFSET - baseOffset);
out.write((byte[]) baseObject, offsetInByteArray, sizeInBytes);
} else {
int dataRemaining = sizeInBytes;
long rowReadPosition = baseOffset;
while (dataRemaining > 0) {
int toTransfer = Math.min(writeBuffer.length, dataRemaining);
Platform.copyMemory(
baseObject, rowReadPosition, writeBuffer, Platform.BYTE_ARRAY_OFFSET, toTransfer);
out.write(writeBuffer, 0, toTransfer);
rowReadPosition += toTransfer;
dataRemaining -= toTransfer;
}
}
}
@Test
public void testCovariance() {
DataFrame df = context.table("testData2");
Double result = df.stat().cov("a", "b");
Assert.assertTrue(Math.abs(result) < 1.0e-6);
}
@Test
public void testCorrelation() {
DataFrame df = context.table("testData2");
Double pearsonCorr = df.stat().corr("a", "b", "pearson");
Assert.assertTrue(Math.abs(pearsonCorr) < 1.0e-6);
}