public static <T> void toBytes(
T serializable, ArrayBackedValueStorage buffer, ISerializerDeserializer<T> serde)
throws HyracksDataException {
buffer.reset();
DataOutput out = buffer.getDataOutput();
serde.serialize(serializable, out);
}
/*
* The following two static methods are expensive. right now, they are used by RSSFeeds and Twitter feed
* TODO: Get rid of them
*/
public static void writeRecord(
AMutableRecord record, DataOutput dataOutput, IARecordBuilder recordBuilder)
throws HyracksDataException {
ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
int numFields = record.getType().getFieldNames().length;
for (int pos = 0; pos < numFields; pos++) {
fieldValue.reset();
IAObject obj = record.getValueByPos(pos);
IDataParser.writeObject(obj, fieldValue.getDataOutput());
recordBuilder.addField(pos, fieldValue);
}
recordBuilder.write(dataOutput, true);
}
private void parseUnorderedList(AUnorderedListType uoltype, DataOutput out)
throws IOException, AsterixException, AdmLexerException {
ArrayBackedValueStorage itemBuffer = getTempBuffer();
UnorderedListBuilder unorderedListBuilder = (UnorderedListBuilder) getUnorderedListBuilder();
IAType itemType = null;
if (uoltype != null) {
itemType = uoltype.getItemType();
}
unorderedListBuilder.reset(uoltype);
int token;
boolean inList = true;
boolean expectingListItem = false;
boolean first = true;
do {
token = admLexer.next();
if (token == AdmLexer.TOKEN_END_RECORD) {
if (admLexer.next() == AdmLexer.TOKEN_END_RECORD) {
if (expectingListItem) {
throw new ParseException("Found END_COLLECTION while expecting a list item.");
} else {
inList = false;
}
} else {
throw new ParseException("Found END_RECORD while expecting a list item.");
}
} else if (token == AdmLexer.TOKEN_COMMA) {
if (first) {
throw new ParseException("Found COMMA before any list item.");
}
if (expectingListItem) {
throw new ParseException("Found COMMA while expecting a list item.");
}
expectingListItem = true;
} else {
expectingListItem = false;
itemBuffer.reset();
admFromLexerStream(token, itemType, itemBuffer.getDataOutput(), false);
unorderedListBuilder.addItem(itemBuffer);
}
first = false;
} while (inList);
unorderedListBuilder.write(out, true);
}
private void parseConstructor(ATypeTag typeTag, IAType objectType, DataOutput out)
throws AsterixException, AdmLexerException, IOException {
final ATypeTag targetTypeTag = getTargetTypeTag(typeTag, objectType);
if (targetTypeTag != null) {
DataOutput dataOutput = out;
if (targetTypeTag != typeTag) {
castBuffer.reset();
dataOutput = castBuffer.getDataOutput();
}
int token = admLexer.next();
if (token == AdmLexer.TOKEN_CONSTRUCTOR_OPEN) {
token = admLexer.next();
if (token == AdmLexer.TOKEN_STRING_LITERAL) {
final String unquoted =
admLexer.getLastTokenImage().substring(1, admLexer.getLastTokenImage().length() - 1);
if (!parseValue(unquoted, typeTag, dataOutput)) {
throw new ParseException(
"Missing deserializer method for constructor: "
+ AdmLexer.tokenKindToString(token)
+ ".");
}
token = admLexer.next();
if (token == AdmLexer.TOKEN_CONSTRUCTOR_CLOSE) {
if (targetTypeTag != typeTag) {
ITypeConvertComputer promoteComputer =
ATypeHierarchy.getTypePromoteComputer(typeTag, targetTypeTag);
// the availability if the promote computer should be consistent with the availability
// of a target type
assert promoteComputer != null;
// do the promotion; note that the type tag field should be skipped
promoteComputer.convertType(
castBuffer.getByteArray(),
castBuffer.getStartOffset() + 1,
castBuffer.getLength() - 1,
out);
}
return;
}
}
}
}
throw new ParseException(
mismatchErrorMessage + objectType.getTypeName() + ". Got " + typeTag + " instead.");
}
protected void processPartialResults(IFrameTupleReference tuple) throws AlgebricksException {
if (skipStep()) {
return;
}
inputVal.reset();
eval.evaluate(tuple);
byte[] serBytes = inputVal.getByteArray();
ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(serBytes[0]);
switch (typeTag) {
case NULL:
{
processNull();
break;
}
case SYSTEM_NULL:
{
// Ignore and return.
break;
}
case RECORD:
{
// Expected.
aggType = ATypeTag.DOUBLE;
int nullBitmapSize = 0;
int offset1 =
ARecordSerializerDeserializer.getFieldOffsetById(
serBytes, SUM_FIELD_ID, nullBitmapSize, false);
sum += ADoubleSerializerDeserializer.getDouble(serBytes, offset1);
int offset2 =
ARecordSerializerDeserializer.getFieldOffsetById(
serBytes, COUNT_FIELD_ID, nullBitmapSize, false);
count += AInt64SerializerDeserializer.getLong(serBytes, offset2);
break;
}
default:
{
throw new AlgebricksException(
"Global-Avg is not defined for values of type "
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(serBytes[0]));
}
}
}
private void parseAndCastNumeric(ATypeTag typeTag, IAType objectType, DataOutput out)
throws AsterixException, IOException {
final ATypeTag targetTypeTag = getTargetTypeTag(typeTag, objectType);
DataOutput dataOutput = out;
if (targetTypeTag != typeTag) {
castBuffer.reset();
dataOutput = castBuffer.getDataOutput();
}
if (targetTypeTag == null || !parseValue(admLexer.getLastTokenImage(), typeTag, dataOutput)) {
throw new ParseException(
mismatchErrorMessage + objectType.getTypeName() + mismatchErrorMessage2 + typeTag);
}
// If two type tags are not the same, either we try to promote or demote source type to the
// target type
if (targetTypeTag != typeTag) {
if (ATypeHierarchy.canPromote(typeTag, targetTypeTag)) {
// can promote typeTag to targetTypeTag
ITypeConvertComputer promoteComputer =
ATypeHierarchy.getTypePromoteComputer(typeTag, targetTypeTag);
if (promoteComputer == null) {
throw new AsterixException(
"Can't cast the " + typeTag + " type to the " + targetTypeTag + " type.");
}
// do the promotion; note that the type tag field should be skipped
promoteComputer.convertType(
castBuffer.getByteArray(),
castBuffer.getStartOffset() + 1,
castBuffer.getLength() - 1,
out);
} else if (ATypeHierarchy.canDemote(typeTag, targetTypeTag)) {
// can demote source type to the target type
ITypeConvertComputer demoteComputer =
ATypeHierarchy.getTypeDemoteComputer(typeTag, targetTypeTag);
if (demoteComputer == null) {
throw new AsterixException(
"Can't cast the " + typeTag + " type to the " + targetTypeTag + " type.");
}
// do the demotion; note that the type tag field should be skipped
demoteComputer.convertType(
castBuffer.getByteArray(),
castBuffer.getStartOffset() + 1,
castBuffer.getLength() - 1,
out);
}
}
}
@Override
public void convertString(UTF8StringPointable stringp, DataOutput dOut)
throws SystemException, IOException {
boolean castable = true;
try {
abvsInner.reset();
CastToIntegerOperation castTo = new CastToIntegerOperation();
castTo.convertString(stringp, dOutInner);
} catch (Exception e) {
castable = false;
}
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) (castable ? 1 : 0));
}
static {
BOOLEAN_TRUE_CONSTANT = new byte[2];
BOOLEAN_TRUE_CONSTANT[0] = ValueTag.XS_BOOLEAN_TAG;
BooleanPointable.setBoolean(BOOLEAN_TRUE_CONSTANT, 1, true);
BOOLEAN_FALSE_CONSTANT = new byte[2];
BOOLEAN_FALSE_CONSTANT[0] = ValueTag.XS_BOOLEAN_TAG;
BooleanPointable.setBoolean(BOOLEAN_FALSE_CONSTANT, 1, false);
ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
SequenceBuilder sb = new SequenceBuilder();
sb.reset(abvs);
try {
sb.finish();
} catch (IOException e) {
throw new RuntimeException(e);
}
EMPTY_SEQUENCE = Arrays.copyOf(abvs.getByteArray(), abvs.getLength());
EMPTY_STRING = new byte[3];
EMPTY_STRING[0] = ValueTag.XS_STRING_TAG;
EMPTY_STRING[1] = 0;
EMPTY_STRING[2] = 0;
}
@Override
public void finish() throws AlgebricksException {
try {
switch (aggType) {
case NULL:
{
out.writeByte(ATypeTag.SERIALIZED_NULL_TYPE_TAG);
break;
}
case SYSTEM_NULL:
{
finishSystemNull();
break;
}
default:
{
out.write(outputVal.getByteArray(), outputVal.getStartOffset(), outputVal.getLength());
break;
}
}
} catch (IOException e) {
throw new AlgebricksException(e);
}
}
private void finishInternal(IPointable result) throws AlgebricksException {
try {
if (!isFinalStage) {
if (!partialAggAsInput) {
combinedResult = combiner.finishPartial();
} else {
combinedResult = combiner.finishPartial2();
}
} else {
combinedResult = combiner.finishFinal();
}
// Writes combined result.
combinedResult.write(resultStorage.getDataOutput());
result.set(resultStorage);
} catch (Exception e) {
throw new AlgebricksException(e);
}
}
@SuppressWarnings("unchecked")
public static void writeObject(IAObject obj, DataOutput dataOutput) throws HyracksDataException {
switch (obj.getType().getTypeTag()) {
case RECORD:
{
IARecordBuilder recordBuilder = new RecordBuilder();
recordBuilder.reset((ARecordType) obj.getType());
recordBuilder.init();
writeRecord((AMutableRecord) obj, dataOutput, recordBuilder);
break;
}
case ORDEREDLIST:
{
OrderedListBuilder listBuilder = new OrderedListBuilder();
listBuilder.reset((AOrderedListType) ((AMutableOrderedList) obj).getType());
IACursor cursor = ((AMutableOrderedList) obj).getCursor();
ArrayBackedValueStorage listItemValue = new ArrayBackedValueStorage();
while (cursor.next()) {
listItemValue.reset();
IAObject item = cursor.get();
writeObject(item, listItemValue.getDataOutput());
listBuilder.addItem(listItemValue);
}
listBuilder.write(dataOutput, true);
break;
}
case UNORDEREDLIST:
{
UnorderedListBuilder listBuilder = new UnorderedListBuilder();
listBuilder.reset((AUnorderedListType) ((AMutableUnorderedList) obj).getType());
IACursor cursor = ((AMutableUnorderedList) obj).getCursor();
ArrayBackedValueStorage listItemValue = new ArrayBackedValueStorage();
while (cursor.next()) {
listItemValue.reset();
IAObject item = cursor.get();
writeObject(item, listItemValue.getDataOutput());
listBuilder.addItem(listItemValue);
}
listBuilder.write(dataOutput, true);
break;
}
default:
AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(obj.getType())
.serialize(obj, dataOutput);
break;
}
}
protected void processDataValues(IFrameTupleReference tuple) throws AlgebricksException {
if (skipStep()) {
return;
}
inputVal.reset();
eval.evaluate(tuple);
ATypeTag typeTag =
EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(inputVal.getByteArray()[0]);
if (typeTag == ATypeTag.NULL) {
processNull();
return;
} else if (aggType == ATypeTag.SYSTEM_NULL) {
aggType = typeTag;
} else if (typeTag != ATypeTag.SYSTEM_NULL && !ATypeHierarchy.isCompatible(typeTag, aggType)) {
throw new AlgebricksException(
"Unexpected type "
+ typeTag
+ " in aggregation input stream. Expected type "
+ aggType
+ ".");
} else if (ATypeHierarchy.canPromote(aggType, typeTag)) {
aggType = typeTag;
}
++count;
switch (typeTag) {
case INT8:
{
byte val = AInt8SerializerDeserializer.getByte(inputVal.getByteArray(), 1);
sum += val;
break;
}
case INT16:
{
short val = AInt16SerializerDeserializer.getShort(inputVal.getByteArray(), 1);
sum += val;
break;
}
case INT32:
{
int val = AInt32SerializerDeserializer.getInt(inputVal.getByteArray(), 1);
sum += val;
break;
}
case INT64:
{
long val = AInt64SerializerDeserializer.getLong(inputVal.getByteArray(), 1);
sum += val;
break;
}
case FLOAT:
{
float val = AFloatSerializerDeserializer.getFloat(inputVal.getByteArray(), 1);
sum += val;
break;
}
case DOUBLE:
{
double val = ADoubleSerializerDeserializer.getDouble(inputVal.getByteArray(), 1);
sum += val;
break;
}
default:
{
throw new NotImplementedException("Cannot compute AVG for values of type " + typeTag);
}
}
inputVal.reset();
}
@SuppressWarnings("unchecked")
@Override
public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
array0.reset();
eval0.evaluate(tuple);
array1.reset();
eval1.evaluate(tuple);
array2.reset();
eval2.evaluate(tuple);
try {
if (array0.getByteArray()[0] == SER_NULL_TYPE_TAG
|| array1.getByteArray()[0] == SER_NULL_TYPE_TAG
|| array2.getByteArray()[0] == SER_NULL_TYPE_TAG) {
nullSerde.serialize(ANull.NULL, dout);
return;
} else if (array0.getByteArray()[0] != SER_STRING_TYPE_TAG
|| array1.getByteArray()[0] != SER_STRING_TYPE_TAG
|| array2.getByteArray()[0] != SER_STRING_TYPE_TAG) {
throw new AlgebricksException(
funcID.getName()
+ ": expects input type (STRING/NULL, STRING/NULL, STRING/NULL), but got ("
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(array0.getByteArray()[0])
+ ", "
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(array1.getByteArray()[0])
+ ", "
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(array2.getByteArray()[0])
+ ".");
}
} catch (HyracksDataException e) {
throw new AlgebricksException(e);
}
strPtr1st.set(array0.getByteArray(), array0.getStartOffset() + 1, array0.getLength());
strPtr2nd.set(array1.getByteArray(), array1.getStartOffset() + 1, array1.getLength());
strPtr3rd.set(array2.getByteArray(), array2.getStartOffset() + 1, array2.getLength());
String res = compute(strPtr1st, strPtr2nd, strPtr3rd);
resultBuffer.setValue(res);
try {
strSerde.serialize(resultBuffer, dout);
} catch (HyracksDataException e) {
throw new AlgebricksException(e);
}
}
@Override
public void init() throws AlgebricksException {
combiner.init(msgList);
resultStorage.reset();
}
@Override
public void step(IFrameTupleReference tuple) throws AlgebricksException {
inputVal.reset();
eval.evaluate(tuple);
ATypeTag typeTag =
EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(inputVal.getByteArray()[0]);
if (typeTag == ATypeTag.NULL) {
processNull();
return;
} else if (aggType == ATypeTag.NULL) {
return;
} else if (aggType == ATypeTag.SYSTEM_NULL) {
if (typeTag == ATypeTag.SYSTEM_NULL) {
// Ignore.
return;
}
// First value encountered. Set type, comparator, and initial value.
aggType = typeTag;
// Set comparator.
IBinaryComparatorFactory cmpFactory =
AqlBinaryComparatorFactoryProvider.INSTANCE.getBinaryComparatorFactory(aggType, isMin);
cmp = cmpFactory.createBinaryComparator();
// Initialize min value.
outputVal.assign(inputVal);
} else if (typeTag != ATypeTag.SYSTEM_NULL && !ATypeHierarchy.isCompatible(typeTag, aggType)) {
throw new AlgebricksException(
"Unexpected type "
+ typeTag
+ " in aggregation input stream. Expected type "
+ aggType
+ ".");
} else {
// If a system_null is encountered locally, it would be an error; otherwise if it is seen
// by a global aggregator, it is simple ignored.
if (typeTag == ATypeTag.SYSTEM_NULL) {
processSystemNull();
return;
}
if (ATypeHierarchy.canPromote(aggType, typeTag)) {
tpc = ATypeHierarchy.getTypePromoteComputer(aggType, typeTag);
aggType = typeTag;
cmp =
AqlBinaryComparatorFactoryProvider.INSTANCE
.getBinaryComparatorFactory(aggType, isMin)
.createBinaryComparator();
if (tpc != null) {
tempValForCasting.reset();
try {
tpc.convertType(
outputVal.getByteArray(),
outputVal.getStartOffset() + 1,
outputVal.getLength() - 1,
tempValForCasting.getDataOutput());
} catch (IOException e) {
throw new AlgebricksException(e);
}
outputVal.reset();
outputVal.assign(tempValForCasting);
}
try {
if (cmp.compare(
inputVal.getByteArray(),
inputVal.getStartOffset(),
inputVal.getLength(),
outputVal.getByteArray(),
outputVal.getStartOffset(),
outputVal.getLength())
< 0) {
outputVal.assign(inputVal);
}
} catch (HyracksDataException e) {
throw new AlgebricksException(e);
}
} else {
tpc = ATypeHierarchy.getTypePromoteComputer(typeTag, aggType);
if (tpc != null) {
tempValForCasting.reset();
try {
tpc.convertType(
inputVal.getByteArray(),
inputVal.getStartOffset() + 1,
inputVal.getLength() - 1,
tempValForCasting.getDataOutput());
} catch (IOException e) {
throw new AlgebricksException(e);
}
try {
if (cmp.compare(
tempValForCasting.getByteArray(),
tempValForCasting.getStartOffset(),
tempValForCasting.getLength(),
outputVal.getByteArray(),
outputVal.getStartOffset(),
outputVal.getLength())
< 0) {
outputVal.assign(tempValForCasting);
}
} catch (HyracksDataException e) {
throw new AlgebricksException(e);
}
} else {
try {
if (cmp.compare(
inputVal.getByteArray(),
inputVal.getStartOffset(),
inputVal.getLength(),
outputVal.getByteArray(),
outputVal.getStartOffset(),
outputVal.getLength())
< 0) {
outputVal.assign(inputVal);
}
} catch (HyracksDataException e) {
throw new AlgebricksException(e);
}
}
}
}
}
@Override
public void init() {
aggType = ATypeTag.SYSTEM_NULL;
outputVal.reset();
tempValForCasting.reset();
}
public class CastableAsIntegerOperation extends AbstractCastableAsOperation {
private ArrayBackedValueStorage abvsInner = new ArrayBackedValueStorage();
private DataOutput dOutInner = abvsInner.getDataOutput();
@Override
public void convertBoolean(BooleanPointable boolp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
@Override
public void convertDecimal(XSDecimalPointable decp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
@Override
public void convertDouble(DoublePointable doublep, DataOutput dOut)
throws SystemException, IOException {
boolean castable = true;
try {
abvsInner.reset();
CastToIntegerOperation castTo = new CastToIntegerOperation();
castTo.convertDouble(doublep, dOutInner);
} catch (Exception e) {
castable = false;
}
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) (castable ? 1 : 0));
}
@Override
public void convertFloat(FloatPointable floatp, DataOutput dOut)
throws SystemException, IOException {
boolean castable = true;
try {
abvsInner.reset();
CastToIntegerOperation castTo = new CastToIntegerOperation();
castTo.convertFloat(floatp, dOutInner);
} catch (Exception e) {
castable = false;
}
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) (castable ? 1 : 0));
}
@Override
public void convertInteger(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
@Override
public void convertString(UTF8StringPointable stringp, DataOutput dOut)
throws SystemException, IOException {
boolean castable = true;
try {
abvsInner.reset();
CastToIntegerOperation castTo = new CastToIntegerOperation();
castTo.convertString(stringp, dOutInner);
} catch (Exception e) {
castable = false;
}
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) (castable ? 1 : 0));
}
@Override
public void convertUntypedAtomic(UTF8StringPointable stringp, DataOutput dOut)
throws SystemException, IOException {
convertString(stringp, dOut);
}
/** Derived Datatypes */
public void convertByte(BytePointable bytep, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertInt(IntegerPointable intp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertLong(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertNegativeInteger(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertNonNegativeInteger(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertNonPositiveInteger(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertPositiveInteger(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertShort(ShortPointable shortp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertUnsignedByte(ShortPointable shortp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertUnsignedInt(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertUnsignedLong(LongPointable longp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
public void convertUnsignedShort(IntegerPointable intp, DataOutput dOut)
throws SystemException, IOException {
dOut.write(ValueTag.XS_BOOLEAN_TAG);
dOut.write((byte) 1);
}
}
public abstract class AbstractAvgAggregateFunction implements ICopyAggregateFunction {
private static final int SUM_FIELD_ID = 0;
private static final int COUNT_FIELD_ID = 1;
private final ARecordType recType;
private DataOutput out;
private ArrayBackedValueStorage inputVal = new ArrayBackedValueStorage();
private ICopyEvaluator eval;
protected ATypeTag aggType;
private double sum;
private long count;
private AMutableDouble aDouble = new AMutableDouble(0);
private AMutableInt64 aInt64 = new AMutableInt64(0);
private ArrayBackedValueStorage avgBytes = new ArrayBackedValueStorage();
private ByteArrayAccessibleOutputStream sumBytes = new ByteArrayAccessibleOutputStream();
private DataOutput sumBytesOutput = new DataOutputStream(sumBytes);
private ByteArrayAccessibleOutputStream countBytes = new ByteArrayAccessibleOutputStream();
private DataOutput countBytesOutput = new DataOutputStream(countBytes);
private ICopyEvaluator evalSum = new AccessibleByteArrayEval(avgBytes.getDataOutput(), sumBytes);
private ICopyEvaluator evalCount =
new AccessibleByteArrayEval(avgBytes.getDataOutput(), countBytes);
private ClosedRecordConstructorEval recordEval;
@SuppressWarnings("unchecked")
private ISerializerDeserializer<ADouble> doubleSerde =
AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);
@SuppressWarnings("unchecked")
private ISerializerDeserializer<AInt64> longSerde =
AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
@SuppressWarnings("unchecked")
private ISerializerDeserializer<ANull> nullSerde =
AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ANULL);
public AbstractAvgAggregateFunction(ICopyEvaluatorFactory[] args, IDataOutputProvider output)
throws AlgebricksException {
eval = args[0].createEvaluator(inputVal);
out = output.getDataOutput();
ARecordType tmpRecType;
try {
tmpRecType =
new ARecordType(
null,
new String[] {"sum", "count"},
new IAType[] {BuiltinType.ADOUBLE, BuiltinType.AINT64},
false);
} catch (AsterixException | HyracksDataException e) {
throw new AlgebricksException(e);
}
recType = tmpRecType;
recordEval =
new ClosedRecordConstructorEval(
recType, new ICopyEvaluator[] {evalSum, evalCount}, avgBytes, out);
}
@Override
public void init() {
aggType = ATypeTag.SYSTEM_NULL;
sum = 0.0;
count = 0;
}
public abstract void step(IFrameTupleReference tuple) throws AlgebricksException;
public abstract void finish() throws AlgebricksException;
public abstract void finishPartial() throws AlgebricksException;
protected abstract void processNull();
protected void processDataValues(IFrameTupleReference tuple) throws AlgebricksException {
if (skipStep()) {
return;
}
inputVal.reset();
eval.evaluate(tuple);
ATypeTag typeTag =
EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(inputVal.getByteArray()[0]);
if (typeTag == ATypeTag.NULL) {
processNull();
return;
} else if (aggType == ATypeTag.SYSTEM_NULL) {
aggType = typeTag;
} else if (typeTag != ATypeTag.SYSTEM_NULL && !ATypeHierarchy.isCompatible(typeTag, aggType)) {
throw new AlgebricksException(
"Unexpected type "
+ typeTag
+ " in aggregation input stream. Expected type "
+ aggType
+ ".");
} else if (ATypeHierarchy.canPromote(aggType, typeTag)) {
aggType = typeTag;
}
++count;
switch (typeTag) {
case INT8:
{
byte val = AInt8SerializerDeserializer.getByte(inputVal.getByteArray(), 1);
sum += val;
break;
}
case INT16:
{
short val = AInt16SerializerDeserializer.getShort(inputVal.getByteArray(), 1);
sum += val;
break;
}
case INT32:
{
int val = AInt32SerializerDeserializer.getInt(inputVal.getByteArray(), 1);
sum += val;
break;
}
case INT64:
{
long val = AInt64SerializerDeserializer.getLong(inputVal.getByteArray(), 1);
sum += val;
break;
}
case FLOAT:
{
float val = AFloatSerializerDeserializer.getFloat(inputVal.getByteArray(), 1);
sum += val;
break;
}
case DOUBLE:
{
double val = ADoubleSerializerDeserializer.getDouble(inputVal.getByteArray(), 1);
sum += val;
break;
}
default:
{
throw new NotImplementedException("Cannot compute AVG for values of type " + typeTag);
}
}
inputVal.reset();
}
protected void finishPartialResults() throws AlgebricksException {
try {
// Double check that count 0 is accounted
if (aggType == ATypeTag.SYSTEM_NULL) {
if (GlobalConfig.DEBUG) {
GlobalConfig.ASTERIX_LOGGER.finest("AVG aggregate ran over empty input.");
}
out.writeByte(ATypeTag.SYSTEM_NULL.serialize());
} else if (aggType == ATypeTag.NULL) {
out.writeByte(ATypeTag.NULL.serialize());
} else {
sumBytes.reset();
aDouble.setValue(sum);
doubleSerde.serialize(aDouble, sumBytesOutput);
countBytes.reset();
aInt64.setValue(count);
longSerde.serialize(aInt64, countBytesOutput);
recordEval.evaluate(null);
}
} catch (IOException e) {
throw new AlgebricksException(e);
}
}
protected void processPartialResults(IFrameTupleReference tuple) throws AlgebricksException {
if (skipStep()) {
return;
}
inputVal.reset();
eval.evaluate(tuple);
byte[] serBytes = inputVal.getByteArray();
ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(serBytes[0]);
switch (typeTag) {
case NULL:
{
processNull();
break;
}
case SYSTEM_NULL:
{
// Ignore and return.
break;
}
case RECORD:
{
// Expected.
aggType = ATypeTag.DOUBLE;
int nullBitmapSize = 0;
int offset1 =
ARecordSerializerDeserializer.getFieldOffsetById(
serBytes, SUM_FIELD_ID, nullBitmapSize, false);
sum += ADoubleSerializerDeserializer.getDouble(serBytes, offset1);
int offset2 =
ARecordSerializerDeserializer.getFieldOffsetById(
serBytes, COUNT_FIELD_ID, nullBitmapSize, false);
count += AInt64SerializerDeserializer.getLong(serBytes, offset2);
break;
}
default:
{
throw new AlgebricksException(
"Global-Avg is not defined for values of type "
+ EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(serBytes[0]));
}
}
}
protected void finishFinalResults() throws AlgebricksException {
try {
if (count == 0 || aggType == ATypeTag.NULL) {
nullSerde.serialize(ANull.NULL, out);
} else {
aDouble.setValue(sum / count);
doubleSerde.serialize(aDouble, out);
}
} catch (IOException e) {
throw new AlgebricksException(e);
}
}
protected boolean skipStep() {
return false;
}
}
private void parseRecord(ARecordType recType, DataOutput out, Boolean datasetRec)
throws IOException, AsterixException, AdmLexerException {
ArrayBackedValueStorage fieldValueBuffer = getTempBuffer();
ArrayBackedValueStorage fieldNameBuffer = getTempBuffer();
IARecordBuilder recBuilder = getRecordBuilder();
BitSet nulls = null;
if (datasetRec) {
if (recType != null) {
nulls = new BitSet(recType.getFieldNames().length);
recBuilder.reset(recType);
} else {
recBuilder.reset(null);
}
} else if (recType != null) {
nulls = new BitSet(recType.getFieldNames().length);
recBuilder.reset(recType);
} else {
recBuilder.reset(null);
}
recBuilder.init();
int token;
boolean inRecord = true;
boolean expectingRecordField = false;
boolean first = true;
Boolean openRecordField = false;
int fieldId = 0;
IAType fieldType = null;
do {
token = admLexer.next();
switch (token) {
case AdmLexer.TOKEN_END_RECORD:
{
if (expectingRecordField) {
throw new ParseException("Found END_RECORD while expecting a record field.");
}
inRecord = false;
break;
}
case AdmLexer.TOKEN_STRING_LITERAL:
{
// we've read the name of the field
// now read the content
fieldNameBuffer.reset();
fieldValueBuffer.reset();
expectingRecordField = false;
if (recType != null) {
String fldName =
admLexer
.getLastTokenImage()
.substring(1, admLexer.getLastTokenImage().length() - 1);
fieldId = recBuilder.getFieldId(fldName);
if (fieldId < 0 && !recType.isOpen()) {
throw new ParseException("This record is closed, you can not add extra fields !!");
} else if (fieldId < 0 && recType.isOpen()) {
aStringFieldName.setValue(
admLexer
.getLastTokenImage()
.substring(1, admLexer.getLastTokenImage().length() - 1));
stringSerde.serialize(aStringFieldName, fieldNameBuffer.getDataOutput());
openRecordField = true;
fieldType = null;
} else {
// a closed field
nulls.set(fieldId);
fieldType = recType.getFieldTypes()[fieldId];
openRecordField = false;
}
} else {
aStringFieldName.setValue(
admLexer
.getLastTokenImage()
.substring(1, admLexer.getLastTokenImage().length() - 1));
stringSerde.serialize(aStringFieldName, fieldNameBuffer.getDataOutput());
openRecordField = true;
fieldType = null;
}
token = admLexer.next();
if (token != AdmLexer.TOKEN_COLON) {
throw new ParseException(
"Unexpected ADM token kind: "
+ AdmLexer.tokenKindToString(token)
+ " while expecting \":\".");
}
token = admLexer.next();
this.admFromLexerStream(token, fieldType, fieldValueBuffer.getDataOutput(), false);
if (openRecordField) {
if (fieldValueBuffer.getByteArray()[0] != ATypeTag.NULL.serialize()) {
recBuilder.addField(fieldNameBuffer, fieldValueBuffer);
}
} else if (NonTaggedFormatUtil.isOptional(recType)) {
if (fieldValueBuffer.getByteArray()[0] != ATypeTag.NULL.serialize()) {
recBuilder.addField(fieldId, fieldValueBuffer);
}
} else {
recBuilder.addField(fieldId, fieldValueBuffer);
}
break;
}
case AdmLexer.TOKEN_COMMA:
{
if (first) {
throw new ParseException("Found COMMA before any record field.");
}
if (expectingRecordField) {
throw new ParseException("Found COMMA while expecting a record field.");
}
expectingRecordField = true;
break;
}
default:
{
throw new ParseException(
"Unexpected ADM token kind: "
+ AdmLexer.tokenKindToString(token)
+ " while parsing record fields.");
}
}
first = false;
} while (inRecord);
if (recType != null) {
nullableFieldId = checkNullConstraints(recType, nulls);
if (nullableFieldId != -1) {
throw new ParseException(
"Field: " + recType.getFieldNames()[nullableFieldId] + " can not be null");
}
}
recBuilder.write(out, true);
}