Exemplo n.º 1
0
  @SuppressWarnings("unchecked")
  public static void writeObject(IAObject obj, DataOutput dataOutput) throws HyracksDataException {
    switch (obj.getType().getTypeTag()) {
      case RECORD:
        {
          IARecordBuilder recordBuilder = new RecordBuilder();
          recordBuilder.reset((ARecordType) obj.getType());
          recordBuilder.init();
          writeRecord((AMutableRecord) obj, dataOutput, recordBuilder);
          break;
        }

      case ORDEREDLIST:
        {
          OrderedListBuilder listBuilder = new OrderedListBuilder();
          listBuilder.reset((AOrderedListType) ((AMutableOrderedList) obj).getType());
          IACursor cursor = ((AMutableOrderedList) obj).getCursor();
          ArrayBackedValueStorage listItemValue = new ArrayBackedValueStorage();
          while (cursor.next()) {
            listItemValue.reset();
            IAObject item = cursor.get();
            writeObject(item, listItemValue.getDataOutput());
            listBuilder.addItem(listItemValue);
          }
          listBuilder.write(dataOutput, true);
          break;
        }

      case UNORDEREDLIST:
        {
          UnorderedListBuilder listBuilder = new UnorderedListBuilder();
          listBuilder.reset((AUnorderedListType) ((AMutableUnorderedList) obj).getType());
          IACursor cursor = ((AMutableUnorderedList) obj).getCursor();
          ArrayBackedValueStorage listItemValue = new ArrayBackedValueStorage();
          while (cursor.next()) {
            listItemValue.reset();
            IAObject item = cursor.get();
            writeObject(item, listItemValue.getDataOutput());
            listBuilder.addItem(listItemValue);
          }
          listBuilder.write(dataOutput, true);
          break;
        }

      default:
        AqlSerializerDeserializerProvider.INSTANCE
            .getSerializerDeserializer(obj.getType())
            .serialize(obj, dataOutput);
        break;
    }
  }
/** Translates a Dataset metadata entity to an ITupleReference and vice versa. */
public class CompactionPolicyTupleTranslator extends AbstractTupleTranslator<CompactionPolicy> {
  // Field indexes of serialized CompactionPolicy in a tuple.
  // Key field.
  public static final int COMPACTION_POLICY_DATAVERSE_NAME_FIELD_INDEX = 0;

  public static final int COMPACTION_POLICY_NAME_FIELD_INDEX = 1;

  // Payload field containing serialized compactionPolicy.
  public static final int COMPACTION_POLICY_PAYLOAD_TUPLE_FIELD_INDEX = 2;

  @SuppressWarnings("unchecked")
  private ISerializerDeserializer<ARecord> recordSerDes =
      AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(
          MetadataRecordTypes.COMPACTION_POLICY_RECORDTYPE);

  public CompactionPolicyTupleTranslator(boolean getTuple) {
    super(getTuple, MetadataPrimaryIndexes.COMPACTION_POLICY_DATASET.getFieldCount());
  }

  @Override
  public CompactionPolicy getMetadataEntityFromTuple(ITupleReference tuple) throws IOException {
    byte[] serRecord = tuple.getFieldData(COMPACTION_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
    int recordStartOffset = tuple.getFieldStart(COMPACTION_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
    int recordLength = tuple.getFieldLength(COMPACTION_POLICY_PAYLOAD_TUPLE_FIELD_INDEX);
    ByteArrayInputStream stream =
        new ByteArrayInputStream(serRecord, recordStartOffset, recordLength);
    DataInput in = new DataInputStream(stream);
    ARecord compactionPolicyRecord = (ARecord) recordSerDes.deserialize(in);
    return createCompactionPolicyFromARecord(compactionPolicyRecord);
  }

  private CompactionPolicy createCompactionPolicyFromARecord(ARecord compactionPolicyRecord) {
    CompactionPolicy compactionPolicy = null;
    String dataverseName =
        ((AString)
                compactionPolicyRecord.getValueByPos(
                    MetadataRecordTypes.COMPACTION_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX))
            .getStringValue();
    String policyName =
        ((AString)
                compactionPolicyRecord.getValueByPos(
                    MetadataRecordTypes.COMPACTION_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX))
            .getStringValue();
    String className =
        ((AString)
                compactionPolicyRecord.getValueByPos(
                    MetadataRecordTypes.COMPACTION_POLICY_ARECORD_CLASSNAME_FIELD_INDEX))
            .getStringValue();

    compactionPolicy = new CompactionPolicy(dataverseName, policyName, className);
    return compactionPolicy;
  }

  @Override
  public ITupleReference getTupleFromMetadataEntity(CompactionPolicy compactionPolicy)
      throws IOException, MetadataException {

    tupleBuilder.reset();
    aString.setValue(compactionPolicy.getDataverseName());
    stringSerde.serialize(aString, tupleBuilder.getDataOutput());
    tupleBuilder.addFieldEndOffset();

    aString.setValue(compactionPolicy.getPolicyName());
    stringSerde.serialize(aString, tupleBuilder.getDataOutput());
    tupleBuilder.addFieldEndOffset();

    recordBuilder.reset(MetadataRecordTypes.COMPACTION_POLICY_RECORDTYPE);

    // write field 0
    fieldValue.reset();
    aString.setValue(compactionPolicy.getDataverseName());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(
        MetadataRecordTypes.COMPACTION_POLICY_ARECORD_DATAVERSE_NAME_FIELD_INDEX, fieldValue);

    // write field 1
    fieldValue.reset();
    aString.setValue(compactionPolicy.getPolicyName());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(
        MetadataRecordTypes.COMPACTION_POLICY_ARECORD_POLICY_NAME_FIELD_INDEX, fieldValue);

    // write field 2
    fieldValue.reset();
    aString.setValue(compactionPolicy.getClassName());
    stringSerde.serialize(aString, fieldValue.getDataOutput());
    recordBuilder.addField(
        MetadataRecordTypes.COMPACTION_POLICY_ARECORD_CLASSNAME_FIELD_INDEX, fieldValue);

    // write record
    recordBuilder.write(tupleBuilder.getDataOutput(), true);
    tupleBuilder.addFieldEndOffset();

    tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
    return tuple;
  }
}
public abstract class AbstractAvgAggregateFunction implements ICopyAggregateFunction {
  private static final int SUM_FIELD_ID = 0;
  private static final int COUNT_FIELD_ID = 1;

  private final ARecordType recType;

  private DataOutput out;
  private ArrayBackedValueStorage inputVal = new ArrayBackedValueStorage();
  private ICopyEvaluator eval;
  protected ATypeTag aggType;
  private double sum;
  private long count;
  private AMutableDouble aDouble = new AMutableDouble(0);
  private AMutableInt64 aInt64 = new AMutableInt64(0);

  private ArrayBackedValueStorage avgBytes = new ArrayBackedValueStorage();
  private ByteArrayAccessibleOutputStream sumBytes = new ByteArrayAccessibleOutputStream();
  private DataOutput sumBytesOutput = new DataOutputStream(sumBytes);
  private ByteArrayAccessibleOutputStream countBytes = new ByteArrayAccessibleOutputStream();
  private DataOutput countBytesOutput = new DataOutputStream(countBytes);
  private ICopyEvaluator evalSum = new AccessibleByteArrayEval(avgBytes.getDataOutput(), sumBytes);
  private ICopyEvaluator evalCount =
      new AccessibleByteArrayEval(avgBytes.getDataOutput(), countBytes);
  private ClosedRecordConstructorEval recordEval;

  @SuppressWarnings("unchecked")
  private ISerializerDeserializer<ADouble> doubleSerde =
      AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);

  @SuppressWarnings("unchecked")
  private ISerializerDeserializer<AInt64> longSerde =
      AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);

  @SuppressWarnings("unchecked")
  private ISerializerDeserializer<ANull> nullSerde =
      AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ANULL);

  public AbstractAvgAggregateFunction(ICopyEvaluatorFactory[] args, IDataOutputProvider output)
      throws AlgebricksException {
    eval = args[0].createEvaluator(inputVal);
    out = output.getDataOutput();

    ARecordType tmpRecType;
    try {
      tmpRecType =
          new ARecordType(
              null,
              new String[] {"sum", "count"},
              new IAType[] {BuiltinType.ADOUBLE, BuiltinType.AINT64},
              false);
    } catch (AsterixException | HyracksDataException e) {
      throw new AlgebricksException(e);
    }

    recType = tmpRecType;
    recordEval =
        new ClosedRecordConstructorEval(
            recType, new ICopyEvaluator[] {evalSum, evalCount}, avgBytes, out);
  }

  @Override
  public void init() {
    aggType = ATypeTag.SYSTEM_NULL;
    sum = 0.0;
    count = 0;
  }

  public abstract void step(IFrameTupleReference tuple) throws AlgebricksException;

  public abstract void finish() throws AlgebricksException;

  public abstract void finishPartial() throws AlgebricksException;

  protected abstract void processNull();

  protected void processDataValues(IFrameTupleReference tuple) throws AlgebricksException {
    if (skipStep()) {
      return;
    }
    inputVal.reset();
    eval.evaluate(tuple);
    ATypeTag typeTag =
        EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(inputVal.getByteArray()[0]);
    if (typeTag == ATypeTag.NULL) {
      processNull();
      return;
    } else if (aggType == ATypeTag.SYSTEM_NULL) {
      aggType = typeTag;
    } else if (typeTag != ATypeTag.SYSTEM_NULL && !ATypeHierarchy.isCompatible(typeTag, aggType)) {
      throw new AlgebricksException(
          "Unexpected type "
              + typeTag
              + " in aggregation input stream. Expected type "
              + aggType
              + ".");
    } else if (ATypeHierarchy.canPromote(aggType, typeTag)) {
      aggType = typeTag;
    }
    ++count;
    switch (typeTag) {
      case INT8:
        {
          byte val = AInt8SerializerDeserializer.getByte(inputVal.getByteArray(), 1);
          sum += val;
          break;
        }
      case INT16:
        {
          short val = AInt16SerializerDeserializer.getShort(inputVal.getByteArray(), 1);
          sum += val;
          break;
        }
      case INT32:
        {
          int val = AInt32SerializerDeserializer.getInt(inputVal.getByteArray(), 1);
          sum += val;
          break;
        }
      case INT64:
        {
          long val = AInt64SerializerDeserializer.getLong(inputVal.getByteArray(), 1);
          sum += val;
          break;
        }
      case FLOAT:
        {
          float val = AFloatSerializerDeserializer.getFloat(inputVal.getByteArray(), 1);
          sum += val;
          break;
        }
      case DOUBLE:
        {
          double val = ADoubleSerializerDeserializer.getDouble(inputVal.getByteArray(), 1);
          sum += val;
          break;
        }
      default:
        {
          throw new NotImplementedException("Cannot compute AVG for values of type " + typeTag);
        }
    }
    inputVal.reset();
  }

  protected void finishPartialResults() throws AlgebricksException {
    try {
      // Double check that count 0 is accounted
      if (aggType == ATypeTag.SYSTEM_NULL) {
        if (GlobalConfig.DEBUG) {
          GlobalConfig.ASTERIX_LOGGER.finest("AVG aggregate ran over empty input.");
        }
        out.writeByte(ATypeTag.SYSTEM_NULL.serialize());
      } else if (aggType == ATypeTag.NULL) {
        out.writeByte(ATypeTag.NULL.serialize());
      } else {
        sumBytes.reset();
        aDouble.setValue(sum);
        doubleSerde.serialize(aDouble, sumBytesOutput);
        countBytes.reset();
        aInt64.setValue(count);
        longSerde.serialize(aInt64, countBytesOutput);
        recordEval.evaluate(null);
      }
    } catch (IOException e) {
      throw new AlgebricksException(e);
    }
  }

  protected void processPartialResults(IFrameTupleReference tuple) throws AlgebricksException {
    if (skipStep()) {
      return;
    }
    inputVal.reset();
    eval.evaluate(tuple);
    byte[] serBytes = inputVal.getByteArray();
    ATypeTag typeTag = EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(serBytes[0]);
    switch (typeTag) {
      case NULL:
        {
          processNull();
          break;
        }
      case SYSTEM_NULL:
        {
          // Ignore and return.
          break;
        }
      case RECORD:
        {
          // Expected.
          aggType = ATypeTag.DOUBLE;
          int nullBitmapSize = 0;
          int offset1 =
              ARecordSerializerDeserializer.getFieldOffsetById(
                  serBytes, SUM_FIELD_ID, nullBitmapSize, false);
          sum += ADoubleSerializerDeserializer.getDouble(serBytes, offset1);
          int offset2 =
              ARecordSerializerDeserializer.getFieldOffsetById(
                  serBytes, COUNT_FIELD_ID, nullBitmapSize, false);
          count += AInt64SerializerDeserializer.getLong(serBytes, offset2);
          break;
        }
      default:
        {
          throw new AlgebricksException(
              "Global-Avg is not defined for values of type "
                  + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(serBytes[0]));
        }
    }
  }

  protected void finishFinalResults() throws AlgebricksException {
    try {
      if (count == 0 || aggType == ATypeTag.NULL) {
        nullSerde.serialize(ANull.NULL, out);
      } else {
        aDouble.setValue(sum / count);
        doubleSerde.serialize(aDouble, out);
      }
    } catch (IOException e) {
      throw new AlgebricksException(e);
    }
  }

  protected boolean skipStep() {
    return false;
  }
}
public abstract class AbstractTripleStringStringEval implements ICopyEvaluator {

  private DataOutput dout;
  private static final byte SER_NULL_TYPE_TAG = ATypeTag.NULL.serialize();
  private static final byte SER_STRING_TYPE_TAG = ATypeTag.STRING.serialize();
  private ArrayBackedValueStorage array0 = new ArrayBackedValueStorage();
  private ArrayBackedValueStorage array1 = new ArrayBackedValueStorage();
  private ArrayBackedValueStorage array2 = new ArrayBackedValueStorage();
  private ICopyEvaluator eval0;
  private ICopyEvaluator eval1;
  private ICopyEvaluator eval2;

  private AMutableString resultBuffer = new AMutableString("");

  @SuppressWarnings("rawtypes")
  private ISerializerDeserializer nullSerde =
      AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ANULL);

  @SuppressWarnings("rawtypes")
  private ISerializerDeserializer strSerde =
      AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);

  private final UTF8StringPointable strPtr1st = new UTF8StringPointable();
  private final UTF8StringPointable strPtr2nd = new UTF8StringPointable();
  private final UTF8StringPointable strPtr3rd = new UTF8StringPointable();

  private final FunctionIdentifier funcID;

  public AbstractTripleStringStringEval(
      DataOutput dout,
      ICopyEvaluatorFactory eval0,
      ICopyEvaluatorFactory eval1,
      ICopyEvaluatorFactory eval2,
      FunctionIdentifier funcID)
      throws AlgebricksException {
    this.dout = dout;
    this.eval0 = eval0.createEvaluator(array0);
    this.eval1 = eval1.createEvaluator(array1);
    this.eval2 = eval2.createEvaluator(array2);
    this.funcID = funcID;
  }

  @SuppressWarnings("unchecked")
  @Override
  public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
    array0.reset();
    eval0.evaluate(tuple);
    array1.reset();
    eval1.evaluate(tuple);
    array2.reset();
    eval2.evaluate(tuple);

    try {
      if (array0.getByteArray()[0] == SER_NULL_TYPE_TAG
          || array1.getByteArray()[0] == SER_NULL_TYPE_TAG
          || array2.getByteArray()[0] == SER_NULL_TYPE_TAG) {
        nullSerde.serialize(ANull.NULL, dout);
        return;
      } else if (array0.getByteArray()[0] != SER_STRING_TYPE_TAG
          || array1.getByteArray()[0] != SER_STRING_TYPE_TAG
          || array2.getByteArray()[0] != SER_STRING_TYPE_TAG) {
        throw new AlgebricksException(
            funcID.getName()
                + ": expects input type (STRING/NULL, STRING/NULL, STRING/NULL), but got ("
                + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(array0.getByteArray()[0])
                + ", "
                + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(array1.getByteArray()[0])
                + ", "
                + EnumDeserializer.ATYPETAGDESERIALIZER.deserialize(array2.getByteArray()[0])
                + ".");
      }
    } catch (HyracksDataException e) {
      throw new AlgebricksException(e);
    }

    strPtr1st.set(array0.getByteArray(), array0.getStartOffset() + 1, array0.getLength());
    strPtr2nd.set(array1.getByteArray(), array1.getStartOffset() + 1, array1.getLength());
    strPtr3rd.set(array2.getByteArray(), array2.getStartOffset() + 1, array2.getLength());

    String res = compute(strPtr1st, strPtr2nd, strPtr3rd);
    resultBuffer.setValue(res);
    try {
      strSerde.serialize(resultBuffer, dout);
    } catch (HyracksDataException e) {
      throw new AlgebricksException(e);
    }
  }

  protected abstract String compute(
      UTF8StringPointable strPtr1st, UTF8StringPointable strPtr2nd, UTF8StringPointable strPtr3rd)
      throws AlgebricksException;
}
 @SuppressWarnings("rawtypes")
 public void testSchemaful() {
   try {
     File file = new File("target/classad-wtih-temporals.adm");
     File expected =
         new File(getClass().getResource("/results/classad-with-temporals.adm").toURI().getPath());
     FileUtils.deleteQuietly(file);
     PrintStream printStream = new PrintStream(Files.newOutputStream(Paths.get(file.toURI())));
     String[] recordFieldNames = {
       "GlobalJobId",
       "Owner",
       "ClusterId",
       "ProcId",
       "RemoteWallClockTime",
       "CompletionDate",
       "QDate",
       "JobCurrentStartDate",
       "JobStartDate",
       "JobCurrentStartExecutingDate"
     };
     IAType[] recordFieldTypes = {
       BuiltinType.ASTRING,
       BuiltinType.ASTRING,
       BuiltinType.AINT32,
       BuiltinType.AINT32,
       BuiltinType.ADURATION,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME,
       BuiltinType.ADATETIME
     };
     ARecordType recordType = new ARecordType("value", recordFieldNames, recordFieldTypes, true);
     int numOfTupleFields = 1;
     ISerializerDeserializer[] serdes = new ISerializerDeserializer[1];
     serdes[0] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recordType);
     IPrinterFactory[] printerFactories = new IPrinterFactory[1];
     printerFactories[0] = AqlADMPrinterFactoryProvider.INSTANCE.getPrinterFactory(recordType);
     // create output descriptor
     IPrinter[] printers = new IPrinter[printerFactories.length];
     for (int i = 0; i < printerFactories.length; i++) {
       printers[i] = printerFactories[i].createPrinter();
     }
     ClassAdObjectPool objectPool = new ClassAdObjectPool();
     String[] files = new String[] {"/classad-with-temporals.classads"};
     ClassAdParser parser =
         new ClassAdParser(recordType, false, false, false, null, null, null, objectPool);
     ArrayTupleBuilder tb = new ArrayTupleBuilder(numOfTupleFields);
     for (String path : files) {
       List<Path> paths = new ArrayList<>();
       paths.add(Paths.get(getClass().getResource(path).toURI()));
       FileSystemWatcher watcher = new FileSystemWatcher(paths, null, false);
       LocalFSInputStream in = new LocalFSInputStream(watcher);
       SemiStructuredRecordReader recordReader = new SemiStructuredRecordReader(in, "[", "]");
       while (recordReader.hasNext()) {
         tb.reset();
         IRawRecord<char[]> record = recordReader.next();
         parser.parse(record, tb.getDataOutput());
         tb.addFieldEndOffset();
         printTuple(tb, printers, printStream);
       }
       recordReader.close();
       printStream.close();
       Assert.assertTrue(FileUtils.contentEquals(file, expected));
     }
   } catch (Throwable th) {
     System.err.println("TEST FAILED");
     th.printStackTrace();
     Assert.assertTrue(false);
   }
   System.err.println("TEST PASSED");
 }