示例#1
0
 /** Writes the given Avro datum into the given record, using the given Avro schema */
 private void extractTree(Object datum, Schema schema, Record outputRecord, String prefix) {
   // RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES, INT, LONG, FLOAT,
   // DOUBLE, BOOLEAN, NULL
   switch (schema.getType()) {
     case RECORD:
       {
         IndexedRecord avroRecord = (IndexedRecord) datum;
         String prefix2 = prefix + "/";
         for (Field field : schema.getFields()) {
           extractTree(
               avroRecord.get(field.pos()),
               field.schema(),
               outputRecord,
               prefix2 + field.name());
         }
         break;
       }
     case ENUM:
       {
         GenericEnumSymbol symbol = (GenericEnumSymbol) datum;
         outputRecord.put(prefix, symbol.toString());
         break;
       }
     case ARRAY:
       {
         Iterator iter = ((Collection) datum).iterator();
         while (iter.hasNext()) {
           extractTree(iter.next(), schema.getElementType(), outputRecord, prefix);
         }
         break;
       }
     case MAP:
       {
         Map<CharSequence, ?> map = (Map<CharSequence, ?>) datum;
         for (Map.Entry<CharSequence, ?> entry : map.entrySet()) {
           extractTree(
               entry.getValue(),
               schema.getValueType(),
               outputRecord,
               prefix + "/" + entry.getKey().toString());
         }
         break;
       }
     case UNION:
       {
         int index = GenericData.get().resolveUnion(schema, datum);
         // String typeName = schema.getTypes().get(index).getName();
         // String prefix2 = prefix + "/" + typeName;
         String prefix2 = prefix;
         extractTree(datum, schema.getTypes().get(index), outputRecord, prefix2);
         break;
       }
     case FIXED:
       {
         GenericFixed fixed = (GenericFixed) datum;
         outputRecord.put(prefix, fixed.bytes());
         // outputRecord.put(prefix, utf8toString(fixed.bytes()));
         break;
       }
     case BYTES:
       {
         ByteBuffer buf = (ByteBuffer) datum;
         int pos = buf.position();
         byte[] bytes = new byte[buf.remaining()];
         buf.get(bytes);
         buf.position(pos); // undo relative read
         outputRecord.put(prefix, bytes);
         // outputRecord.put(prefix, utf8toString(bytes));
         break;
       }
     case STRING:
       {
         outputRecord.put(prefix, datum.toString());
         break;
       }
     case INT:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case LONG:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case FLOAT:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case DOUBLE:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case BOOLEAN:
       {
         outputRecord.put(prefix, datum);
         break;
       }
     case NULL:
       {
         break;
       }
     default:
       throw new MorphlineRuntimeException("Unknown Avro schema type: " + schema.getType());
   }
 }
示例#2
0
  @SuppressWarnings("rawtypes")
  @Test
  public void testCommit() throws IOException, ClassNotFoundException, SQLException {
    // connection part
    TJDBCConnectionDefinition connectionDefinition = new TJDBCConnectionDefinition();
    TJDBCConnectionProperties connectionProperties =
        DBTestUtils.createCommonJDBCConnectionProperties(allSetting, connectionDefinition);

    JDBCSourceOrSink sourceOrSink = new JDBCSourceOrSink();
    sourceOrSink.initialize(null, connectionProperties);

    ValidationResult result = sourceOrSink.validate(container);
    assertTrue(result.getStatus() == ValidationResult.Result.OK);

    // output part
    TJDBCOutputDefinition outputDefinition = new TJDBCOutputDefinition();
    TJDBCOutputProperties outputProperties =
        (TJDBCOutputProperties) outputDefinition.createRuntimeProperties();

    outputProperties.main.schema.setValue(DBTestUtils.createTestSchema());
    outputProperties.updateOutputSchemas();

    outputProperties.tableSelection.tablename.setValue(DBTestUtils.getTablename());

    outputProperties.dataAction.setValue(DataAction.INSERT);

    outputProperties.referencedComponent.componentInstanceId.setValue(refComponentId);

    JDBCSink sink = new JDBCSink();
    sink.initialize(container, outputProperties);

    WriteOperation writerOperation = sink.createWriteOperation();
    writerOperation.initialize(container);
    JDBCOutputInsertWriter writer =
        (JDBCOutputInsertWriter) writerOperation.createWriter(container);

    try {
      writer.open("wid");

      IndexedRecord r1 = new GenericData.Record(outputProperties.main.schema.getValue());
      r1.put(0, 4);
      r1.put(1, "xiaoming");
      writer.write(r1);

      DBTestUtils.assertSuccessRecord(writer, r1);

      IndexedRecord r2 = new GenericData.Record(outputProperties.main.schema.getValue());
      r2.put(0, 5);
      r2.put(1, "xiaobai");
      writer.write(r2);

      DBTestUtils.assertSuccessRecord(writer, r2);

      writer.close();
    } finally {
      writer.close();
    }

    // commit part
    TJDBCCommitDefinition commitDefinition = new TJDBCCommitDefinition();
    TJDBCCommitProperties commitProperties =
        (TJDBCCommitProperties) commitDefinition.createRuntimeProperties();

    commitProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
    commitProperties.closeConnection.setValue(false);

    JDBCCommitSourceOrSink commitSourceOrSink = new JDBCCommitSourceOrSink();
    commitSourceOrSink.initialize(container, commitProperties);
    commitSourceOrSink.validate(container);

    int count = -1;

    // create another session and check if the data is inserted
    try (Connection conn = JdbcRuntimeUtils.createConnection(allSetting);
        Statement statement = conn.createStatement();
        ResultSet resultset = statement.executeQuery("select count(*) from TEST")) {
      if (resultset.next()) {
        count = resultset.getInt(1);
      }
    }

    Assert.assertEquals(5, count);

    try (java.sql.Connection refConnection =
        (java.sql.Connection)
            container.getComponentData(refComponentId, ComponentConstants.CONNECTION_KEY)) {
      assertTrue(refConnection != null);
      Assert.assertTrue(!refConnection.isClosed());
    }
  }