/** Writes the given Avro datum into the given record, using the given Avro schema */
private void extractTree(Object datum, Schema schema, Record outputRecord, String prefix) {
// RECORD, ENUM, ARRAY, MAP, UNION, FIXED, STRING, BYTES, INT, LONG, FLOAT,
// DOUBLE, BOOLEAN, NULL
switch (schema.getType()) {
case RECORD:
{
IndexedRecord avroRecord = (IndexedRecord) datum;
String prefix2 = prefix + "/";
for (Field field : schema.getFields()) {
extractTree(
avroRecord.get(field.pos()),
field.schema(),
outputRecord,
prefix2 + field.name());
}
break;
}
case ENUM:
{
GenericEnumSymbol symbol = (GenericEnumSymbol) datum;
outputRecord.put(prefix, symbol.toString());
break;
}
case ARRAY:
{
Iterator iter = ((Collection) datum).iterator();
while (iter.hasNext()) {
extractTree(iter.next(), schema.getElementType(), outputRecord, prefix);
}
break;
}
case MAP:
{
Map<CharSequence, ?> map = (Map<CharSequence, ?>) datum;
for (Map.Entry<CharSequence, ?> entry : map.entrySet()) {
extractTree(
entry.getValue(),
schema.getValueType(),
outputRecord,
prefix + "/" + entry.getKey().toString());
}
break;
}
case UNION:
{
int index = GenericData.get().resolveUnion(schema, datum);
// String typeName = schema.getTypes().get(index).getName();
// String prefix2 = prefix + "/" + typeName;
String prefix2 = prefix;
extractTree(datum, schema.getTypes().get(index), outputRecord, prefix2);
break;
}
case FIXED:
{
GenericFixed fixed = (GenericFixed) datum;
outputRecord.put(prefix, fixed.bytes());
// outputRecord.put(prefix, utf8toString(fixed.bytes()));
break;
}
case BYTES:
{
ByteBuffer buf = (ByteBuffer) datum;
int pos = buf.position();
byte[] bytes = new byte[buf.remaining()];
buf.get(bytes);
buf.position(pos); // undo relative read
outputRecord.put(prefix, bytes);
// outputRecord.put(prefix, utf8toString(bytes));
break;
}
case STRING:
{
outputRecord.put(prefix, datum.toString());
break;
}
case INT:
{
outputRecord.put(prefix, datum);
break;
}
case LONG:
{
outputRecord.put(prefix, datum);
break;
}
case FLOAT:
{
outputRecord.put(prefix, datum);
break;
}
case DOUBLE:
{
outputRecord.put(prefix, datum);
break;
}
case BOOLEAN:
{
outputRecord.put(prefix, datum);
break;
}
case NULL:
{
break;
}
default:
throw new MorphlineRuntimeException("Unknown Avro schema type: " + schema.getType());
}
}
@SuppressWarnings("rawtypes")
@Test
public void testCommit() throws IOException, ClassNotFoundException, SQLException {
// connection part
TJDBCConnectionDefinition connectionDefinition = new TJDBCConnectionDefinition();
TJDBCConnectionProperties connectionProperties =
DBTestUtils.createCommonJDBCConnectionProperties(allSetting, connectionDefinition);
JDBCSourceOrSink sourceOrSink = new JDBCSourceOrSink();
sourceOrSink.initialize(null, connectionProperties);
ValidationResult result = sourceOrSink.validate(container);
assertTrue(result.getStatus() == ValidationResult.Result.OK);
// output part
TJDBCOutputDefinition outputDefinition = new TJDBCOutputDefinition();
TJDBCOutputProperties outputProperties =
(TJDBCOutputProperties) outputDefinition.createRuntimeProperties();
outputProperties.main.schema.setValue(DBTestUtils.createTestSchema());
outputProperties.updateOutputSchemas();
outputProperties.tableSelection.tablename.setValue(DBTestUtils.getTablename());
outputProperties.dataAction.setValue(DataAction.INSERT);
outputProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
JDBCSink sink = new JDBCSink();
sink.initialize(container, outputProperties);
WriteOperation writerOperation = sink.createWriteOperation();
writerOperation.initialize(container);
JDBCOutputInsertWriter writer =
(JDBCOutputInsertWriter) writerOperation.createWriter(container);
try {
writer.open("wid");
IndexedRecord r1 = new GenericData.Record(outputProperties.main.schema.getValue());
r1.put(0, 4);
r1.put(1, "xiaoming");
writer.write(r1);
DBTestUtils.assertSuccessRecord(writer, r1);
IndexedRecord r2 = new GenericData.Record(outputProperties.main.schema.getValue());
r2.put(0, 5);
r2.put(1, "xiaobai");
writer.write(r2);
DBTestUtils.assertSuccessRecord(writer, r2);
writer.close();
} finally {
writer.close();
}
// commit part
TJDBCCommitDefinition commitDefinition = new TJDBCCommitDefinition();
TJDBCCommitProperties commitProperties =
(TJDBCCommitProperties) commitDefinition.createRuntimeProperties();
commitProperties.referencedComponent.componentInstanceId.setValue(refComponentId);
commitProperties.closeConnection.setValue(false);
JDBCCommitSourceOrSink commitSourceOrSink = new JDBCCommitSourceOrSink();
commitSourceOrSink.initialize(container, commitProperties);
commitSourceOrSink.validate(container);
int count = -1;
// create another session and check if the data is inserted
try (Connection conn = JdbcRuntimeUtils.createConnection(allSetting);
Statement statement = conn.createStatement();
ResultSet resultset = statement.executeQuery("select count(*) from TEST")) {
if (resultset.next()) {
count = resultset.getInt(1);
}
}
Assert.assertEquals(5, count);
try (java.sql.Connection refConnection =
(java.sql.Connection)
container.getComponentData(refComponentId, ComponentConstants.CONNECTION_KEY)) {
assertTrue(refConnection != null);
Assert.assertTrue(!refConnection.isClosed());
}
}