@Override
public void append(String tempFile, String committedFile) throws ConnectException {
try {
acquireLease();
WALEntry key = new WALEntry(tempFile);
WALEntry value = new WALEntry(committedFile);
writer.append(key, value);
writer.hsync();
} catch (IOException e) {
throw new ConnectException(e);
}
}
@Test
public void testeAppend() throws Exception {
Map<String, String> props = createProps();
HdfsSinkConnectorConfig connectorConfig = new HdfsSinkConnectorConfig(props);
String topicsDir = connectorConfig.getString(HdfsSinkConnectorConfig.TOPICS_DIR_CONFIG);
String topic = "topic";
int partition = 0;
TopicPartition topicPart = new TopicPartition(topic, partition);
Path file = new Path(FileUtils.logFileName(url, topicsDir, topicPart));
WALFile.Writer writer = WALFile.createWriter(conf, WALFile.Writer.file(file));
WALEntry key1 = new WALEntry("key1");
WALEntry val1 = new WALEntry("val1");
WALEntry key2 = new WALEntry("key2");
WALEntry val2 = new WALEntry("val2");
writer.append(key1, val1);
writer.append(key2, val2);
writer.close();
verify2Values(file);
writer =
WALFile.createWriter(conf, WALFile.Writer.file(file), WALFile.Writer.appendIfExists(true));
WALEntry key3 = new WALEntry("key3");
WALEntry val3 = new WALEntry("val3");
WALEntry key4 = new WALEntry("key4");
WALEntry val4 = new WALEntry("val4");
writer.append(key3, val3);
writer.append(key4, val4);
writer.hsync();
writer.close();
verifyAll4Values(file);
fs.deleteOnExit(file);
}
@Override
public void close() throws ConnectException {
try {
if (writer != null) {
writer.close();
writer = null;
}
if (reader != null) {
reader.close();
reader = null;
}
} catch (IOException e) {
throw new ConnectException("Error closing " + logFile, e);
}
}