public void acquireLease() throws ConnectException {
long sleepIntervalMs = 1000L;
long MAX_SLEEP_INTERVAL_MS = 16000L;
while (sleepIntervalMs < MAX_SLEEP_INTERVAL_MS) {
try {
if (writer == null) {
writer =
WALFile.createWriter(
conf, Writer.file(new Path(logFile)), Writer.appendIfExists(true));
log.info("Successfully acquired lease for {}", logFile);
}
break;
} catch (RemoteException e) {
if (e.getClassName().equals(leaseException)) {
log.info("Cannot acquire lease on WAL {}", logFile);
try {
Thread.sleep(sleepIntervalMs);
} catch (InterruptedException ie) {
throw new ConnectException(ie);
}
sleepIntervalMs = sleepIntervalMs * 2;
} else {
throw new ConnectException(e);
}
} catch (IOException e) {
throw new ConnectException("Error creating writer for log file " + logFile, e);
}
}
if (sleepIntervalMs >= MAX_SLEEP_INTERVAL_MS) {
throw new ConnectException("Cannot acquire lease after timeout, will retry.");
}
}
@Test
public void testeAppend() throws Exception {
Map<String, String> props = createProps();
HdfsSinkConnectorConfig connectorConfig = new HdfsSinkConnectorConfig(props);
String topicsDir = connectorConfig.getString(HdfsSinkConnectorConfig.TOPICS_DIR_CONFIG);
String topic = "topic";
int partition = 0;
TopicPartition topicPart = new TopicPartition(topic, partition);
Path file = new Path(FileUtils.logFileName(url, topicsDir, topicPart));
WALFile.Writer writer = WALFile.createWriter(conf, WALFile.Writer.file(file));
WALEntry key1 = new WALEntry("key1");
WALEntry val1 = new WALEntry("val1");
WALEntry key2 = new WALEntry("key2");
WALEntry val2 = new WALEntry("val2");
writer.append(key1, val1);
writer.append(key2, val2);
writer.close();
verify2Values(file);
writer =
WALFile.createWriter(conf, WALFile.Writer.file(file), WALFile.Writer.appendIfExists(true));
WALEntry key3 = new WALEntry("key3");
WALEntry val3 = new WALEntry("val3");
WALEntry key4 = new WALEntry("key4");
WALEntry val4 = new WALEntry("val4");
writer.append(key3, val3);
writer.append(key4, val4);
writer.hsync();
writer.close();
verifyAll4Values(file);
fs.deleteOnExit(file);
}