Esempio n. 1
0
 @Override
 public void append(String tempFile, String committedFile) throws ConnectException {
   try {
     acquireLease();
     WALEntry key = new WALEntry(tempFile);
     WALEntry value = new WALEntry(committedFile);
     writer.append(key, value);
     writer.hsync();
   } catch (IOException e) {
     throw new ConnectException(e);
   }
 }
  @Test
  public void testeAppend() throws Exception {
    Map<String, String> props = createProps();
    HdfsSinkConnectorConfig connectorConfig = new HdfsSinkConnectorConfig(props);

    String topicsDir = connectorConfig.getString(HdfsSinkConnectorConfig.TOPICS_DIR_CONFIG);
    String topic = "topic";
    int partition = 0;
    TopicPartition topicPart = new TopicPartition(topic, partition);

    Path file = new Path(FileUtils.logFileName(url, topicsDir, topicPart));

    WALFile.Writer writer = WALFile.createWriter(conf, WALFile.Writer.file(file));

    WALEntry key1 = new WALEntry("key1");
    WALEntry val1 = new WALEntry("val1");

    WALEntry key2 = new WALEntry("key2");
    WALEntry val2 = new WALEntry("val2");

    writer.append(key1, val1);
    writer.append(key2, val2);
    writer.close();

    verify2Values(file);

    writer =
        WALFile.createWriter(conf, WALFile.Writer.file(file), WALFile.Writer.appendIfExists(true));

    WALEntry key3 = new WALEntry("key3");
    WALEntry val3 = new WALEntry("val3");

    WALEntry key4 = new WALEntry("key4");
    WALEntry val4 = new WALEntry("val4");

    writer.append(key3, val3);
    writer.append(key4, val4);
    writer.hsync();
    writer.close();

    verifyAll4Values(file);

    fs.deleteOnExit(file);
  }
Esempio n. 3
0
 @Override
 public void close() throws ConnectException {
   try {
     if (writer != null) {
       writer.close();
       writer = null;
     }
     if (reader != null) {
       reader.close();
       reader = null;
     }
   } catch (IOException e) {
     throw new ConnectException("Error closing " + logFile, e);
   }
 }