// @Test
public void testLifecycle() throws InterruptedException, LifecycleException {
LOG.debug("Starting...");
Context context = new Context();
context.put("hdfs.path", testPath);
context.put("hdfs.filePrefix", "pageview");
Configurables.configure(sink, context);
sink.setChannel(new MemoryChannel());
sink.start();
sink.stop();
}
@Before
public void setUp() {
LOG.debug("Starting...");
testPath =
"file:///Jorson-Linux:10000/tmp/flume-test"; // + Calendar.getInstance().getTimeInMillis() +
// "." + Thread.currentThread().getId();
sink = new HDFSEventSink();
sink.setName("HdfsSink-" + UUID.randomUUID().toString());
dirCleanup();
}
public void doTestTextBatchAppend() throws Exception {
LOG.debug("Starting...");
final long rollCount = 10;
final long batchSize = 2;
final String fileName = "PageView";
String newPath = testPath + "/singleTextBucket";
int totalEvents = 0;
int i = 1, j = 1;
// clear the test directory
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
Path dirPath = new Path(newPath);
fs.delete(dirPath, true);
fs.mkdirs(dirPath);
Context context = new Context();
context.put("hdfs.path", newPath);
context.put("hdfs.rollCount", String.valueOf(rollCount));
context.put("hdfs.batchSize", String.valueOf(batchSize));
context.put("hdfs.filePrefix", "pageview");
Channel channel = new MemoryChannel();
Configurables.configure(channel, context);
sink.setChannel(channel);
sink.start();
Calendar eventDate = Calendar.getInstance();
Date currentDate = new Date();
Map<String, String> header = new HashMap<String, String>();
header.put("topic", "PageView");
List<String> bodies = Lists.newArrayList();
// 将测试的事件推入到通道中
for (i = 1; i <= (rollCount * 10) / batchSize; i++) {
Transaction txn = channel.getTransaction();
txn.begin();
for (j = 1; j <= batchSize; j++) {
header.put("timestamp", String.valueOf(currentDate.getTime()));
Event event = new SimpleEvent();
eventDate.clear();
eventDate.set(2014, i, i, i, 0);
String body = "Test." + i + "." + j;
event.setHeaders(header);
event.setBody(body.getBytes());
bodies.add(body);
channel.put(event);
totalEvents++;
}
txn.commit();
txn.close();
// execute sink to process the events
sink.process();
}
sink.stop();
FileStatus[] dirStat = fs.listStatus(dirPath);
Path fList[] = FileUtil.stat2Paths(dirStat);
long expectedFiles = totalEvents / rollCount;
if (totalEvents % rollCount > 0) {
expectedFiles++;
}
Assert.assertEquals(
"num files wrong, found: " + Lists.newArrayList(fList), expectedFiles, fList.length);
// 检查所有写入文件的内容
verifyOutputTextFiles(fs, conf, dirPath.toUri().getPath(), fileName, bodies);
}
