private void configure(ClusterHdfsDSource hdfsClusterSource, String dirLocation) {
hdfsClusterSource.hdfsUri = miniDFS.getURI().toString();
hdfsClusterSource.hdfsDirLocations = Arrays.asList(dirLocation);
hdfsClusterSource.hdfsConfigs = new HashMap<String, String>();
hdfsClusterSource.hdfsConfigs.put("x", "X");
hdfsClusterSource.dataFormat = DataFormat.TEXT;
hdfsClusterSource.textMaxLineLen = 1024;
}
@Test
public void testGetHdfsConfiguration() throws Exception {
ClusterHdfsDSource dSource = new ForTestClusterHdfsDSource();
configure(dSource, dir.toString());
ClusterHdfsSource clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
try {
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
Assert.assertNotNull(clusterHdfsSource.getConfiguration());
assertEquals("X", clusterHdfsSource.getConfiguration().get("x"));
} finally {
clusterHdfsSource.destroy();
}
}
@Test
public void testWrongHDFSDirLocation() throws Exception {
ClusterHdfsDSource dSource = new ForTestClusterHdfsDSource();
configure(dSource, dir.toUri().getPath());
dSource.hdfsUri = "/pathwithnoschemeorauthority";
ClusterHdfsSource clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
try {
List<ConfigIssue> issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 1, issues.size());
assertTrue(String.valueOf(issues), issues.get(0).toString().contains("HADOOPFS_02"));
dSource.hdfsUri = "file://localhost:8020/";
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 1, issues.size());
assertTrue(String.valueOf(issues), issues.get(0).toString().contains("HADOOPFS_12"));
dSource.hdfsUri = "hdfs:///noauthority";
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 1, issues.size());
assertTrue(String.valueOf(issues), issues.get(0).toString().contains("HADOOPFS_13"));
dSource.hdfsUri = "hdfs://localhost:8020";
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 1, issues.size());
assertTrue(String.valueOf(issues), issues.get(0).toString().contains("HADOOPFS_11"));
dSource.hdfsUri = miniDFS.getURI().toString();
dSource.hdfsDirLocations = Arrays.asList("/pathdoesnotexist");
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 1, issues.size());
assertTrue(String.valueOf(issues), issues.get(0).toString().contains("HADOOPFS_10"));
dSource.hdfsUri = miniDFS.getURI().toString();
dSource.hdfsDirLocations = Arrays.asList(dir.toUri().getPath());
FileSystem fs = miniDFS.getFileSystem();
Path someFile = new Path(new Path(dir.toUri()), "/someFile");
fs.create(someFile).close();
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 0, issues.size());
dSource.hdfsUri = null;
dSource.hdfsConfigs.put(
CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, miniDFS.getURI().toString());
someFile = new Path(new Path(dir.toUri()), "/someFile2");
fs.create(someFile).close();
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 0, issues.size());
Path dummyFile = new Path(new Path(dir.toUri()), "/dummyFile");
fs.create(dummyFile).close();
dSource.hdfsUri = miniDFS.getURI().toString();
dSource.hdfsDirLocations = Arrays.asList(dummyFile.toUri().getPath());
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 1, issues.size());
assertTrue(String.valueOf(issues), issues.get(0).toString().contains("HADOOPFS_15"));
Path emptyDir = new Path(dir.toUri().getPath(), "emptyDir");
fs.mkdirs(emptyDir);
dSource.hdfsUri = miniDFS.getURI().toString();
dSource.hdfsDirLocations = Arrays.asList(emptyDir.toUri().getPath());
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 1, issues.size());
assertTrue(String.valueOf(issues), issues.get(0).toString().contains("HADOOPFS_16"));
Path path1 = new Path(emptyDir, "path1");
fs.create(path1).close();
dSource.hdfsUri = miniDFS.getURI().toString();
dSource.hdfsDirLocations = Arrays.asList(emptyDir.toUri().getPath());
clusterHdfsSource = (ClusterHdfsSource) dSource.createSource();
issues =
clusterHdfsSource.init(
null,
ContextInfoCreator.createSourceContext(
"myInstance", false, OnRecordError.TO_ERROR, ImmutableList.of("lane")));
assertEquals(String.valueOf(issues), 0, issues.size());
} finally {
clusterHdfsSource.destroy();
}
}