private int readFile() throws IllegalArgumentException, IOException {
int count = 0;
final FileSystem fs = FileSystem.get(MapReduceTestUtils.getConfiguration());
final FileStatus[] fss =
fs.listStatus(
new Path(
TestUtils.TEMP_DIR
+ File.separator
+ MapReduceTestEnvironment.HDFS_BASE_DIRECTORY
+ "/t1/pairs"));
for (final FileStatus ifs : fss) {
if (ifs.isFile() && ifs.getPath().toString().matches(".*part-r-0000[0-9]")) {
try (SequenceFile.Reader reader =
new SequenceFile.Reader(
MapReduceTestUtils.getConfiguration(), Reader.file(ifs.getPath()))) {
final Text key = new Text();
final Text val = new Text();
while (reader.next(key, val)) {
count++;
System.err.println(key + "\t" + val);
}
}
}
}
return count;
}
private void runNN(final DistributableQuery query) throws Exception {
final NNJobRunner jobRunner = new NNJobRunner();
// final int res = 1;
// GeoWaveMain.main(new String[] {
// "analytic",
// "nn",
// "--query.adapters",
// "testnn",
// "--query.index",
// new
// SpatialDimensionalityTypeProvider().createPrimaryIndex().getId().getString(),
// "-emn",
// Integer.toString(MIN_INPUT_SPLITS),
// "-emx",
// Integer.toString(MAX_INPUT_SPLITS),
// "-pmd",
// "0.2",
// "-pdt",
// "0.2,0.2",
// "-pc",
// OrthodromicDistancePartitioner.class.toString(),
// "-oop",
// hdfsBaseDirectory + "/t1/pairs",
// "-hdfsbase",
// hdfsBaseDirectory + "/t1",
// "-orc",
// "3",
// "-ofc",
// SequenceFileOutputFormatConfiguration.class.toString(),
// "-ifc",
// GeoWaveInputFormatConfiguration.class.toString(),
// "foo"
// });
final int res =
jobRunner.run(
MapReduceTestUtils.getConfiguration(),
new PropertyManagement(
new ParameterEnum[] {
ExtractParameters.Extract.QUERY,
ExtractParameters.Extract.MIN_INPUT_SPLIT,
ExtractParameters.Extract.MAX_INPUT_SPLIT,
PartitionParameters.Partition.MAX_DISTANCE,
PartitionParameters.Partition.DISTANCE_THRESHOLDS,
PartitionParameters.Partition.PARTITIONER_CLASS,
StoreParam.INPUT_STORE,
OutputParameters.Output.HDFS_OUTPUT_PATH,
MapReduceParameters.MRConfig.HDFS_BASE_DIR,
OutputParameters.Output.REDUCER_COUNT,
OutputParameters.Output.OUTPUT_FORMAT,
InputParameters.Input.INPUT_FORMAT
},
new Object[] {
query,
Integer.toString(MapReduceTestUtils.MIN_INPUT_SPLITS),
Integer.toString(MapReduceTestUtils.MAX_INPUT_SPLITS),
0.2,
"0.2,0.2",
OrthodromicDistancePartitioner.class,
new PersistableStore(dataStorePluginOptions),
TestUtils.TEMP_DIR
+ File.separator
+ MapReduceTestEnvironment.HDFS_BASE_DIRECTORY
+ "/t1/pairs",
TestUtils.TEMP_DIR
+ File.separator
+ MapReduceTestEnvironment.HDFS_BASE_DIRECTORY
+ "/t1",
3,
SequenceFileOutputFormatConfiguration.class,
GeoWaveInputFormatConfiguration.class
}));
Assert.assertEquals(0, res);
Assert.assertTrue(readFile() > 0);
// for travis-ci to run, we want to limit the memory consumption
System.gc();
}
