/**
* Runs a range query on the local machine by iterating over the whole file.
*
* @param fs - FileSystem that contains input file
* @param file - path to the input file
* @param queryRange - The range to look in
* @param shape - An instance of the shape stored in file
* @param output - Output is sent to this collector. If <code>null</code>, output is not collected
* and only the number of results is returned.
* @return number of results found
* @throws IOException
*/
public static <S extends Shape> long rangeQueryLocal(
FileSystem fs, Path file, Shape queryRange, S shape, ResultCollector<S> output)
throws IOException {
long file_size = fs.getFileStatus(file).getLen();
ShapeRecordReader<S> shapeReader = new ShapeRecordReader<S>(fs.open(file), 0, file_size);
long resultCount = 0;
Prism cell = shapeReader.createKey();
while (shapeReader.next(cell, shape)) {
if (shape.isIntersected(queryRange)) {
boolean report_result;
if (cell.isValid()) {
// Check for duplicate avoidance
Prism intersection_mbr = queryRange.getMBR().getIntersection(shape.getMBR());
report_result =
cell.contains(intersection_mbr.t1, intersection_mbr.x1, intersection_mbr.y1);
} else {
report_result = true;
}
if (report_result) {
resultCount++;
if (output != null) {
output.collect(shape);
}
}
}
}
shapeReader.close();
return resultCount;
}
@Override
public void configure(JobConf job) {
super.configure(job);
try {
String queryShapeClassName = job.get(QUERY_SHAPE_CLASS);
Class<? extends Shape> queryShapeClass =
Class.forName(queryShapeClassName).asSubclass(Shape.class);
queryShape = queryShapeClass.newInstance();
queryShape.fromText(new Text(job.get(QUERY_SHAPE)));
queryMbr = queryShape.getMBR();
queryField = job.get(QUERY_FIELD);
} catch (ClassNotFoundException e) {
e.printStackTrace();
} catch (InstantiationException e) {
e.printStackTrace();
} catch (IllegalAccessException e) {
e.printStackTrace();
}
}
/**
* Performs a range query using MapReduce
*
* @param fs
* @param inputFile
* @param queryRange
* @param shape
* @param output
* @return
* @throws IOException
*/
public static long rangeQueryMapReduce(
FileSystem fs,
Path inputFile,
Path userOutputPath,
Shape queryShape,
Shape shape,
boolean overwrite,
boolean background,
QueryInput query)
throws IOException {
JobConf job = new JobConf(FileMBR.class);
FileSystem outFs = inputFile.getFileSystem(job);
Path outputPath = userOutputPath;
if (outputPath == null) {
do {
outputPath =
new Path(
inputFile.toUri().getPath() + ".rangequery_" + (int) (Math.random() * 1000000));
} while (outFs.exists(outputPath));
} else {
if (outFs.exists(outputPath)) {
if (overwrite) {
outFs.delete(outputPath, true);
} else {
throw new RuntimeException("Output path already exists and -overwrite flag is not set");
}
}
}
job.setJobName("RangeQuery");
job.setClass(SpatialSite.FilterClass, RangeFilter.class, BlockFilter.class);
RangeFilter.setQueryRange(job, queryShape); // Set query range for
// filter
ClusterStatus clusterStatus = new JobClient(job).getClusterStatus();
job.setNumMapTasks(clusterStatus.getMaxMapTasks() * 5);
job.setNumReduceTasks(3);
// Decide which map function to use depending on how blocks are indexed
// And also which input format to use
if (SpatialSite.isRTree(fs, inputFile)) {
// RTree indexed file
LOG.info("Searching an RTree indexed file");
job.setInputFormat(RTreeInputFormat.class);
} else {
// A file with no local index
LOG.info("Searching a non local-indexed file");
job.setInputFormat(ShapeInputFormat.class);
}
GlobalIndex<Partition> gIndex = SpatialSite.getGlobalIndex(fs, inputFile);
// if (gIndex != null && gIndex.isReplicated()){
// job.setMapperClass(RangeQueryMap.class);
Class<?> OutputKey = NullWritable.class;
try {
Class<?> c = shape.getClass();
Field f = c.getDeclaredField(query.field);
f.setAccessible(true);
if (f.getType().equals(Integer.TYPE)) {
OutputKey = IntWritable.class;
} else if (f.getType().equals(Double.TYPE)) {
OutputKey = DoubleWritable.class;
} else if (f.getType().equals(Long.TYPE)) {
OutputKey = LongWritable.class;
}
} catch (SecurityException e) {
e.printStackTrace();
} catch (NoSuchFieldException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
job.setMapOutputKeyClass(OutputKey);
switch (query.type) {
case Distinct:
job.setMapperClass(DistinctQueryMap.class);
job.setReducerClass(DistinctQueryReduce.class);
job.setMapOutputValueClass(NullWritable.class);
break;
case Distribution:
job.setMapperClass(DistributionQueryMap.class);
job.setReducerClass(DistributionQueryReduce.class);
job.setMapOutputValueClass(IntWritable.class);
break;
default:
break;
}
// }
// else
// job.setMapperClass(RangeQueryMapNoDupAvoidance.class);
// Set query range for the map function
job.set(QUERY_SHAPE_CLASS, queryShape.getClass().getName());
job.set(QUERY_SHAPE, queryShape.toText(new Text()).toString());
job.set(QUERY_FIELD, query.field);
// Set shape class for the SpatialInputFormat
SpatialSite.setShapeClass(job, shape.getClass());
job.setOutputFormat(TextOutputFormat.class);
ShapeInputFormat.setInputPaths(job, inputFile);
TextOutputFormat.setOutputPath(job, outputPath);
// Submit the job
if (!background) {
RunningJob runningJob = JobClient.runJob(job);
Counters counters = runningJob.getCounters();
Counter outputRecordCounter = counters.findCounter(Task.Counter.MAP_OUTPUT_RECORDS);
final long resultCount = outputRecordCounter.getValue();
// If outputPath not set by user, automatically delete it
if (userOutputPath == null) outFs.delete(outputPath, true);
return resultCount;
} else {
JobClient jc = new JobClient(job);
lastRunningJob = jc.submitJob(job);
return -1;
}
}
/** Map function for non-indexed blocks */
public void map(
final Prism cellMbr,
final Writable value,
final OutputCollector<Writable, IntWritable> output,
Reporter reporter)
throws IOException {
if (value instanceof Shape) {
try {
Shape shape = (Shape) value;
Class<?> c = shape.getClass();
Field f;
f = c.getDeclaredField(queryField);
f.setAccessible(true);
if (shape.isIntersected(queryShape)) {
boolean report_result = false;
if (cellMbr.isValid()) {
// Check for duplicate avoidance using reference
// point
// technique
double reference_t = Math.max(queryMbr.t1, shape.getMBR().t1);
double reference_x = Math.max(queryMbr.x1, shape.getMBR().x1);
double reference_y = Math.max(queryMbr.y1, shape.getMBR().y1);
report_result = cellMbr.contains(reference_t, reference_x, reference_y);
} else {
// A heap block, report right away
report_result = true;
}
if (report_result) {
Writable result = null;
if (f.getType().equals(Integer.TYPE)) {
try {
result = new IntWritable((int) f.get(shape));
} catch (IllegalArgumentException | IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else if (f.getType().equals(Long.TYPE)) {
try {
result = new LongWritable((int) f.get(shape));
} catch (IllegalArgumentException | IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else if (f.getType().equals(Double.TYPE)) {
try {
result = new DoubleWritable((int) f.get(shape));
} catch (IllegalArgumentException | IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
output.collect(result, one);
}
}
} catch (IllegalArgumentException | NoSuchFieldException | SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
} else if (value instanceof RTree) {
RTree<Shape> shapes = (RTree<Shape>) value;
if (shapes.columnar) {
shapes.searchColumnar(
queryMbr,
new ResultCollector<Writable>() {
@Override
public void collect(Writable shape) {
try {
output.collect(shape, one);
} catch (IOException e) {
e.printStackTrace();
}
}
},
queryField);
} else {
shapes.search(
queryMbr,
new ResultCollector<Shape>() {
@Override
public void collect(Shape shape) {
try {
Class<?> c = shape.getClass();
Field f;
f = c.getDeclaredField(queryField);
f.setAccessible(true);
Writable result = null;
if (f.getType().equals(Integer.class)) {
result = new IntWritable((int) f.get(shape));
} else if (f.getType().equals(Integer.class)) {
result = new IntWritable((int) f.get(shape));
} else if (f.getType().equals(Integer.class)) {
result = new IntWritable((int) f.get(shape));
}
output.collect(result, one);
} catch (IOException e) {
e.printStackTrace();
} catch (SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (NoSuchFieldException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
},
queryField);
}
}
}