示例#1
0
  /**
   * Runs a range query on the local machine by iterating over the whole file.
   *
   * @param fs - FileSystem that contains input file
   * @param file - path to the input file
   * @param queryRange - The range to look in
   * @param shape - An instance of the shape stored in file
   * @param output - Output is sent to this collector. If <code>null</code>, output is not collected
   *     and only the number of results is returned.
   * @return number of results found
   * @throws IOException
   */
  public static <S extends Shape> long rangeQueryLocal(
      FileSystem fs, Path file, Shape queryRange, S shape, ResultCollector<S> output)
      throws IOException {
    long file_size = fs.getFileStatus(file).getLen();
    ShapeRecordReader<S> shapeReader = new ShapeRecordReader<S>(fs.open(file), 0, file_size);

    long resultCount = 0;
    Prism cell = shapeReader.createKey();

    while (shapeReader.next(cell, shape)) {
      if (shape.isIntersected(queryRange)) {
        boolean report_result;
        if (cell.isValid()) {
          // Check for duplicate avoidance
          Prism intersection_mbr = queryRange.getMBR().getIntersection(shape.getMBR());
          report_result =
              cell.contains(intersection_mbr.t1, intersection_mbr.x1, intersection_mbr.y1);
        } else {
          report_result = true;
        }
        if (report_result) {
          resultCount++;
          if (output != null) {
            output.collect(shape);
          }
        }
      }
    }
    shapeReader.close();
    return resultCount;
  }
示例#2
0
 @Override
 public void configure(JobConf job) {
   super.configure(job);
   try {
     String queryShapeClassName = job.get(QUERY_SHAPE_CLASS);
     Class<? extends Shape> queryShapeClass =
         Class.forName(queryShapeClassName).asSubclass(Shape.class);
     queryShape = queryShapeClass.newInstance();
     queryShape.fromText(new Text(job.get(QUERY_SHAPE)));
     queryMbr = queryShape.getMBR();
     queryField = job.get(QUERY_FIELD);
   } catch (ClassNotFoundException e) {
     e.printStackTrace();
   } catch (InstantiationException e) {
     e.printStackTrace();
   } catch (IllegalAccessException e) {
     e.printStackTrace();
   }
 }
示例#3
0
  /**
   * Performs a range query using MapReduce
   *
   * @param fs
   * @param inputFile
   * @param queryRange
   * @param shape
   * @param output
   * @return
   * @throws IOException
   */
  public static long rangeQueryMapReduce(
      FileSystem fs,
      Path inputFile,
      Path userOutputPath,
      Shape queryShape,
      Shape shape,
      boolean overwrite,
      boolean background,
      QueryInput query)
      throws IOException {
    JobConf job = new JobConf(FileMBR.class);

    FileSystem outFs = inputFile.getFileSystem(job);
    Path outputPath = userOutputPath;
    if (outputPath == null) {
      do {
        outputPath =
            new Path(
                inputFile.toUri().getPath() + ".rangequery_" + (int) (Math.random() * 1000000));
      } while (outFs.exists(outputPath));
    } else {
      if (outFs.exists(outputPath)) {
        if (overwrite) {
          outFs.delete(outputPath, true);
        } else {
          throw new RuntimeException("Output path already exists and -overwrite flag is not set");
        }
      }
    }

    job.setJobName("RangeQuery");
    job.setClass(SpatialSite.FilterClass, RangeFilter.class, BlockFilter.class);
    RangeFilter.setQueryRange(job, queryShape); // Set query range for
    // filter

    ClusterStatus clusterStatus = new JobClient(job).getClusterStatus();
    job.setNumMapTasks(clusterStatus.getMaxMapTasks() * 5);
    job.setNumReduceTasks(3);

    // Decide which map function to use depending on how blocks are indexed
    // And also which input format to use
    if (SpatialSite.isRTree(fs, inputFile)) {
      // RTree indexed file
      LOG.info("Searching an RTree indexed file");
      job.setInputFormat(RTreeInputFormat.class);
    } else {
      // A file with no local index
      LOG.info("Searching a non local-indexed file");
      job.setInputFormat(ShapeInputFormat.class);
    }

    GlobalIndex<Partition> gIndex = SpatialSite.getGlobalIndex(fs, inputFile);
    // if (gIndex != null && gIndex.isReplicated()){
    // job.setMapperClass(RangeQueryMap.class);

    Class<?> OutputKey = NullWritable.class;
    try {
      Class<?> c = shape.getClass();
      Field f = c.getDeclaredField(query.field);
      f.setAccessible(true);
      if (f.getType().equals(Integer.TYPE)) {
        OutputKey = IntWritable.class;
      } else if (f.getType().equals(Double.TYPE)) {
        OutputKey = DoubleWritable.class;
      } else if (f.getType().equals(Long.TYPE)) {
        OutputKey = LongWritable.class;
      }
    } catch (SecurityException e) {
      e.printStackTrace();
    } catch (NoSuchFieldException e) {
      // TODO Auto-generated catch block
      e.printStackTrace();
    }

    job.setMapOutputKeyClass(OutputKey);
    switch (query.type) {
      case Distinct:
        job.setMapperClass(DistinctQueryMap.class);
        job.setReducerClass(DistinctQueryReduce.class);
        job.setMapOutputValueClass(NullWritable.class);
        break;
      case Distribution:
        job.setMapperClass(DistributionQueryMap.class);
        job.setReducerClass(DistributionQueryReduce.class);
        job.setMapOutputValueClass(IntWritable.class);
        break;
      default:
        break;
    }
    // }
    // else
    // job.setMapperClass(RangeQueryMapNoDupAvoidance.class);

    // Set query range for the map function
    job.set(QUERY_SHAPE_CLASS, queryShape.getClass().getName());
    job.set(QUERY_SHAPE, queryShape.toText(new Text()).toString());
    job.set(QUERY_FIELD, query.field);

    // Set shape class for the SpatialInputFormat
    SpatialSite.setShapeClass(job, shape.getClass());

    job.setOutputFormat(TextOutputFormat.class);

    ShapeInputFormat.setInputPaths(job, inputFile);
    TextOutputFormat.setOutputPath(job, outputPath);

    // Submit the job
    if (!background) {
      RunningJob runningJob = JobClient.runJob(job);
      Counters counters = runningJob.getCounters();
      Counter outputRecordCounter = counters.findCounter(Task.Counter.MAP_OUTPUT_RECORDS);
      final long resultCount = outputRecordCounter.getValue();

      // If outputPath not set by user, automatically delete it
      if (userOutputPath == null) outFs.delete(outputPath, true);

      return resultCount;
    } else {
      JobClient jc = new JobClient(job);
      lastRunningJob = jc.submitJob(job);
      return -1;
    }
  }
示例#4
0
    /** Map function for non-indexed blocks */
    public void map(
        final Prism cellMbr,
        final Writable value,
        final OutputCollector<Writable, IntWritable> output,
        Reporter reporter)
        throws IOException {
      if (value instanceof Shape) {
        try {
          Shape shape = (Shape) value;
          Class<?> c = shape.getClass();
          Field f;
          f = c.getDeclaredField(queryField);
          f.setAccessible(true);

          if (shape.isIntersected(queryShape)) {
            boolean report_result = false;
            if (cellMbr.isValid()) {
              // Check for duplicate avoidance using reference
              // point
              // technique
              double reference_t = Math.max(queryMbr.t1, shape.getMBR().t1);
              double reference_x = Math.max(queryMbr.x1, shape.getMBR().x1);
              double reference_y = Math.max(queryMbr.y1, shape.getMBR().y1);
              report_result = cellMbr.contains(reference_t, reference_x, reference_y);
            } else {
              // A heap block, report right away
              report_result = true;
            }

            if (report_result) {
              Writable result = null;
              if (f.getType().equals(Integer.TYPE)) {
                try {
                  result = new IntWritable((int) f.get(shape));
                } catch (IllegalArgumentException | IllegalAccessException e) {
                  // TODO Auto-generated catch block
                  e.printStackTrace();
                }
              } else if (f.getType().equals(Long.TYPE)) {
                try {
                  result = new LongWritable((int) f.get(shape));
                } catch (IllegalArgumentException | IllegalAccessException e) {
                  // TODO Auto-generated catch block
                  e.printStackTrace();
                }
              } else if (f.getType().equals(Double.TYPE)) {
                try {
                  result = new DoubleWritable((int) f.get(shape));
                } catch (IllegalArgumentException | IllegalAccessException e) {
                  // TODO Auto-generated catch block
                  e.printStackTrace();
                }
              }
              output.collect(result, one);
            }
          }
        } catch (IllegalArgumentException | NoSuchFieldException | SecurityException e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      } else if (value instanceof RTree) {
        RTree<Shape> shapes = (RTree<Shape>) value;
        if (shapes.columnar) {
          shapes.searchColumnar(
              queryMbr,
              new ResultCollector<Writable>() {
                @Override
                public void collect(Writable shape) {
                  try {
                    output.collect(shape, one);
                  } catch (IOException e) {
                    e.printStackTrace();
                  }
                }
              },
              queryField);
        } else {
          shapes.search(
              queryMbr,
              new ResultCollector<Shape>() {
                @Override
                public void collect(Shape shape) {
                  try {
                    Class<?> c = shape.getClass();
                    Field f;
                    f = c.getDeclaredField(queryField);
                    f.setAccessible(true);
                    Writable result = null;
                    if (f.getType().equals(Integer.class)) {
                      result = new IntWritable((int) f.get(shape));
                    } else if (f.getType().equals(Integer.class)) {
                      result = new IntWritable((int) f.get(shape));
                    } else if (f.getType().equals(Integer.class)) {
                      result = new IntWritable((int) f.get(shape));
                    }
                    output.collect(result, one);

                  } catch (IOException e) {
                    e.printStackTrace();
                  } catch (SecurityException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                  } catch (IllegalArgumentException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                  } catch (NoSuchFieldException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                  } catch (IllegalAccessException e) {
                    // TODO Auto-generated catch block
                    e.printStackTrace();
                  }
                }
              },
              queryField);
        }
      }
    }