public void testFormat() throws Exception {
    localFs = FileSystem.getLocal(defaultConf);
    localFs.delete(workDir, true);

    Job job = new Job(new Configuration(defaultConf));
    Path file = new Path(workDir, "test.txt");

    int seed = new Random().nextInt();
    Random random = new Random(seed);

    // for a variety of lengths
    for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
      // create a file with length entries
      Writer writer = new OutputStreamWriter(localFs.create(file));
      try {
        MyClass mc = new MyClass();
        for (int i = 0; i < length; i++) {
          mc.s = Integer.toString(i);
          mc.v = i;
          byte[] raw = MessagePack.pack(mc);
          byte[] b64e = base64_.encodeBase64(raw);
          byte[] b64d = base64_.decode(b64e);
          MyClass mc2 = MessagePack.unpack(b64d, mc.getClass());
          assertEquals(mc.s, mc2.s);
          assertEquals(mc.v, mc2.v);

          writer.write(base64_.encodeToString(raw));
        }
      } finally {
        writer.close();
      }
      checkFormat(job);
    }
  }
Пример #2
0
  public static void extractQueryFeatures2HDFS(String filename, Job job) throws IOException {

    // Read the local image.jpg as a Mat
    Mat query_mat_float =
        Highgui.imread(LOCAL_USER_DIR + ID + INPUT + "/" + filename, CvType.CV_32FC3);
    // Convert RGB to GRAY
    Mat query_gray = new Mat();
    Imgproc.cvtColor(query_mat_float, query_gray, Imgproc.COLOR_RGB2GRAY);
    // Convert the float type to unsigned integer(required by SIFT)
    Mat query_mat_byte = new Mat();
    query_gray.convertTo(query_mat_byte, CvType.CV_8UC3);
    //        // Resize the image to 1/FACTOR both width and height
    //        Mat query_mat_byte = FeatureExtraction.resize(query_mat_byte);

    // Extract the feature from the (Mat)image
    Mat query_features = FeatureExtraction.extractFeature(query_mat_byte);

    System.out.println(PREFIX + "Extracting the query image feature...");
    System.out.println("query_mat(float,color):" + query_mat_float);
    System.out.println("query_mat(float,gray):" + query_gray);
    System.out.println("query_mat(byte,gray):" + query_mat_byte);
    System.out.println("query_mat_features:" + query_features);
    System.out.println();

    // Store the feature to the hdfs in order to use it later in different map tasks
    System.out.println(PREFIX + "Generating the feature file for the query image in HDFS...");
    FileSystem fs = FileSystem.get(job.getConfiguration());
    String featureFileName = filename.substring(0, filename.lastIndexOf(".")) + ".json";
    FSDataOutputStream fsDataOutputStream =
        fs.create(new Path(HDFS_HOME + USER + ID + INPUT + "/" + featureFileName));
    BufferedWriter bw =
        new BufferedWriter(new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8));
    bw.write(FeatureExtraction.mat2json(query_features));
    bw.close();
    System.out.println(PREFIX + "Query feature extraction finished...");
    System.out.println();
  }