Example #1
0
  private void indexFiles(ArrayList<String> images, DocumentBuilder builder, String indexPath)
      throws IOException {
    // eventually check if the directory is there or not ...
    IndexWriter iw = LuceneUtils.createIndexWriter(testIndex, false);
    int count = 0;
    long time = System.currentTimeMillis();
    for (String identifier : images) {
      // TODO: cut toes from the image ... -> doesn't work out very well. Stable at first,
      // decreasing then.
      // TODO: Joint Histogram ...
      // TODO: LSA / PCA on the vectors ...-> this looks like a job for me :-D
      // TODO: local features ...
      Document doc = null;
      if (cutImages) {
        BufferedImage bimg =
            ImageUtils.cropImage(ImageIO.read(new FileInputStream(identifier)), 0, 0, 200, 69);
        doc = builder.createDocument(bimg, identifier);
      } else doc = builder.createDocument(new FileInputStream(identifier), identifier);
      iw.addDocument(doc);
      count++;
      if (count % 100 == 0) {
        int percent = (int) Math.floor(((double) count * 100.0) / (double) images.size());
        double timeTemp = (double) (System.currentTimeMillis() - time) / 1000d;
        int secsLeft =
            (int) Math.round(((timeTemp / (double) count) * (double) images.size()) - timeTemp);
        System.out.println(percent + "% finished (" + count + " files), " + secsLeft + " s left");
      }
    }
    long timeTaken = (System.currentTimeMillis() - time);
    float sec = ((float) timeTaken) / 1000f;

    System.out.println(sec + " seconds taken, " + (timeTaken / count) + " ms per image.");
    iw.commit();
    iw.close();
  }
  public double testIndexing() throws IOException, IllegalAccessException, InstantiationException {
    LocalitySensitiveHashing.generateHashFunctions();
    LocalitySensitiveHashing.readHashFunctions();
    DocumentBuilder builder = new ChainedDocumentBuilder();
    ((ChainedDocumentBuilder) builder).addBuilder(DocumentBuilderFactory.getCEDDDocumentBuilder());

    //        System.out.println("-< Getting files to index >--------------");
    ArrayList<String> images = FileUtils.getAllImages(new File(testExtensive), true);
    //        System.out.println("-< Indexing " + images.size() + " files >--------------");

    IndexWriter iw =
        LuceneUtils.createIndexWriter(indexPath, true, LuceneUtils.AnalyzerType.WhitespaceAnalyzer);
    int count = 0;
    long time = System.currentTimeMillis();
    for (String identifier : images) {
      CEDD cedd = new CEDD();
      cedd.extract(ImageIO.read(new FileInputStream(identifier)));
      Document doc = new Document();
      doc.add(new Field(DocumentBuilder.FIELD_NAME_CEDD, cedd.getByteArrayRepresentation()));
      doc.add(
          new Field(
              DocumentBuilder.FIELD_NAME_IDENTIFIER,
              identifier,
              Field.Store.YES,
              Field.Index.NOT_ANALYZED));
      int[] hashes = LocalitySensitiveHashing.generateHashes(cedd.getDoubleHistogram());
      StringBuilder hash = new StringBuilder(512);
      for (int i = 0; i < hashes.length; i++) {
        hash.append(hashes[i]);
        hash.append(' ');
      }
      //            System.out.println("hash = " + hash);
      doc.add(new Field("hash", hash.toString(), Field.Store.YES, Field.Index.ANALYZED));
      iw.addDocument(doc);
      count++;
      //            if (count % 100 == 0) System.out.println(count + " files indexed.");
    }
    long timeTaken = (System.currentTimeMillis() - time);
    float sec = ((float) timeTaken) / 1000f;

    //        System.out.println(sec + " seconds taken, " + (timeTaken / count) + " ms per image.");
    iw.close();

    return testSearch();
  }
Example #3
0
  private void indexFiles(ArrayList<String> images, DocumentBuilder builder, String indexPath)
      throws IOException {
    //        System.out.println(">> Indexing " + images.size() + " files.");
    //        DocumentBuilder builder = DocumentBuilderFactory.getExtensiveDocumentBuilder();
    //        DocumentBuilder builder = DocumentBuilderFactory.getFastDocumentBuilder();
    IndexWriter iw = LuceneUtils.createIndexWriter(indexPath, true);
    int count = 0;
    long time = System.currentTimeMillis();
    for (String identifier : images) {
      Document doc = builder.createDocument(new FileInputStream(identifier), identifier);
      iw.addDocument(doc);
      count++;
      if (count % 100 == 0) System.out.println(count + " files indexed.");
      //            if (count == 200) break;
    }
    long timeTaken = (System.currentTimeMillis() - time);
    float sec = ((float) timeTaken) / 1000f;

    System.out.println(sec + " seconds taken, " + (timeTaken / count) + " ms per image.");
    iw.commit();
    iw.close();
  }
Example #4
0
  protected void setUp() throws Exception {
    super.setUp();
    // set to all queries ... approach "leave one out"
    sampleQueries = new int[1000];
    for (int i = 0; i < sampleQueries.length; i++) {
      sampleQueries[i] = i;
    }
    indexPath += "-" + System.currentTimeMillis() % (1000 * 60 * 60 * 24 * 7);
    // Setting up DocumentBuilder:
    //        parallelIndexer = new ParallelIndexer(8, indexPath, testExtensive);
    parallelIndexer =
        new ParallelIndexer(8, indexPath, testExtensive, true) {
          @Override
          public void addBuilders(ChainedDocumentBuilder builder) {
            //                builder.addBuilder(DocumentBuilderFactory.getCEDDDocumentBuilder());
            //
            // builder.addBuilder(DocumentBuilderFactory.getAutoColorCorrelogramDocumentBuilder());
            builder.addBuilder(DocumentBuilderFactory.getColorLayoutBuilder());
            builder.addBuilder(DocumentBuilderFactory.getEdgeHistogramBuilder());
            //                builder.addBuilder(DocumentBuilderFactory.getFCTHDocumentBuilder());
            builder.addBuilder(DocumentBuilderFactory.getJCDDocumentBuilder());
            //
            // builder.addBuilder(DocumentBuilderFactory.getJointHistogramDocumentBuilder());
            builder.addBuilder(DocumentBuilderFactory.getOpponentHistogramDocumentBuilder());
            builder.addBuilder(DocumentBuilderFactory.getPHOGDocumentBuilder());
            //
            // builder.addBuilder(DocumentBuilderFactory.getColorHistogramDocumentBuilder());
            builder.addBuilder(DocumentBuilderFactory.getScalableColorBuilder());

            //
            // builder.addBuilder(DocumentBuilderFactory.getLuminanceLayoutDocumentBuilder());
            //
            // builder.addBuilder(DocumentBuilderFactory.getJpegCoefficientHistogramDocumentBuilder());
            //
            // builder.addBuilder(DocumentBuilderFactory.getColorHistogramDocumentBuilder());
            //               builder.addBuilder(DocumentBuilderFactory.getGaborDocumentBuilder());
            //               builder.addBuilder(DocumentBuilderFactory.getTamuraDocumentBuilder());
            //               builder.addBuilder(DocumentBuilderFactory.getScalableColorBuilder());
            //               builder.addBuilder(new GenericDocumentBuilder(RankAndOpponent.class,
            // "jop"));
            //               builder.addBuilder(new
            // GenericFastDocumentBuilder(FuzzyOpponentHistogram.class, "opHist"));
            //               builder.addBuilder(new SurfDocumentBuilder());
            //               builder.addBuilder(new MSERDocumentBuilder());
            //               builder.addBuilder(new SiftDocumentBuilder());

            //                builder.addBuilder(new GenericDocumentBuilder(SPCEDD.class,
            // "spcedd"));
            //                builder.addBuilder(new GenericDocumentBuilder(SPFCTH.class,
            // "spfcth"));
            //                builder.addBuilder(new GenericDocumentBuilder(SPJCD.class, "spjcd"));
            //                builder.addBuilder(new GenericDocumentBuilder(SPACC.class, "spacc"));
            //                builder.addBuilder(new
            // GenericDocumentBuilder(LocalBinaryPatterns.class, "lbp"));
            //                builder.addBuilder(new
            // GenericDocumentBuilder(BinaryPatternsPyramid.class, "whog"));
            //                builder.addBuilder(new
            // GenericDocumentBuilder(LocalBinaryPatternsAndOpponent.class, "jhl"));
            //                builder.addBuilder(new
            // GenericDocumentBuilder(RotationInvariantLocalBinaryPatterns.class, "rlbp"));
            //                builder.addBuilder(new GenericDocumentBuilder(SPLBP.class, "splbp"));

          }
        };
  }
Example #5
0
 public void computeMAP(ImageSearcher searcher, String prefix, IndexReader reader)
     throws IOException {
   Pattern p = Pattern.compile("([0-9]+).jpg");
   double map = 0;
   double errorRate = 0d;
   double precision10 = 0d;
   double[] pr10cat = new double[10];
   double[] pr10cnt = new double[10];
   for (int i = 0; i < pr10cat.length; i++) {
     pr10cat[i] = 0d;
     pr10cnt[i] = 0d;
   }
   long sum = 0, ms = 0;
   for (int i = 0; i < sampleQueries.length; i++) {
     int id = sampleQueries[i];
     String file = testExtensive + "/" + id + ".jpg";
     ms = System.currentTimeMillis();
     ImageSearchHits hits = searcher.search(findDoc(reader, id + ".jpg"), reader);
     sum += (System.currentTimeMillis() - ms);
     int goodOnes = 0;
     double avgPrecision = 0d;
     double precision10temp = 0d;
     int countResults = 0;
     for (int j = 0; j < hits.length(); j++) {
       Document d = hits.doc(j);
       String hitsId = d.getValues(DocumentBuilder.FIELD_NAME_IDENTIFIER)[0];
       Matcher matcher = p.matcher(hitsId);
       if (matcher.find()) hitsId = matcher.group(1);
       else fail("Did not get the number ...");
       int testID = Integer.parseInt(hitsId);
       if (testID != id) countResults++;
       if ((testID != id) && ((int) Math.floor(id / 100) == (int) Math.floor(testID / 100))) {
         goodOnes++;
         // Only if there is a change in recall
         avgPrecision += (double) goodOnes / (double) countResults;
         //                    System.out.print("x");
         if (j <= 10) {
           precision10temp += 1d;
         }
       } else {
         if (j == 1) { // error rate
           errorRate++;
         }
       }
     } // end for loop iterating results.
     //            if (avgPrecision<=0) {
     //                System.out.println("avgPrecision = " + avgPrecision);
     //                System.out.println("goodOnes = " + goodOnes);
     //            }
     assertTrue("Check if average precision is > 0", avgPrecision > 0);
     assertTrue("Check if goodOnes is > 0", goodOnes > 0);
     avgPrecision = avgPrecision / goodOnes;
     precision10 += precision10temp / 10d;
     // precision @ 10 for each category ...
     pr10cat[(int) Math.floor(id / 100)] += precision10temp / 10d;
     pr10cnt[(int) Math.floor(id / 100)] += 1d;
     map += avgPrecision;
   }
   map = map / sampleQueries.length;
   errorRate = errorRate / sampleQueries.length;
   precision10 = precision10 / sampleQueries.length;
   System.out.print(prefix + "\t");
   System.out.print(String.format("%.5f", map) + '\t');
   System.out.print(String.format("%.5f", precision10) + '\t');
   System.out.print(String.format("%.5f", errorRate) + '\t');
   // precision@10 per category
   for (int i = 0; i < pr10cat.length; i++) {
     double v = 0;
     if (pr10cnt[i] > 0) v = pr10cat[i] / pr10cnt[i];
     //            System.out.print(i + ": ");
     System.out.printf("%.5f\t", v);
   }
   System.out.printf("%2.3f\t", (double) sum / (double) sampleQueries.length);
   System.out.println();
 }