Ejemplo n.º 1
0
  public void run(String[] args)
      throws ParseException, TransformationException, IOException, AnalysisException {
    Options options = new Options();
    options.addOption("input", true, "input path");
    options.addOption("output", true, "output path");
    options.addOption("ext", true, "extension");
    CommandLineParser parser = new DefaultParser();
    CommandLine line = parser.parse(options, args);
    String inDir = line.getOptionValue("input");
    String outDir = line.getOptionValue("output");
    String extension = line.getOptionValue("ext");

    File dir = new File(inDir);

    for (File f : FileUtils.listFiles(dir, new String[] {extension}, true)) {
      TrueVizToBxDocumentReader tvReader = new TrueVizToBxDocumentReader();
      List<BxPage> pages = tvReader.read(new FileReader(f));
      BxDocument doc = new BxDocument().setPages(pages);
      doc.setFilename(f.getName());

      BxDocument rewritten = transform(doc);

      File f2 = new File(outDir + doc.getFilename());
      BxDocumentToTrueVizWriter wrt = new BxDocumentToTrueVizWriter();
      boolean created = f2.createNewFile();
      if (!created) {
        throw new IOException("Cannot create file: ");
      }
      FileWriter fw = new FileWriter(f2);
      wrt.write(fw, Lists.newArrayList(rewritten));
      fw.flush();
      fw.close();
    }
  }
Ejemplo n.º 2
0
  public static void main(String[] args)
      throws ParseException, IOException, TransformationException, AnalysisException,
          CloneNotSupportedException {
    Options options = new Options();

    CommandLineParser parser = new GnuParser();
    CommandLine line = parser.parse(options, args);

    if (args.length != 2) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.printHelp(" [-options] input-directory extension", options);
      System.exit(1);
    }
    String inputDirPath = line.getArgs()[0];
    File inputDirFile = new File(inputDirPath);

    Integer docIdx = 0;

    HierarchicalReadingOrderResolver ror = new HierarchicalReadingOrderResolver();
    DocumentsIterator iter = new DocumentsIterator(inputDirPath, line.getArgs()[1]);

    FeatureVectorBuilder<BxZone, BxPage> metaVectorBuilder =
        SVMMetadataZoneClassifier.getFeatureVectorBuilder();
    FeatureVectorBuilder<BxZone, BxPage> initialVectorBuilder =
        SVMInitialZoneClassifier.getFeatureVectorBuilder();

    SampleFilter metaSamplesFilter = new SampleFilter(BxZoneLabelCategory.CAT_METADATA);

    FileWriter initialStream = new FileWriter("initial_" + inputDirFile.getName() + ".dat");
    BufferedWriter svmInitialFile = new BufferedWriter(initialStream);

    FileWriter metaStream = new FileWriter("meta_" + inputDirFile.getName() + ".dat");
    BufferedWriter svmMetaFile = new BufferedWriter(metaStream);

    for (BxDocument doc : iter) {
      System.out.println(docIdx + ": " + doc.getFilename());
      String filename = doc.getFilename();
      doc = ror.resolve(doc);
      doc.setFilename(filename);

      for (BxZone zone : doc.asZones()) {
        if (zone.getLabel() != null) {
          if (zone.getLabel().getCategory() != BxZoneLabelCategory.CAT_METADATA) {
            zone.setLabel(zone.getLabel().getGeneralLabel());
          }
        } else {
          zone.setLabel(BxZoneLabel.OTH_UNKNOWN);
        }
      }
      List<TrainingSample<BxZoneLabel>> newMetaSamples =
          BxDocsToTrainingSamplesConverter.getZoneTrainingSamples(
              doc, metaVectorBuilder, BxZoneLabel.getIdentityMap());
      newMetaSamples = metaSamplesFilter.pickElements(newMetaSamples);

      List<TrainingSample<BxZoneLabel>> newInitialSamples =
          BxDocsToTrainingSamplesConverter.getZoneTrainingSamples(
              doc, initialVectorBuilder, BxZoneLabel.getLabelToGeneralMap());

      for (TrainingSample<BxZoneLabel> sample : newMetaSamples) {
        toLibSVM(sample, svmMetaFile);
      }
      for (TrainingSample<BxZoneLabel> sample : newInitialSamples) {
        toLibSVM(sample, svmInitialFile);
      }
      ++docIdx;
    }
    svmInitialFile.close();
    svmMetaFile.close();
  }