private void fillLists(
      String zipFileName,
      List<BxDocument> documents,
      List<DocumentContentStructure> headerStructures)
      throws IOException, TransformationException, JDOMException, URISyntaxException {
    ZipFile zipFile = new ZipFile(new File(this.getClass().getResource(zipFileName).toURI()));
    List<ZipEntry> entries = getEntries(zipFile);

    HierarchicalReadingOrderResolver roa = new HierarchicalReadingOrderResolver();
    TrueVizToBxDocumentReader bxReader = new TrueVizToBxDocumentReader();
    HTMLToDocContentStructReader dcsReader = new HTMLToDocContentStructReader();

    for (ZipEntry ze : entries) {
      if (ze.getName().matches("^.*/" + sourceDir + ".*$")) {
        InputStream xis = zipFile.getInputStream(ze);
        InputStreamReader xisr = new InputStreamReader(xis);

        System.out.println(ze.getName());
        List<BxPage> pages = bxReader.read(xisr);
        documents.add(roa.resolve(new BxDocument().setPages(pages)));
      }
      if (ze.getName().matches("^.*/" + structureDir + ".*$")) {
        InputStream cis = zipFile.getInputStream(ze);
        InputStreamReader cisr = new InputStreamReader(cis);

        DocumentContentStructure hs = dcsReader.read(cisr);
        headerStructures.add(hs);
      }
    }
  }
Ejemplo n.º 2
0
  public static void main(String[] args)
      throws ParseException, IOException, TransformationException, AnalysisException,
          CloneNotSupportedException {
    Options options = new Options();

    CommandLineParser parser = new GnuParser();
    CommandLine line = parser.parse(options, args);

    if (args.length != 2) {
      HelpFormatter formatter = new HelpFormatter();
      formatter.printHelp(" [-options] input-directory extension", options);
      System.exit(1);
    }
    String inputDirPath = line.getArgs()[0];
    File inputDirFile = new File(inputDirPath);

    Integer docIdx = 0;

    HierarchicalReadingOrderResolver ror = new HierarchicalReadingOrderResolver();
    DocumentsIterator iter = new DocumentsIterator(inputDirPath, line.getArgs()[1]);

    FeatureVectorBuilder<BxZone, BxPage> metaVectorBuilder =
        SVMMetadataZoneClassifier.getFeatureVectorBuilder();
    FeatureVectorBuilder<BxZone, BxPage> initialVectorBuilder =
        SVMInitialZoneClassifier.getFeatureVectorBuilder();

    SampleFilter metaSamplesFilter = new SampleFilter(BxZoneLabelCategory.CAT_METADATA);

    FileWriter initialStream = new FileWriter("initial_" + inputDirFile.getName() + ".dat");
    BufferedWriter svmInitialFile = new BufferedWriter(initialStream);

    FileWriter metaStream = new FileWriter("meta_" + inputDirFile.getName() + ".dat");
    BufferedWriter svmMetaFile = new BufferedWriter(metaStream);

    for (BxDocument doc : iter) {
      System.out.println(docIdx + ": " + doc.getFilename());
      String filename = doc.getFilename();
      doc = ror.resolve(doc);
      doc.setFilename(filename);

      for (BxZone zone : doc.asZones()) {
        if (zone.getLabel() != null) {
          if (zone.getLabel().getCategory() != BxZoneLabelCategory.CAT_METADATA) {
            zone.setLabel(zone.getLabel().getGeneralLabel());
          }
        } else {
          zone.setLabel(BxZoneLabel.OTH_UNKNOWN);
        }
      }
      List<TrainingSample<BxZoneLabel>> newMetaSamples =
          BxDocsToTrainingSamplesConverter.getZoneTrainingSamples(
              doc, metaVectorBuilder, BxZoneLabel.getIdentityMap());
      newMetaSamples = metaSamplesFilter.pickElements(newMetaSamples);

      List<TrainingSample<BxZoneLabel>> newInitialSamples =
          BxDocsToTrainingSamplesConverter.getZoneTrainingSamples(
              doc, initialVectorBuilder, BxZoneLabel.getLabelToGeneralMap());

      for (TrainingSample<BxZoneLabel> sample : newMetaSamples) {
        toLibSVM(sample, svmMetaFile);
      }
      for (TrainingSample<BxZoneLabel> sample : newInitialSamples) {
        toLibSVM(sample, svmInitialFile);
      }
      ++docIdx;
    }
    svmInitialFile.close();
    svmMetaFile.close();
  }