コード例 #1
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void duplicateRow() {
   Row row = new Row(5);
   row.flagged = true;
   Row duplicateRow = row.dup();
   Assert.assertTrue(duplicateRow.flagged);
 }
コード例 #2
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void nonBlankCell() {
   Row row = new Row(5);
   row.setCell(0, new Cell("I'm not empty", null));
   Assert.assertFalse(row.isCellBlank(0));
   row.setCell(3, new Cell("I'm not empty", null));
   Assert.assertFalse(row.isCellBlank(3));
 }
コード例 #3
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void saveRow() {
   Row row = new Row(5);
   row.setCell(0, new Cell("I'm not empty", null));
   row.save(writer, options);
   Assert.assertEquals(
       writer.getBuffer().toString(),
       "{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}]}");
 }
コード例 #4
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void toStringTest() {
   Row row = new Row(5);
   row.setCell(0, new Cell(1, null));
   row.setCell(1, new Cell(2, null));
   row.setCell(2, new Cell(3, null));
   row.setCell(3, new Cell(4, null));
   row.setCell(4, new Cell(5, null));
   Assert.assertEquals(row.toString(), "1,2,3,4,5,");
 }
コード例 #5
0
    @Test
    public void canParseSample(){
        RunTest(getSample());

        log(project);
        assertProjectCreated(project, 4, 6);

        Row row = project.rows.get(0);
        Assert.assertNotNull(row);
        Assert.assertNotNull(row.getCell(1));
        Assert.assertEquals(row.getCell(1).value, "Author 1, The");
    }
コード例 #6
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void saveRowWithRecordIndex() {
   Row row = new Row(5);
   row.setCell(0, new Cell("I'm not empty", null));
   when(options.containsKey("rowIndex")).thenReturn(true);
   when(options.get("rowIndex")).thenReturn(0);
   when(options.containsKey("recordIndex")).thenReturn(true);
   when(options.get("recordIndex")).thenReturn(1);
   row.save(writer, options);
   Assert.assertEquals(
       writer.getBuffer().toString(),
       "{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}],\"i\":0,\"j\":1}");
 }
コード例 #7
0
    @Test
    public void testCanParseLineBreak(){

        RunTest(getSampleWithLineBreak());

        log(project);
        assertProjectCreated(project, 4, 6);

        Row row = project.rows.get(3);
        Assert.assertNotNull(row);
        Assert.assertEquals(row.cells.size(), 4);
        Assert.assertNotNull(row.getCell(1));
        Assert.assertEquals(row.getCell(1).value, "With line\n break");
    }
コード例 #8
0
    @Test
    public void canParseSampleWithDuplicateNestedElements(){
        RunTest(getSampleWithDuplicateNestedElements());

        log(project);
        assertProjectCreated(project, 4, 12);

        Row row = project.rows.get(0);
        Assert.assertNotNull(row);
        Assert.assertEquals(row.cells.size(), 4);
        Assert.assertNotNull(row.getCell(1));
        Assert.assertEquals(row.getCell(1).value, "Author 1, The");
        Assert.assertEquals(project.rows.get(1).getCell(1).value, "Author 1, Another");
    }
コード例 #9
0
  protected static void addImportRecordToProject(ImportRecord record, Project project) {
    if (record.rows.size() > 0) {
      for (List<Cell> row : record.rows) {
        Row realRow = new Row(row.size());
        int cellCount = 0;

        for (int c = 0; c < row.size(); c++) {
          Cell cell = row.get(c);
          if (cell != null) {
            realRow.setCell(c, cell);
            cellCount++;
          }
        }

        if (cellCount > 0) {
          project.rows.add(realRow);
        }
      }
    }
  }
コード例 #10
0
  protected void extendRow(
      Row row, DataExtension dataExtension, int extensionRowIndex, Map<String, Recon> reconMap) {
    Object[] values = dataExtension.data[extensionRowIndex];
    for (int c = 0; c < values.length; c++) {
      Object value = values[c];
      Cell cell = null;

      if (value instanceof ReconCandidate) {
        ReconCandidate rc = (ReconCandidate) value;
        Recon recon;
        if (reconMap.containsKey(rc.id)) {
          recon = reconMap.get(rc.id);
        } else {

          if (rc.id.equals("")) {
            cell = new Cell(rc.name, null);
          } else {

            recon =
                new DBpediaDataExtensionReconConfig(new DBpediaType(rc.name, rc.id))
                    .createNewRecon(_historyEntryID);
            recon.addCandidate(rc);
            recon.service = "dbpedia-extension";
            recon.match = rc;
            recon.matchRank = 0;
            recon.judgment = Judgment.Matched;
            recon.judgmentAction = "auto";
            recon.judgmentBatchSize = 1;

            reconMap.put(rc.id, recon);
            cell = new Cell(rc.name, recon);
          }
        }

      } else {
        cell = new Cell((Serializable) value, null);
      }

      row.setCell(_firstNewCellIndex + c, cell);
    }
  }
コード例 #11
0
  protected void extendRow(
      Row row, DataExtension dataExtension, int extensionRowIndex, Map<String, Recon> reconMap) {
    Object[] values = dataExtension.data[extensionRowIndex];
    for (int c = 0; c < values.length; c++) {
      Object value = values[c];
      Cell cell = null;

      if (value instanceof ReconCandidate) {
        ReconCandidate rc = (ReconCandidate) value;
        Recon recon;
        if (reconMap.containsKey(rc.id)) {
          recon = reconMap.get(rc.id);
        } else {
          if (rc.id.equals("")) {
            cell = new Cell(rc.name, null);
          } else {
            // TODO: maybe zemanta type will be needed
            // what if there are many types? - preview links?
            // maybe recon fields have to be populated with data from results
            recon = new ZemantaDataExtensionReconConfig().createNewRecon(_historyEntryID);
            recon.addCandidate(rc);
            recon.service = "zemanta";
            recon.match = rc;
            recon.matchRank = 0;
            recon.judgment = Judgment.Matched;
            recon.judgmentAction = "auto";
            recon.judgmentBatchSize = 1;

            reconMap.put(rc.id, recon);
            cell = new Cell(rc.name, recon);
          }
        }
      } else {
        cell = new Cell((Serializable) value, null);
      }

      row.setCell(_firstNewCellIndex + c, cell);
    }
  }
コード例 #12
0
  @Override
  public void apply(Project project) {
    synchronized (project) {
      if (_firstNewCellIndex < 0) {
        _firstNewCellIndex = project.columnModel.allocateNewCellIndex();
        for (int i = 1; i < _columnNames.size(); i++) {
          project.columnModel.allocateNewCellIndex();
        }

        _oldRows = new ArrayList<Row>(project.rows);

        _newRows = new ArrayList<Row>(project.rows.size());

        int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
        int keyCellIndex =
            project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
        int index = 0;

        int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
        DataExtension dataExtension =
            index < _rowIndices.size() ? _dataExtensions.get(index) : null;

        index++;

        Map<String, Recon> reconMap = new HashMap<String, Recon>();

        for (int r = 0; r < _oldRows.size(); r++) {
          Row oldRow = _oldRows.get(r);
          if (r < rowIndex) {
            _newRows.add(oldRow.dup());
            continue;
          }

          if (dataExtension == null || dataExtension.data.length == 0) {
            _newRows.add(oldRow);
          } else {
            Row firstNewRow = oldRow.dup();
            extendRow(firstNewRow, dataExtension, 0, reconMap);
            _newRows.add(firstNewRow);

            int r2 = r + 1;
            for (int subR = 1; subR < dataExtension.data.length; subR++) {
              if (r2 < project.rows.size()) {
                Row oldRow2 = project.rows.get(r2);
                if (oldRow2.isCellBlank(cellIndex) && oldRow2.isCellBlank(keyCellIndex)) {

                  Row newRow = oldRow2.dup();
                  extendRow(newRow, dataExtension, subR, reconMap);

                  _newRows.add(newRow);
                  r2++;

                  continue;
                }
              }

              Row newRow = new Row(cellIndex + _columnNames.size());
              extendRow(newRow, dataExtension, subR, reconMap);

              _newRows.add(newRow);
            }

            r = r2 - 1; // r will be incremented by the for loop anyway
          }

          rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
          dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
          index++;
        }
      }

      project.rows.clear();
      project.rows.addAll(_newRows);

      for (int i = 0; i < _columnNames.size(); i++) {
        String name = _columnNames.get(i);
        int cellIndex = _firstNewCellIndex + i;

        Column column = new Column(cellIndex, name);
        column.setReconConfig(new DBpediaDataExtensionReconConfig(_columnTypes.get(i)));
        column.setReconStats(ReconStats.create(project, cellIndex));

        try {
          project.columnModel.addColumn(_columnInsertIndex + i, column, true);

          // the column might have been renamed to avoid collision
          _columnNames.set(i, column.getName());
        } catch (ModelException e) {
          // won't get here since we set the avoid collision flag
        }
      }

      project.update();
    }
  }
コード例 #13
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void notEmptyRow() {
   Row row = new Row(1);
   row.setCell(0, new Cell("I'm not empty", null));
   Assert.assertFalse(row.isEmpty());
 }
コード例 #14
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void getStarredField() {
   Row row = new Row(5);
   row.starred = true;
   Assert.assertTrue((Boolean) row.getField("starred", options));
 }
コード例 #15
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void blankCell() {
   Row row = new Row(5);
   Assert.assertTrue(row.isCellBlank(0));
 }
コード例 #16
0
  @Override
  public void save(Writer writer, Properties options) throws IOException {
    writer.write("baseColumnName=");
    writer.write(_baseColumnName);
    writer.write('\n');
    writer.write("columnInsertIndex=");
    writer.write(Integer.toString(_columnInsertIndex));
    writer.write('\n');
    writer.write("columnNameCount=");
    writer.write(Integer.toString(_columnNames.size()));
    writer.write('\n');
    for (String name : _columnNames) {
      writer.write(name);
      writer.write('\n');
    }
    writer.write("columnTypeCount=");
    writer.write(Integer.toString(_columnTypes.size()));
    writer.write('\n');
    for (DBpediaType type : _columnTypes) {
      try {
        JSONWriter jsonWriter = new JSONWriter(writer);

        type.write(jsonWriter, options);
      } catch (JSONException e) {
        // ???
      }
      writer.write('\n');
    }
    writer.write("rowIndexCount=");
    writer.write(Integer.toString(_rowIndices.size()));
    writer.write('\n');
    for (Integer rowIndex : _rowIndices) {
      writer.write(rowIndex.toString());
      writer.write('\n');
    }
    writer.write("dataExtensionCount=");
    writer.write(Integer.toString(_dataExtensions.size()));
    writer.write('\n');
    for (DataExtension dataExtension : _dataExtensions) {
      if (dataExtension == null) {
        writer.write('\n');
        continue;
      }

      writer.write(Integer.toString(dataExtension.data.length));
      writer.write('\n');

      for (Object[] values : dataExtension.data) {
        for (Object value : values) {
          if (value == null) {
            writer.write("null");
          } else if (value instanceof ReconCandidate) {
            try {
              JSONWriter jsonWriter = new JSONWriter(writer);
              ((ReconCandidate) value).write(jsonWriter, options);
            } catch (JSONException e) {
              // ???
            }
          } else if (value instanceof String) {
            writer.write(JSONObject.quote((String) value));
          } else {
            writer.write(value.toString());
          }
          writer.write('\n');
        }
      }
    }

    writer.write("firstNewCellIndex=");
    writer.write(Integer.toString(_firstNewCellIndex));
    writer.write('\n');

    writer.write("newRowCount=");
    writer.write(Integer.toString(_newRows.size()));
    writer.write('\n');
    for (Row row : _newRows) {
      row.save(writer, options);
      writer.write('\n');
    }
    writer.write("oldRowCount=");
    writer.write(Integer.toString(_oldRows.size()));
    writer.write('\n');
    for (Row row : _oldRows) {
      row.save(writer, options);
      writer.write('\n');
    }
    writer.write("/ec/\n"); // end of change marker
  }
コード例 #17
0
  public static Change load(LineNumberReader reader, Pool pool) throws Exception {
    String baseColumnName = null;
    int columnInsertIndex = -1;

    List<String> columnNames = null;
    List<DBpediaType> columnTypes = null;

    List<Integer> rowIndices = null;
    List<DataExtension> dataExtensions = null;

    List<Row> oldRows = null;
    List<Row> newRows = null;

    int firstNewCellIndex = -1;

    String line;
    while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
      int equal = line.indexOf('=');
      CharSequence field = line.subSequence(0, equal);
      String value = line.substring(equal + 1);

      if ("baseColumnName".equals(field)) {
        baseColumnName = value;
      } else if ("columnInsertIndex".equals(field)) {
        columnInsertIndex = Integer.parseInt(value);
      } else if ("firstNewCellIndex".equals(field)) {
        firstNewCellIndex = Integer.parseInt(value);
      } else if ("rowIndexCount".equals(field)) {
        int count = Integer.parseInt(value);

        rowIndices = new ArrayList<Integer>(count);
        for (int i = 0; i < count; i++) {
          line = reader.readLine();
          if (line != null) {
            rowIndices.add(Integer.parseInt(line));
          }
        }
      } else if ("columnNameCount".equals(field)) {
        int count = Integer.parseInt(value);

        columnNames = new ArrayList<String>(count);
        for (int i = 0; i < count; i++) {
          line = reader.readLine();
          if (line != null) {
            columnNames.add(line);
          }
        }
      } else if ("columnTypeCount".equals(field)) {
        int count = Integer.parseInt(value);

        columnTypes = new ArrayList<DBpediaType>(count);
        for (int i = 0; i < count; i++) {
          line = reader.readLine();
          columnTypes.add(DBpediaType.load(ParsingUtilities.evaluateJsonStringToObject(line)));
        }
      } else if ("dataExtensionCount".equals(field)) {
        int count = Integer.parseInt(value);

        dataExtensions = new ArrayList<DataExtension>(count);
        for (int i = 0; i < count; i++) {
          line = reader.readLine();

          if (line == null) {
            continue;
          }

          if (line.length() == 0) {
            dataExtensions.add(null);
            continue;
          }

          int rowCount = Integer.parseInt(line);
          Object[][] data = new Object[rowCount][];

          for (int r = 0; r < rowCount; r++) {
            Object[] row = new Object[columnNames.size()];
            for (int c = 0; c < columnNames.size(); c++) {
              line = reader.readLine();

              row[c] = ReconCandidate.loadStreaming(line);
            }

            data[r] = row;
          }

          dataExtensions.add(new DataExtension(data));
        }
      } else if ("oldRowCount".equals(field)) {
        int count = Integer.parseInt(value);

        oldRows = new ArrayList<Row>(count);
        for (int i = 0; i < count; i++) {
          line = reader.readLine();
          if (line != null) {
            oldRows.add(Row.load(line, pool));
          }
        }
      } else if ("newRowCount".equals(field)) {
        int count = Integer.parseInt(value);

        newRows = new ArrayList<Row>(count);
        for (int i = 0; i < count; i++) {
          line = reader.readLine();
          if (line != null) {
            newRows.add(Row.load(line, pool));
          }
        }
      }
    }

    DBpediaDataExtensionChange change =
        new DBpediaDataExtensionChange(
            baseColumnName,
            columnInsertIndex,
            columnNames,
            columnTypes,
            rowIndices,
            dataExtensions,
            firstNewCellIndex,
            oldRows,
            newRows);

    return change;
  }
コード例 #18
0
ファイル: RowTests.java プロジェクト: Solenn1/google-refine
 @Test
 public void emptyRow() {
   Row row = new Row(5);
   Assert.assertTrue(row.isEmpty());
 }