@Test
public void duplicateRow() {
Row row = new Row(5);
row.flagged = true;
Row duplicateRow = row.dup();
Assert.assertTrue(duplicateRow.flagged);
}
@Test
public void nonBlankCell() {
Row row = new Row(5);
row.setCell(0, new Cell("I'm not empty", null));
Assert.assertFalse(row.isCellBlank(0));
row.setCell(3, new Cell("I'm not empty", null));
Assert.assertFalse(row.isCellBlank(3));
}
@Test
public void saveRow() {
Row row = new Row(5);
row.setCell(0, new Cell("I'm not empty", null));
row.save(writer, options);
Assert.assertEquals(
writer.getBuffer().toString(),
"{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}]}");
}
@Test
public void toStringTest() {
Row row = new Row(5);
row.setCell(0, new Cell(1, null));
row.setCell(1, new Cell(2, null));
row.setCell(2, new Cell(3, null));
row.setCell(3, new Cell(4, null));
row.setCell(4, new Cell(5, null));
Assert.assertEquals(row.toString(), "1,2,3,4,5,");
}
@Test
public void canParseSample(){
RunTest(getSample());
log(project);
assertProjectCreated(project, 4, 6);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "Author 1, The");
}
@Test
public void saveRowWithRecordIndex() {
Row row = new Row(5);
row.setCell(0, new Cell("I'm not empty", null));
when(options.containsKey("rowIndex")).thenReturn(true);
when(options.get("rowIndex")).thenReturn(0);
when(options.containsKey("recordIndex")).thenReturn(true);
when(options.get("recordIndex")).thenReturn(1);
row.save(writer, options);
Assert.assertEquals(
writer.getBuffer().toString(),
"{\"flagged\":false,\"starred\":false,\"cells\":[{\"v\":\"I'm not empty\"}],\"i\":0,\"j\":1}");
}
@Test
public void testCanParseLineBreak(){
RunTest(getSampleWithLineBreak());
log(project);
assertProjectCreated(project, 4, 6);
Row row = project.rows.get(3);
Assert.assertNotNull(row);
Assert.assertEquals(row.cells.size(), 4);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "With line\n break");
}
@Test
public void canParseSampleWithDuplicateNestedElements(){
RunTest(getSampleWithDuplicateNestedElements());
log(project);
assertProjectCreated(project, 4, 12);
Row row = project.rows.get(0);
Assert.assertNotNull(row);
Assert.assertEquals(row.cells.size(), 4);
Assert.assertNotNull(row.getCell(1));
Assert.assertEquals(row.getCell(1).value, "Author 1, The");
Assert.assertEquals(project.rows.get(1).getCell(1).value, "Author 1, Another");
}
protected static void addImportRecordToProject(ImportRecord record, Project project) {
if (record.rows.size() > 0) {
for (List<Cell> row : record.rows) {
Row realRow = new Row(row.size());
int cellCount = 0;
for (int c = 0; c < row.size(); c++) {
Cell cell = row.get(c);
if (cell != null) {
realRow.setCell(c, cell);
cellCount++;
}
}
if (cellCount > 0) {
project.rows.add(realRow);
}
}
}
}
protected void extendRow(
Row row, DataExtension dataExtension, int extensionRowIndex, Map<String, Recon> reconMap) {
Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) {
Object value = values[c];
Cell cell = null;
if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value;
Recon recon;
if (reconMap.containsKey(rc.id)) {
recon = reconMap.get(rc.id);
} else {
if (rc.id.equals("")) {
cell = new Cell(rc.name, null);
} else {
recon =
new DBpediaDataExtensionReconConfig(new DBpediaType(rc.name, rc.id))
.createNewRecon(_historyEntryID);
recon.addCandidate(rc);
recon.service = "dbpedia-extension";
recon.match = rc;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
reconMap.put(rc.id, recon);
cell = new Cell(rc.name, recon);
}
}
} else {
cell = new Cell((Serializable) value, null);
}
row.setCell(_firstNewCellIndex + c, cell);
}
}
protected void extendRow(
Row row, DataExtension dataExtension, int extensionRowIndex, Map<String, Recon> reconMap) {
Object[] values = dataExtension.data[extensionRowIndex];
for (int c = 0; c < values.length; c++) {
Object value = values[c];
Cell cell = null;
if (value instanceof ReconCandidate) {
ReconCandidate rc = (ReconCandidate) value;
Recon recon;
if (reconMap.containsKey(rc.id)) {
recon = reconMap.get(rc.id);
} else {
if (rc.id.equals("")) {
cell = new Cell(rc.name, null);
} else {
// TODO: maybe zemanta type will be needed
// what if there are many types? - preview links?
// maybe recon fields have to be populated with data from results
recon = new ZemantaDataExtensionReconConfig().createNewRecon(_historyEntryID);
recon.addCandidate(rc);
recon.service = "zemanta";
recon.match = rc;
recon.matchRank = 0;
recon.judgment = Judgment.Matched;
recon.judgmentAction = "auto";
recon.judgmentBatchSize = 1;
reconMap.put(rc.id, recon);
cell = new Cell(rc.name, recon);
}
}
} else {
cell = new Cell((Serializable) value, null);
}
row.setCell(_firstNewCellIndex + c, cell);
}
}
@Override
public void apply(Project project) {
synchronized (project) {
if (_firstNewCellIndex < 0) {
_firstNewCellIndex = project.columnModel.allocateNewCellIndex();
for (int i = 1; i < _columnNames.size(); i++) {
project.columnModel.allocateNewCellIndex();
}
_oldRows = new ArrayList<Row>(project.rows);
_newRows = new ArrayList<Row>(project.rows.size());
int cellIndex = project.columnModel.getColumnByName(_baseColumnName).getCellIndex();
int keyCellIndex =
project.columnModel.columns.get(project.columnModel.getKeyColumnIndex()).getCellIndex();
int index = 0;
int rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
DataExtension dataExtension =
index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
Map<String, Recon> reconMap = new HashMap<String, Recon>();
for (int r = 0; r < _oldRows.size(); r++) {
Row oldRow = _oldRows.get(r);
if (r < rowIndex) {
_newRows.add(oldRow.dup());
continue;
}
if (dataExtension == null || dataExtension.data.length == 0) {
_newRows.add(oldRow);
} else {
Row firstNewRow = oldRow.dup();
extendRow(firstNewRow, dataExtension, 0, reconMap);
_newRows.add(firstNewRow);
int r2 = r + 1;
for (int subR = 1; subR < dataExtension.data.length; subR++) {
if (r2 < project.rows.size()) {
Row oldRow2 = project.rows.get(r2);
if (oldRow2.isCellBlank(cellIndex) && oldRow2.isCellBlank(keyCellIndex)) {
Row newRow = oldRow2.dup();
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
r2++;
continue;
}
}
Row newRow = new Row(cellIndex + _columnNames.size());
extendRow(newRow, dataExtension, subR, reconMap);
_newRows.add(newRow);
}
r = r2 - 1; // r will be incremented by the for loop anyway
}
rowIndex = index < _rowIndices.size() ? _rowIndices.get(index) : _oldRows.size();
dataExtension = index < _rowIndices.size() ? _dataExtensions.get(index) : null;
index++;
}
}
project.rows.clear();
project.rows.addAll(_newRows);
for (int i = 0; i < _columnNames.size(); i++) {
String name = _columnNames.get(i);
int cellIndex = _firstNewCellIndex + i;
Column column = new Column(cellIndex, name);
column.setReconConfig(new DBpediaDataExtensionReconConfig(_columnTypes.get(i)));
column.setReconStats(ReconStats.create(project, cellIndex));
try {
project.columnModel.addColumn(_columnInsertIndex + i, column, true);
// the column might have been renamed to avoid collision
_columnNames.set(i, column.getName());
} catch (ModelException e) {
// won't get here since we set the avoid collision flag
}
}
project.update();
}
}
@Test
public void notEmptyRow() {
Row row = new Row(1);
row.setCell(0, new Cell("I'm not empty", null));
Assert.assertFalse(row.isEmpty());
}
@Test
public void getStarredField() {
Row row = new Row(5);
row.starred = true;
Assert.assertTrue((Boolean) row.getField("starred", options));
}
@Test
public void blankCell() {
Row row = new Row(5);
Assert.assertTrue(row.isCellBlank(0));
}
@Override
public void save(Writer writer, Properties options) throws IOException {
writer.write("baseColumnName=");
writer.write(_baseColumnName);
writer.write('\n');
writer.write("columnInsertIndex=");
writer.write(Integer.toString(_columnInsertIndex));
writer.write('\n');
writer.write("columnNameCount=");
writer.write(Integer.toString(_columnNames.size()));
writer.write('\n');
for (String name : _columnNames) {
writer.write(name);
writer.write('\n');
}
writer.write("columnTypeCount=");
writer.write(Integer.toString(_columnTypes.size()));
writer.write('\n');
for (DBpediaType type : _columnTypes) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
type.write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
writer.write('\n');
}
writer.write("rowIndexCount=");
writer.write(Integer.toString(_rowIndices.size()));
writer.write('\n');
for (Integer rowIndex : _rowIndices) {
writer.write(rowIndex.toString());
writer.write('\n');
}
writer.write("dataExtensionCount=");
writer.write(Integer.toString(_dataExtensions.size()));
writer.write('\n');
for (DataExtension dataExtension : _dataExtensions) {
if (dataExtension == null) {
writer.write('\n');
continue;
}
writer.write(Integer.toString(dataExtension.data.length));
writer.write('\n');
for (Object[] values : dataExtension.data) {
for (Object value : values) {
if (value == null) {
writer.write("null");
} else if (value instanceof ReconCandidate) {
try {
JSONWriter jsonWriter = new JSONWriter(writer);
((ReconCandidate) value).write(jsonWriter, options);
} catch (JSONException e) {
// ???
}
} else if (value instanceof String) {
writer.write(JSONObject.quote((String) value));
} else {
writer.write(value.toString());
}
writer.write('\n');
}
}
}
writer.write("firstNewCellIndex=");
writer.write(Integer.toString(_firstNewCellIndex));
writer.write('\n');
writer.write("newRowCount=");
writer.write(Integer.toString(_newRows.size()));
writer.write('\n');
for (Row row : _newRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("oldRowCount=");
writer.write(Integer.toString(_oldRows.size()));
writer.write('\n');
for (Row row : _oldRows) {
row.save(writer, options);
writer.write('\n');
}
writer.write("/ec/\n"); // end of change marker
}
public static Change load(LineNumberReader reader, Pool pool) throws Exception {
String baseColumnName = null;
int columnInsertIndex = -1;
List<String> columnNames = null;
List<DBpediaType> columnTypes = null;
List<Integer> rowIndices = null;
List<DataExtension> dataExtensions = null;
List<Row> oldRows = null;
List<Row> newRows = null;
int firstNewCellIndex = -1;
String line;
while ((line = reader.readLine()) != null && !"/ec/".equals(line)) {
int equal = line.indexOf('=');
CharSequence field = line.subSequence(0, equal);
String value = line.substring(equal + 1);
if ("baseColumnName".equals(field)) {
baseColumnName = value;
} else if ("columnInsertIndex".equals(field)) {
columnInsertIndex = Integer.parseInt(value);
} else if ("firstNewCellIndex".equals(field)) {
firstNewCellIndex = Integer.parseInt(value);
} else if ("rowIndexCount".equals(field)) {
int count = Integer.parseInt(value);
rowIndices = new ArrayList<Integer>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
rowIndices.add(Integer.parseInt(line));
}
}
} else if ("columnNameCount".equals(field)) {
int count = Integer.parseInt(value);
columnNames = new ArrayList<String>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
columnNames.add(line);
}
}
} else if ("columnTypeCount".equals(field)) {
int count = Integer.parseInt(value);
columnTypes = new ArrayList<DBpediaType>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
columnTypes.add(DBpediaType.load(ParsingUtilities.evaluateJsonStringToObject(line)));
}
} else if ("dataExtensionCount".equals(field)) {
int count = Integer.parseInt(value);
dataExtensions = new ArrayList<DataExtension>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line == null) {
continue;
}
if (line.length() == 0) {
dataExtensions.add(null);
continue;
}
int rowCount = Integer.parseInt(line);
Object[][] data = new Object[rowCount][];
for (int r = 0; r < rowCount; r++) {
Object[] row = new Object[columnNames.size()];
for (int c = 0; c < columnNames.size(); c++) {
line = reader.readLine();
row[c] = ReconCandidate.loadStreaming(line);
}
data[r] = row;
}
dataExtensions.add(new DataExtension(data));
}
} else if ("oldRowCount".equals(field)) {
int count = Integer.parseInt(value);
oldRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
oldRows.add(Row.load(line, pool));
}
}
} else if ("newRowCount".equals(field)) {
int count = Integer.parseInt(value);
newRows = new ArrayList<Row>(count);
for (int i = 0; i < count; i++) {
line = reader.readLine();
if (line != null) {
newRows.add(Row.load(line, pool));
}
}
}
}
DBpediaDataExtensionChange change =
new DBpediaDataExtensionChange(
baseColumnName,
columnInsertIndex,
columnNames,
columnTypes,
rowIndices,
dataExtensions,
firstNewCellIndex,
oldRows,
newRows);
return change;
}
@Test
public void emptyRow() {
Row row = new Row(5);
Assert.assertTrue(row.isEmpty());
}