private void parseAndAddEntry(String type) {
/**
* Morten Alver 13 Aug 2006: Trying to make the parser more robust. If an exception is thrown
* when parsing an entry, drop the entry and try to resume parsing. Add a warning for the user.
*/
try {
BibEntry entry = parseEntry(type);
boolean duplicateKey = database.insertEntry(entry);
entry.setParsedSerialization(dumpTextReadSoFarToString());
if (duplicateKey) {
parserResult.addDuplicateKey(entry.getCiteKey());
} else if ((entry.getCiteKey() == null) || entry.getCiteKey().isEmpty()) {
parserResult.addWarning(
Localization.lang("Empty BibTeX key")
+ ": "
+ entry.getAuthorTitleYear(40)
+ " ("
+ Localization.lang("Grouping may not work for this entry.")
+ ")");
}
} catch (IOException ex) {
LOGGER.warn("Could not parse entry", ex);
parserResult.addWarning(
Localization.lang("Error occurred when parsing entry")
+ ": '"
+ ex.getMessage()
+ "'. "
+ Localization.lang("Skipped entry."));
}
}
@Test
public void roundTripTest() throws IOException {
// @formatter:off
String bibtexEntry =
"@Article{test,"
+ OS.NEWLINE
+ " Author = {Foo Bar},"
+ OS.NEWLINE
+ " Journal = {International Journal of Something},"
+ OS.NEWLINE
+ " Note = {some note},"
+ OS.NEWLINE
+ " Number = {1}"
+ OS.NEWLINE
+ "}";
// @formatter:on
// read in bibtex string
ParserResult result = BibtexParser.parse(new StringReader(bibtexEntry));
Collection<BibEntry> entries = result.getDatabase().getEntries();
BibEntry entry = entries.iterator().next();
// write out bibtex string
StringWriter stringWriter = new StringWriter();
writer.write(entry, stringWriter, BibDatabaseMode.BIBTEX);
String actual = stringWriter.toString();
assertEquals(bibtexEntry, actual);
}
@Test
public void monthFieldSpecialSyntax() throws IOException {
// @formatter:off
String bibtexEntry =
"@Article{test,"
+ OS.NEWLINE
+ " Author = {Foo Bar},"
+ OS.NEWLINE
+ " Month = mar,"
+ OS.NEWLINE
+ " Number = {1}"
+ OS.NEWLINE
+ "}";
// @formatter:on
// read in bibtex string
ParserResult result = BibtexParser.parse(new StringReader(bibtexEntry));
Collection<BibEntry> entries = result.getDatabase().getEntries();
BibEntry entry = entries.iterator().next();
// modify month field
Set<String> fields = entry.getFieldNames();
assertTrue(fields.contains("month"));
assertEquals("#mar#", entry.getFieldOptional("month").get());
// write out bibtex string
StringWriter stringWriter = new StringWriter();
writer.write(entry, stringWriter, BibDatabaseMode.BIBTEX);
String actual = stringWriter.toString();
assertEquals(bibtexEntry, actual);
}
@Test
public void roundTripWithAppendedNewlines() throws IOException {
// @formatter:off
String bibtexEntry =
"@Article{test,"
+ OS.NEWLINE
+ " Author = {Foo Bar},"
+ OS.NEWLINE
+ " Journal = {International Journal of Something},"
+ OS.NEWLINE
+ " Note = {some note},"
+ OS.NEWLINE
+ " Number = {1}"
+ OS.NEWLINE
+ "}\n\n";
// @formatter:on
// read in bibtex string
ParserResult result = BibtexParser.parse(new StringReader(bibtexEntry));
Collection<BibEntry> entries = result.getDatabase().getEntries();
BibEntry entry = entries.iterator().next();
// write out bibtex string
StringWriter stringWriter = new StringWriter();
writer.write(entry, stringWriter, BibDatabaseMode.BIBTEX);
String actual = stringWriter.toString();
// Only one appending newline is written by the writer, the rest by FileActions. So, these
// should be removed here.
assertEquals(bibtexEntry.substring(0, bibtexEntry.length() - 1), actual);
}
@Test
public void testEntryTypeChange() throws IOException {
// @formatter:off
String expected =
OS.NEWLINE
+ "@Article{test,"
+ OS.NEWLINE
+ " author = {BlaBla},"
+ OS.NEWLINE
+ " journal = {International Journal of Something},"
+ OS.NEWLINE
+ " number = {1},"
+ OS.NEWLINE
+ " note = {some note},"
+ OS.NEWLINE
+ " howpublished = {asdf},"
+ OS.NEWLINE
+ "}"
+ OS.NEWLINE;
// @formatter:on
// read in bibtex string
ParserResult result = BibtexParser.parse(new StringReader(expected));
Collection<BibEntry> entries = result.getDatabase().getEntries();
BibEntry entry = entries.iterator().next();
// modify entry
entry.setType("inproceedings");
// write out bibtex string
StringWriter stringWriter = new StringWriter();
writer.write(entry, stringWriter, BibDatabaseMode.BIBTEX);
String actual = stringWriter.toString();
// @formatter:off
String expectedNewEntry =
OS.NEWLINE
+ "@InProceedings{test,"
+ OS.NEWLINE
+ " author = {BlaBla},"
+ OS.NEWLINE
+ " number = {1},"
+ OS.NEWLINE
+ " note = {some note},"
+ OS.NEWLINE
+ " howpublished = {asdf},"
+ OS.NEWLINE
+ " journal = {International Journal of Something},"
+ OS.NEWLINE
+ "}"
+ OS.NEWLINE;
// @formatter:on
assertEquals(expectedNewEntry, actual);
}
@Test
public void roundTripWithCamelCasingInTheOriginalEntryAndResultInLowerCase() throws IOException {
// @formatter:off
String bibtexEntry =
OS.NEWLINE
+ "@Article{test,"
+ OS.NEWLINE
+ " Author = {Foo Bar},"
+ OS.NEWLINE
+ " Journal = {International Journal of Something},"
+ OS.NEWLINE
+ " Note = {some note},"
+ OS.NEWLINE
+ " Number = {1},"
+ OS.NEWLINE
+ " HowPublished = {asdf},"
+ OS.NEWLINE
+ "}";
// @formatter:on
// read in bibtex string
ParserResult result = BibtexParser.parse(new StringReader(bibtexEntry));
Collection<BibEntry> entries = result.getDatabase().getEntries();
BibEntry entry = entries.iterator().next();
// modify entry
entry.setField("author", "BlaBla");
// write out bibtex string
StringWriter stringWriter = new StringWriter();
writer.write(entry, stringWriter, BibDatabaseMode.BIBTEX);
String actual = stringWriter.toString();
// @formatter:off
String expected =
OS.NEWLINE
+ "@Article{test,"
+ OS.NEWLINE
+ " author = {BlaBla},"
+ OS.NEWLINE
+ " journal = {International Journal of Something},"
+ OS.NEWLINE
+ " number = {1},"
+ OS.NEWLINE
+ " note = {some note},"
+ OS.NEWLINE
+ " howpublished = {asdf},"
+ OS.NEWLINE
+ "}"
+ OS.NEWLINE;
// @formatter:on
assertEquals(expected, actual);
}
@Test
public void roundTripWithPrecedingCommentAndModificationTest() throws IOException {
// @formatter:off
String bibtexEntry =
"% Some random comment that should stay here"
+ OS.NEWLINE
+ "@Article{test,"
+ OS.NEWLINE
+ " Author = {Foo Bar},"
+ OS.NEWLINE
+ " Journal = {International Journal of Something},"
+ OS.NEWLINE
+ " Note = {some note},"
+ OS.NEWLINE
+ " Number = {1}"
+ OS.NEWLINE
+ "}";
// @formatter:on
// read in bibtex string
ParserResult result = BibtexParser.parse(new StringReader(bibtexEntry));
Collection<BibEntry> entries = result.getDatabase().getEntries();
BibEntry entry = entries.iterator().next();
// change the entry
entry.setField("author", "John Doe");
// write out bibtex string
StringWriter stringWriter = new StringWriter();
writer.write(entry, stringWriter, BibDatabaseMode.BIBTEX);
String actual = stringWriter.toString();
// @formatter:off
String expected =
"% Some random comment that should stay here"
+ OS.NEWLINE
+ OS.NEWLINE
+ "@Article{test,"
+ OS.NEWLINE
+ " author = {John Doe},"
+ OS.NEWLINE
+ " journal = {International Journal of Something},"
+ OS.NEWLINE
+ " number = {1},"
+ OS.NEWLINE
+ " note = {some note},"
+ OS.NEWLINE
+ "}"
+ OS.NEWLINE;
// @formatter:on
assertEquals(expected, actual);
}
private String testSingleWrite(String bibtexEntry) throws IOException {
// read in bibtex string
ParserResult result = BibtexParser.parse(new StringReader(bibtexEntry));
Collection<BibEntry> entries = result.getDatabase().getEntries();
BibEntry entry = entries.iterator().next();
// write out bibtex string
StringWriter stringWriter = new StringWriter();
writer.write(entry, stringWriter, BibDatabaseMode.BIBTEX);
String actual = stringWriter.toString();
assertEquals(bibtexEntry, actual);
return actual;
}
private void parseBibtexString() throws IOException {
BibtexString bibtexString = parseString();
bibtexString.setParsedSerialization(dumpTextReadSoFarToString());
try {
database.addString(bibtexString);
} catch (KeyCollisionException ex) {
parserResult.addWarning(
Localization.lang("Duplicate string name") + ": " + bibtexString.getName());
}
}
private ParserResult parseFileContent() throws IOException {
Map<String, String> meta = new HashMap<>();
while (!eof) {
boolean found = consumeUncritically('@');
if (!found) {
break;
}
skipWhitespace();
// Try to read the entry type
String entryType = parseTextToken().toLowerCase().trim();
if ("preamble".equals(entryType)) {
database.setPreamble(parsePreamble());
// Consume new line which signals end of preamble
skipOneNewline();
// the preamble is saved verbatim anyways, so the text read so far can be dropped
dumpTextReadSoFarToString();
} else if ("string".equals(entryType)) {
parseBibtexString();
} else if ("comment".equals(entryType)) {
parseJabRefComment(meta);
} else {
// Not a comment, preamble, or string. Thus, it is an entry
parseAndAddEntry(entryType);
}
skipWhitespace();
}
// Instantiate meta data:
try {
parserResult.setMetaData(MetaData.parse(meta));
} catch (ParseException exception) {
parserResult.addWarning(exception.getLocalizedMessage());
}
parseRemainingContent();
return parserResult;
}
private void parseJabRefComment(Map<String, String> meta) throws IOException {
StringBuilder buffer = parseBracketedTextExactly();
/**
* Metadata are used to store Bibkeeper-specific information in .bib files.
*
* <p>Metadata are stored in bibtex files in the format
*
* @comment{jabref-meta: type:data0;data1;data2;...}
* <p>Each comment that starts with the META_FLAG is stored in the meta HashMap, with type
* as key. Unluckily, the old META_FLAG bibkeeper-meta: was used in JabRef 1.0 and 1.1, so
* we need to support it as well. At least for a while. We'll always save with the new one.
*/
String comment = buffer.toString().replaceAll("[\\x0d\\x0a]", "");
if (comment
.substring(0, Math.min(comment.length(), MetaData.META_FLAG.length()))
.equals(MetaData.META_FLAG)) {
if (comment.substring(0, MetaData.META_FLAG.length()).equals(MetaData.META_FLAG)) {
String rest = comment.substring(MetaData.META_FLAG.length());
int pos = rest.indexOf(':');
if (pos > 0) {
// We remove all line breaks in the metadata - these
// will have been inserted
// to prevent too long lines when the file was
// saved, and are not part of the data.
meta.put(rest.substring(0, pos), rest.substring(pos + 1));
// meta comments are always re-written by JabRef and not stored in the file
dumpTextReadSoFarToString();
}
}
} else if (comment
.substring(0, Math.min(comment.length(), CustomEntryType.ENTRYTYPE_FLAG.length()))
.equals(CustomEntryType.ENTRYTYPE_FLAG)) {
// A custom entry type can also be stored in a
// "@comment"
Optional<CustomEntryType> typ = CustomEntryType.parse(comment);
if (typ.isPresent()) {
entryTypes.put(typ.get().getName(), typ.get());
} else {
parserResult.addWarning(
Localization.lang("Ill-formed entrytype comment in bib file") + ": " + comment);
}
// custom entry types are always re-written by JabRef and not stored in the file
dumpTextReadSoFarToString();
} else {
// FIXME: user comments are simply dropped
// at least, we log that we ignored the comment
LOGGER.info("Dropped comment from database: " + comment);
}
}
/**
* This method should be performed if the major/minor versions recorded in the ParserResult are
* less than or equal to 2.2.
*
* @param pr
* @return true if the file was written by a jabref version <=2.2
*/
@Override
public boolean isActionNecessary(ParserResult pr) {
// Find out which actions should be offered:
// Only offer to change Preferences if file column is not already visible:
offerChangeSettings =
!Globals.prefs.getBoolean(JabRefPreferences.FILE_COLUMN) || !showsFileInGenFields();
// Only offer to upgrade links if the pdf/ps fields are used:
offerChangeDatabase = linksFound(pr.getDatabase(), FileLinksUpgradeWarning.FIELDS_TO_LOOK_FOR);
// If the "file" directory is not set, offer to migrate pdf/ps dir:
offerSetFileDir =
!Globals.prefs.hasKey(Globals.FILE_FIELD + Globals.DIR_SUFFIX)
&& (Globals.prefs.hasKey("pdfDirectory") || Globals.prefs.hasKey("psDirectory"));
// First check if this warning is disabled:
return Globals.prefs.getBoolean(JabRefPreferences.SHOW_FILE_LINKS_UPGRADE_WARNING)
&& isThereSomethingToBeDone();
}
/**
* This method performs the actual changes.
*
* @param panel
* @param pr
* @param fileDir The path to the file directory to set, or null if it should not be set.
*/
private void makeChanges(
BasePanel panel,
ParserResult pr,
boolean upgradePrefs,
boolean upgradeDatabase,
String fileDir) {
if (upgradeDatabase) {
// Update file links links in the database:
NamedCompound ce =
Util.upgradePdfPsToFile(pr.getDatabase(), FileLinksUpgradeWarning.FIELDS_TO_LOOK_FOR);
panel.undoManager.addEdit(ce);
panel.markBaseChanged();
}
if (fileDir != null) {
Globals.prefs.put(Globals.FILE_FIELD + Globals.DIR_SUFFIX, fileDir);
}
if (upgradePrefs) {
// Exchange table columns:
Globals.prefs.putBoolean(JabRefPreferences.FILE_COLUMN, Boolean.TRUE);
// Modify General fields if necessary:
// If we don't find the file field, insert it at the bottom of the first tab:
if (!showsFileInGenFields()) {
String gfs = Globals.prefs.get(JabRefPreferences.CUSTOM_TAB_FIELDS + "0");
StringBuilder sb = new StringBuilder(gfs);
if (!gfs.isEmpty()) {
sb.append(";");
}
sb.append(Globals.FILE_FIELD);
Globals.prefs.put(JabRefPreferences.CUSTOM_TAB_FIELDS + "0", sb.toString());
Globals.prefs.updateEntryEditorTabList();
panel.frame().removeCachedEntryEditors();
}
panel.frame().setupAllTables();
}
}
private static BibtexEntry bibtexString2BibtexEntry(String s) throws IOException {
ParserResult result = BibtexParser.parse(new StringReader(s));
Collection<BibtexEntry> c = result.getDatabase().getEntries();
Assert.assertEquals(1, c.size());
return c.iterator().next();
}
/**
* Tries to restore the key
*
* @return rest of key on success, otherwise empty string
* @throws IOException on Reader-Error
*/
private String fixKey() throws IOException {
StringBuilder key = new StringBuilder();
int lookaheadUsed = 0;
char currentChar;
// Find a char which ends key (','&&'\n') or entryfield ('='):
do {
currentChar = (char) read();
key.append(currentChar);
lookaheadUsed++;
} while ((currentChar != ',')
&& (currentChar != '\n')
&& (currentChar != '=')
&& (lookaheadUsed < BibtexParser.LOOKAHEAD));
// Consumed a char too much, back into reader and remove from key:
unread(currentChar);
key.deleteCharAt(key.length() - 1);
// Restore if possible:
switch (currentChar) {
case '=':
// Get entryfieldname, push it back and take rest as key
key = key.reverse();
boolean matchedAlpha = false;
for (int i = 0; i < key.length(); i++) {
currentChar = key.charAt(i);
/// Skip spaces:
if (!matchedAlpha && (currentChar == ' ')) {
continue;
}
matchedAlpha = true;
// Begin of entryfieldname (e.g. author) -> push back:
unread(currentChar);
if ((currentChar == ' ') || (currentChar == '\n')) {
/*
* found whitespaces, entryfieldname completed -> key in
* keybuffer, skip whitespaces
*/
StringBuilder newKey = new StringBuilder();
for (int j = i; j < key.length(); j++) {
currentChar = key.charAt(j);
if (!Character.isWhitespace(currentChar)) {
newKey.append(currentChar);
}
}
// Finished, now reverse newKey and remove whitespaces:
parserResult.addWarning(
Localization.lang("Line %0: Found corrupted BibTeX key.", String.valueOf(line)));
key = newKey.reverse();
}
}
break;
case ',':
parserResult.addWarning(
Localization.lang(
"Line %0: Found corrupted BibTeX key (contains whitespaces).",
String.valueOf(line)));
break;
case '\n':
parserResult.addWarning(
Localization.lang(
"Line %0: Found corrupted BibTeX key (comma missing).", String.valueOf(line)));
break;
default:
// No more lookahead, give up:
unreadBuffer(key);
return "";
}
return removeWhitespaces(key).toString();
}