@Test
public void testIsValidNavigonUrl() throws IOException {
ParserResult result = read(TEST_PATH + "from-nmn.txt");
assertEquals(NmnUrlFormat.class, result.getFormat().getClass());
ParserResult plainResult = read(TEST_PATH + "from-nmn-plain.txt");
assertEquals(NmnUrlFormat.class, plainResult.getFormat().getClass());
}
/**
* Import an entry from an OAI2 archive. The BibtexEntry provided has to have the field
* OAI2_IDENTIFIER_FIELD set to the search string.
*
* @param key The OAI2 key to fetch from ArXiv.
* @return The imnported BibtexEntry or null if none.
*/
private BibtexDatabase importInspireEntries(String key, OutputPrinter frame) {
String url = constructUrl(key);
try {
HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
conn.setRequestProperty("User-Agent", "Jabref");
InputStream inputStream = conn.getInputStream();
INSPIREBibtexFilterReader reader =
new INSPIREBibtexFilterReader(new InputStreamReader(inputStream));
ParserResult pr = BibtexParser.parse(reader);
return pr.getDatabase();
} catch (IOException e) {
frame.showMessage(
Localization.lang("An Exception ocurred while accessing '%0'", url) + "\n\n" + e,
Localization.lang(getKeyName()),
JOptionPane.ERROR_MESSAGE);
} catch (RuntimeException e) {
frame.showMessage(
Localization.lang(
"An Error occurred while fetching from INSPIRE source (%0):", new String[] {url})
+ "\n\n"
+ e.getMessage(),
Localization.lang(getKeyName()),
JOptionPane.ERROR_MESSAGE);
}
return null;
}
@Test
@Ignore
public void testAddEntrysFromFiles() throws Exception {
ParserResult result =
BibtexParser.parse(new FileReader(ImportDataTest.UNLINKED_FILES_TEST_BIB));
BibtexDatabase database = result.getDatabase();
List<File> files = new ArrayList<File>();
files.add(ImportDataTest.FILE_NOT_IN_DATABASE);
files.add(ImportDataTest.NOT_EXISTING_PDF);
EntryFromFileCreatorManager manager = new EntryFromFileCreatorManager();
List<String> errors = manager.addEntrysFromFiles(files, database, null, true);
/** One file doesn't exist, so adding it as an entry should lead to an error message. */
Assert.assertEquals(1, errors.size());
boolean file1Found = false;
boolean file2Found = false;
for (BibtexEntry entry : database.getEntries()) {
String filesInfo = entry.getField("file");
if (filesInfo.contains(files.get(0).getName())) {
file1Found = true;
}
if (filesInfo.contains(files.get(1).getName())) {
file2Found = true;
}
}
Assert.assertTrue(file1Found);
Assert.assertFalse(file2Found);
}
/**
* After a JabRef version number has been parsed and put into _pr, parse the version number to
* determine the JabRef major and minor version number
*/
private void setMajorMinorVersions() {
String v = _pr.getJabrefVersion();
Pattern p = Pattern.compile("([0-9]+)\\.([0-9]+).*");
Matcher m = p.matcher(v);
if (!m.matches()) return;
if (m.groupCount() >= 2) {
_pr.setJabrefMajorVersion(Integer.parseInt(m.group(1)));
_pr.setJabrefMinorVersion(Integer.parseInt(m.group(2)));
}
}
/* (non-Javadoc)
* @see junit.framework.TestCase#setUp()
*/
@Before
public void setUp() throws Exception {
ParserResult result =
BibtexParser.parse(new FileReader(ImportDataTest.UNLINKED_FILES_TEST_BIB));
database = result.getDatabase();
entries = database.getEntries();
entry1 = database.getEntryByKey("entry1");
entry2 = database.getEntryByKey("entry2");
}
private BibtexDatabase parseBibtexDatabase(List<String> id, boolean abs) throws IOException {
if (id.isEmpty()) {
return null;
}
URLConnection conn;
try {
conn = new URL(IEEEXploreFetcher.IMPORT_URL).openConnection();
} catch (MalformedURLException e) {
e.printStackTrace();
return null;
}
conn.setDoInput(true);
conn.setDoOutput(true);
conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
conn.setRequestProperty("Referer", searchUrl);
PrintWriter out = new PrintWriter(conn.getOutputStream());
String recordIds = "";
for (String anId : id) {
recordIds += anId + " ";
}
recordIds = recordIds.trim();
String citation = abs ? "citation-abstract" : "citation-only";
String content =
"recordIds="
+ recordIds.replaceAll(" ", "%20")
+ "&fromPageName=&citations-format="
+ citation
+ "&download-format=download-bibtex";
System.out.println(content);
out.write(content);
out.flush();
out.close();
BufferedReader bufr = new BufferedReader(new InputStreamReader(conn.getInputStream()));
StringBuilder sb = new StringBuilder();
char[] buffer = new char[256];
while (true) {
int bytesRead = bufr.read(buffer);
if (bytesRead == -1) {
break;
}
for (int i = 0; i < bytesRead; i++) {
sb.append(buffer[i]);
}
}
System.out.println(sb);
ParserResult results = new BibtexParser(bufr).parse();
bufr.close();
return results.getDatabase();
}
public void checkEntryTypes(ParserResult _pr) {
for (BibtexEntry be : _db.getEntries()) {
if (be.getType() instanceof UnknownEntryType) {
// Look up the unknown type name in our map of parsed types:
Object o = entryTypes.get(be.getType().getName().toLowerCase());
if (o != null) {
BibtexEntryType type = (BibtexEntryType) o;
be.setType(type);
} else {
// System.out.println("Unknown entry type:
// "+be.getType().getName());
_pr.addWarning(
Globals.lang("unknown entry type")
+ ": "
+ be.getType().getName()
+ ". "
+ Globals.lang("Type set to 'other'")
+ ".");
be.setType(BibtexEntryType.OTHER);
}
}
}
}
void readRouteCharacteristics(
String testFileName,
RouteCharacteristics characteristics,
int characteristicsCount,
int[] positionCount)
throws IOException {
ParserResult result = read(testFileName);
List<BaseRoute> routes = getRouteCharacteristics(result.getAllRoutes(), characteristics);
if (routes == null || characteristicsCount == 0) {
assertNull(routes);
} else {
assertEquals(characteristicsCount, routes.size());
for (int i = 0; i < routes.size(); i++) {
BaseRoute route = routes.get(i);
assertEquals(characteristics, route.getCharacteristics());
assertEquals(positionCount[i], route.getPositionCount());
}
}
}
/** Adds a key->value assignment to the values HashMap */
private void createKeyValueAssignment(String line, int lineNr) {
String[] split = line.split("=");
if (split.length == 1) {
split = new String[] {split[0], ""};
}
if (split[0].matches(PlotConstants.KEY_INT_GRID_WIDTH)) {
parserResult.addPlotGridValue(split[0], new KeyValue(split[0], split[1], lineNr));
} else {
tempPlotValuesCache.put(split[0], new KeyValue(split[0], split[1], lineNr));
}
}
public ParserResult parse(String source) {
List<String> inputList = Arrays.asList(source.split("\n", -1));
ListIterator<String> inputIterator = inputList.listIterator();
while (inputIterator.hasNext()) {
String line = inputIterator.next();
if (line.isEmpty() || line.matches(PlotConstants.REGEX_COMMENT)) {
/* ignore empty lines and comments */
} else if (line.matches(PlotConstants.REGEX_PLOT)) {
parserResult.addPlotState(createPlotStateObject(line.split(" "), inputIterator));
} else if (line.matches(PlotConstants.REGEX_PLOT_ADD)) {
List<PlotState> plotStates = parserResult.getPlotStateList();
if (plotStates.isEmpty()) {
// if no plotStates, create a new one
parserResult.addPlotState(createPlotStateObject(line.split(" "), inputIterator));
} else {
// if plots exist, add new plotState to last plotState
PlotState last = plotStates.get(plotStates.size() - 1);
last.addSubPlot(createPlotStateObject(line.split(" "), inputIterator));
}
} else if (line.matches(PlotConstants.REGEX_DATA)) {
createDatasetObject(line.split(" "), inputIterator);
} else if (line.matches(PlotConstants.REGEX_DATA_GUESS)) {
inputIterator
.previous(); // Must go 1 step back to avoid skipping the first line in
// createDatasetObject
createDatasetObject(new String[] {PlotConstants.DATA}, inputIterator);
} else if (line.matches(PlotConstants.REGEX_VALUE_ASSIGNMENT)) {
createKeyValueAssignment(line, inputIterator.nextIndex());
} else {
throw new ParserException(
"Invalid line: " + line + "(line: " + inputIterator.nextIndex() + ")");
}
}
analyseDatasets();
addDatasetsToPlotStates();
return parserResult;
}
/** Is called after parsing everything to fill datasets in each plotState Object */
private void addDatasetsToPlotStates() {
if (datasetList.isEmpty()) {
throw new ParserException("You must specify at least one dataset.");
}
int actualAutoDatasetNr = 0;
for (PlotState plotState : parserResult.getPlotStateList()) {
actualAutoDatasetNr = addDataset(plotState, actualAutoDatasetNr);
// also add datasets to subplots
for (PlotState subPlotState : plotState.getSubplots()) {
log.info("Add dataset for subplot");
actualAutoDatasetNr = addDataset(subPlotState, actualAutoDatasetNr);
}
}
}
ParserResult read(String testFileName) throws IOException {
File source = new File(testFileName);
ParserResult result = parser.read(source);
assertNotNull(result.getFormat());
assertNotNull(result.getAllRoutes());
assertTrue(result.getAllRoutes().size() > 0);
assertNotNull("Cannot read route from " + source, result.getTheRoute());
assertTrue(result.getTheRoute().getPositionCount() > 0);
return result;
}
/**
* Tries to restore the key
*
* @return rest of key on success, otherwise empty string
* @throws IOException on Reader-Error
*/
private String fixKey() throws IOException {
StringBuilder key = new StringBuilder();
int lookahead_used = 0;
char currentChar;
// Find a char which ends key (','&&'\n') or entryfield ('='):
do {
currentChar = (char) read();
key.append(currentChar);
lookahead_used++;
} while ((currentChar != ',' && currentChar != '\n' && currentChar != '=')
&& (lookahead_used < LOOKAHEAD));
// Consumed a char too much, back into reader and remove from key:
unread(currentChar);
key.deleteCharAt(key.length() - 1);
// Restore if possible:
switch (currentChar) {
case '=':
// Get entryfieldname, push it back and take rest as key
key = key.reverse();
boolean matchedAlpha = false;
for (int i = 0; i < key.length(); i++) {
currentChar = key.charAt(i);
/// Skip spaces:
if (!matchedAlpha && currentChar == ' ') {
continue;
}
matchedAlpha = true;
// Begin of entryfieldname (e.g. author) -> push back:
unread(currentChar);
if (currentChar == ' ' || currentChar == '\n') {
/*
* found whitespaces, entryfieldname completed -> key in
* keybuffer, skip whitespaces
*/
StringBuilder newKey = new StringBuilder();
for (int j = i; j < key.length(); j++) {
currentChar = key.charAt(j);
if (!Character.isWhitespace(currentChar)) {
newKey.append(currentChar);
}
}
// Finished, now reverse newKey and remove whitespaces:
_pr.addWarning(
Globals.lang("Line %0: Found corrupted BibTeX-key.", String.valueOf(line)));
key = newKey.reverse();
}
}
break;
case ',':
_pr.addWarning(
Globals.lang(
"Line %0: Found corrupted BibTeX-key (contains whitespaces).",
String.valueOf(line)));
case '\n':
_pr.addWarning(
Globals.lang(
"Line %0: Found corrupted BibTeX-key (comma missing).", String.valueOf(line)));
break;
default:
// No more lookahead, give up:
unreadBuffer(key);
return "";
}
return removeWhitespaces(key).toString();
}
/**
* Will parse the BibTex-Data found when reading from reader.
*
* <p>The reader will be consumed.
*
* <p>Multiple calls to parse() return the same results
*
* @return ParserResult
* @throws IOException
*/
public ParserResult parse() throws IOException {
// If we already parsed this, just return it.
if (_pr != null) return _pr;
_db = new BibtexDatabase(); // Bibtex related contents.
_meta = new HashMap<String, String>(); // Metadata in comments for Bibkeeper.
entryTypes = new HashMap<String, BibtexEntryType>(); // To store custem entry types parsed.
_pr = new ParserResult(_db, _meta, entryTypes);
// First see if we can find the version number of the JabRef version that
// wrote the file:
String versionNum = readJabRefVersionNumber();
if (versionNum != null) {
_pr.setJabrefVersion(versionNum);
setMajorMinorVersions();
} else {
// No version number found. However, we have only
}
skipWhitespace();
try {
while (!_eof) {
boolean found = consumeUncritically('@');
if (!found) break;
skipWhitespace();
String entryType = parseTextToken();
BibtexEntryType tp = BibtexEntryType.getType(entryType);
boolean isEntry = (tp != null);
// Util.pr(tp.getName());
if (!isEntry) {
// The entry type name was not recognized. This can mean
// that it is a string, preamble, or comment. If so,
// parse and set accordingly. If not, assume it is an entry
// with an unknown type.
if (entryType.toLowerCase().equals("preamble")) {
_db.setPreamble(parsePreamble());
} else if (entryType.toLowerCase().equals("string")) {
BibtexString bs = parseString();
try {
_db.addString(bs);
} catch (KeyCollisionException ex) {
_pr.addWarning(Globals.lang("Duplicate string name") + ": " + bs.getName());
// ex.printStackTrace();
}
} else if (entryType.toLowerCase().equals("comment")) {
StringBuffer commentBuf = parseBracketedTextExactly();
/**
* Metadata are used to store Bibkeeper-specific information in .bib files.
*
* <p>Metadata are stored in bibtex files in the format
*
* @comment{jabref-meta: type:data0;data1;data2;...}
* <p>Each comment that starts with the META_FLAG is stored in the meta HashMap,
* with type as key. Unluckily, the old META_FLAG bibkeeper-meta: was used in JabRef
* 1.0 and 1.1, so we need to support it as well. At least for a while. We'll always
* save with the new one.
*/
String comment = commentBuf.toString().replaceAll("[\\x0d\\x0a]", "");
if (comment
.substring(0, Math.min(comment.length(), GUIGlobals.META_FLAG.length()))
.equals(GUIGlobals.META_FLAG)
|| comment
.substring(0, Math.min(comment.length(), GUIGlobals.META_FLAG_OLD.length()))
.equals(GUIGlobals.META_FLAG_OLD)) {
String rest;
if (comment.substring(0, GUIGlobals.META_FLAG.length()).equals(GUIGlobals.META_FLAG))
rest = comment.substring(GUIGlobals.META_FLAG.length());
else rest = comment.substring(GUIGlobals.META_FLAG_OLD.length());
int pos = rest.indexOf(':');
if (pos > 0) _meta.put(rest.substring(0, pos), rest.substring(pos + 1));
// We remove all line breaks in the metadata - these
// will have been inserted
// to prevent too long lines when the file was
// saved, and are not part of the data.
}
/**
* A custom entry type can also be stored in a
*
* @comment:
*/
if (comment
.substring(0, Math.min(comment.length(), GUIGlobals.ENTRYTYPE_FLAG.length()))
.equals(GUIGlobals.ENTRYTYPE_FLAG)) {
CustomEntryType typ = CustomEntryType.parseEntryType(comment);
entryTypes.put(typ.getName().toLowerCase(), typ);
}
} else {
// The entry type was not recognized. This may mean that
// it is a custom entry type whose definition will
// appear
// at the bottom of the file. So we use an
// UnknownEntryType
// to remember the type name by.
tp = new UnknownEntryType(entryType.toLowerCase());
// System.out.println("unknown type: "+entryType);
isEntry = true;
}
}
if (isEntry) // True if not comment, preamble or string.
{
/**
* Morten Alver 13 Aug 2006: Trying to make the parser more robust. If an exception is
* thrown when parsing an entry, drop the entry and try to resume parsing. Add a warning
* for the user.
*
* <p>An alternative solution is to try rescuing the entry for which parsing failed, by
* returning the entry with the exception and adding it before parsing is continued.
*/
try {
BibtexEntry be = parseEntry(tp);
boolean duplicateKey = _db.insertEntry(be);
if (duplicateKey) // JZTODO lyrics
_pr.addWarning(
Globals.lang("duplicate BibTeX key")
+ ": "
+ be.getCiteKey()
+ " ("
+ Globals.lang("grouping may not work for this entry")
+ ")");
else if (be.getCiteKey() == null || be.getCiteKey().equals("")) {
_pr.addWarning(
Globals.lang("empty BibTeX key")
+ ": "
+ be.getAuthorTitleYear(40)
+ " ("
+ Globals.lang("grouping may not work for this entry")
+ ")");
}
} catch (IOException ex) {
ex.printStackTrace();
_pr.addWarning(
Globals.lang("Error occured when parsing entry")
+ ": '"
+ ex.getMessage()
+ "'. "
+ Globals.lang("Skipped entry."));
}
}
skipWhitespace();
}
// Before returning the database, update entries with unknown type
// based on parsed type definitions, if possible.
checkEntryTypes(_pr);
return _pr;
} catch (KeyCollisionException kce) {
// kce.printStackTrace();
throw new IOException("Duplicate ID in bibtex file: " + kce.toString());
}
}
public static void main(String[] args) throws Exception {
boolean createSampleSet = Boolean.parseBoolean(args[2]);
int sampleSetSize = 0;
if (createSampleSet) {
sampleSetSize = Integer.parseInt(args[3]);
}
boolean dryRun = Boolean.parseBoolean(args[1]);
File basePath = new File(args[0]);
if (basePath.exists()) {
File artistsFile = null;
File labelsFile = null;
File mastersFile = null;
File releasesFile = null;
for (File child : basePath.listFiles()) {
if (child.isFile()) {
String filename = child.getName().toLowerCase();
if (filename.endsWith("_artists.xml")) {
artistsFile = child;
} else if (filename.endsWith("_labels.xml")) {
labelsFile = child;
} else if (filename.endsWith("_masters.xml")) {
mastersFile = child;
} else if (filename.endsWith("_releases.xml")) {
releasesFile = child;
}
}
}
if (artistsFile == null
|| labelsFile == null
|| mastersFile == null
|| releasesFile == null) {
throw new IllegalStateException("Not all needed files found");
}
List<Artist> artists = null;
ParserResult<Artist> artistParserResult = null;
try (FileInputStream is = new FileInputStream(artistsFile)) {
StaxParser<Artist> parser = new ArtistStaxParser(dryRun);
artistParserResult = parser.parse(is);
artists = artistParserResult.getResult();
}
List<Label> labels = null;
ParserResult<Label> labelParserResult = null;
try (FileInputStream is = new FileInputStream(labelsFile)) {
StaxParser<Label> parser = new LabelStaxParser(dryRun);
labelParserResult = parser.parse(is);
labels = labelParserResult.getResult();
}
List<Master> masters = null;
ParserResult<Master> masterParserResult = null;
try (FileInputStream is = new FileInputStream(mastersFile)) {
StaxParser<Master> parser = new MasterStaxParser(dryRun);
masterParserResult = parser.parse(is);
masters = masterParserResult.getResult();
}
List<Release> releases = null;
ParserResult<Release> releaseParserResult = null;
try (FileInputStream is = new FileInputStream(releasesFile)) {
StaxParser<Release> parser = new ReleaseStaxParser(dryRun, sampleSetSize);
releaseParserResult = parser.parse(is);
releases = releaseParserResult.getResult();
}
System.out.println("Found " + artistParserResult);
System.out.println("Found " + labelParserResult);
System.out.println("Found " + masterParserResult);
System.out.println("Found " + releaseParserResult);
if (createSampleSet) {
System.out.println("Collecting data for sample dataset...");
Set<Artist> neededArtist = new HashSet<>(artists.size() / 1000);
for (Release release : releases) {
for (ArtistRef ref : release.getArtists()) {
Artist artist = findArtist(ref.getArtistId(), artists);
if (artist != null) {
neededArtist.add(artist);
}
}
for (ArtistRef ref : release.getExtraArtists()) {
Artist artist = findArtist(ref.getArtistId(), artists);
if (artist != null) {
neededArtist.add(artist);
}
}
}
File file = new File(basePath, "dataset.sample");
FileOutputStream fos = new FileOutputStream(file);
SerializationServiceBuilder builder = new SerializationServiceBuilder();
SerializationService ss =
builder
.setAllowUnsafe(true)
.setClassLoader(StaxParser.class.getClassLoader())
.setEnableCompression(true)
.setPartitioningStrategy(new DefaultPartitioningStrategy())
.setCheckClassDefErrors(true)
.setByteOrder(ByteOrder.BIG_ENDIAN)
.build();
BufferedOutputStream bos = new BufferedOutputStream(fos);
ObjectDataOutputStream out = new ObjectDataOutputStream(bos, ss, ByteOrder.BIG_ENDIAN);
System.out.println("Writing " + neededArtist.size() + " artists to sample dataset...");
out.writeInt(neededArtist.size());
for (Artist artist : neededArtist) {
out.writeObject(artist);
}
System.out.println("Writing " + labels.size() + " labels to sample dataset...");
out.writeInt(labels.size());
for (Label label : labels) {
out.writeObject(label);
}
System.out.println("Writing " + masters.size() + " masters to sample dataset...");
out.writeInt(masters.size());
for (Master master : masters) {
out.writeObject(master);
}
System.out.println("Writing " + releases.size() + " releases to sample dataset...");
out.writeInt(releases.size());
for (Release release : releases) {
out.writeObject(release);
}
out.flush();
out.close();
}
}
}
@Test
public void testReadWithFormatList() throws IOException {
List<NavigationFormat> formats = new ArrayList<>();
ParserResult result1 = parser.read(new File(TEST_PATH + "from.itn"), formats);
assertFalse(result1.isSuccessful());
formats.add(new TomTom8RouteFormat());
ParserResult result2 = parser.read(new File(TEST_PATH + "from.itn"), formats);
assertTrue(result2.isSuccessful());
assertEquals(0, result2.getTheRoute().getPositions().size());
assertEquals(1, result2.getAllRoutes().size());
assertEquals(result2.getFormat().getClass(), TomTom8RouteFormat.class);
formats.add(new TomTom5RouteFormat());
ParserResult result3 = parser.read(new File(TEST_PATH + "from.itn"), formats);
assertTrue(result3.isSuccessful());
assertEquals(46, result3.getTheRoute().getPositions().size());
assertEquals(1, result3.getAllRoutes().size());
assertEquals(result3.getFormat().getClass(), TomTom5RouteFormat.class);
}
@Test
public void testIsValidGoogleMapsUrl() throws IOException {
ParserResult result = read(TEST_PATH + "from-googlemaps.url");
assertEquals(GoogleMapsUrlFormat.class, result.getFormat().getClass());
}