/**
* Takes an XML file containing parameterised queries + parameter queries and substitutes
* parameters for values, then saves the resulting queries
*
* @param queryFileName the XML query file
* @param queryMixFile
* @param ignoreFile
*/
public void generateQueries(String queryFileName, String queryMixFile, String ignoreFile) {
File queryFile = new File(queryFileName);
try {
DocumentBuilderFactory docBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder docBuilder = docBuilderFactory.newDocumentBuilder();
Document doc = docBuilder.parse(queryFile);
XPath xpath = XPathFactory.newInstance().newXPath();
XPathExpression expr = xpath.compile("/queries/query");
Object result = expr.evaluate(doc, XPathConstants.NODESET);
NodeList nodes = (NodeList) result;
// for each query, generate a map of parameter names and values
// then replace the parameters in the final query with the respective (randomly
// selected) values
for (int i = 0; i < nodes.getLength(); i++) {
queryCount = i + 1;
String completeQuery = generateCompleteQuery(nodes.item(i), doc);
saveCompleteQuery(completeQuery);
}
saveHelperFiles(queryMixFile, ignoreFile);
} catch (Exception e) {
e.printStackTrace();
}
}
private NodeList findNodes(Document doc, Node operation) {
List<Node> xpaths = getChildNodes(operation, "xpath");
if (xpaths.isEmpty()) {
return null;
}
String xpathExpression = xpaths.get(0).getTextContent();
if (xpathExpression == null) {
return null;
}
XPathFactory xPathfactory = XPathFactory.newInstance();
XPath xpath = xPathfactory.newXPath();
NodeList nl = null;
try {
XPathExpression expr = xpath.compile(xpathExpression);
nl = (NodeList) expr.evaluate(doc, XPathConstants.NODESET);
} catch (XPathExpressionException ex) {
Utils.onError(new Error.WrongXpathExpression(xpathExpression));
}
return nl;
}
/**
* extract the XPath from the content. the return value type is passed in input using one of the
* {@link XPathConstants}. See also {@link XPathExpression#evaluate(Object item, QName
* returnType)} ;
*
* @param ns
* @param xpathExpression
* @param content
* @param returnType
* @param charset
* @return the result
*/
public static Object extractXPath(
Map<String, String> ns,
String xpathExpression,
String content,
QName returnType,
String charset) {
if (null == ns) {
ns = new HashMap<String, String>();
}
String ch = charset;
if (ch == null) {
ch = Charset.defaultCharset().name();
}
Document doc = toDocument(content, charset);
XPathExpression expr = toExpression(ns, xpathExpression);
try {
Object o = expr.evaluate(doc, returnType);
return o;
} catch (XPathExpressionException e) {
throw new IllegalArgumentException("xPath expression cannot be executed: " + xpathExpression);
}
}
public Credentials authenticate() throws Exception {
HttpURLConnection urlConnection =
new ConnBuilder(authUrl)
.addHeader(HttpHeaders.CONTENT_TYPE_HEADER, "application/json")
.addHeader(HttpHeaders.ACCEPT_HEADER, "application/xml")
.getConnection();
StringBuilder jsonBuilder = new StringBuilder();
jsonBuilder
.append("{\"auth\": {\"tenantName\": \"")
.append(tenant)
.append("\", \"passwordCredentials\": {\"username\": \"")
.append(username)
.append("\", \"password\": \"")
.append(password)
.append("\"}}}");
HttpResponse response =
Utils.doOperation(urlConnection, jsonBuilder.toString().getBytes(), true);
if (response.isSuccessCode()) {
DocumentBuilderFactory documentBuilderFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = documentBuilderFactory.newDocumentBuilder();
Document doc = builder.parse(new ByteArrayInputStream(response.payload));
String authToken = (String) tokenIdExpression.evaluate(doc, XPathConstants.STRING);
String storageUrl = (String) publicUrlExpression.evaluate(doc, XPathConstants.STRING);
log.trace("Authentication successful");
return new Credentials(authToken, storageUrl);
} else {
throw new IllegalStateException(
"Error authenticating to the service. Please check your credentials. Code = "
+ response.code);
}
}
/**
* generates a complete query from a parameterised query + parameter queries
*
* @param namedQuery the current query
* @param doc the XML file containing the queries
* @return a complete query
* @throws XPathExpressionException
*/
protected String generateCompleteQuery(Node namedQuery, Document doc)
throws XPathExpressionException {
XPath xpath = XPathFactory.newInstance().newXPath();
String queryName = namedQuery.getAttributes().getNamedItem("id").getTextContent();
currentQueryName = queryName;
System.out.println("* Generating query: " + currentQueryName);
// get the parameterised query
Node queryStringNode = namedQuery.getChildNodes().item(1); // TODO: get it by name
String queryString = queryStringNode.getTextContent();
// this returns all paramquery nodes
XPathExpression xGetParamNode = xpath.compile("//query[@id='" + queryName + "']/parameter");
NodeList paramQueries = (NodeList) xGetParamNode.evaluate(doc, XPathConstants.NODESET);
Map<String, String> paramMap = new HashMap<String, String>();
for (int j = 0; j < paramQueries.getLength(); j++) {
// this is one parameter query item
Node paramNode = paramQueries.item(j);
// get the list of parameter names for which we query
XPathExpression xGetParamNames = xpath.compile("paramname");
NodeList paramNames = (NodeList) xGetParamNames.evaluate(paramNode, XPathConstants.NODESET);
XPathExpression xGetParamQueryNodes = xpath.compile("paramvaluesquery");
NodeList paramQueryNodes =
(NodeList) xGetParamQueryNodes.evaluate(paramNode, XPathConstants.NODESET);
// the query to obtain the parameters
String paramQueryString = paramQueryNodes.item(0).getTextContent();
populateParamMap(paramMap, paramNames, paramQueryString);
}
return substituteQueryParameters(queryString, paramMap);
}
public static void main(String[] args) {
try {
DocumentBuilderFactory db = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = db.newDocumentBuilder();
Document dom = builder.parse("data/geographic-area-data.html");
XPathFactory xpath = XPathFactory.newInstance();
XPath path = xpath.newXPath();
XPathExpression table =
path.compile("//div[@id='mw-content-text']/table[contains(@class,'wikitable')]/tr");
NodeList wikiData = (NodeList) table.evaluate(dom, XPathConstants.NODESET);
NodeList children;
String currentData, cleanData;
/* Open output stream */
FileWriter fstream = new FileWriter("data/parsed.yaml");
BufferedWriter out = new BufferedWriter(fstream);
for (int i = 0; i < wikiData.getLength(); i++) {
if (i == 0) {
continue;
}
out.write(new Integer(i).toString() + ":\n");
children = wikiData.item(i).getChildNodes();
for (int j = 0; j < children.getLength(); j++) {
currentData = (String) children.item(j).getTextContent();
switch (j) {
case 0:
/* Current Data is empty */
break;
case 1:
cleanData = decompose(currentData).trim().replaceAll("[^a-zA-Z\\s]+", "");
out.write("\t\"Geographic entity\": \"" + cleanData + "\"\n");
break;
case 2:
/* Current Data is empty */
break;
case 3:
cleanData = decompose(currentData).trim().replaceAll(",", "");
out.write("\t\"Area\": \"" + cleanData + "\"\n");
break;
case 4:
/* Current Data is empty */
break;
case 5:
cleanData = decompose(currentData).trim();
out.write("\t\"Notes\": \"" + cleanData + "\"\n");
break;
case 6:
/* Current Data is empty */
break;
default:
/* System.out.println("[" + j + "] Hit default case statement. Current Data is: " + currentData); */
break;
}
}
}
/* Close output stream */
out.close();
} catch (Exception e) {
System.out.println(e);
}
}
protected Document parse(InputStream inputStream) throws LibraryException {
Map<Integer, SongBean> songMap = new HashMap<Integer, SongBean>();
Map<Integer, AlbumBean> albumMap = new HashMap<Integer, AlbumBean>();
Map<Integer, AlbumBean> secondaryAlbumMap = new HashMap<Integer, AlbumBean>();
warningList.clear();
try {
// Create a builder factory
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
factory.setValidating(false);
factory.setNamespaceAware(true);
factory.setIgnoringElementContentWhitespace(true);
factory.setIgnoringComments(true);
// Create the builder and parse the file
DocumentBuilder builder = factory.newDocumentBuilder();
// Set an error listener and parse the document
builder.setErrorHandler(new iTradeTunesLibraryErrorHandler());
builder.setEntityResolver(new iTradeTunesLibraryResolver());
Document document = builder.parse(inputStream);
synchronized (libraryList) {
// Query the library document and build the library list
XPath xPath = XPathFactory.newInstance().newXPath();
XPathExpression xPathExpression = xPath.compile(XPATH_LIBRARY_LIST);
NodeList nodelist = (NodeList) xPathExpression.evaluate(document, XPathConstants.NODESET);
// Process the elements in the nodelist
SongBean song = null;
for (int i = 0; i < nodelist.getLength(); i++) {
boolean isTrackID = false;
// Get element and child nodes
Element elem = (Element) nodelist.item(i);
NodeList list = elem.getChildNodes();
// Get node value
Node childKey = list.item(0);
String key = childKey.getNodeValue();
// Check if we have to create a new bean
if (SongBean.NAME_TRACK_ID.equals(key)) {
isTrackID = true;
SongBean previousSong = song;
song = new SongBean();
if (previousSong != null
&& !("AAC audio file".equals(previousSong.getKind())
|| "MPEG audio file".equals(previousSong.getKind()))) {
songMap.remove(previousSong.getTrack_ID());
} else {
// Add an album bean
addOrUpdateAlbum(albumMap, previousSong, false);
}
}
// The first parameter is the key
String prop = childKey.getNodeValue().replace(' ', '_');
// The second parameter is the value
i++;
// Get element and child nodes
elem = (Element) nodelist.item(i);
// Check for boolean properties
Object value = null;
// Get node value
list = elem.getChildNodes();
childKey = list.item(0);
value = (childKey == null) ? elem.getNodeName() : childKey.getNodeValue();
if (isTrackID) {
isTrackID = false;
}
// Set the property of the song bean
Statement stmt = new Statement(song, "set" + prop, new Object[] {value});
try {
stmt.execute();
} catch (Exception e) {
// Ignore that field, we do not have it in our bean
}
// If the property is the track ID, add the song to the hash
// map
if (SongBean.NAME_TRACK_ID.equals(key)) {
int trackID = Integer.valueOf((String) value);
songMap.put(trackID, song);
}
}
// Update album for last song
addOrUpdateAlbum(albumMap, song, false);
// Check the album map for inconsistencies
Iterator<AlbumBean> albums = albumMap.values().iterator();
while (albums.hasNext()) {
AlbumBean album = albums.next();
if (album.checkConsistency()) {
libraryList.add(album);
album.setHashCode();
} else {
// Add an inconsistent album only using the album title
SongBean[] songs = album.getSongs();
for (int i = 0; i < songs.length; i++) {
addOrUpdateAlbum(secondaryAlbumMap, songs[i], true);
}
}
}
// Check secondary album map for consistency
albums = secondaryAlbumMap.values().iterator();
while (albums.hasNext()) {
AlbumBean album = albums.next();
if (album.checkConsistency()) {
libraryList.add(album);
album.setHashCode();
} else {
// This album cannot be matched
// TODO: Add to warning message
}
}
setChanged();
}
return document;
} catch (IOException ioe) {
// Log an expected connect exception
throw new LibraryException(ioe);
} catch (SAXException se) {
// Catch all other exceptions
throw new LibraryException(se);
} catch (ParserConfigurationException pce) {
// Catch all other exceptions
Utils.logSevere(pce);
throw new LibraryException(pce);
} catch (XPathExpressionException xpe) {
// Catch all other exceptions
Utils.logSevere(xpe);
throw new LibraryException(xpe);
} catch (NumberFormatException nfe) {
// Catch all other exceptions
throw new LibraryException(nfe);
}
}