/** Process the links to downloadable files */
private Element processDownloadLinks(
ServiceContext context, String id, String siteURL, List elems, Element response) {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
for (Iterator<Object> iter = elems.iterator(); iter.hasNext(); ) {
Object ob = iter.next();
if (ob instanceof Element) {
Element elem = (Element) ob;
elem = (Element) elem.clone();
// --- get file name and href
String fname = elem.getAttributeValue("name");
String href = elem.getAttributeValue("href");
if (href != null) {
boolean local = href.startsWith(siteURL);
boolean found = false;
long size = 0;
String dateModified = "";
String linkPieces[] = href.split("\\&");
// local file (possibly)
if (linkPieces.length > 0) {
if (local) {
String access = "private";
for (int i = 0; i < linkPieces.length; i++) {
String lp = linkPieces[i];
if (lp.startsWith("access=")) {
access = lp.substring(lp.indexOf('=') + 1);
}
}
File dir = new File(Lib.resource.getDir(context, access, id));
File file = new File(dir, fname);
if (file.exists()) {
size = file.length();
Date date = new Date(file.lastModified());
dateModified = sdf.format(date);
found = true;
}
}
}
elem.setAttribute("size", size + "");
elem.setAttribute("datemodified", dateModified);
elem.setAttribute("found", found + "");
elem.setAttribute("local", local + "");
elem.setAttribute("download", "true");
response.addContent(elem);
} else {
context.info("Unknown download link: " + Xml.getString(elem));
}
}
}
return response;
}
private Element doSearch(XmlRequest request, Search s) throws OperationAbortedEx {
request.setAddress(params.getServletPath() + "/srv/en/" + Geonet.Service.XML_SEARCH);
try {
log.info("Searching on : " + params.name);
Element response = request.execute(s.createRequest());
if (log.isDebugEnabled()) log.debug("Search results:\n" + Xml.getString(response));
return response;
} catch (Exception e) {
log.warning("Raised exception when searching : " + e);
throw new OperationAbortedEx("Raised exception when searching", e);
}
}
/**
* Does CSW GetRecordById request. If validation is requested and the metadata does not validate,
* null is returned.
*
* @param uuid uuid of metadata to request
* @return metadata the metadata
*/
private Element retrieveMetadata(String uuid) {
request.clearIds();
request.addId(uuid);
try {
if (log.isDebugEnabled())
log.debug("Getting record from : " + request.getHost() + " (uuid:" + uuid + ")");
Element response = request.execute();
if (log.isDebugEnabled()) log.debug("Record got:\n" + Xml.getString(response));
@SuppressWarnings("unchecked")
List<Element> list = response.getChildren();
// --- maybe the metadata has been removed
if (list.size() == 0) return null;
response = list.get(0);
response = (Element) response.detach();
// validate it here if requested
if (params.validate) {
if (!dataMan.validate(response)) {
log.info("Ignoring invalid metadata with uuid " + uuid);
result.doesNotValidate++;
return null;
}
}
if (params.rejectDuplicateResource) {
if (foundDuplicateForResource(uuid, response)) {
return null;
}
}
return response;
} catch (Exception e) {
log.warning("Raised exception while getting record : " + e);
e.printStackTrace();
result.unretrievable++;
// --- we don't raise any exception here. Just try to go on
return null;
}
}
public Z3950ServerResults harvest() throws Exception {
Set<String> newUuids = new HashSet<String>();
int groupSize = 10;
log.info("Retrieving remote metadata information:" + params.uuid);
Z3950ServerResults serverResults = new Z3950ServerResults();
// --- Clean all before harvest : Remove/Add mechanism
localUuids = new UUIDMapper(dbms, params.uuid);
// --- remove old metadata
for (String uuid : localUuids.getUUIDs()) {
String id = localUuids.getID(uuid);
if (log.isDebugEnabled()) log.debug(" - Removing old metadata before update with id: " + id);
dataMan.deleteMetadataGroup(context, dbms, id);
serverResults.locallyRemoved++;
}
if (serverResults.locallyRemoved > 0) dbms.commit();
// --- Search remote node
MetaSearcher s = searchMan.newSearcher(SearchManager.Z3950, Geonet.File.SEARCH_Z3950_CLIENT);
ServiceConfig config = new ServiceConfig();
Element request = new Element("request");
// --- Z39.50 servers from harvest params
for (String id : params.getRepositories()) {
request.addContent(new Element(Geonet.SearchResult.SERVERS).setText(id));
}
// --- Z39.50 query from harvest params
request.addContent(new Element(Geonet.SearchResult.ZQUERY).setText(params.query));
// --- don't get html presentations (get them later)
request.addContent(new Element(Geonet.SearchResult.SERVERHTML).setText("off"));
// --- set timeout to be 100 seconds
request.addContent(new Element(Geonet.SearchResult.TIMEOUT).setText("100"));
// --- set hitsPerPage
request.addContent(new Element(Geonet.SearchResult.HITS_PER_PAGE).setText(groupSize + ""));
// --- do the search
s.search(context, request, config);
if (s.getSize() == 0) {
log.error("Search failed or returned 0 results, trying again");
s.search(context, request, config);
if (s.getSize() == 0) {
throw new Exception("Bad luck, Search failed or returned 0 results");
}
}
if (log.isDebugEnabled()) log.debug("Search returned " + s.getSize() + " hits");
// -- process the hits in groups of groupSize
int numberOfHits = Math.min(Integer.parseInt(params.maximumHits), s.getSize());
// -- add from and to placeholders to request
request.addContent(new Element("from"));
request.addContent(new Element("to"));
Element categories = Lib.local.retrieve(dbms, "Categories");
if (log.isDebugEnabled()) log.debug("categories " + Xml.getString(categories));
Element repositories = new Info().getZRepositories(context, settingMan);
if (log.isDebugEnabled()) log.debug("repos " + Xml.getString(repositories));
// -- build a map of collection code versus repository name for
// -- assigning the categories
Map<String, String> codes = new HashMap<String, String>();
Map<String, String> catCodes = new HashMap<String, String>();
// -- add new category for each repository
boolean addcateg = false;
for (String repo : params.getRepositories()) {
Element repoElem = Xml.selectElement(repositories, "record[id='" + repo + "']");
if (repoElem != null) {
Element repoId = repoElem.getChild("id");
String repoName = repoElem.getChildText("name");
codes.put(
repoId.getAttributeValue("serverCode") + ":" + repoId.getAttributeValue("code"),
repoName);
// create a result holder for this repository
serverResults.getServerResult(repoName);
// sanitize the name of the category
String categName = repoName.replaceAll("[^\\w]", "");
categName = categName.toLowerCase();
catCodes.put(
repoId.getAttributeValue("serverCode") + ":" + repoId.getAttributeValue("code"),
categName);
if (Xml.selectElement(categories, "record[name='" + categName + "']") == null) {
int newId = context.getSerialFactory().getSerial(dbms, "Categories");
dbms.execute("INSERT INTO Categories(id, name) VALUES (?, ?)", newId, categName);
Lib.local.insert(dbms, "Categories", newId, repoName);
addcateg = true;
}
}
}
if (addcateg) dbms.commit();
// --- return only maximum hits as directed by the harvest params
int nrGroups = (numberOfHits / groupSize) + 1;
for (int i = 1; i <= nrGroups; i++) {
int lower = ((i - 1) * groupSize) + 1;
int upper = Math.min((i * groupSize), numberOfHits);
request.getChild("from").setText("" + lower);
request.getChild("to").setText("" + upper);
// --- Loading results
List<Document> list = s.presentDocuments(context, request, config);
// --- Loading categories and groups
localCateg = new CategoryMapper(dbms);
localGroups = new GroupMapper(dbms);
if (log.isDebugEnabled())
log.debug(
"There are "
+ (list.size() - 1)
+ " children in the results ("
+ lower
+ " to "
+ upper
+ ")");
boolean transformIt = false;
String thisXslt = context.getAppPath() + Geonet.Path.IMPORT_STYLESHEETS + "/";
if (!params.importXslt.equals("none")) {
thisXslt = thisXslt + params.importXslt;
transformIt = true;
}
// --- For each record....
for (Document doc : list) {
Element md = doc.getRootElement();
String eName = md.getQualifiedName();
if (eName.equals("summary")) continue;
// -- Remove existing geonet:info children as for example
// -- GeoNetwork Z39.50 server return when full mode
// -- an extra element with server info not needed
// -- once harvested
String colCode = "";
Element info = md.getChild(Edit.RootChild.INFO, Edit.NAMESPACE);
if (info != null) {
String serverCode = info.getChildText("server");
int colPos = serverCode.indexOf(':');
if (colPos != -1) {
colCode = serverCode.substring(0, colPos) + ":" + info.getChildText("collection");
}
}
md.removeChildren(Edit.RootChild.INFO, Edit.NAMESPACE);
String repoName = codes.get(colCode);
if (log.isDebugEnabled()) log.debug("Processing record from server " + repoName);
HarvestResult result = serverResults.getServerResult(repoName);
result.totalMetadata++;
if (eName.equals("error")) {
log.error("JZKit could not retrieve record - returned " + Xml.getString(md));
result.unretrievable++;
continue;
}
// transform using importxslt if not none
if (transformIt) {
try {
if (log.isDebugEnabled()) log.debug("Before transform: " + Xml.getString(md));
md = Xml.transform(md, thisXslt);
if (log.isDebugEnabled()) log.debug("After transform: " + Xml.getString(md));
} catch (Exception e) {
System.out.println("Cannot transform XML, ignoring. Error was: " + e.getMessage());
result.badFormat++;
continue; // skip this one
}
}
// detect schema, extract uuid and add
String schema = dataMan.autodetectSchema(md, null);
if (schema == null) {
log.warning("Skipping metadata with unknown schema.");
result.unknownSchema++;
continue;
}
String uuid = null;
try {
uuid = dataMan.extractUUID(schema, md);
} catch (Exception e) {
log.error("Unable to extract UUID: " + e.getMessage());
e.printStackTrace();
}
if (uuid == null || uuid.equals("")) {
log.warning("Skipping metadata due to failure extracting uuid (uuid null or empty).");
result.unretrievable++;
continue;
}
log.info(" - Adding metadata with " + uuid);
// --- generate a new metadata id
int id = context.getSerialFactory().getSerial(dbms, "Metadata");
// TODO end confusion about datatypes
String id$ = Integer.toString(id);
String docType = "";
if (!transformIt && (doc.getDocType() != null)) {
docType = Xml.getString(doc.getDocType());
}
// --- check for duplicate uuid - violates constraints on metadata table
// --- if we attempt insert
boolean alreadyAdded = !newUuids.add(uuid);
boolean alreadyInDb = (dataMan.getMetadataId(dbms, uuid) != null);
if (alreadyAdded || alreadyInDb) {
log.error("Uuid " + uuid + " already exists in this set/database - cannot insert");
result.couldNotInsert++;
continue;
}
//
// insert metadata
//
try {
String groupOwner = "1", isTemplate = "n", title = null;
int owner = 1;
String category = null, createDate = new ISODate().toString(), changeDate = createDate;
boolean ufo = false, indexImmediate = false;
dataMan.insertMetadata(
context,
dbms,
schema,
md,
id,
uuid,
owner,
groupOwner,
params.uuid,
isTemplate,
docType,
title,
category,
createDate,
changeDate,
ufo,
indexImmediate);
} catch (Exception e) {
log.error("Unable to insert metadata " + e.getMessage());
e.printStackTrace();
result.couldNotInsert++;
continue;
}
addPrivileges(id$, params.getPrivileges(), localGroups, dataMan, context, dbms, log);
addCategories(
id$,
params.getCategories(),
localCateg,
dataMan,
dbms,
context,
log,
catCodes.get(colCode));
dataMan.setTemplateExt(dbms, id, "n", null);
dataMan.setHarvestedExt(dbms, id, params.uuid, params.name);
// validate it here if requested
if (params.validate) {
Document docVal;
if (!transformIt && (doc.getDocType() != null)) {
docVal = new Document(md, (DocType) doc.getDocType().detach());
} else {
docVal = new Document(md);
}
if (!dataMan.doValidate(dbms, schema, id$, docVal, context.getLanguage())) {
result.doesNotValidate++;
}
}
dataMan.indexMetadata(dbms, id$);
result.addedMetadata++;
}
}
dbms.commit();
return serverResults;
}
public String toString() {
return Xml.getString(getQuery());
}
public InformationFragment[] getFragment(
int startingFragment, int count, RecordFormatSpecification spec) throws IRResultSetException {
if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
Log.debug(
Geonet.Z3950_SERVER,
"Request for fragment start:" + startingFragment + ", count:" + count);
InformationFragment fragment[] = new InformationFragment[count];
ExplicitRecordFormatSpecification rec_spec =
new ExplicitRecordFormatSpecification("xml", null, "f");
try {
// build fragment data
int from = startingFragment;
int to = startingFragment + count - 1;
Element request = new Element("request");
request.addContent(new Element("from").setText(from + ""));
request.addContent(new Element("to").setText(to + ""));
ServiceConfig config = new ServiceConfig();
if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
Log.debug(Geonet.Z3950_SERVER, "Search request:\n" + Xml.getString(request));
// get result set
Element result = this.metasearcher.present(this.srvxtx, request, config);
if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
Log.debug(Geonet.Z3950_SERVER, "Search result:\n" + Xml.getString(result));
// remove summary
result.removeChildren("summary");
@SuppressWarnings("unchecked")
List<Element> list = result.getChildren();
if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
Log.debug(Geonet.Z3950_SERVER, "Set name asked:" + spec);
// save other records to fragment
for (int i = 0; i < count; i++) {
Element md = list.get(0);
md.detach();
if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
Log.debug(Geonet.Z3950_SERVER, "Returning fragment:\n" + Xml.getString(md));
// add metadata
// fragment[i] = new DOMTree("geonetwork", "geonetwork", null, getRecord(md),rec_spec );
// fragment[i].setHitNo(startingFragment+i);
DOMOutputter outputter = new DOMOutputter();
Document doc = new Document(md);
org.w3c.dom.Document doc2 = outputter.output(doc);
fragment[i] =
new InformationFragmentImpl(
startingFragment + i, "geonetwork", "geonetwork", null, doc2, rec_spec);
// fragment[i] = new
// InformationFragmentImpl(startingFragment+i,"geonetwork","geonetwork",null,doc,rec_spec);
// System.err.println(fragment[i]);
}
this.srvxtx.getResourceManager().close();
if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
Log.debug(Geonet.Z3950_SERVER, "Fragment returned");
} catch (Throwable e) {
try {
this.srvxtx.getResourceManager().abort();
} catch (Exception e2) {
e2.printStackTrace();
}
if (Log.isDebugEnabled(Geonet.Z3950_SERVER))
Log.debug(Geonet.Z3950_SERVER, "Exception: " + e.getClass().getName() + " " + e);
// e.printStackTrace();
}
return fragment;
}