/**
* Creates a filter from a JSON object
*
* @param json The json to extract "field","operation" (optional) and "regex" from
* @throws HarvesterException if
* <ul>
* <li>field is missing
* <li>operation is not a valid operation
* <li>regex is missing or not a valid regex string
* </ul>
*/
public Filter(JsonSimple json) throws HarvesterException {
field = json.getString(null, "field");
if (field == null) {
throw new HarvesterException(
"In a filter definition, missing the mandatory attribute 'field'");
}
String matchTypeStr = json.getString("ANY", "multi");
try {
type = MultiMatchType.valueOf(matchTypeStr);
} catch (Exception e) {
throw new HarvesterException(
"In a filter definition, invalid filter match type '"
+ matchTypeStr
+ "', valid values are "
+ MultiMatchType.values());
}
String regexStr = json.getString(null, "regex");
if (regexStr == null) {
throw new HarvesterException(
"In a filter definition, missing the mandatory attribute 'regex'");
}
try {
regex = Pattern.compile(regexStr, Pattern.MULTILINE);
} catch (PatternSyntaxException e) {
throw new HarvesterException(
"In a filter definition, provided regex was invalid " + e.getMessage(), e);
}
}
/**
* Merge the newly processed data with an (possible) existing data already present, also convert
* the completed JSON merge into a Stream for storage.
*
* @param dataJson an instantiated JSON object containing data to store
* @param metaJson an instantiated JSON object containing metadata to store
* @param existing an instantiated JsonSimple object with any existing data
* @throws IOException if any character encoding issues effect the Stream
*/
private InputStream streamMergedJson(
JsonObject dataJson, JsonObject metaJson, JsonSimple existing) throws IOException {
// Overwrite and/or create only nodes we consider new data
existing.getJsonObject().put("recordIDPrefix", idPrefix);
JsonObject existingData = existing.writeObject("data");
existingData.putAll(dataJson);
JsonObject existingMeta = existing.writeObject("metadata");
existingMeta.putAll(metaJson);
// Turn into a stream to return
String jsonString = existing.toString(true);
return IOUtils.toInputStream(jsonString, "UTF-8");
}
/**
* Gets a string list from a JsonSimple object. Convenience method to return an empty list instead
* of null if the node was not found.
*
* @param json a JsonSimple object
* @param path path to the node
* @return string list found at node, or empty if not found
*/
private List<String> getStringList(JsonSimple json, Object... path) {
List<String> list = json.getStringList(path);
if (list == null) {
list = Collections.emptyList();
}
return list;
}
/** Basic constructor, should be run automatically by Tapestry. */
public PortalSecurityManagerImpl() throws IOException {
// Get system configuration
config = new JsonSimpleConfig();
// For all SSO providers configured
sso = new LinkedHashMap<String, SSOInterface>();
for (String ssoId : config.getStringList("sso", "plugins")) {
// Instantiate from the ServiceLoader
SSOInterface valid = getSSOProvider(ssoId);
if (valid == null) {
log.error("Invalid SSO Implementation requested: '{}'", ssoId);
} else {
// Store valid implementations
sso.put(ssoId, valid);
log.info("SSO Provider instantiated: '{}'", ssoId);
}
}
defaultPortal = config.getString(PortalManager.DEFAULT_PORTAL_NAME, "portal", "defaultView");
serverUrlBase = config.getString(null, "urlBase");
ssoLoginUrl = serverUrlBase + defaultPortal + SSO_LOGIN_PAGE;
// Get exclusions Strings from config
excStarts = config.getStringList("sso", "urlExclusions", "startsWith");
excEnds = config.getStringList("sso", "urlExclusions", "endsWith");
excEquals = config.getStringList("sso", "urlExclusions", "equals");
// Trust tokens
Map<String, JsonSimple> tokenMap = config.getJsonSimpleMap("sso", "trustTokens");
tokens = new HashMap<String, String>();
tokenExpiry = new HashMap<String, Long>();
for (String key : tokenMap.keySet()) {
JsonSimple tok = tokenMap.get(key);
String publicKey = tok.getString(null, "publicKey");
String privateKey = tok.getString(null, "privateKey");
String expiry = tok.getString(TRUST_TOKEN_EXPIRY, "expiry");
if (publicKey != null && privateKey != null) {
// Valid key
tokens.put(publicKey, privateKey);
tokenExpiry.put(publicKey, Long.valueOf(expiry));
} else {
log.error("Invalid token data: '{}'", key);
}
}
}
/** Reset the transformer in preparation for a new object */
private void reset() throws TransformerException {
if (firstRun) {
firstRun = false;
// Output directory
String outputPath = config.getString(null, "outputPath");
if (outputPath == null) {
throw new TransformerException("Output path not specified!");
}
outputDir = new File(outputPath);
outputDir.mkdirs();
// Rendition exclusions
excludeList =
Arrays.asList(StringUtils.split(config.getString(null, "excludeRenditionExt"), ','));
// Conversion Service URL
convertUrl = config.getString(null, "url");
if (convertUrl == null) {
throw new TransformerException("No ICE URL provided!");
}
}
// Priority
Boolean testResponse = itemConfig.getBoolean(null, "priority");
if (testResponse != null) {
// We found it in item config
priority = testResponse;
} else {
// Try system config
priority = config.getBoolean(true, "priority");
}
// Clear the old SAX reader
reader = new SafeSAXReader();
// Remove the last object
thumbnails = null;
previews = null;
}
/**
* Main render method to send the file to ICE service
*
* @param sourceFile : File to be rendered
* @return file returned by ICE service
* @throws TransformerException
*/
private File render(File sourceFile) throws TransformerException {
log.info("Converting {}...", sourceFile);
String filename = sourceFile.getName();
String basename = FilenameUtils.getBaseName(filename);
String ext = FilenameUtils.getExtension(filename);
int status = HttpStatus.SC_OK;
// Grab our config
JsonObject object = itemConfig.getObject("resize");
Map<String, JsonSimple> resizeConfig = JsonSimple.toJavaMap(object);
if (resizeConfig == null || resizeConfig.isEmpty()) {
// Try system config instead
object = config.getObject("resize");
resizeConfig = JsonSimple.toJavaMap(object);
if (resizeConfig == null || resizeConfig.isEmpty()) {
throw new TransformerException("No resizing configuration found.");
}
}
String resizeJson = "";
for (String key : resizeConfig.keySet()) {
JsonSimple j = resizeConfig.get(key);
resizeJson += "\"" + key + "\":" + j.toString() + ",";
}
PostMethod post = new PostMethod(convertUrl);
try {
Part[] parts = {
new StringPart("zip", "on"),
new StringPart("dc", "on"),
new StringPart("toc", "on"),
new StringPart("pdflink", "on"),
new StringPart("addThumbnail", "on"),
new StringPart("pathext", ext),
new StringPart("template", getTemplate()),
new StringPart(
"multipleImageOptions", "{" + StringUtils.substringBeforeLast(resizeJson, ",") + "}"),
new StringPart("mode", "download"),
new FilePart("file", sourceFile)
};
post.setRequestEntity(new MultipartRequestEntity(parts, post.getParams()));
BasicHttpClient client = new BasicHttpClient(convertUrl);
log.debug("Using conversion URL: {}", convertUrl);
status = client.executeMethod(post);
log.debug("HTTP status: {} {}", status, HttpStatus.getStatusText(status));
} catch (IOException ioe) {
throw new TransformerException("Failed to send ICE conversion request", ioe);
}
try {
if (status != HttpStatus.SC_OK) {
String xmlError = post.getResponseBodyAsString();
log.debug("Error: {}", xmlError);
throw new TransformerException(xmlError);
}
String type = post.getResponseHeader("Content-Type").getValue();
if ("application/zip".equals(type)) {
filename = basename + ".zip";
} else if (type.startsWith("image/")) {
filename = basename + "_thumbnail.jpg";
} else if ("video/x-flv".equals(type)) {
filename = basename + ".flv";
} else if ("audio/mpeg".equals(type)) {
filename = basename + ".mp3";
}
File outputFile = new File(outputDir, filename);
if (outputFile.exists()) {
outputFile.delete();
}
InputStream in = post.getResponseBodyAsStream();
FileOutputStream out = new FileOutputStream(outputFile);
IOUtils.copy(in, out);
in.close();
out.close();
log.debug("ICE output file: {}", outputFile);
return outputFile;
} catch (IOException ioe) {
throw new TransformerException("Failed to process ICE output", ioe);
}
}
/**
* Initialise the CSV harvester plugin.
*
* @throws HarvesterException if an error occurred
*/
@Override
public void init() throws HarvesterException {
JsonSimple options = new JsonSimple(getJsonConfig().getObject("harvester", "csv"));
String filePath = options.getString(null, "fileLocation");
if (filePath == null) {
throw new HarvesterException("No data file provided!");
}
File csvDataFile = new File(filePath);
if (csvDataFile == null || !csvDataFile.exists()) {
throw new HarvesterException("Could not find CSV file '" + filePath + "'");
}
filename = csvDataFile.getName();
idPrefix = options.getString("", "recordIDPrefix");
maxRows = options.getInteger(-1, "maxRows");
delimiter = options.getString(String.valueOf(DEFAULT_DELIMITER), "delimiter").charAt(0);
ignoredFields = getStringList(options, "ignoreFields");
includedFields = getStringList(options, "includedFields");
multiValueFields = getStringList(options, "multiValueFields");
multiValueFieldDelimiter =
options
.getString(
String.valueOf(DEFAULT_MULTI_VALUE_FIELD_DELIMITER), "multiValueFieldDelimiter")
.charAt(0);
payloadId = options.getString(DEFAULT_PAYLOAD_ID, "payloadId");
batchSize = options.getInteger(DEFAULT_BATCH_SIZE, "batchSize");
hasMore = true;
if (delimiter == multiValueFieldDelimiter) {
throw new HarvesterException(
"Cannot parse CSV: The requested delimiters for the CSV and multivalue fields are the same: "
+ delimiter);
}
try {
// open the CSV file for reading
Reader fileReader = new InputStreamReader(new FileInputStream(csvDataFile), "UTF-8");
// char delimiter = options.getString(String.valueOf(DEFAULT_DELIMITER),
// "delimiter").charAt(0);
csvReader = new CSVReader(fileReader, delimiter);
// configure the data fields
if (options.getBoolean(true, "headerRow")) {
dataFields = Arrays.asList(csvReader.readNext());
} else {
dataFields = getStringList(options, "headerList");
}
// check that the specified id column is valid
idColumn = options.getString(null, "idColumn");
if (idColumn != null && !dataFields.contains(idColumn)) {
throw new HarvesterException(
"ID column '" + idColumn + "' was invalid or not found in the data!");
}
// load filters, all filters must pass for the row to be considered
filters = new HashMap<String, List<Filter>>();
List<JsonSimple> filterConfig = options.getJsonSimpleList("filters");
if (filterConfig != null) {
for (JsonSimple singleFilterConfig : filterConfig) {
Filter filter = new Filter(singleFilterConfig);
String field = filter.getField();
if (!dataFields.contains(field)) {
throw new HarvesterException("Filter column '" + field + "' was not found in the data");
}
List<Filter> existingFilters = filters.get(field);
if (existingFilters == null) {
existingFilters = new ArrayList<Filter>();
filters.put(field, existingFilters);
}
existingFilters.add(filter);
}
}
} catch (IOException ioe) {
throw new HarvesterException(ioe);
}
}
