/**
* Creates a filter from a JSON object
*
* @param json The json to extract "field","operation" (optional) and "regex" from
* @throws HarvesterException if
* <ul>
* <li>field is missing
* <li>operation is not a valid operation
* <li>regex is missing or not a valid regex string
* </ul>
*/
public Filter(JsonSimple json) throws HarvesterException {
field = json.getString(null, "field");
if (field == null) {
throw new HarvesterException(
"In a filter definition, missing the mandatory attribute 'field'");
}
String matchTypeStr = json.getString("ANY", "multi");
try {
type = MultiMatchType.valueOf(matchTypeStr);
} catch (Exception e) {
throw new HarvesterException(
"In a filter definition, invalid filter match type '"
+ matchTypeStr
+ "', valid values are "
+ MultiMatchType.values());
}
String regexStr = json.getString(null, "regex");
if (regexStr == null) {
throw new HarvesterException(
"In a filter definition, missing the mandatory attribute 'regex'");
}
try {
regex = Pattern.compile(regexStr, Pattern.MULTILINE);
} catch (PatternSyntaxException e) {
throw new HarvesterException(
"In a filter definition, provided regex was invalid " + e.getMessage(), e);
}
}
/** Basic constructor, should be run automatically by Tapestry. */
public PortalSecurityManagerImpl() throws IOException {
// Get system configuration
config = new JsonSimpleConfig();
// For all SSO providers configured
sso = new LinkedHashMap<String, SSOInterface>();
for (String ssoId : config.getStringList("sso", "plugins")) {
// Instantiate from the ServiceLoader
SSOInterface valid = getSSOProvider(ssoId);
if (valid == null) {
log.error("Invalid SSO Implementation requested: '{}'", ssoId);
} else {
// Store valid implementations
sso.put(ssoId, valid);
log.info("SSO Provider instantiated: '{}'", ssoId);
}
}
defaultPortal = config.getString(PortalManager.DEFAULT_PORTAL_NAME, "portal", "defaultView");
serverUrlBase = config.getString(null, "urlBase");
ssoLoginUrl = serverUrlBase + defaultPortal + SSO_LOGIN_PAGE;
// Get exclusions Strings from config
excStarts = config.getStringList("sso", "urlExclusions", "startsWith");
excEnds = config.getStringList("sso", "urlExclusions", "endsWith");
excEquals = config.getStringList("sso", "urlExclusions", "equals");
// Trust tokens
Map<String, JsonSimple> tokenMap = config.getJsonSimpleMap("sso", "trustTokens");
tokens = new HashMap<String, String>();
tokenExpiry = new HashMap<String, Long>();
for (String key : tokenMap.keySet()) {
JsonSimple tok = tokenMap.get(key);
String publicKey = tok.getString(null, "publicKey");
String privateKey = tok.getString(null, "privateKey");
String expiry = tok.getString(TRUST_TOKEN_EXPIRY, "expiry");
if (publicKey != null && privateKey != null) {
// Valid key
tokens.put(publicKey, privateKey);
tokenExpiry.put(publicKey, Long.valueOf(expiry));
} else {
log.error("Invalid token data: '{}'", key);
}
}
}
/**
* Initialise the CSV harvester plugin.
*
* @throws HarvesterException if an error occurred
*/
@Override
public void init() throws HarvesterException {
JsonSimple options = new JsonSimple(getJsonConfig().getObject("harvester", "csv"));
String filePath = options.getString(null, "fileLocation");
if (filePath == null) {
throw new HarvesterException("No data file provided!");
}
File csvDataFile = new File(filePath);
if (csvDataFile == null || !csvDataFile.exists()) {
throw new HarvesterException("Could not find CSV file '" + filePath + "'");
}
filename = csvDataFile.getName();
idPrefix = options.getString("", "recordIDPrefix");
maxRows = options.getInteger(-1, "maxRows");
delimiter = options.getString(String.valueOf(DEFAULT_DELIMITER), "delimiter").charAt(0);
ignoredFields = getStringList(options, "ignoreFields");
includedFields = getStringList(options, "includedFields");
multiValueFields = getStringList(options, "multiValueFields");
multiValueFieldDelimiter =
options
.getString(
String.valueOf(DEFAULT_MULTI_VALUE_FIELD_DELIMITER), "multiValueFieldDelimiter")
.charAt(0);
payloadId = options.getString(DEFAULT_PAYLOAD_ID, "payloadId");
batchSize = options.getInteger(DEFAULT_BATCH_SIZE, "batchSize");
hasMore = true;
if (delimiter == multiValueFieldDelimiter) {
throw new HarvesterException(
"Cannot parse CSV: The requested delimiters for the CSV and multivalue fields are the same: "
+ delimiter);
}
try {
// open the CSV file for reading
Reader fileReader = new InputStreamReader(new FileInputStream(csvDataFile), "UTF-8");
// char delimiter = options.getString(String.valueOf(DEFAULT_DELIMITER),
// "delimiter").charAt(0);
csvReader = new CSVReader(fileReader, delimiter);
// configure the data fields
if (options.getBoolean(true, "headerRow")) {
dataFields = Arrays.asList(csvReader.readNext());
} else {
dataFields = getStringList(options, "headerList");
}
// check that the specified id column is valid
idColumn = options.getString(null, "idColumn");
if (idColumn != null && !dataFields.contains(idColumn)) {
throw new HarvesterException(
"ID column '" + idColumn + "' was invalid or not found in the data!");
}
// load filters, all filters must pass for the row to be considered
filters = new HashMap<String, List<Filter>>();
List<JsonSimple> filterConfig = options.getJsonSimpleList("filters");
if (filterConfig != null) {
for (JsonSimple singleFilterConfig : filterConfig) {
Filter filter = new Filter(singleFilterConfig);
String field = filter.getField();
if (!dataFields.contains(field)) {
throw new HarvesterException("Filter column '" + field + "' was not found in the data");
}
List<Filter> existingFilters = filters.get(field);
if (existingFilters == null) {
existingFilters = new ArrayList<Filter>();
filters.put(field, existingFilters);
}
existingFilters.add(filter);
}
}
} catch (IOException ioe) {
throw new HarvesterException(ioe);
}
}
