/**
* Create job arguments that are used by CrowdClient to build a POST request for a new job on
* Crowdflower
*/
void createArgumentMaps() {
argumentMap = new LinkedMultiValueMap<String, String>();
includedCountries = new Vector<String>();
excludedCountries = new Vector<String>();
Iterator<Map.Entry<String, JsonNode>> jsonRootIt = template.fields();
for (Map.Entry<String, JsonNode> elt; jsonRootIt.hasNext(); ) {
elt = jsonRootIt.next();
JsonNode currentNode = elt.getValue();
String currentKey = elt.getKey();
if (currentNode.isContainerNode()) {
// special processing for these arrays:
if (currentKey.equals(includedCountriesKey) || currentKey.equals(excludedCountriesKey)) {
Iterator<JsonNode> jsonSubNodeIt = currentNode.elements();
for (JsonNode subElt; jsonSubNodeIt.hasNext(); ) {
subElt = jsonSubNodeIt.next();
(currentKey.equals(includedCountriesKey) ? includedCountries : excludedCountries)
.addElement(subElt.path(countryCodeKey).asText());
}
}
} else if (!currentNode.isNull() && argumentFilterSet.contains(currentKey)) {
argumentMap.add(jobKey + "[" + currentKey + "]", currentNode.asText());
}
if (currentKey == idKey) {
this.id = currentNode.asText();
}
}
}
protected UIEntity fieldsToUIEntity(Iterator<Map.Entry<String, JsonNode>> fields) {
UIEntity entity = new UIEntity();
while (fields.hasNext()) {
Map.Entry<String, JsonNode> field = fields.next();
JsonNode fieldValue = field.getValue();
if (fieldValue instanceof ObjectNode) {
entity.put(field.getKey(), fieldsToUIEntity(field.getValue().fields()));
} else if (fieldValue instanceof ArrayNode) {
Iterator<JsonNode> elements = fieldValue.elements();
List<Object> listValues = new ArrayList<>();
while (elements.hasNext()) {
JsonNode nextNode = elements.next();
if (nextNode instanceof ObjectNode) {
listValues.add(fieldsToUIEntity(nextNode.fields()));
} else {
listValues.add(nextNode.asText());
}
}
entity.put(field.getKey(), listValues);
} else if (!(field.getValue() instanceof NullNode)) {
entity.put(field.getKey(), field.getValue().asText());
}
}
return entity;
}
public static void firstSituation(String ok) throws IOException {
byte[] jsonData = ok.getBytes();
ObjectMapper objectMapper = new ObjectMapper();
JsonNode rootNode = objectMapper.readTree(jsonData);
JsonNode idNode = rootNode.path("id");
System.out.println("id = " + idNode.asInt());
JsonNode phoneNosNode = rootNode.path("phoneNumbers");
Iterator<JsonNode> elements = phoneNosNode.elements();
while (elements.hasNext()) {
JsonNode phone = elements.next();
System.out.println("phoneNumbers = " + phone.asLong());
}
}
/**
* Get a list of tv certification.
*
* @return
* @throws MovieDbException
*/
public ResultsMap<String, List<Certification>> getTvCertification() throws MovieDbException {
URL url = new ApiUrl(apiKey, MethodBase.CERTIFICATION).subMethod(MethodSub.TV_LIST).buildUrl();
String webpage = httpTools.getRequest(url);
try {
JsonNode node = MAPPER.readTree(webpage);
Map<String, List<Certification>> results =
MAPPER.readValue(
node.elements().next().traverse(),
new TypeReference<Map<String, List<Certification>>>() {});
return new ResultsMap<>(results);
} catch (IOException ex) {
throw new MovieDbException(
ApiExceptionType.MAPPING_FAILED, "Failed to get TV certifications", url, ex);
}
}
public void load(JsonNode json) {
setPluginName(json.get("plugin_name").asText());
setName(json.get("name").asText());
setId(json.get("id").asText());
setHadoopVersion(json.get("hadoop_version").asText());
JsonNode node_groups = json.get("node_groups");
Iterator<JsonNode> elements = node_groups.elements();
NodeTemplateCount t = new NodeTemplateCount();
ArrayList<NodeTemplateCount> temp = new ArrayList<NodeTemplateCount>();
while (elements.hasNext()) {
JsonNode node = elements.next();
logger.info("node template count" + node.toString());
t.setCount(node.get("count").asInt());
t.setNode(node.get("name").asText());
temp.add(t);
}
logger.info("Before setNodeTemplatelist");
setNodeTemplatelist(temp);
logger.info("After setNodeTemplatelist");
}
@Override
public void onMessage(Message message) {
try {
String payload = ((TextMessage) message).getText();
JsonNode rootNode = objectMapper.readTree(payload);
JsonNode metricData = rootNode.get("metricData");
// Get <rid>.status.code metrics
String tenant = metricData.get("tenantId").textValue();
JsonNode data = metricData.get("data");
List<SingleAvail> outer = new ArrayList<>();
Iterator<JsonNode> items = data.elements();
while (items.hasNext()) {
JsonNode item = items.next();
String source = item.get("source").textValue();
if (source.endsWith(".status.code")) {
int code = item.get("value").intValue();
String id = source.substring(0, source.indexOf("."));
long timestamp = item.get("timestamp").longValue();
String avail = computeAvail(code);
SingleAvail ar = new SingleAvail(tenant, id, timestamp, avail);
outer.add(ar);
}
}
if (!outer.isEmpty()) {
availPublisher.publish(outer);
}
} catch (Exception e) {
Log.LOG.eCouldNotHandleBusMessage(e);
}
}
private static void addTimeHorizons(JsonNode document, ExternalConfigurationModel model)
throws Exception {
JsonNode node = StreamAggregatorUtils.readJsonValue(document, "timeHorizons");
if (node != null) {
Iterator<JsonNode> timeHorizonValues = node.elements();
while (timeHorizonValues.hasNext()) {
String t = timeHorizonValues.next().asText();
String timeHorizonName = null;
int granularity = -1;
// process parameterised time horizons
if (t.contains("MINUTES_GROUPED")) {
String[] items = t.split("\\(");
timeHorizonName = items[0];
granularity = Integer.parseInt(items[1].replaceAll("\\)", ""));
} else {
timeHorizonName = t;
}
try {
TimeHorizon th = TimeHorizon.valueOf(timeHorizonName);
if (th.equals(TimeHorizon.MINUTES_GROUPED) && granularity == -1) {
throw new InvalidConfigurationException(
"Unable to create Grouped Minutes Time Horizon without configuration of Granularity using notation MINUTES_GROUPED(<granularity in minutes>)");
} else {
if (th.equals(TimeHorizon.MINUTES_GROUPED)) {
th.setGranularity(granularity);
}
}
model.addTimeHorizon(th);
} catch (Exception e) {
throw new Exception(String.format("Unable to configure Time Horizon %s", t), e);
}
}
}
}
public static List<ExternalConfigurationModel> buildFromConfig(String configFilePath)
throws Exception {
List<ExternalConfigurationModel> response = new ArrayList<>();
// reference the config file as a full path
File configFile = new File(configFilePath);
if (!configFile.exists()) {
// try to load the file from the classpath
InputStream classpathConfig =
ExternalConfigurationModel.class.getClassLoader().getResourceAsStream(configFilePath);
if (classpathConfig != null && classpathConfig.available() > 0) {
configFile =
new File(
ExternalConfigurationModel.class
.getResource((configFilePath.startsWith("/") ? "" : "/") + configFilePath)
.toURI());
LOG.info(String.format("Loaded Configuration %s from Classpath", configFilePath));
} else {
if (configFilePath.startsWith("s3://")) {
AmazonS3 s3Client = new AmazonS3Client(new DefaultAWSCredentialsProviderChain());
TransferManager tm = new TransferManager(s3Client);
// parse the config path to get the bucket name and prefix
final String s3ProtoRegex = "s3:\\/\\/";
String bucket = configFilePath.replaceAll(s3ProtoRegex, "").split("/")[0];
String prefix =
configFilePath.replaceAll(String.format("%s%s\\/", s3ProtoRegex, bucket), "");
// download the file using TransferManager
configFile = File.createTempFile(configFilePath, null);
Download download = tm.download(bucket, prefix, configFile);
download.waitForCompletion();
// shut down the transfer manager
tm.shutdownNow();
LOG.info(
String.format(
"Loaded Configuration from Amazon S3 %s/%s to %s",
bucket, prefix, configFile.getAbsolutePath()));
} else {
// load the file from external URL
try {
configFile = File.createTempFile(configFilePath, null);
FileUtils.copyURLToFile(new URL(configFilePath), configFile, 1000, 1000);
LOG.info(
String.format(
"Loaded Configuration from %s to %s",
configFilePath, configFile.getAbsolutePath()));
} catch (IOException e) {
// handle the timeouts and so on with a generalised
// config
// file not found handler later
}
}
}
} else {
LOG.info(String.format("Loaded Configuration from Filesystem %s", configFilePath));
}
// if we haven't been able to load a config file, then bail
if (configFile == null || !configFile.exists()) {
throw new InvalidConfigurationException(
String.format("Unable to Load Config File from %s", configFilePath));
}
JsonNode document = StreamAggregatorUtils.asJsonNode(configFile);
ExternalConfigurationModel config = null;
Iterator<JsonNode> i = document.elements();
while (i.hasNext()) {
config = new ExternalConfigurationModel();
JsonNode section = i.next();
// set generic properties
config.setNamespace(StreamAggregatorUtils.readValueAsString(section, "namespace"));
config.setDateFormat(StreamAggregatorUtils.readValueAsString(section, "dateFormat"));
addTimeHorizons(section, config);
setAggregatorType(section, config);
// set the label items
JsonNode labelItems = StreamAggregatorUtils.readJsonValue(section, "labelItems");
if (labelItems != null && labelItems.size() > 0) {
Iterator<JsonNode> iterator = labelItems.elements();
while (iterator.hasNext()) {
JsonNode n = iterator.next();
config.addLabelItems(n.asText());
}
}
config.setLabelAttributeAlias(
StreamAggregatorUtils.readValueAsString(section, "labelAttributeAlias"));
config.setDateItem(StreamAggregatorUtils.readValueAsString(section, "dateItem"));
config.setDateAttributeAlias(
StreamAggregatorUtils.readValueAsString(section, "dateAttributeAlias"));
JsonNode summaryItems = StreamAggregatorUtils.readJsonValue(section, "summaryItems");
if (summaryItems != null && summaryItems.size() > 0) {
Iterator<JsonNode> iterator = summaryItems.elements();
while (iterator.hasNext()) {
JsonNode n = iterator.next();
config.addSummaryItem(n.asText());
}
}
config.setTableName(StreamAggregatorUtils.readValueAsString(section, "tableName"));
String readIO = StreamAggregatorUtils.readValueAsString(section, "readIOPS");
if (readIO != null) config.setReadIOPs(Long.parseLong(readIO));
String writeIO = StreamAggregatorUtils.readValueAsString(section, "writeIOPS");
if (writeIO != null) config.setWriteIOPs(Long.parseLong(writeIO));
// configure tolerance of data extraction problems
String failOnDataExtraction =
StreamAggregatorUtils.readValueAsString(section, "failOnDataExtraction");
if (failOnDataExtraction != null)
config.setFailOnDataExtraction(Boolean.parseBoolean(failOnDataExtraction));
// configure whether metrics should be emitted
String emitMetrics = StreamAggregatorUtils.readValueAsString(section, "emitMetrics");
String metricsEmitterClassname =
StreamAggregatorUtils.readValueAsString(section, "metricsEmitterClass");
if (emitMetrics != null || metricsEmitterClassname != null) {
if (metricsEmitterClassname != null) {
config.setMetricsEmitter(
(Class<IMetricsEmitter>)
ClassLoader.getSystemClassLoader().loadClass(metricsEmitterClassname));
} else {
config.setEmitMetrics(Boolean.parseBoolean(emitMetrics));
}
}
// configure the data store class
String dataStoreClass = StreamAggregatorUtils.readValueAsString(section, "IDataStore");
if (dataStoreClass != null) {
Class<IDataStore> dataStore =
(Class<IDataStore>) ClassLoader.getSystemClassLoader().loadClass(dataStoreClass);
config.setDataStore(dataStore);
}
// get the data extractor configuration, so we know what other json
// elements to retrieve from the configuration document
String useExtractor = null;
try {
useExtractor = StreamAggregatorUtils.readValueAsString(section, "dataExtractor");
config.setDataExtractor(DataExtractor.valueOf(useExtractor));
} catch (Exception e) {
throw new Exception(
String.format("Unable to configure aggregator with Data Extractor %s", useExtractor));
}
switch (config.getDataExtractor()) {
case CSV:
configureStringCommon(section, config);
configureCsv(section, config);
break;
case JSON:
configureStringCommon(section, config);
break;
case OBJECT:
configureObject(section, config);
break;
case REGEX:
configureRegex(section, config);
}
response.add(config);
}
return response;
}
{
super(1, nodecursor);
_contents = jsonnode.elements();
}