/**
* 20141022.
*
* @param jObj the j obj
* @param projectionStr the projection str
* @return the FQDN value list cms
* @throws JSONException the JSON exception
*/
static List<String> getFQDNValueListCMS(JSONObject jObj, String projectionStr)
throws JSONException {
final List<String> labelList = new ArrayList<String>();
if (!jObj.has("result")) {
logger.error(
"!!CMS_ERROR! result key is not in jOBJ in getFQDNValueListCMS!!: \njObj:"
+ PcStringUtils.renderJson(jObj));
return labelList;
}
JSONArray jArr = (JSONArray) jObj.get("result");
if (jArr == null || jArr.length() == 0) {
return labelList;
}
for (int i = 0; i < jArr.length(); ++i) {
JSONObject agentObj = jArr.getJSONObject(i);
// properties can be null
if (!agentObj.has(projectionStr)) {
continue;
}
String label = (String) agentObj.get(projectionStr);
if (label != null && !label.trim().isEmpty()) {
labelList.add(label);
}
}
return labelList;
}
@Override
public boolean doWork(Message message) {
try {
// either create and insert a new payload:
// JSONObject json_out = new JSONObject();
// json_out.put("processed", true);
// message.setPayload(json_out);
// or work on the existing one:
JSONObject tweet = message.getPayload();
if (processTweet(tweet)) {
JSONObject result = new JSONObject();
result.put("tweet_id", tweet.getLong("id"));
result.put("date_posted", tweet.getLong("date_posted"));
if (tweet.has("images")) result.put("images", tweet.getJSONArray("images"));
if (tweet.has("coordinates") && !tweet.isNull("coordinates"))
result.put("coordinates", tweet.getJSONObject("coordinates"));
message.setPayload(result);
return true;
} else {
return false;
}
} catch (JSONException e) {
log.error("JSONException: " + e);
logConn.error("JSONException: " + e);
return false;
}
}
@Test
public void testAppDataPush() throws Exception {
final String topic = "xyz";
final List<String> messages = new ArrayList<>();
EmbeddedWebSocketServer server = new EmbeddedWebSocketServer(0);
server.setWebSocket(
new WebSocket.OnTextMessage() {
@Override
public void onMessage(String data) {
messages.add(data);
}
@Override
public void onOpen(WebSocket.Connection connection) {}
@Override
public void onClose(int closeCode, String message) {}
});
try {
server.start();
int port = server.getPort();
TestGeneratorInputOperator o1 = dag.addOperator("o1", TestGeneratorInputOperator.class);
GenericTestOperator o2 = dag.addOperator("o2", GenericTestOperator.class);
dag.addStream("o1.outport", o1.outport, o2.inport1);
dag.setAttribute(LogicalPlan.METRICS_TRANSPORT, new AutoMetricBuiltInTransport(topic));
dag.setAttribute(LogicalPlan.GATEWAY_CONNECT_ADDRESS, "localhost:" + port);
dag.setAttribute(LogicalPlan.PUBSUB_CONNECT_TIMEOUT_MILLIS, 2000);
StramLocalCluster lc = new StramLocalCluster(dag);
StreamingContainerManager dnmgr = lc.dnmgr;
StramAppContext appContext = new StramTestSupport.TestAppContext(dag.getAttributes());
AppDataPushAgent pushAgent = new AppDataPushAgent(dnmgr, appContext);
pushAgent.init();
pushAgent.pushData();
Thread.sleep(1000);
Assert.assertTrue(messages.size() > 0);
pushAgent.close();
JSONObject message = new JSONObject(messages.get(0));
Assert.assertEquals(topic, message.getString("topic"));
Assert.assertEquals("publish", message.getString("type"));
JSONObject data = message.getJSONObject("data");
Assert.assertTrue(StringUtils.isNotBlank(data.getString("appId")));
Assert.assertTrue(StringUtils.isNotBlank(data.getString("appUser")));
Assert.assertTrue(StringUtils.isNotBlank(data.getString("appName")));
JSONObject logicalOperators = data.getJSONObject("logicalOperators");
for (String opName : new String[] {"o1", "o2"}) {
JSONObject opObj = logicalOperators.getJSONObject(opName);
Assert.assertTrue(opObj.has("totalTuplesProcessed"));
Assert.assertTrue(opObj.has("totalTuplesEmitted"));
Assert.assertTrue(opObj.has("tuplesProcessedPSMA"));
Assert.assertTrue(opObj.has("tuplesEmittedPSMA"));
Assert.assertTrue(opObj.has("latencyMA"));
}
} finally {
server.stop();
}
}
private HashMap<String, String[]> getCriteria(JSONArray criterias) {
HashMap<String, String[]> criteriaValues = new HashMap<String, String[]>();
try {
for (int i = 0; i < criterias.length(); i++) {
JSONObject criteria = criterias.getJSONObject(i);
if (!criteria.has("fieldName")
&& criteria.has("criteria")
&& criteria.has("_constructor")) {
// nested criteria, eval it recursively
JSONArray cs = criteria.getJSONArray("criteria");
HashMap<String, String[]> c = getCriteria(cs);
for (String k : c.keySet()) {
criteriaValues.put(k, c.get(k));
}
continue;
}
final String operator = criteria.getString("operator");
final String fieldName = criteria.getString("fieldName");
String[] criterion;
if (operator.equals(AdvancedQueryBuilder.OPERATOR_EXISTS)
&& criteria.has(AdvancedQueryBuilder.EXISTS_QUERY_KEY)) {
String value = "";
JSONArray values = criteria.getJSONArray("value");
for (int v = 0; v < values.length(); v++) {
value += value.length() > 0 ? ", " : "";
value += "'" + values.getString(v) + "'";
}
String qry =
criteria
.getString(AdvancedQueryBuilder.EXISTS_QUERY_KEY)
.replace(AdvancedQueryBuilder.EXISTS_VALUE_HOLDER, value);
if (criteriaValues.containsKey(fieldName)) {
// assuming it is possible to have more than one query for exists in same field, storing
// them as array
String[] originalCriteria = criteriaValues.get(fieldName);
List<String> newCriteria = new ArrayList<String>(Arrays.asList(originalCriteria));
newCriteria.add(qry);
criteriaValues.put(fieldName, newCriteria.toArray(new String[newCriteria.size()]));
} else {
criteriaValues.put(
fieldName, new String[] {AdvancedQueryBuilder.EXISTS_QUERY_KEY, qry});
}
} else {
criterion = new String[] {operator, criteria.getString("value")};
criteriaValues.put(fieldName, criterion);
}
}
} catch (JSONException e) {
log.error("Error getting criteria for custom query selector", e);
}
if (criteriaValues.isEmpty()) {
return null;
}
return criteriaValues;
}
/**
* Creates a topic model from a JSON value.
*
* <p>Both topic serialization formats are supported: 1) canonic format -- contains entire topic
* models. 2) compact format -- contains the topic value only (simple or composite).
*/
private TopicModel createTopicModel(String childTypeUri, Object value) {
if (value instanceof JSONObject) {
JSONObject val = (JSONObject) value;
// we detect the canonic format by checking for a mandatory topic property
// ### TODO: "type_uri" should not be regarded mandatory. It would simplify update requests.
// ### Can we use another heuristic for detection: "value" exists OR "composite" exists?
if (val.has("type_uri")) {
// canonic format
return new TopicModel(val);
} else {
// compact format (composite topic)
return new TopicModel(childTypeUri, new CompositeValueModel(val));
}
} else {
// compact format (simple topic or topic reference)
if (value instanceof String) {
String val = (String) value;
if (val.startsWith(REF_ID_PREFIX)) {
return new TopicReferenceModel(refTopicId(val)); // topic reference by-ID
} else if (val.startsWith(REF_URI_PREFIX)) {
return new TopicReferenceModel(refTopicUri(val)); // topic reference by-URI
} else if (val.startsWith(DEL_PREFIX)) {
return new TopicDeletionModel(delTopicId(val)); // topic deletion reference
}
}
// compact format (simple topic)
return new TopicModel(childTypeUri, new SimpleValue(value));
}
}
private boolean parseCriteria(JSONObject jsonCriteria) throws JSONException {
// a constructor so the content is an advanced criteria
if (jsonCriteria.has("_constructor") || hasOrAndOperator(jsonCriteria)) {
return parseAdvancedCriteria(jsonCriteria);
}
return parseSingleClause(jsonCriteria);
}
public static FinancialAccounting fromJSON(JSONObject jObj) throws JSONException, ParseException {
if (jObj.has("result") && nullStringToNull(jObj, "result") != null) {
jObj = jObj.getJSONObject("result");
}
final FinancialAccounting financialAccounting =
new FinancialAccounting.Builder().revision(jObj.getLong("revision")).build();
if (!jObj.isNull("items")) {
final List<FinancialAccountingItem> items = new ArrayList<FinancialAccountingItem>();
final JSONArray jItems = jObj.getJSONArray("items");
for (int i = 0; i < jItems.length(); i++) {
final JSONObject jItem = jItems.getJSONObject(i);
final FinancialAccountingItem financialAccountingItem =
FinancialAccountingItem.fromJSON(jItem);
items.add(financialAccountingItem);
}
financialAccounting.setItems(items);
}
return financialAccounting;
}
private boolean hasOrAndOperator(JSONObject jsonCriteria) throws JSONException {
if (!jsonCriteria.has(OPERATOR_KEY)) {
return mainOperatorIsAnd;
}
return OPERATOR_OR.equals(jsonCriteria.get(OPERATOR_KEY))
|| OPERATOR_AND.equals(jsonCriteria.get(OPERATOR_KEY));
}
private boolean parseStructuredClause(JSONArray clauses, String hqlOperator)
throws JSONException {
boolean doOr = OPERATOR_OR.equals(hqlOperator);
boolean doAnd = OPERATOR_AND.equals(hqlOperator);
for (int i = 0; i < clauses.length(); i++) {
final JSONObject clause = clauses.getJSONObject(i);
if (clause.has(VALUE_KEY)
&& clause.get(VALUE_KEY) != null
&& clause.getString(VALUE_KEY).equals("")) {
continue;
}
final boolean clauseResult = parseCriteria(clause);
if (doOr && clauseResult) {
return true;
}
if (doAnd && !clauseResult) {
return false;
}
}
if (doOr) {
return false;
} else if (doAnd) {
return true;
}
return mainOperatorIsAnd;
}
private void readCriteria(Map<String, String> parameters) throws JSONException {
JSONArray criteriaArray = (JSONArray) JsonUtils.buildCriteria(parameters).get("criteria");
selectedIds = new ArrayList<String>();
selectedChValues = new HashMap<String, List<CharacteristicValue>>();
nameFilter = null;
searchKeyFilter = null;
variantCreated = null;
for (int i = 0; i < criteriaArray.length(); i++) {
JSONObject criteria = criteriaArray.getJSONObject(i);
// Basic advanced criteria handling
if (criteria.has("_constructor")
&& "AdvancedCriteria".equals(criteria.getString("_constructor"))
&& criteria.has("criteria")) {
JSONArray innerCriteriaArray = new JSONArray(criteria.getString("criteria"));
criteria = innerCriteriaArray.getJSONObject(0);
}
String fieldName = criteria.getString("fieldName");
// String operatorName = criteria.getString("operator");
String value = criteria.getString("value");
if (fieldName.equals("name")) {
nameFilter = value;
} else if (fieldName.equals("searchKey")) {
searchKeyFilter = value;
} else if (fieldName.equals("id")) {
selectedIds.add(value);
} else if (fieldName.equals("variantCreated")) {
variantCreated = criteria.getBoolean("value");
} else if (fieldName.equals("characteristicDescription")) {
JSONArray values = new JSONArray(value);
// All values belong to the same characteristicId, get the first one.
String strCharacteristicId = null;
List<CharacteristicValue> chValueIds = new ArrayList<CharacteristicValue>();
for (int j = 0; j < values.length(); j++) {
CharacteristicValue chValue =
OBDal.getInstance().get(CharacteristicValue.class, values.getString(j));
chValueIds.add(chValue);
if (strCharacteristicId == null) {
strCharacteristicId = (String) DalUtil.getId(chValue.getCharacteristic());
}
}
selectedChValues.put(strCharacteristicId, chValueIds);
}
}
}
private JSONObject getFieldUnisex(JSONObject json, String attributeName) throws JSONException {
final JSONObject fieldsJson = json.getJSONObject(FIELDS);
final JSONObject fieldJson = fieldsJson.getJSONObject(attributeName);
if (fieldJson.has(VALUE_ATTR)) {
return fieldJson.getJSONObject(VALUE_ATTR); // pre 5.0 way
} else {
return fieldJson; // JIRA 5.0 way
}
}
/** @return the population. */
public Long getPopulation() {
try {
if (data.has(ToponymProperty.population.name())) {
return data.getLong(ToponymProperty.population.name());
} else {
return null;
}
} catch (JSONException e) {
throw new IllegalStateException(
String.format("Unable to parse %s form %s", ToponymProperty.population, data));
}
}
public Double getScore() {
try {
if (data.has(ToponymProperty.score.name())) {
return data.getDouble(ToponymProperty.score.name());
} else {
return null;
}
} catch (JSONException e) {
throw new IllegalStateException(
String.format("Unable to parse %s form %s", ToponymProperty.score, data));
}
}
/** @return the elevation in meter. */
public Integer getElevation() {
try {
if (data.has(ToponymProperty.elevation.name())) {
return data.getInt(ToponymProperty.elevation.name());
} else {
return null;
}
} catch (JSONException e) {
throw new IllegalStateException(
String.format("Unable to parse %s form %s", ToponymProperty.elevation, data));
}
}
private void setAttributeRec(int i, String[] parts, JSONObject attrs, Object value)
throws JSONException {
if (i == parts.length - 1) {
attrs.put(parts[i], value);
return;
}
if (attrs.has(parts[i])) {
setAttributeRec(i + 1, parts, attrs.getJSONObject(parts[i]), value);
} else {
attrs.put(parts[i], new JSONObject());
setAttributeRec(i + 1, parts, attrs.getJSONObject(parts[i]), value);
}
}
/**
* alternate names of this place as a list of string arrays with two entries the first entry is
* the label and the second represents the language
*
* @return the alternateNames as comma separated list
*/
public List<String[]> getAlternateNames() {
try {
if (data.has(ToponymProperty.alternateNames.name())) {
List<String[]> parsedNames = new ArrayList<String[]>();
JSONArray altNames = data.getJSONArray(ToponymProperty.alternateNames.name());
for (int i = 0; i < altNames.length(); i++) {
JSONObject altName = altNames.getJSONObject(i);
if (altName.has("name")) {
parsedNames.add(
new String[] {
altName.getString("name"), altName.has("lang") ? altName.getString("lang") : null
});
} // else ignore alternate names without a name
}
return parsedNames;
} else {
return Collections.emptyList();
}
} catch (JSONException e) {
throw new IllegalStateException(
String.format("Unable to parse %s form %s", ToponymProperty.alternateNames, data));
}
}
private String assertEntityIsRegistered(final String query, String... arg) throws Exception {
waitFor(
2000,
new Predicate() {
@Override
public boolean evaluate() throws Exception {
JSONArray results = dgiCLient.search(query);
return results.length() == 1;
}
});
String column = (arg.length > 0) ? arg[0] : "_col_0";
JSONArray results = dgiCLient.search(query);
JSONObject row = results.getJSONObject(0);
if (row.has("__guid")) {
return row.getString("__guid");
} else if (row.has("$id$")) {
return row.getJSONObject("$id$").getString("id");
} else {
return row.getJSONObject(column).getString("id");
}
}
@Test
public void testLineage() throws Exception {
String table1 = createTable(false);
String db2 = createDatabase();
String table2 = tableName();
String query = String.format("create table %s.%s as select * from %s", db2, table2, table1);
runCommand(query);
String table1Id = assertTableIsRegistered(DEFAULT_DB, table1);
String table2Id = assertTableIsRegistered(db2, table2);
String datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, db2, table2);
JSONObject response = dgiCLient.getInputGraph(datasetName);
JSONObject vertices = response.getJSONObject("values").getJSONObject("vertices");
Assert.assertTrue(vertices.has(table1Id));
Assert.assertTrue(vertices.has(table2Id));
datasetName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, table1);
response = dgiCLient.getOutputGraph(datasetName);
vertices = response.getJSONObject("values").getJSONObject("vertices");
Assert.assertTrue(vertices.has(table1Id));
Assert.assertTrue(vertices.has(table2Id));
}
public void cargarDatosGrupo() {
JSONObject respuesta =
new IGrupo().listarMiembrosDeGrupo(sesion.getTokenId(), grupoSeleccionado);
JSONObject respuesta2 =
new IGrupo().tecnicosFueraDelGrupo(sesion.getTokenId(), grupoSeleccionado);
try {
if (respuesta.has("jefe")) {
JSONObject objetoJefe = respuesta.getJSONObject("jefe");
nuevoAdministrador =
objetoJefe.getString("nombre") + " " + objetoJefe.getString("apellido");
if (!objetoJefe.getString("nombre").equals(""))
nuevoAdministrador = nuevoAdministrador + " - ";
nuevoAdministrador = nuevoAdministrador + objetoJefe.getString("correo");
} else nuevoAdministrador = "No tiene administrador.";
listaTecnicosGrupo.clear();
JSONArray objetoIntegantes = respuesta.getJSONArray("grupo");
if (objetoIntegantes != null) {
for (int i = 0; i < objetoIntegantes.length(); i++) {
JSONObject ob = objetoIntegantes.getJSONObject(i);
NombreMail b = new NombreMail();
b.setMail(ob.getString("correo"));
b.setNombre(ob.getString("nombre") + " " + ob.getString("apellido"));
listaTecnicosGrupo.add(b);
}
}
listaRestoTecnicos.clear();
JSONArray objetoTecnicos = respuesta2.getJSONArray("Tecnicos");
if (objetoTecnicos != null) {
for (int i = 0; i < objetoTecnicos.length(); i++) {
JSONObject ob = objetoTecnicos.getJSONObject(i);
NombreMail b = new NombreMail();
b.setMail(ob.getString("correo"));
b.setNombre(ob.getString("nombre") + " " + ob.getString("apellido"));
listaRestoTecnicos.add(b);
}
}
} catch (JSONException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/** 结束活动 */
public synchronized void finish(
Session session, String personExpression, String username, String userid) throws Exception {
// 解析传递过来的对象属性
JSONArray array = new JSONArray(personExpression);
JSONObject firstObj = array.getJSONObject(0);
final String data = firstObj.getString("data");
JSONObject dataObj = new JSONObject(data);
// 把前台传过来的，活动对应的人员，转换成json串
state = "结束";
// 设置完成人，完成时间
this.setPerson(username);
this.setUserid(userid);
this.setFinishTime(new Date());
// 根据每个转移创建任务和活动
ProcessDef procDef = ProcessDefManager.getInstance().getProcessDef(processName);
ActivityDef actDef = procDef.getActivity(defid);
for (DiversionDef divDef : actDef.getSplits()) {
ActivityDef tailDef = divDef.getTail();
// 给流程中的变量赋值
this.process.putVar(dataObj);
// 如果不满足表达式条件，继续下一个
String expression = divDef.getExpression();
if (expression != null
&& !expression.equals("")
&& !this.process.getExpressionValue(expression)) {
continue;
}
// 根据人员表达式，产生人员对照表
String exp = tailDef.getPersonExpression();
if (dataObj.has(tailDef.getName())) {
String pExp = dataObj.getString(tailDef.getName());
exp = pExp;
}
PersonService.Run(exp, session);
ActivityInstance actIns = new ActivityInstance(tailDef, process, exp, username, userid, this);
session.save(actIns);
}
session.update(this);
}
/**
* Create a Base Bucket instance
*
* @param bucketName Bucket name
* @param namespace Namespace where Bucket should reside
* @param repGroup Volume
* @return Source ID of the Bucket created
*/
public String createBucket(String bucketName, String namespace, String repGroup) {
_log.debug("ECSApi:createBucket Create bucket initiated for : {}", bucketName);
String id = null;
ClientResponse clientResp = null;
String body =
" { \"name\": \""
+ bucketName
+ "\", "
+ "\"vpool\": \""
+ repGroup
+ "\", \"namespace\": \""
+ namespace
+ "\"} ";
try {
clientResp = post(URI_CREATE_BUCKET, body);
} catch (Exception e) {
_log.error("Error occured while bucket base creation : {}", bucketName, e);
} finally {
if (null == clientResp) {
throw ECSException.exceptions.storageAccessFailed(
_baseUrl.resolve(URI_CREATE_BUCKET), 500, "no response from ECS");
} else if (clientResp.getStatus() != 200) {
String response = String.format("%1$s", (clientResp == null) ? "" : clientResp);
throw ECSException.exceptions.storageAccessFailed(
_baseUrl.resolve(URI_CREATE_BUCKET), clientResp.getStatus(), response);
} else {
// extract bucket id
JSONObject jObj = clientResp.getEntity(JSONObject.class);
if (jObj.has("id")) {
try {
id = jObj.getString("id");
} catch (JSONException e) {
throw ECSException.exceptions.storageAccessFailed(
_baseUrl.resolve(URI_CREATE_BUCKET),
clientResp.getStatus(),
"Unable to extract source ID of the bucket");
}
}
}
closeResponse(clientResp);
}
return id;
}
TaskAttemptInfo(JSONObject jsonObject) throws JSONException {
super(jsonObject);
Preconditions.checkArgument(
jsonObject
.getString(Constants.ENTITY_TYPE)
.equalsIgnoreCase(Constants.TEZ_TASK_ATTEMPT_ID));
taskAttemptId = StringInterner.weakIntern(jsonObject.optString(Constants.ENTITY));
// Parse additional Info
final JSONObject otherInfoNode = jsonObject.getJSONObject(Constants.OTHER_INFO);
startTime = otherInfoNode.optLong(Constants.START_TIME);
endTime = otherInfoNode.optLong(Constants.FINISH_TIME);
diagnostics = otherInfoNode.optString(Constants.DIAGNOSTICS);
creationTime = otherInfoNode.optLong(Constants.CREATION_TIME);
creationCausalTA =
StringInterner.weakIntern(otherInfoNode.optString(Constants.CREATION_CAUSAL_ATTEMPT));
allocationTime = otherInfoNode.optLong(Constants.ALLOCATION_TIME);
containerId = StringInterner.weakIntern(otherInfoNode.optString(Constants.CONTAINER_ID));
String id = otherInfoNode.optString(Constants.NODE_ID);
nodeId = StringInterner.weakIntern((id != null) ? (id.split(":")[0]) : "");
logUrl = otherInfoNode.optString(Constants.COMPLETED_LOGS_URL);
status = StringInterner.weakIntern(otherInfoNode.optString(Constants.STATUS));
container = new Container(containerId, nodeId);
if (otherInfoNode.has(Constants.LAST_DATA_EVENTS)) {
List<DataDependencyEvent> eventInfo =
Utils.parseDataEventDependencyFromJSON(
otherInfoNode.optJSONObject(Constants.LAST_DATA_EVENTS));
long lastTime = 0;
for (DataDependencyEvent item : eventInfo) {
// check these are in time order
Preconditions.checkState(lastTime < item.getTimestamp());
lastTime = item.getTimestamp();
lastDataEvents.add(item);
}
}
terminationCause =
StringInterner.weakIntern(otherInfoNode.optString(ATSConstants.TASK_ATTEMPT_ERROR_ENUM));
executionTimeInterval = (endTime > startTime) ? (endTime - startTime) : 0;
}
private CSVEntry processResponses(PersistenceManager pm, List<Response> responseJDOs) {
CSVEntry entry = CSVCache.getInstance().getCSV(csvId);
if (entry == null) {
entry =
new CSVEntry(
"Response Identifier;Timestamp;Item Identifier;Response Item Identifier; Account; "
+ "Image Url; Video Url; Width; Height; Latitude; Longitude; Audio Url;Text; Value \n",
CSVEntry.BUILDING_STATUS);
}
String csv = entry.getCSV();
for (Response responseJDO : responseJDOs) {
String value = responseJDO.getResponseValue();
String imageUrl = "";
String audioUrl = "";
String videoUrl = "";
String width = "";
String height = "";
String text = "";
String valueString = "";
String lat = responseJDO.getLat() == null ? "" : "" + responseJDO.getLat().doubleValue();
String lng = responseJDO.getLng() == null ? "" : "" + responseJDO.getLng().doubleValue();
try {
org.codehaus.jettison.json.JSONObject json =
new org.codehaus.jettison.json.JSONObject(value);
if (json.has("imageUrl")) imageUrl = json.getString("imageUrl");
if (json.has("audioUrl")) audioUrl = json.getString("audioUrl");
if (json.has("videoUrl")) videoUrl = json.getString("videoUrl");
if (json.has("width")) width = "" + json.getInt("width");
if (json.has("height")) height = "" + json.getInt("height");
if (json.has("text")) text = json.getString("text");
if (json.has("value")) valueString = json.getString("value");
} catch (JSONException e) {
e.printStackTrace();
}
//
// {"imageUrl":"http:\/\/streetlearn.appspot.com\/\/uploadService\/2288002\/2:113116978440346588314\/image1537716519.jpg","width":3264,"height":1840}
// {"answer":"3","isCorrect":true}
csv +=
responseJDO.getResponseId()
+ ";"
+ responseJDO.getTimestamp()
+ ";"
+ responseJDO.getGeneralItemId()
+ ";"
+ responseJDO.getResponseItemId()
+ ";"
+ responseJDO.getUserEmail()
+ ";"
+ imageUrl
+ ";"
+ videoUrl
+ ";"
+ width
+ ";"
+ height
+ ";"
+ lat
+ ";"
+ lng
+ ";"
+ audioUrl
+ ";"
+ text
+ ";"
+ valueString
+ "\n";
}
entry.setCSV(csv);
return entry;
}
private boolean parseSingleClause(JSONObject jsonCriteria) throws JSONException {
String operator = jsonCriteria.getString(OPERATOR_KEY);
if (operator.equals(OPERATOR_BETWEEN)
|| operator.equals(OPERATOR_BETWEENINCLUSIVE)
|| operator.equals(OPERATOR_IBETWEEN)
|| operator.equals(OPERATOR_IBETWEENINCLUSIVE)) {
return parseBetween(jsonCriteria, operator, true);
}
Object value = jsonCriteria.has(VALUE_KEY) ? jsonCriteria.get(VALUE_KEY) : null;
if (operator.equals(OPERATOR_EXISTS)) {
// not supported
return mainOperatorIsAnd;
}
String fieldName = jsonCriteria.getString(FIELD_NAME_KEY);
// translate to a OR for each value
if (value instanceof JSONArray) {
final JSONArray jsonArray = (JSONArray) value;
final JSONObject advancedCriteria = new JSONObject();
advancedCriteria.put(OPERATOR_KEY, OPERATOR_OR);
final JSONArray subCriteria = new JSONArray();
for (int i = 0; i < jsonArray.length(); i++) {
final JSONObject subCriterion = new JSONObject();
subCriterion.put(OPERATOR_KEY, operator);
subCriterion.put(FIELD_NAME_KEY, fieldName);
subCriterion.put(VALUE_KEY, jsonArray.get(i));
subCriteria.put(i, subCriterion);
}
advancedCriteria.put(CRITERIA_KEY, subCriteria);
return parseAdvancedCriteria(advancedCriteria);
}
// Retrieves the UTC time zone offset of the client
if (jsonCriteria.has("minutesTimezoneOffset")) {
int clientMinutesTimezoneOffset =
Integer.parseInt(jsonCriteria.get("minutesTimezoneOffset").toString());
Calendar now = Calendar.getInstance();
// Obtains the UTC time zone offset of the server
int serverMinutesTimezoneOffset =
(now.get(Calendar.ZONE_OFFSET) + now.get(Calendar.DST_OFFSET)) / (1000 * 60);
// Obtains the time zone offset between the server and the client
clientUTCMinutesTimeZoneDiff = clientMinutesTimezoneOffset;
UTCServerMinutesTimeZoneDiff = serverMinutesTimezoneOffset;
}
if (operator.equals(OPERATOR_ISNULL) || operator.equals(OPERATOR_NOTNULL)) {
value = null;
}
// if a comparison is done on an equal date then replace
// with a between start time and end time on that date
if (operator.equals(OPERATOR_EQUALS) || operator.equals(OPERATOR_EQUALSFIELD)) {
Object curValue = recordMap.get(fieldName);
if (curValue instanceof Date) {
if (operator.equals(OPERATOR_EQUALS)) {
return parseSimpleClause(fieldName, OPERATOR_GREATEROREQUAL, value)
&& parseSimpleClause(fieldName, OPERATOR_LESSOREQUAL, value);
} else {
return parseSimpleClause(fieldName, OPERATOR_GREATEROREQUALFIELD, value)
&& parseSimpleClause(fieldName, OPERATOR_LESSOREQUALFIElD, value);
}
}
}
return parseSimpleClause(fieldName, operator, value);
}
public JSONObject describeOperator(String clazz) throws Exception {
TypeGraphVertex tgv = typeGraph.getTypeGraphVertex(clazz);
if (tgv.isInstantiable()) {
JSONObject response = new JSONObject();
JSONArray inputPorts = new JSONArray();
JSONArray outputPorts = new JSONArray();
// Get properties from ASM
JSONObject operatorDescriptor = describeClassByASM(clazz);
JSONArray properties = operatorDescriptor.getJSONArray("properties");
properties = enrichProperties(clazz, properties);
JSONArray portTypeInfo = operatorDescriptor.getJSONArray("portTypeInfo");
List<CompactFieldNode> inputPortfields = typeGraph.getAllInputPorts(clazz);
List<CompactFieldNode> outputPortfields = typeGraph.getAllOutputPorts(clazz);
try {
for (CompactFieldNode field : inputPortfields) {
JSONObject inputPort = setFieldAttributes(clazz, field);
if (!inputPort.has("optional")) {
inputPort.put(
"optional",
false); // input port that is not annotated is default to be not optional
}
if (!inputPort.has(SCHEMA_REQUIRED_KEY)) {
inputPort.put(SCHEMA_REQUIRED_KEY, false);
}
inputPorts.put(inputPort);
}
for (CompactFieldNode field : outputPortfields) {
JSONObject outputPort = setFieldAttributes(clazz, field);
if (!outputPort.has("optional")) {
outputPort.put(
"optional", true); // output port that is not annotated is default to be optional
}
if (!outputPort.has("error")) {
outputPort.put("error", false);
}
if (!outputPort.has(SCHEMA_REQUIRED_KEY)) {
outputPort.put(SCHEMA_REQUIRED_KEY, false);
}
outputPorts.put(outputPort);
}
response.put("name", clazz);
response.put("properties", properties);
response.put(PORT_TYPE_INFO_KEY, portTypeInfo);
response.put("inputPorts", inputPorts);
response.put("outputPorts", outputPorts);
OperatorClassInfo oci = classInfo.get(clazz);
if (oci != null) {
if (oci.comment != null) {
String[] descriptions;
// first look for a <p> tag
String keptPrefix = "<p>";
descriptions = oci.comment.split("<p>", 2);
if (descriptions.length == 0) {
keptPrefix = "";
// if no <p> tag, then look for a blank line
descriptions = oci.comment.split("\n\n", 2);
}
if (descriptions.length > 0) {
response.put("shortDesc", descriptions[0]);
}
if (descriptions.length > 1) {
response.put("longDesc", keptPrefix + descriptions[1]);
}
}
response.put("category", oci.tags.get("@category"));
String displayName = oci.tags.get("@displayName");
if (displayName == null) {
displayName = decamelizeClassName(ClassUtils.getShortClassName(clazz));
}
response.put("displayName", displayName);
String tags = oci.tags.get("@tags");
if (tags != null) {
JSONArray tagArray = new JSONArray();
for (String tag : StringUtils.split(tags, ',')) {
tagArray.put(tag.trim().toLowerCase());
}
response.put("tags", tagArray);
}
String doclink = oci.tags.get("@doclink");
if (doclink != null) {
response.put("doclink", doclink + "?" + getDocName(clazz));
} else if (clazz.startsWith("com.datatorrent.lib.")
|| clazz.startsWith("com.datatorrent.contrib.")) {
response.put("doclink", DT_OPERATOR_DOCLINK_PREFIX + "?" + getDocName(clazz));
}
}
} catch (JSONException ex) {
throw new RuntimeException(ex);
}
return response;
} else {
throw new UnsupportedOperationException();
}
}
/**
* Enrich portClassHier with class/interface names that map to a list of parent
* classes/interfaces. For any class encountered, find its parents too.<br>
* Also find the port types which have assignable schema classes.
*
* @param oper Operator to work on
* @param portClassHierarchy In-Out param that contains a mapping of class/interface to its
* parents
* @param portTypesWithSchemaClasses Json that will contain all the ports which have any schema
* classes.
*/
public void buildAdditionalPortInfo(
JSONObject oper, JSONObject portClassHierarchy, JSONObject portTypesWithSchemaClasses) {
try {
JSONArray ports = oper.getJSONArray(OperatorDiscoverer.PORT_TYPE_INFO_KEY);
for (int i = 0; i < ports.length(); i++) {
JSONObject port = ports.getJSONObject(i);
String portType = port.optString("type");
if (portType == null) {
// skipping if port type is null
continue;
}
if (typeGraph.size() == 0) {
buildTypeGraph();
}
try {
// building port class hierarchy
LinkedList<String> queue = Lists.newLinkedList();
queue.add(portType);
while (!queue.isEmpty()) {
String currentType = queue.remove();
if (portClassHierarchy.has(currentType)) {
// already present in the json so we skip.
continue;
}
List<String> immediateParents = typeGraph.getParents(currentType);
if (immediateParents == null) {
portClassHierarchy.put(currentType, Lists.<String>newArrayList());
continue;
}
portClassHierarchy.put(currentType, immediateParents);
queue.addAll(immediateParents);
}
} catch (JSONException e) {
LOG.warn("building port type hierarchy {}", portType, e);
}
// finding port types with schema classes
if (portTypesWithSchemaClasses.has(portType)) {
// already present in the json so skipping
continue;
}
if (portType.equals("byte")
|| portType.equals("short")
|| portType.equals("char")
|| portType.equals("int")
|| portType.equals("long")
|| portType.equals("float")
|| portType.equals("double")
|| portType.equals("java.lang.String")
|| portType.equals("java.lang.Object")) {
// ignoring primitives, strings and object types as this information is needed only for
// complex types.
continue;
}
if (port.has("typeArgs")) {
// ignoring any type with generics
continue;
}
boolean hasSchemaClasses = false;
List<String> instantiableDescendants = typeGraph.getInstantiableDescendants(portType);
if (instantiableDescendants != null) {
for (String descendant : instantiableDescendants) {
try {
if (typeGraph.isInstantiableBean(descendant)) {
hasSchemaClasses = true;
break;
}
} catch (JSONException ex) {
LOG.warn("checking descendant is instantiable {}", descendant);
}
}
}
portTypesWithSchemaClasses.put(portType, hasSchemaClasses);
}
} catch (JSONException e) {
// should not reach this
LOG.error("JSON Exception {}", e);
throw new RuntimeException(e);
}
}
/**
* Executes the call to the REST Service and processes the response.
*
* @return The service response.
*/
public UserInfoResponse exec() {
// Prepare request parameters
initClientRequest();
clientRequest.header("Content-Type", MediaType.APPLICATION_FORM_URLENCODED);
clientRequest.setHttpMethod(getHttpMethod());
if (getRequest().getAuthorizationMethod() == null
|| getRequest().getAuthorizationMethod()
== AuthorizationMethod.AUTHORIZATION_REQUEST_HEADER_FIELD) {
if (StringUtils.isNotBlank(getRequest().getAccessToken())) {
clientRequest.header("Authorization", "Bearer " + getRequest().getAccessToken());
}
} else if (getRequest().getAuthorizationMethod()
== AuthorizationMethod.FORM_ENCODED_BODY_PARAMETER) {
if (StringUtils.isNotBlank(getRequest().getAccessToken())) {
clientRequest.formParameter("access_token", getRequest().getAccessToken());
}
} else if (getRequest().getAuthorizationMethod() == AuthorizationMethod.URL_QUERY_PARAMETER) {
if (StringUtils.isNotBlank(getRequest().getAccessToken())) {
clientRequest.queryParameter("access_token", getRequest().getAccessToken());
}
}
// Call REST Service and handle response
try {
if (getRequest().getAuthorizationMethod() == null
|| getRequest().getAuthorizationMethod()
== AuthorizationMethod.AUTHORIZATION_REQUEST_HEADER_FIELD
|| getRequest().getAuthorizationMethod() == AuthorizationMethod.URL_QUERY_PARAMETER) {
clientResponse = clientRequest.get(String.class);
} else if (getRequest().getAuthorizationMethod()
== AuthorizationMethod.FORM_ENCODED_BODY_PARAMETER) {
clientResponse = clientRequest.post(String.class);
}
int status = clientResponse.getStatus();
setResponse(new UserInfoResponse(status));
String entity = clientResponse.getEntity(String.class);
getResponse().setEntity(entity);
getResponse().setHeaders(clientResponse.getHeaders());
if (StringUtils.isNotBlank(entity)) {
List<String> contentType = clientResponse.getHeaders().get("Content-Type");
if (contentType != null && contentType.contains("application/jwt")) {
String[] jwtParts = entity.split("\\.");
if (jwtParts.length == 5) {
byte[] sharedSymmetricKey =
sharedKey != null ? sharedKey.getBytes(Util.UTF8_STRING_ENCODING) : null;
Jwe jwe = Jwe.parse(entity, rsaPrivateKey, sharedSymmetricKey);
getResponse().setClaims(jwe.getClaims().toMap());
} else {
Jwt jwt = Jwt.parse(entity);
JwsValidator jwtValidator = new JwsValidator(jwt, sharedKey, jwksUri, null);
if (jwtValidator.validateSignature()) {
getResponse().setClaims(jwt.getClaims().toMap());
}
}
} else {
try {
JSONObject jsonObj = new JSONObject(entity);
if (jsonObj.has("error")) {
getResponse()
.setErrorType(UserInfoErrorResponseType.fromString(jsonObj.getString("error")));
jsonObj.remove("error");
}
if (jsonObj.has("error_description")) {
getResponse().setErrorDescription(jsonObj.getString("error_description"));
jsonObj.remove("error_description");
}
if (jsonObj.has("error_uri")) {
getResponse().setErrorUri(jsonObj.getString("error_uri"));
jsonObj.remove("error_uri");
}
for (Iterator<String> iterator = jsonObj.keys(); iterator.hasNext(); ) {
String key = iterator.next();
List<String> values = new ArrayList<String>();
JSONArray jsonArray = jsonObj.optJSONArray(key);
if (jsonArray != null) {
for (int i = 0; i < jsonArray.length(); i++) {
String value = jsonArray.optString(i);
if (value != null) {
values.add(value);
}
}
} else {
String value = jsonObj.optString(key);
if (value != null) {
values.add(value);
}
}
getResponse().getClaims().put(key, values);
}
} catch (JSONException e) {
e.printStackTrace();
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
closeConnection();
}
return getResponse();
}
/**
* TODO : will be removed .
*
* @param url the url
* @param projectionStr the projection str
* @return the node list complete url for cms
*/
public List<String> getNodeListCompleteURLForCMS(String url, String projectionStr) {
List<String> nodes = new ArrayList<String>();
try {
int indexNextUrlStartPos = url.indexOf("/repositories/");
String cmsQueryUrlPrefix = url.substring(0, indexNextUrlStartPos);
// add 1st
// updated 201501 for adding auth header.
JSONObject jsonObject = readJsonFromUrlWithCmsHeader(url);
nodes.addAll(getFQDNValueListCMS(jsonObject, projectionStr));
JSONObject jsonObjectNext = jsonObject;
Boolean hasMore = false;
String hasMoreNextUrl = null;
String KEY_HAS_MORE = "hasmore";
String KEY_NEXT_PARENT = "next";
String KEY_NEXT_URL = "url";
do {
/** 20141103: BEGIN: for loop to check if has more. */
hasMore = false;
hasMoreNextUrl = null;
if (jsonObjectNext.has(KEY_HAS_MORE)) {
hasMore = jsonObjectNext.getBoolean(KEY_HAS_MORE);
if (jsonObjectNext.has(KEY_NEXT_PARENT)
&& jsonObjectNext.getJSONObject(KEY_NEXT_PARENT).has(KEY_NEXT_URL)) {
hasMoreNextUrl = jsonObjectNext.getJSONObject(KEY_NEXT_PARENT).getString(KEY_NEXT_URL);
}
// final check
if (hasMoreNextUrl != null) {
String nextUrlComplete = cmsQueryUrlPrefix + hasMoreNextUrl;
// 201501 add here too.
jsonObjectNext = readJsonFromUrlWithCmsHeader(nextUrlComplete);
nodes.addAll(getFQDNValueListCMS(jsonObjectNext, projectionStr));
logger.info(
"CMS query: hasMore==true. Found next round in query with URL:" + nextUrlComplete);
}
}
} while (hasMore);
/** 20141103: END: for loop to check if has more. */
// filtering duplicated nodes:
int removedDuplicatedNodeCount = PcTargetHostsUtils.removeDuplicateNodeList(nodes);
if (removedDuplicatedNodeCount > 0) {
logger.info(" Removed duplicated node #: " + removedDuplicatedNodeCount);
}
logger.info("List size: " + nodes.size());
} catch (Exception e) {
throw new TargetHostsLoadException("error when reading " + url, e);
}
return nodes;
}
