/**
* @param operator
* @param parent
* @return
* @throws JSONException
* @throws JsonParseException
* @throws JsonMappingException
* @throws IOException
* @throws Exception assumption: each operator only has one parent but may have many children
*/
Op extractOp(JSONObject operator)
throws JSONException, JsonParseException, JsonMappingException, IOException, Exception {
String[] names = JSONObject.getNames(operator);
if (names.length != 1) {
throw new Exception("Expect only one operator in " + operator.toString());
} else {
String opName = names[0];
JSONObject attrObj = (JSONObject) operator.get(opName);
Map<String, String> attrs = new TreeMap<>();
List<Op> children = new ArrayList<>();
String id = null;
String outputVertexName = null;
for (String attrName : JSONObject.getNames(attrObj)) {
if (attrName.equals("children")) {
Object childrenObj = attrObj.get(attrName);
if (childrenObj instanceof JSONObject) {
if (((JSONObject) childrenObj).length() != 0) {
children.add(extractOp((JSONObject) childrenObj));
}
} else if (childrenObj instanceof JSONArray) {
if (((JSONArray) childrenObj).length() != 0) {
JSONArray array = ((JSONArray) childrenObj);
for (int index = 0; index < array.length(); index++) {
children.add(extractOp(array.getJSONObject(index)));
}
}
} else {
throw new Exception(
"Unsupported operator "
+ this.name
+ "'s children operator is neither a jsonobject nor a jsonarray");
}
} else {
if (attrName.equals("OperatorId:")) {
id = attrObj.get(attrName).toString();
} else if (attrName.equals("outputname:")) {
outputVertexName = attrObj.get(attrName).toString();
} else {
if (!attrObj.get(attrName).toString().isEmpty()) {
attrs.put(attrName, attrObj.get(attrName).toString());
}
}
}
}
Op op = new Op(opName, id, outputVertexName, children, attrs, operator, this, parser);
if (!children.isEmpty()) {
for (Op child : children) {
child.parent = op;
}
} else {
this.rootOps.add(op);
}
return op;
}
}
/**
* Jason object/array object compare
*
* @param obj1
* @param obj2
* @return
* @throws JSONException
*/
public static boolean jsonsEqual(Object obj1, Object obj2) throws JSONException {
if (!obj1.getClass().equals(obj2.getClass())) {
return false;
}
if (obj1 instanceof JSONObject) {
JSONObject jsonObj1 = (JSONObject) obj1;
JSONObject jsonObj2 = (JSONObject) obj2;
String[] names = JSONObject.getNames(jsonObj1);
String[] names2 = JSONObject.getNames(jsonObj2);
if (names.length != names2.length) {
return false;
}
for (String fieldName : names) {
Object obj1FieldValue = jsonObj1.get(fieldName);
Object obj2FieldValue = jsonObj2.get(fieldName);
if (!jsonsEqual(obj1FieldValue, obj2FieldValue)) {
return false;
}
}
} else if (obj1 instanceof JSONArray) {
JSONArray obj1Array = (JSONArray) obj1;
JSONArray obj2Array = (JSONArray) obj2;
if (obj1Array.length() != obj2Array.length()) {
return false;
}
for (int i = 0; i < obj1Array.length(); i++) {
boolean matchFound = false;
for (int j = 0; j < obj2Array.length(); j++) {
if (jsonsEqual(obj1Array.get(i), obj2Array.get(j))) {
matchFound = true;
break;
}
}
if (!matchFound) {
return false;
}
}
} else {
if (!obj1.equals(obj2)) {
return false;
}
}
return true;
}
/**
* Compares 2 JSONObjects for equality. Ignores property order and only matches on defined
* properties and property values.
*
* @param jsonObject1
* @param jsonObject2
* @return
*/
public static boolean isEqual(final JSONObject jsonObject1, final JSONObject jsonObject2) {
if (jsonObject1 == null || JSONObject.getNames(jsonObject1) == null) {
// if object 1 is null or empty -> object 2 should also be null or empty
return (jsonObject2 == null || JSONObject.getNames(jsonObject2) == null);
} else if (jsonObject2 == null || JSONObject.getNames(jsonObject2) == null) {
return false;
}
final List<String> objectProperties1 = Arrays.asList(JSONObject.getNames(jsonObject1));
Collections.sort(objectProperties1);
final List<String> objectProperties2 = Arrays.asList(JSONObject.getNames(jsonObject2));
Collections.sort(objectProperties2);
// compare sorted propertynames
if (!objectProperties1.equals(objectProperties2)) {
log.debug(
"Object:\n", objectProperties1, "didn't have same properties as:\n", objectProperties2);
return false;
}
try {
for (String key : objectProperties1) {
final Object value1 = jsonObject1.get(key);
final Object value2 = jsonObject2.get(key);
if (value1 instanceof JSONObject) {
if (!(value2 instanceof JSONObject)) {
log.debug(value1, "was a JSONObject unlike", value2);
return false;
} else if (!isEqual((JSONObject) value1, (JSONObject) value2)) {
log.debug("JSONObject recursion was not equal");
return false;
}
} else if (value1 instanceof JSONArray) {
if (!(value2 instanceof JSONArray)) {
log.debug(value1, "was a JSONArray unlike", value2);
return false;
}
if (!isEqual((JSONArray) value1, (JSONArray) value2)) {
log.debug("JSONArrays were not equal");
return false;
}
} else if (value1 == null) {
if (value2 != null) {
log.debug("value1 was <null>, but value2 was:" + value2);
return false;
}
} else if (!value1.equals(value2)) {
log.debug("Values were not equal:", value1, "!=", value2);
return false;
}
}
} catch (Exception ex) {
log.warn(ex, "Error comparing JSONObjects");
return false;
}
return true;
}
private static boolean deepEqualObjects(final JSONObject jsonObj1, final JSONObject jsonObj2) {
String[] obj1Names = JSONObject.getNames(jsonObj1);
if (!Arrays.equals(obj1Names, JSONObject.getNames(jsonObj2))) {
return false;
}
if (obj1Names == null) {
return true;
}
for (String name : obj1Names) {
if (!deepEquals(jsonObj1.get(name), jsonObj2.get(name))) {
return false;
}
}
return true;
}
/**
* Overrides values in base data and returns a new object as the merged result.
*
* @param baseData
* @param overrides
* @return merged result
*/
public static JSONObject merge(final JSONObject baseData, final JSONObject overrides) {
if (baseData == null) {
return merge(new JSONObject(), overrides);
}
// copy existing values so we don't leak mutable references
final JSONObject result = createJSONObject(baseData.toString());
// TODO: maybe do the same for overrides?
if (overrides == null || overrides.length() == 0) {
// JSONObject.getNames() on empty object returns null so early exit here
return result;
}
try {
for (String key : JSONObject.getNames(overrides)) {
Object val = overrides.opt(key);
if (val instanceof JSONObject) {
val = merge(result.optJSONObject(key), (JSONObject) val);
}
result.put(key, val);
}
} catch (Exception ex) {
log.warn(ex, "Error merging objects from:", overrides, "- to:", baseData);
}
return result;
}
private JSONObject cloneJSON(JSONObject jsonObject) throws JSONException {
String[] names = JSONObject.getNames(jsonObject);
if (names == null) {
names = EMPTY_NAMES;
}
return new JSONObject(jsonObject, names);
}
/**
* Creates a new instance of a GameItem with the given data
*
* @param inventory The inventory this item is contained in
* @param itemData The data specifying this item
* @throws WebApiException on Web API errors
*/
public GameItem(GameInventory inventory, JSONObject itemData) throws SteamCondenserException {
this.inventory = inventory;
try {
this.defindex = itemData.getInt("defindex");
this.backpackPosition = (int) itemData.getLong("inventory") & 0xffff;
this.count = itemData.getInt("quantity");
this.craftable = !itemData.optBoolean("flag_cannot_craft");
this.id = itemData.getInt("id");
this.itemClass = this.getSchemaData().getString("item_class");
this.itemSet =
this.inventory
.getItemSchema()
.getItemSets()
.get(this.getSchemaData().optString("item_set"));
this.level = itemData.getInt("level");
this.name = this.getSchemaData().getString("item_name");
this.preliminary = (itemData.getLong("inventory") & 0x40000000) != 0;
this.originalId = itemData.getInt("original_id");
this.quality = this.inventory.getItemSchema().getQualities().get(itemData.getInt("quality"));
this.tradeable = !itemData.optBoolean("flag_cannot_trade");
this.type = this.getSchemaData().getString("item_type_name");
if (itemData.has("origin")) {
this.origin = this.inventory.getItemSchema().getOrigins().get(itemData.getInt("origin"));
}
JSONArray attributesData = this.getSchemaData().optJSONArray("attributes");
if (attributesData == null) {
attributesData = new JSONArray();
}
if (itemData.has("attributes")) {
JSONArray itemAttributes = itemData.getJSONArray("attributes");
for (int i = 0; i < itemAttributes.length(); i++) {
attributesData.put(itemAttributes.get(i));
}
}
this.attributes = new ArrayList<JSONObject>();
for (int i = 0; i < attributesData.length(); i++) {
JSONObject attributeData = attributesData.getJSONObject(i);
Object attributeKey = attributeData.opt("defindex");
if (attributeKey == null) {
attributeKey = attributeData.opt("name");
}
if (attributeKey != null) {
JSONObject schemaAttributeData =
inventory.getItemSchema().getAttributes().get(attributeKey);
for (String key : JSONObject.getNames(schemaAttributeData)) {
attributeData.put(key, schemaAttributeData.get(key));
}
this.attributes.add(attributeData);
}
}
} catch (JSONException e) {
throw new WebApiException("Could not parse JSON data.", e);
}
}
private void updateDoc(JSONObject object, Document doc) {
String[] names = JSONObject.getNames(object);
for (String name : names) {
IndexableField fieldtype = checkFieldType(name, object, doc.getField(name));
if (doc.getField(name) == null) {
doc.add(fieldtype);
}
}
}
/**
* Constructor, decodes encrypted passwords and makes them available to the program.
*
* @throws Throwable If something went very wrong.
*/
private WikiGen() throws Throwable {
Cipher c = Cipher.getInstance("AES");
c.init(Cipher.DECRYPT_MODE, new SecretKeySpec(Files.readAllBytes(findConfig(pf)), "AES"));
JSONObject jo =
new JSONObject(new String(c.doFinal(Files.readAllBytes(findConfig(px))), "UTF-8"));
for (String s : JSONObject.getNames(jo)) {
JSONObject entry = jo.getJSONObject(s);
master.put(s, entry.getString("pass"));
}
}
/**
* @throws JSONException
* @throws JsonParseException
* @throws JsonMappingException
* @throws IOException
* @throws Exception We assume that there is a single top-level Map Operator Tree or a Reduce
* Operator Tree in a vertex
*/
public void extractOpTree()
throws JSONException, JsonParseException, JsonMappingException, IOException, Exception {
if (vertexObject.length() != 0) {
for (String key : JSONObject.getNames(vertexObject)) {
if (key.equals("Map Operator Tree:")) {
extractOp(vertexObject.getJSONArray(key).getJSONObject(0));
} else if (key.equals("Reduce Operator Tree:") || key.equals("Processor Tree:")) {
extractOp(vertexObject.getJSONObject(key));
} else if (key.equals("Join:")) {
// this is the case when we have a map-side SMB join
// one input of the join is treated as a dummy vertex
JSONArray array = vertexObject.getJSONArray(key);
for (int index = 0; index < array.length(); index++) {
JSONObject mpOpTree = array.getJSONObject(index);
Vertex v = new Vertex(null, mpOpTree, parser);
v.extractOpTree();
v.dummy = true;
mergeJoinDummyVertexs.add(v);
}
} else if (key.equals("Merge File Operator")) {
JSONObject opTree = vertexObject.getJSONObject(key);
if (opTree.has("Map Operator Tree:")) {
extractOp(opTree.getJSONArray("Map Operator Tree:").getJSONObject(0));
} else {
throw new Exception("Merge File Operator does not have a Map Operator Tree");
}
} else if (key.equals("Execution mode:")) {
executionMode = " " + vertexObject.getString(key);
} else if (key.equals("tagToInput:")) {
JSONObject tagToInput = vertexObject.getJSONObject(key);
for (String tag : JSONObject.getNames(tagToInput)) {
this.tagToInput.put(tag, (String) tagToInput.get(tag));
}
} else if (key.equals("tag:")) {
this.tag = vertexObject.getString(key);
} else {
throw new Exception("Unsupported operator tree in vertex " + this.name);
}
}
}
}
/**
* 讲json串转换为map
*
* @param dataFormat
* @return Map<String, String>
*/
private Map<String, String> convertStr2Map(String dataFormat) {
try {
JSONObject json = new JSONObject(dataFormat);
Map<String, String> rs = Maps.newHashMap();
for (String str : JSONObject.getNames(json)) {
rs.put(str, json.getString(str));
}
return rs;
} catch (JSONException e) {
throw new IllegalArgumentException("数据格式必须为Json格式， dataFormat = " + dataFormat);
}
}
// build search database
private static JSONObject buildInfoJSONObject() throws JSONException {
JSONObject resultJSON = new JSONObject();
for (String name : JSONObject.getNames(restaurants)) {
JSONObject detail = new JSONObject();
JSONObject restaurant = restaurants.getJSONObject(name);
detail.put(BUSINESS_NAME, (String) restaurant.get(BUSINESS_NAME));
detail.put(ADDRESS, (String) restaurant.get(ADDRESS));
resultJSON.put(name, detail);
}
return resultJSON;
}
public static Map<String, String> convertParams(JSONObject obj) {
Map<String, String> paramMap = new HashMap<String, String>();
String[] names = JSONObject.getNames(obj);
if (names != null) {
for (String name : names) {
String val = obj.optString(name, null);
if (val != null) {
paramMap.put(name, val);
}
}
}
return paramMap;
}
private Dictionary<String, Object> asDictionary(JSONObject object) throws JSONException {
Dictionary<String, Object> toReturn = new Hashtable<String, Object>();
String[] names = JSONObject.getNames(object);
for (String key : names) {
Object value = object.get(key);
if (value instanceof JSONArray) {
value = asSet(key, (JSONArray) value);
} else if (value instanceof JSONObject) {
value = asDictionary((JSONObject) value);
}
toReturn.put(key, value);
}
return toReturn;
}
private JdbcServiceConfig asConfig(JSONObject object) throws JSONException {
JdbcServiceConfig toReturn = new JdbcServiceConfig();
String[] names = JSONObject.getNames(object);
for (String key : names) {
Object value = object.get(key);
if (value instanceof JSONArray) {
value = asSet(key, (JSONArray) value);
} else if (value instanceof JSONObject) {
value = asDictionary((JSONObject) value);
}
toReturn.put(key, value);
}
return toReturn;
}
@Override
public void parsePacket(JSONObject packet) {
if (packet.has("lista")) {
JSONObject tmp = packet.getJSONObject("lista");
listOfUsers = tmp;
String[] array = tmp.getNames(tmp);
if (array.length == 0) return;
for (int i = 0; i < array.length; i++) listaUsuarios.addElement(array[i].toString());
list.setSelectedIndex(0);
}
}
@Override
public String genUpdateSetSQL(String table, JSONObject joModel, List<Object> values) {
StringBuffer sbSQL = new StringBuffer("UPDATE \"").append(table).append("\" SET ");
String[] columns = JSONObject.getNames(joModel);
try {
for (String col : columns) {
sbSQL.append("\"").append(col.toUpperCase()).append("\" = ?, ");
values.add(joModel.get(col));
}
} catch (JSONException e) {
throw new RuntimeException(e);
}
sbSQL.setLength(sbSQL.length() - 2);
return sbSQL.toString();
}
@Override
public String genWhereEqAnd(JSONObject joWhere, List<Object> outValues) {
StringBuffer sbSQL = new StringBuffer();
String[] columns = JSONObject.getNames(joWhere);
try {
for (String column : columns) {
sbSQL.append("\"").append(column.toUpperCase()).append("\" = ? AND ");
outValues.add(joWhere.get(column));
}
} catch (JSONException e) {
throw new RuntimeException(e);
}
sbSQL.setLength(sbSQL.length() - 5); // 去掉and
return sbSQL.toString();
}
/**
* Get the values of each field of an event, as an Array.
*
* @param messageFlowId ID of the message flow
* @param event Current event
* @param payloadsMap Payloads Map
* @param eventIndex Index of the current event
* @return Array of values of the fields in the event
*/
private Object[] getFieldValues(
JSONObject eventsAggregated,
JSONObject event,
Map<Integer, Map<String, String>> payloadsMap,
int eventIndex,
long timestamp) {
Map<String, Object> extendedRowVals = new LinkedHashMap<String, Object>();
String[] commonColumns = null;
try {
// Iterate over new (split) fields and add them
for (int j = 0; j < outputColumns.size(); j++) {
String fieldName = outputColumns.get(j);
if (fieldName.equals(AnalyticsConstants.TIMESTAMP_FIELD)) {
this.timestampIndex = j;
extendedRowVals.put(fieldName, timestamp);
} else if (fieldName.equalsIgnoreCase(AnalyticsConstants.COMPONENT_INDEX)) {
extendedRowVals.put(fieldName, eventIndex);
} else if (event.has(fieldName)) {
if (event.isNull(fieldName)
&& payloadsMap != null
&& payloadsMap.containsKey(eventIndex)) {
extendedRowVals.put(fieldName, payloadsMap.get(eventIndex).get(fieldName));
} else {
extendedRowVals.put(fieldName, event.get(fieldName));
}
} else {
extendedRowVals.put(fieldName, null);
}
}
// Iterate over common fields to all events, and add them
commonColumns = JSONObject.getNames(eventsAggregated);
for (int k = 0; k < commonColumns.length; k++) {
String fieldName = commonColumns[k];
if (!fieldName.equalsIgnoreCase(AnalyticsConstants.DATA_COLUMN)
|| !fieldName.equalsIgnoreCase(AnalyticsConstants.JSON_FIELD_EVENTS)) {
extendedRowVals.put(fieldName, eventsAggregated.get(fieldName));
}
}
return extendedRowVals.values().toArray();
} catch (JSONException e) {
throw new RuntimeException(
"Error occured while splitting the record to rows: " + e.getMessage(), e);
}
}
private Map<String, List<String>> gatherFieldErrors(JSONObject object) throws JSONException {
Map<String, List<String>> allErrors = new LinkedHashMap<String, List<String>>();
JSONObject jsonFields = object.getJSONObject("fields");
for (String fieldName : JSONObject.getNames(jsonFields)) {
JSONArray jsonErrorsForField = jsonFields.getJSONArray(fieldName);
List<String> errorsForField = new ArrayList<String>(jsonErrorsForField.length());
for (int i = 0; i < jsonErrorsForField.length(); i++)
errorsForField.add(jsonErrorsForField.getString(i));
allErrors.put(fieldName, errorsForField);
}
return allErrors;
}
public static DataAccessObject updateFromJSON(DataAccessObject dao, JSONObject json)
throws Exception {
Class<? extends DataAccessObject> c = dao.getClass();
for (String fieldName : JSONObject.getNames(json)) {
try {
Field field = c.getSuperclass().getDeclaredField(fieldName);
Object value = getValue(json.get(fieldName), field.getType());
if (value != null)
c.getMethod(
"set" + fieldName.substring(0, 1).toUpperCase() + fieldName.substring(1),
field.getType())
.invoke(dao, value);
} catch (NoSuchFieldException e) {
// Ignore and just move on
// if (!fieldName.startsWith("$$"))
// throw e;
}
}
return dao;
}
private static Map<String, Object> parse(JSONObject obj) {
Map<String, Object> m = new HashMap<String, Object>();
String[] names = JSONObject.getNames(obj);
if (names == null) return m;
for (String key : names) {
try {
Object value = obj.get(key);
if (value == JSONObject.NULL) value = null;
else if (value instanceof JSONArray) value = parse((JSONArray) value);
else if (value instanceof JSONObject) value = parse((JSONObject) value);
m.put(key, value);
} catch (JSONException e) {
e.printStackTrace();
}
}
return m;
}
private JSONObject build() {
this.typeMap = new HashMap<String, String>();
this.result = new JSONObject();
this.result.put("$schema", "http://json-schema.org/draft-04/schema#");
this.definitions = new JSONObject();
buildTypeDefinition(baseObject);
if (hasReferenceToRoot) this.result.put("$ref", "#/definitions/type1");
else {
JSONObject roottype = (JSONObject) this.definitions.remove("type1");
for (String key : JSONObject.getNames(roottype)) result.put(key, roottype.get(key));
}
// only add definitions if there are any types left.
if (definitions.keys().hasNext()) this.result.put("definitions", definitions);
return result;
}
@Override
public @Nonnull Iterable<Lightbulb> listBulbs() throws CommunicationException {
HueMethod method = new HueMethod(this);
JSONObject list = method.get("lights");
if (list == null) {
return Collections.emptyList();
}
ArrayList<Lightbulb> matches = new ArrayList<Lightbulb>();
for (String id : JSONObject.getNames(list)) {
try {
JSONObject item = list.getJSONObject(id);
String name = (item.has("name") ? item.getString("name") : id);
matches.add(new HueBulb(this, id, name));
} catch (JSONException e) {
throw new HueException(e);
}
}
return matches;
}
@Override
public String genInsertSQL(String table, JSONObject joModel, List<Object> values) {
// 获得对象属性名称
String[] columns = JSONObject.getNames(joModel);
String[] vh = new String[joModel.length()];
Arrays.fill(vh, "?");
try {
// 遍历列
for (String column : columns) {
values.add(joModel.get(column)); // 添加值
}
} catch (Exception e) {
throw new RuntimeException(e);
}
return new StringBuffer("INSERT INTO \"")
.append(table.toUpperCase())
.append("\" (\"")
.append(StringUtils.join(columns, "\", \"").toUpperCase())
.append("\") VALUES (")
.append(StringUtils.join(vh, ", "))
.append(")")
.toString();
}
public HTweet(JSONObject jsonObj) throws JSONException {
super(jsonObj, JSONObject.getNames(jsonObj));
}
private JSONObject convertXMLToJSON(String xmlData) throws JSONException, JAXBException {
// Unmarshall the xml into a GraphML
ByteArrayInputStream baiStream = new ByteArrayInputStream(xmlData.getBytes());
JAXBContext jc = JAXBContext.newInstance(GraphMLUtil.GRAPHML_CLASSES);
Unmarshaller unmarshaller = jc.createUnmarshaller();
GraphML graphML = (GraphML) unmarshaller.unmarshal(baiStream);
// eventually check version for compatibility
s_logger.info("Importing chart version: {}", graphML.getversion());
Graph graph = graphML.getGraph();
JSONObject toReturn = new JSONObject(),
miscData = null,
columnJSON,
nodeJSON = null,
clusterJSON;
String cachedClusterJSON = null;
int columnIdx = -1, rowIdx = -1;
List<List<JSONObject>> allColumns = new ArrayList<List<JSONObject>>();
List<JSONObject> currColumn, outColumns = new ArrayList<JSONObject>();
// first, set up the column and misc JSON data contained in the graph
for (GraphDataXML data : graph.getdata()) {
if (data.getkey().startsWith("column")) {
columnJSON = XML.toJSONObject(data.getvalue());
columnIdx = Integer.parseInt(data.getkey().substring(6));
while (columnIdx >= outColumns.size()) {
outColumns.add(new JSONObject());
}
outColumns.set(columnIdx, columnJSON);
allColumns.add(new ArrayList<JSONObject>());
} else if (data.getkey().equals("miscData")) {
miscData = XML.toJSONObject(data.getvalue());
}
}
// next, iterate through the graph nodes and place the JSONObject for each into it's proper
// row/column
for (GraphNode node : graph.getnode()) {
for (GraphData data : node.getdata()) { // parse the goodies from each file node
if (data.getkey().equals("column")) {
columnIdx = Integer.parseInt(data.getvalue());
} else if (data.getkey().equals("row")) {
rowIdx = Integer.parseInt(data.getvalue());
}
}
for (GraphDataXML data : node.getnodedata()) { // parse the goodies from each data node
if (data.getkey().equals("fileJSON")) {
nodeJSON = XML.toJSONObject(data.getvalue());
} else if (data.getkey().equals("clusterJSON")) {
cachedClusterJSON = data.getvalue();
}
}
if (nodeJSON != null
&& nodeJSON.has("clusterUIObject")
&& !nodeJSON.get("clusterUIObject").toString().equals("null")) {
clusterJSON = nodeJSON.getJSONObject("clusterUIObject"); // do annoying cleanup
insertJSONArray(clusterJSON, "children");
insertJSONArray(clusterJSON.getJSONObject("spec"), "members");
}
if (cachedClusterJSON != null) {
PermitSet permits = new PermitSet();
try {
List<String> entityIds = new LinkedList<String>();
List<FL_Cluster> allClusters = ClusterHelper.listFromJson(cachedClusterJSON);
;
for (FL_Cluster cluster : allClusters) {
entityIds.addAll(cluster.getMembers());
for (FL_Property property : cluster.getProperties()) {
if (!(property.getRange() instanceof FL_SingletonRange)) continue;
FL_SingletonRange range = (FL_SingletonRange) property.getRange();
if (!range.getType().equals(FL_PropertyType.STRING)) continue;
property.setRange(
new SingletonRangeHelper(
StringEscapeUtils.unescapeXml((String) range.getValue())));
}
}
final ContextReadWrite contextRW =
contextCache.getReadWrite(nodeJSON.getString("xfId"), permits);
contextRW.getContext().addClusters(allClusters);
contextRW
.getContext()
.addEntities(entityAccess.getEntities(entityIds, FL_LevelOfDetail.SUMMARY));
contextRW.setSimplifiedContext(allClusters);
} catch (IOException e) {
throw new ResourceException(
Status.CLIENT_ERROR_BAD_REQUEST, "Exception during cluster cache processing.", e);
} finally {
permits.revoke();
}
}
currColumn = allColumns.get(columnIdx);
while (rowIdx >= currColumn.size()) {
currColumn.add(new JSONObject());
}
currColumn.set(rowIdx, nodeJSON);
}
// place the files as children in the outgoing columns array
for (int i = 0; i < allColumns.size(); i++) {
columnJSON = outColumns.get(i);
columnJSON.put("children", new JSONArray(allColumns.get(i)));
}
// finally, place the child columns and misc data in the resulting JSON object
toReturn.put("children", new JSONArray(outColumns));
for (String dataKey : JSONObject.getNames(miscData)) {
toReturn.put(dataKey, miscData.get(dataKey));
}
return toReturn;
}
private DocumentPojo buildDocument(
SyndEntry entry, SourcePojo source, LinkedList<String> duplicateSources) {
String tmpURL = this.cleanUrlStart(entry.getLink().toString());
// (can't return null because called from code which checks this)
// create the feed pojo
DocumentPojo doc = new DocumentPojo();
doc.setUrl(tmpURL);
doc.setCreated(new Date());
doc.setModified(new Date());
// Strip out html if it is present
if (entry.getTitle() != null) doc.setTitle(entry.getTitle().replaceAll("\\<.*?\\>", "").trim());
if (entry.getDescription() != null)
doc.setDescription(entry.getDescription().getValue().replaceAll("\\<.*?\\>", "").trim());
if (entry.getPublishedDate() != null) {
doc.setPublishedDate(entry.getPublishedDate());
} else {
doc.setPublishedDate(new Date());
}
// Clone from an existing source if we can:
if (!duplicateSources.isEmpty()
&& (null == doc.getUpdateId())) { // (can't duplicate updating document)
doc.setDuplicateFrom(duplicateSources.getFirst());
}
// GeoRSS
GeoRSSModule geoRSSModule =
GeoRSSUtils.getGeoRSS(entry); // currently does not handle <georss:circle>
if (null != geoRSSModule) {
if (null != geoRSSModule.getPosition()) {
double lat = geoRSSModule.getPosition().getLatitude();
double lon = geoRSSModule.getPosition().getLongitude();
GeoPojo gp = new GeoPojo();
gp.lat = lat;
gp.lon = lon;
doc.setDocGeo(gp);
}
if (null != geoRSSModule.getGeometry()) {
AbstractGeometry ag = geoRSSModule.getGeometry();
if (ag.getClass().equals(new LineString().getClass())) { // <georss:line>
LineString ls = ((LineString) geoRSSModule.getGeometry());
double latAvg = 0.0;
double lonAvg = 0.0;
int length = ls.getPositionList().size();
for (int i = 0; i < length; i++) {
latAvg += ls.getPositionList().getLatitude(i);
lonAvg += ls.getPositionList().getLongitude(i);
}
latAvg = latAvg / length;
lonAvg = lonAvg / length;
GeoPojo gp = new GeoPojo();
gp.lat = latAvg;
gp.lon = lonAvg;
doc.setDocGeo(gp);
} else if (ag.getClass().equals(new Polygon().getClass())) // <georss:polygon>
{
Polygon poly = ((Polygon) geoRSSModule.getGeometry());
AbstractRing ar = poly.getExterior();
LinearRing lr = (LinearRing) ar;
double latAvg = 0.0;
double lonAvg = 0.0;
int length = lr.getPositionList().size();
for (int i = 0; i < length; i++) {
latAvg += lr.getPositionList().getLatitude(i);
lonAvg += lr.getPositionList().getLongitude(i);
}
latAvg = latAvg / length;
lonAvg = lonAvg / length;
GeoPojo gp = new GeoPojo();
gp.lat = latAvg;
gp.lon = lonAvg;
doc.setDocGeo(gp);
} else if (ag.getClass().equals(new Envelope().getClass())) { // <georss:box>
Envelope env = ((Envelope) geoRSSModule.getGeometry());
double latAvg = (env.getMaxLatitude() + env.getMinLatitude()) / 2;
double lonAvg = (env.getMaxLongitude() + env.getMinLongitude()) / 2;
GeoPojo gp = new GeoPojo();
gp.lat = latAvg;
gp.lon = lonAvg;
doc.setDocGeo(gp);
}
}
} // end if GeoRSS
// Arbitrary other metadata:
if (null != entry.getForeignMarkup()) {
JSONObject rssMetadata = new JSONObject();
@SuppressWarnings("unchecked")
List<Element> fms = (List<Element>) entry.getForeignMarkup();
for (Element fm : fms) {
try {
JSONObject subObj = XML.toJSONObject(new XMLOutputter().outputString(fm));
if (1 == subObj.length()) {
for (String name : JSONObject.getNames(subObj)) {
rssMetadata.put(name, subObj.get(name));
}
} else { // (this will never happen in practice?)
rssMetadata.put(fm.getName(), subObj);
}
} catch (JSONException e) {
} // (do nothing just carry on)
}
if (!fms.isEmpty()) {
doc.addToMetadata(
"_FEED_METADATA_", XmlToMetadataParser.convertJsonObjectToLinkedHashMap(rssMetadata));
}
} // TESTED (longs converted to string, eg edgar:assistantDirector from
// "http.www.sec.gov.archives.edgar.usgaap.rss.xml")
return doc;
}
/**
* Called when AVL data is read from URL. Processes the JSON data and calls processAvlReport() for
* each AVL report.
*/
@Override
protected Collection<AvlReport> processData(InputStream in) throws Exception {
// Get the JSON string containing the AVL data
String jsonStr = getJsonString(in);
try {
// Convert JSON string to a JSON object
JSONObject jsonObj = new JSONObject(jsonStr);
// The JSON feed is really odd. Instead of having an
// array of vehicles there is a separate JSON object
// for each vehicle, and the name of the object is the
// trip ID. Therefore need to get names of all the objects
// so that each one can be accessed by name.
String tripNames[] = JSONObject.getNames(jsonObj);
// The return value for the method
Collection<AvlReport> avlReportsReadIn = new ArrayList<AvlReport>();
// For each vehicle...
for (String tripName : tripNames) {
JSONObject v = jsonObj.getJSONObject(tripName);
// Create the AvlReport from the JSON data
String vehicleId = Integer.toString(v.getInt("vehicle_id"));
Double lat = v.getDouble("vehicle_lat");
Double lon = v.getDouble("vehicle_lon");
// Guessing that the speed is in mph since getting values up to 58.
// Therefore need to convert to m/s.
float speed = (float) v.getDouble("vehicle_speed") * Geo.MPH_TO_MPS;
// Not sure if bearing is the same as heading.
float heading = (float) v.getDouble("vehicle_bearing");
// The time is really strange. It is off by 4-5 hours for some
// strange reason. It appears to be 4 hours off during daylight
// savings time but 5 hours during normal winter hours. Yikes!
// Therefore using the parameter keolisFeedAvlTimeOffset to
// specify the offset.
long gpsTime =
v.getLong("vehicle_timestamp") * Time.MS_PER_SEC
+ keolisFeedAvlTimeOffset.getValue() * Time.HOUR_IN_MSECS;
// Create the AvlReport
AvlReport avlReport = new AvlReport(vehicleId, gpsTime, lat, lon, speed, heading, "Keolis");
// Need to set assignment info separately. Unfortunately the
// trip ID in the Keolis feed doesn't correspond exactly to the
// trip IDs in the GTFS data. In Keolis feed it will be
// something like "CR-FRAMINGHAM-Weekday-515" but in GTFS it will
// be "CR-Worcester-CR-Weekday-Worcester-Jun15-515". Therefore
// it is best to just use the trip short name, such as "515".
String tripShortName = tripName.substring(tripName.lastIndexOf('-') + 1);
// Actually set the assignment
avlReport.setAssignment(tripShortName, AssignmentType.TRIP_SHORT_NAME);
logger.debug("From KeolisAvlModule {}", avlReport);
if (shouldProcessAvl) {
avlReportsReadIn.add(avlReport);
}
}
// Return all the AVL reports read in
return avlReportsReadIn;
} catch (JSONException e) {
logger.error("Error parsing JSON. {}. {}", e.getMessage(), jsonStr, e);
return new ArrayList<AvlReport>();
}
}
public Object analyseSingleWeibo(String id, int mode) {
boolean calcRepostList = (mode == 0) || (mode == 2);
boolean calcKeyword = (mode == 0) || (mode == 2);
boolean calcKeyUser = (mode == 0) || (mode == 1);
boolean simple = mode == 1;
Map<String, Object> rsMap = Maps.newHashMap();
try {
JSONObject json = elasticSearch.analyseSingleWeibo(id, calcRepostList || calcKeyword);
String[] names = JSONObject.getNames(json);
for (String string : names) {
rsMap.put(string, json.get(string));
}
JSONObject time = (JSONObject) json.get("time");
JSONObject top = (JSONObject) json.get("top");
JSONObject contents = (JSONObject) json.get("content");
if (calcKeyUser) {
List<Weibo> keyuserArray = Lists.newArrayList();
for (Iterator iter = top.keys(); iter.hasNext(); ) {
String mid = iter.next().toString();
if (mid.startsWith("si")) {
mid = mid.substring(2);
}
Weibo msg = findWeiboById(Long.parseLong(mid));
// BSONObject msg = mongoDBClient.findWeiboById(mid);
for (int i = 0; i < keyuserArray.size(); i++) {
int max = i;
for (int j = i + 1; j < keyuserArray.size(); j++) {
if (keyuserArray.get(j).getRepostCount() > keyuserArray.get(max).getRepostCount()) {
max = j;
}
}
if (max != i) {
Weibo tmp = keyuserArray.get(max);
keyuserArray.set(max, keyuserArray.get(i));
keyuserArray.set(i, tmp);
}
}
keyuserArray.add(msg);
}
rsMap.put("keyuser", keyuserArray);
}
if (calcKeyword) {
Weibo w = findWeiboById(Long.parseLong(id));
rsMap.put("senti", 0);
rsMap.put("keyword", KeyWordMapJson);
if (ok) {
ok = true;
try {
int senti = (int) (10 * SentimentAnalysis.calcSentiScore(w.getContent()));
rsMap.put("senti", senti);
} catch (Exception e) {
rsMap.put("senti", 0);
}
try {
StringBuilder sb = new StringBuilder(w.getContent());
for (Iterator iter = contents.keys(); iter.hasNext(); ) {
String mid = iter.next().toString();
String ww = contents.get(mid).toString();
ww = StringUtils.substringBefore(ww, "//");
if (ww.length() > 5) {
sb.append(ww);
}
}
String map = KeywordExtractor.extractKeyword(sb.toString());
if (map.equals("0")) {
throw new RuntimeException();
}
rsMap.put("keyword", JSON.parse(map));
ok = true;
} catch (Throwable e) {
e.printStackTrace();
}
}
}
if (calcRepostList && (elasticSearch instanceof CrawlerIndex)) {
Weibo ww = elasticSearch.getWeibo(id);
((CrawlerIndex) elasticSearch).findAllReposts(ww);
rsMap.put("repostList", ww);
}
} catch (JSONException e) {
e.printStackTrace();
}
return rsMap;
}