private static boolean parseCategory(
JsonElement categoryElement, Map<String, IConfigPropertyHolder> properties, Gson gson) {
if (!(categoryElement instanceof JsonObject)) return true;
JsonObject categoryNode = categoryElement.getAsJsonObject();
boolean missingFields = false;
for (Map.Entry<String, IConfigPropertyHolder> e : properties.entrySet()) {
JsonElement propertyValue = categoryNode.get(e.getKey());
missingFields |= parseProperty(propertyValue, e.getValue(), gson);
}
return missingFields;
}
private static boolean parseProperty(
JsonElement propertyElement, IConfigPropertyHolder property, Gson gson) {
if (!(propertyElement instanceof JsonObject)) return true;
JsonObject propertyNode = propertyElement.getAsJsonObject();
JsonElement value = propertyNode.get(VALUE_TAG);
if (value == null) return true;
try {
Object parsedValue = gson.fromJson(value, property.getType());
property.setValue(parsedValue);
} catch (Exception e) {
Log.warn(e, "Failed to parse value of field %s:%s", property.category(), property.name());
return true;
}
JsonElement comment = propertyNode.get(COMMENT_TAG);
final String expectedComment = property.comment();
if (comment == null) {
return !Strings.isNullOrEmpty(expectedComment);
} else if (comment.isJsonPrimitive()) {
String commentValue = comment.getAsString();
return !expectedComment.equals(commentValue);
}
return true;
}
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
@Override
public RefObject deserialize(
JsonElement jsonElement, Type type, JsonDeserializationContext context)
throws JsonParseException {
if (jsonElement.isJsonPrimitive()) {
RefObject refObject = new RefObject();
refObject.setValue(jsonElement.getAsString());
return refObject;
}
return context.deserialize(jsonElement, JsonRefObject.class);
}
/**
* Get the configurations from the configurations File as a {@link JsonElement}
*
* @return {@link JsonElement} The configuration tree contained inside the configuration File. If
* the File doesn't exist or an error occurred, the default configuration tree is returned
*/
private JsonElement getConfigurationTreeFromFile() {
if (mConfigurationFile.exists()) { // Check, if the configuration File exists on disk
JsonParser parser = new JsonParser();
try {
JsonElement result = parser.parse(new FileReader(mConfigurationFile));
if (result.isJsonObject()) { // Check if content is a valid JsonObject
return result;
}
} catch (FileNotFoundException e) {
e.printStackTrace();
}
}
return mDefaultConfigurations; // If the configurations File doesn't exist on disk, return the
// default configurations
}
@Then("^I parse the string and print keys and values$")
public void i_parse_the_string_and_print_keys_and_values() throws Throwable {
System.out.println(json);
JsonParser parser = new JsonParser();
JsonObject myobject = (JsonObject) parser.parse(json);
// Accessing the value of "desc"
System.out.println(myobject.get("desc"));
// Deserializing the value into JSONObject
JsonObject descValue = (JsonObject) myobject.get("desc");
// Printing the someKey value using JsonObject
System.out.println("SomeKey Value--" + descValue.get("someKey"));
// Retrieving the JSON Element -- JsonElement can represent a string, array or other data types
JsonElement someElement = descValue.get("someKey");
// Printing a value again
System.out.println("SomeKey Value--" + someElement.getAsString());
}
@Override
public OutputDataRaw deserialize(
JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
final JsonObject object = json.getAsJsonObject();
final OutputDataRaw dataRaw = new OutputDataRaw();
final JsonElement png = object.get("image/png");
if (png != null) {
dataRaw.png = png.getAsString();
}
dataRaw.html = getStringOrArray("text/html", object);
dataRaw.svg = getStringOrArray("image/svg+xml", object);
dataRaw.jpeg = getStringOrArray("image/jpeg", object);
dataRaw.latex = getStringOrArray("text/latex", object);
dataRaw.text = getStringOrArray("text/plain", object);
return dataRaw;
}
private static boolean loadConfig(
JsonElement parsed, Table<String, String, IConfigPropertyHolder> properties) {
if (!parsed.isJsonObject()) return true;
Gson gson = new Gson();
JsonObject rootNode = parsed.getAsJsonObject();
boolean missingFields = false;
for (Map.Entry<String, Map<String, IConfigPropertyHolder>> e : properties.rowMap().entrySet()) {
JsonElement categoryTmp = rootNode.get(e.getKey());
missingFields |= parseCategory(categoryTmp, e.getValue(), gson);
}
return missingFields;
}
private JsonArray getBuildsArray(CIJob job) throws PhrescoException {
try {
String jenkinsUrl = "http://" + job.getJenkinsUrl() + ":" + job.getJenkinsPort() + "/ci/";
String jobNameUtf8 = job.getName().replace(" ", "%20");
String buildsJsonUrl = jenkinsUrl + "job/" + jobNameUtf8 + "/api/json";
String jsonResponse = getJsonResponse(buildsJsonUrl);
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(jsonResponse);
JsonObject jsonObject = jsonElement.getAsJsonObject();
JsonElement element = jsonObject.get(FrameworkConstants.CI_JOB_JSON_BUILDS);
JsonArray jsonArray = element.getAsJsonArray();
return jsonArray;
} catch (Exception e) {
throw new PhrescoException(e);
}
}
private int getTotalBuilds(CIJob job) throws PhrescoException {
try {
S_LOGGER.debug("Entering Method CIManagerImpl.getTotalBuilds(CIJob job)");
S_LOGGER.debug("getCIBuilds() JobName = " + job.getName());
JsonArray jsonArray = getBuildsArray(job);
Gson gson = new Gson();
CIBuild ciBuild = null;
if (jsonArray.size() > 0) {
ciBuild = gson.fromJson(jsonArray.get(0), CIBuild.class);
String buildUrl = ciBuild.getUrl();
String jenkinsUrl = job.getJenkinsUrl() + ":" + job.getJenkinsPort();
// display the jenkins running url in ci
buildUrl = buildUrl.replaceAll("localhost:" + job.getJenkinsPort(), jenkinsUrl);
// list
String response = getJsonResponse(buildUrl + API_JSON);
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(response);
JsonObject jsonObject = jsonElement.getAsJsonObject();
JsonElement resultJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_RESULT);
JsonArray asJsonArray =
jsonObject.getAsJsonArray(FrameworkConstants.CI_JOB_BUILD_ARTIFACTS);
// when build result is not known
if (jsonObject.get(FrameworkConstants.CI_JOB_BUILD_RESULT).toString().equals(STRING_NULL)) {
// it indicates the job is in progress and not yet completed
return -1;
// when build is success and build zip relative path is unknown
} else if (resultJson.getAsString().equals(CI_SUCCESS_FLAG) && asJsonArray.size() < 1) {
return -1;
} else {
return jsonArray.size();
}
} else {
return -1; // When the project is build first time,
}
} catch (ClientHandlerException ex) {
S_LOGGER.error(ex.getLocalizedMessage());
throw new PhrescoException(ex);
}
}
@Nullable
private static ArrayList<String> getStringOrArray(String name, JsonObject object) {
final JsonElement jsonElement = object.get(name);
final ArrayList<String> strings = Lists.newArrayList();
if (jsonElement == null) return null;
if (jsonElement.isJsonArray()) {
final JsonArray array = jsonElement.getAsJsonArray();
for (JsonElement element : array) {
strings.add(element.getAsString());
}
} else {
strings.add(jsonElement.getAsString());
}
return strings;
}
/**
* Method to recursively convert a Json tree, consisting of {@link JsonElement}s, to a to a
* configuration tree, consisting of {@link ConfigurationElement}s
*
* @param parentConfiguration {@link ConfigurationElement} The parent configuration of the new
* configurations added in this method call
* @param parentJsonElement {@link JsonElement} The parent of the {@link JsonElement}s which will
* be converted to {@link ConfigurationElement}s
*/
private void recConvertJsonTreeToConfigurationTree(
ConfigurationElement parentConfiguration, JsonElement parentJsonElement) {
if (parentJsonElement.isJsonObject()) {
Set<Map.Entry<String, JsonElement>> childConfigurations =
((JsonObject) parentJsonElement).entrySet();
for (Map.Entry<String, JsonElement> childConfiguration : childConfigurations) {
if (childConfiguration.getValue().isJsonObject()) {
ConfigurationElement newConfigurationSection =
new ConfigurationElement(childConfiguration.getKey());
parentConfiguration.addChildConfiguration(newConfigurationSection);
recConvertJsonTreeToConfigurationTree(
newConfigurationSection, childConfiguration.getValue());
} else {
parentConfiguration.addChildConfiguration(
new ConfigurationElement(
childConfiguration.getKey(),
new JsonConfigurationValue(childConfiguration.getValue())));
}
}
}
}
@Override
public CellOutputRaw deserialize(
JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
final JsonObject object = json.getAsJsonObject();
final CellOutputRaw cellOutputRaw = new CellOutputRaw();
final JsonElement ename = object.get("ename");
if (ename != null) {
cellOutputRaw.ename = ename.getAsString();
}
final JsonElement name = object.get("name");
if (name != null) {
cellOutputRaw.name = name.getAsString();
}
final JsonElement evalue = object.get("evalue");
if (evalue != null) {
cellOutputRaw.evalue = evalue.getAsString();
}
final JsonElement data = object.get("data");
if (data != null) {
cellOutputRaw.data = gson.fromJson(data, OutputDataRaw.class);
}
final JsonElement count = object.get("execution_count");
if (count != null) {
cellOutputRaw.execution_count = count.getAsInt();
}
final JsonElement outputType = object.get("output_type");
if (outputType != null) {
cellOutputRaw.output_type = outputType.getAsString();
}
final JsonElement png = object.get("png");
if (png != null) {
cellOutputRaw.png = png.getAsString();
}
final JsonElement stream = object.get("stream");
if (stream != null) {
cellOutputRaw.stream = stream.getAsString();
}
final JsonElement jpeg = object.get("jpeg");
if (jpeg != null) {
cellOutputRaw.jpeg = jpeg.getAsString();
}
cellOutputRaw.html = getStringOrArray("html", object);
cellOutputRaw.latex = getStringOrArray("latex", object);
cellOutputRaw.svg = getStringOrArray("svg", object);
final JsonElement promptNumber = object.get("prompt_number");
if (promptNumber != null) {
cellOutputRaw.prompt_number = promptNumber.getAsInt();
}
cellOutputRaw.text = getStringOrArray("text", object);
cellOutputRaw.traceback = getStringOrArray("traceback", object);
final JsonElement metadata = object.get("metadata");
if (metadata != null) {
cellOutputRaw.metadata = gson.fromJson(metadata, Map.class);
}
return cellOutputRaw;
}
@Override
public IpnbCellRaw deserialize(
JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
final JsonObject object = json.getAsJsonObject();
final IpnbCellRaw cellRaw = new IpnbCellRaw();
final JsonElement cell_type = object.get("cell_type");
if (cell_type != null) {
cellRaw.cell_type = cell_type.getAsString();
}
final JsonElement count = object.get("execution_count");
if (count != null) {
cellRaw.execution_count = count.isJsonNull() ? null : count.getAsInt();
}
final JsonElement metadata = object.get("metadata");
if (metadata != null) {
cellRaw.metadata = gson.fromJson(metadata, Map.class);
}
final JsonElement level = object.get("level");
if (level != null) {
cellRaw.level = level.getAsInt();
}
final JsonElement outputsElement = object.get("outputs");
if (outputsElement != null) {
final JsonArray outputs = outputsElement.getAsJsonArray();
cellRaw.outputs = Lists.newArrayList();
for (JsonElement output : outputs) {
cellRaw.outputs.add(gson.fromJson(output, CellOutputRaw.class));
}
}
cellRaw.source = getStringOrArray("source", object);
cellRaw.input = getStringOrArray("input", object);
final JsonElement language = object.get("language");
if (language != null) {
cellRaw.language = language.getAsString();
}
final JsonElement number = object.get("prompt_number");
if (number != null) {
cellRaw.prompt_number = number.getAsInt();
}
return cellRaw;
}
private void setBuildStatus(CIBuild ciBuild, CIJob job) throws PhrescoException {
S_LOGGER.debug("Entering Method CIManagerImpl.setBuildStatus(CIBuild ciBuild)");
S_LOGGER.debug("setBuildStatus() url = " + ciBuild.getUrl());
String buildUrl = ciBuild.getUrl();
String jenkinsUrl = job.getJenkinsUrl() + ":" + job.getJenkinsPort();
buildUrl =
buildUrl.replaceAll(
"localhost:" + job.getJenkinsPort(),
jenkinsUrl); // display the jenkins running url in ci list
String response = getJsonResponse(buildUrl + API_JSON);
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(response);
JsonObject jsonObject = jsonElement.getAsJsonObject();
JsonElement resultJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_RESULT);
JsonElement idJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_ID);
JsonElement timeJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_TIME_STAMP);
JsonArray asJsonArray = jsonObject.getAsJsonArray(FrameworkConstants.CI_JOB_BUILD_ARTIFACTS);
if (jsonObject
.get(FrameworkConstants.CI_JOB_BUILD_RESULT)
.toString()
.equals(STRING_NULL)) { // when build is result is not known
ciBuild.setStatus(INPROGRESS);
} else if (resultJson.getAsString().equals(CI_SUCCESS_FLAG)
&& asJsonArray.size()
< 1) { // when build is success and zip relative path is not added in json
ciBuild.setStatus(INPROGRESS);
} else {
ciBuild.setStatus(resultJson.getAsString());
// download path
for (JsonElement jsonArtElement : asJsonArray) {
String buildDownloadZip =
jsonArtElement
.getAsJsonObject()
.get(FrameworkConstants.CI_JOB_BUILD_DOWNLOAD_PATH)
.toString();
if (buildDownloadZip.endsWith(CI_ZIP)) {
if (debugEnabled) {
S_LOGGER.debug("download artifact " + buildDownloadZip);
}
ciBuild.setDownload(buildDownloadZip);
}
}
}
ciBuild.setId(idJson.getAsString());
String dispFormat = DD_MM_YYYY_HH_MM_SS;
ciBuild.setTimeStamp(getDate(timeJson.getAsString(), dispFormat));
}
@Override
public DateTime deserialize(JsonElement json, Type type, JsonDeserializationContext context)
throws JsonParseException {
return new DateTime(json.getAsString());
}
public Object __parse_response__(String resp) throws RemoteException {
JsonParser parser = new JsonParser();
JsonElement main_response = (JsonElement) parser.parse(resp);
if (main_response.isJsonArray()) {
List<Object> res = new Vector<Object>();
for (Object o : main_response.getAsJsonArray()) {
res.add(__parse_response__(gson.toJson(o)));
}
return res;
}
JsonObject response = main_response.getAsJsonObject();
if (response.has("error")) {
JsonObject e = response.get("error").getAsJsonObject().get("data").getAsJsonObject();
throw new RemoteException(e.get("exception").getAsString(), e.get("message").getAsString());
}
JsonElement value = response.get("result");
String valuestr = gson.toJson(response.get("result"));
if (valuestr.contains("hash:")) {
Class<? extends RpcClient> klass = this.getClass();
try {
Constructor<? extends RpcClient> m =
klass.getDeclaredConstructor(String.class, String.class);
String new_endpoint = gson.fromJson(value, String.class);
return m.newInstance(this.base_endpoint, new_endpoint.replace("hash:", ""));
} catch (NoSuchMethodException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
} catch (IllegalArgumentException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
} catch (InstantiationException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
} catch (IllegalAccessException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
} catch (InvocationTargetException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return null;
}
} else if (valuestr.contains("funcs")) {
return gson.fromJson(valuestr, Interface.class);
}
if (value.isJsonPrimitive()) {
JsonPrimitive val = value.getAsJsonPrimitive();
if (val.isBoolean()) {
return val.getAsBoolean();
} else if (val.isNumber()) {
return val.getAsNumber();
} else if (val.isString()) {
DateTimeFormatter dtparser = ISODateTimeFormat.dateHourMinuteSecond();
try {
return dtparser.parseDateTime(val.getAsString());
} catch (Exception ex) {
}
return val.getAsString();
} else {
return null;
}
} else if (value.isJsonNull()) {
return null;
} else if (value.isJsonObject() && !value.getAsJsonObject().has("__meta__")) {
return gson.fromJson(value, HashMap.class);
} else if (value.isJsonArray()) {
if (value.getAsJsonArray().size() == 0) return new LinkedList();
JsonElement obj = value.getAsJsonArray().get(0);
if (obj.isJsonObject()) {
if (!obj.getAsJsonObject().has("__meta__")) {
return gson.fromJson(value, LinkedList.class);
}
}
}
return __resolve_references__(valuestr);
}