private void setBuildStatus(CIBuild ciBuild, CIJob job) throws PhrescoException {
S_LOGGER.debug("Entering Method CIManagerImpl.setBuildStatus(CIBuild ciBuild)");
S_LOGGER.debug("setBuildStatus() url = " + ciBuild.getUrl());
String buildUrl = ciBuild.getUrl();
String jenkinsUrl = job.getJenkinsUrl() + ":" + job.getJenkinsPort();
buildUrl =
buildUrl.replaceAll(
"localhost:" + job.getJenkinsPort(),
jenkinsUrl); // display the jenkins running url in ci list
String response = getJsonResponse(buildUrl + API_JSON);
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(response);
JsonObject jsonObject = jsonElement.getAsJsonObject();
JsonElement resultJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_RESULT);
JsonElement idJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_ID);
JsonElement timeJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_TIME_STAMP);
JsonArray asJsonArray = jsonObject.getAsJsonArray(FrameworkConstants.CI_JOB_BUILD_ARTIFACTS);
if (jsonObject
.get(FrameworkConstants.CI_JOB_BUILD_RESULT)
.toString()
.equals(STRING_NULL)) { // when build is result is not known
ciBuild.setStatus(INPROGRESS);
} else if (resultJson.getAsString().equals(CI_SUCCESS_FLAG)
&& asJsonArray.size()
< 1) { // when build is success and zip relative path is not added in json
ciBuild.setStatus(INPROGRESS);
} else {
ciBuild.setStatus(resultJson.getAsString());
// download path
for (JsonElement jsonArtElement : asJsonArray) {
String buildDownloadZip =
jsonArtElement
.getAsJsonObject()
.get(FrameworkConstants.CI_JOB_BUILD_DOWNLOAD_PATH)
.toString();
if (buildDownloadZip.endsWith(CI_ZIP)) {
if (debugEnabled) {
S_LOGGER.debug("download artifact " + buildDownloadZip);
}
ciBuild.setDownload(buildDownloadZip);
}
}
}
ciBuild.setId(idJson.getAsString());
String dispFormat = DD_MM_YYYY_HH_MM_SS;
ciBuild.setTimeStamp(getDate(timeJson.getAsString(), dispFormat));
}
@Nullable
private static ArrayList<String> getStringOrArray(String name, JsonObject object) {
final JsonElement jsonElement = object.get(name);
final ArrayList<String> strings = Lists.newArrayList();
if (jsonElement == null) return null;
if (jsonElement.isJsonArray()) {
final JsonArray array = jsonElement.getAsJsonArray();
for (JsonElement element : array) {
strings.add(element.getAsString());
}
} else {
strings.add(jsonElement.getAsString());
}
return strings;
}
private static boolean parseProperty(
JsonElement propertyElement, IConfigPropertyHolder property, Gson gson) {
if (!(propertyElement instanceof JsonObject)) return true;
JsonObject propertyNode = propertyElement.getAsJsonObject();
JsonElement value = propertyNode.get(VALUE_TAG);
if (value == null) return true;
try {
Object parsedValue = gson.fromJson(value, property.getType());
property.setValue(parsedValue);
} catch (Exception e) {
Log.warn(e, "Failed to parse value of field %s:%s", property.category(), property.name());
return true;
}
JsonElement comment = propertyNode.get(COMMENT_TAG);
final String expectedComment = property.comment();
if (comment == null) {
return !Strings.isNullOrEmpty(expectedComment);
} else if (comment.isJsonPrimitive()) {
String commentValue = comment.getAsString();
return !expectedComment.equals(commentValue);
}
return true;
}
@Override
public CellOutputRaw deserialize(
JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
final JsonObject object = json.getAsJsonObject();
final CellOutputRaw cellOutputRaw = new CellOutputRaw();
final JsonElement ename = object.get("ename");
if (ename != null) {
cellOutputRaw.ename = ename.getAsString();
}
final JsonElement name = object.get("name");
if (name != null) {
cellOutputRaw.name = name.getAsString();
}
final JsonElement evalue = object.get("evalue");
if (evalue != null) {
cellOutputRaw.evalue = evalue.getAsString();
}
final JsonElement data = object.get("data");
if (data != null) {
cellOutputRaw.data = gson.fromJson(data, OutputDataRaw.class);
}
final JsonElement count = object.get("execution_count");
if (count != null) {
cellOutputRaw.execution_count = count.getAsInt();
}
final JsonElement outputType = object.get("output_type");
if (outputType != null) {
cellOutputRaw.output_type = outputType.getAsString();
}
final JsonElement png = object.get("png");
if (png != null) {
cellOutputRaw.png = png.getAsString();
}
final JsonElement stream = object.get("stream");
if (stream != null) {
cellOutputRaw.stream = stream.getAsString();
}
final JsonElement jpeg = object.get("jpeg");
if (jpeg != null) {
cellOutputRaw.jpeg = jpeg.getAsString();
}
cellOutputRaw.html = getStringOrArray("html", object);
cellOutputRaw.latex = getStringOrArray("latex", object);
cellOutputRaw.svg = getStringOrArray("svg", object);
final JsonElement promptNumber = object.get("prompt_number");
if (promptNumber != null) {
cellOutputRaw.prompt_number = promptNumber.getAsInt();
}
cellOutputRaw.text = getStringOrArray("text", object);
cellOutputRaw.traceback = getStringOrArray("traceback", object);
final JsonElement metadata = object.get("metadata");
if (metadata != null) {
cellOutputRaw.metadata = gson.fromJson(metadata, Map.class);
}
return cellOutputRaw;
}
private static void addFolder(FileSystem fs, Path p, JsonArray succeeded, JsonArray failed) {
try {
if (fs == null) return;
for (FileStatus file : fs.listStatus(p)) {
Path pfs = file.getPath();
if (file.isDir()) {
addFolder(fs, pfs, succeeded, failed);
} else {
Key k = Key.make(pfs.toString());
long size = file.getLen();
Value val = null;
if (pfs.getName().endsWith(Extensions.JSON)) {
JsonParser parser = new JsonParser();
JsonObject json = parser.parse(new InputStreamReader(fs.open(pfs))).getAsJsonObject();
JsonElement v = json.get(Constants.VERSION);
if (v == null) throw new RuntimeException("Missing version");
JsonElement type = json.get(Constants.TYPE);
if (type == null) throw new RuntimeException("Missing type");
Class c = Class.forName(type.getAsString());
OldModel model = (OldModel) c.newInstance();
model.fromJson(json);
} else if (pfs.getName().endsWith(Extensions.HEX)) { // Hex file?
FSDataInputStream s = fs.open(pfs);
int sz = (int) Math.min(1L << 20, size); // Read up to the 1st meg
byte[] mem = MemoryManager.malloc1(sz);
s.readFully(mem);
// Convert to a ValueArray (hope it fits in 1Meg!)
ValueArray ary = new ValueArray(k, 0).read(new AutoBuffer(mem));
val = new Value(k, ary, Value.HDFS);
} else if (size >= 2 * ValueArray.CHUNK_SZ) {
val =
new Value(
k,
new ValueArray(k, size),
Value.HDFS); // ValueArray byte wrapper over a large file
} else {
val = new Value(k, (int) size, Value.HDFS); // Plain Value
val.setdsk();
}
DKV.put(k, val);
Log.info("PersistHdfs: DKV.put(" + k + ")");
JsonObject o = new JsonObject();
o.addProperty(Constants.KEY, k.toString());
o.addProperty(Constants.FILE, pfs.toString());
o.addProperty(Constants.VALUE_SIZE, file.getLen());
succeeded.add(o);
}
}
} catch (Exception e) {
Log.err(e);
JsonObject o = new JsonObject();
o.addProperty(Constants.FILE, p.toString());
o.addProperty(Constants.ERROR, e.getMessage());
failed.add(o);
}
}
@Override
public IpnbCellRaw deserialize(
JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
final JsonObject object = json.getAsJsonObject();
final IpnbCellRaw cellRaw = new IpnbCellRaw();
final JsonElement cell_type = object.get("cell_type");
if (cell_type != null) {
cellRaw.cell_type = cell_type.getAsString();
}
final JsonElement count = object.get("execution_count");
if (count != null) {
cellRaw.execution_count = count.isJsonNull() ? null : count.getAsInt();
}
final JsonElement metadata = object.get("metadata");
if (metadata != null) {
cellRaw.metadata = gson.fromJson(metadata, Map.class);
}
final JsonElement level = object.get("level");
if (level != null) {
cellRaw.level = level.getAsInt();
}
final JsonElement outputsElement = object.get("outputs");
if (outputsElement != null) {
final JsonArray outputs = outputsElement.getAsJsonArray();
cellRaw.outputs = Lists.newArrayList();
for (JsonElement output : outputs) {
cellRaw.outputs.add(gson.fromJson(output, CellOutputRaw.class));
}
}
cellRaw.source = getStringOrArray("source", object);
cellRaw.input = getStringOrArray("input", object);
final JsonElement language = object.get("language");
if (language != null) {
cellRaw.language = language.getAsString();
}
final JsonElement number = object.get("prompt_number");
if (number != null) {
cellRaw.prompt_number = number.getAsInt();
}
return cellRaw;
}
@Override
public RefObject deserialize(
JsonElement jsonElement, Type type, JsonDeserializationContext context)
throws JsonParseException {
if (jsonElement.isJsonPrimitive()) {
RefObject refObject = new RefObject();
refObject.setValue(jsonElement.getAsString());
return refObject;
}
return context.deserialize(jsonElement, JsonRefObject.class);
}
@Then("^I parse the string and print keys and values$")
public void i_parse_the_string_and_print_keys_and_values() throws Throwable {
System.out.println(json);
JsonParser parser = new JsonParser();
JsonObject myobject = (JsonObject) parser.parse(json);
// Accessing the value of "desc"
System.out.println(myobject.get("desc"));
// Deserializing the value into JSONObject
JsonObject descValue = (JsonObject) myobject.get("desc");
// Printing the someKey value using JsonObject
System.out.println("SomeKey Value--" + descValue.get("someKey"));
// Retrieving the JSON Element -- JsonElement can represent a string, array or other data types
JsonElement someElement = descValue.get("someKey");
// Printing a value again
System.out.println("SomeKey Value--" + someElement.getAsString());
}
@Override
public OutputDataRaw deserialize(
JsonElement json, Type typeOfT, JsonDeserializationContext context)
throws JsonParseException {
final JsonObject object = json.getAsJsonObject();
final OutputDataRaw dataRaw = new OutputDataRaw();
final JsonElement png = object.get("image/png");
if (png != null) {
dataRaw.png = png.getAsString();
}
dataRaw.html = getStringOrArray("text/html", object);
dataRaw.svg = getStringOrArray("image/svg+xml", object);
dataRaw.jpeg = getStringOrArray("image/jpeg", object);
dataRaw.latex = getStringOrArray("text/latex", object);
dataRaw.text = getStringOrArray("text/plain", object);
return dataRaw;
}
private int getTotalBuilds(CIJob job) throws PhrescoException {
try {
S_LOGGER.debug("Entering Method CIManagerImpl.getTotalBuilds(CIJob job)");
S_LOGGER.debug("getCIBuilds() JobName = " + job.getName());
JsonArray jsonArray = getBuildsArray(job);
Gson gson = new Gson();
CIBuild ciBuild = null;
if (jsonArray.size() > 0) {
ciBuild = gson.fromJson(jsonArray.get(0), CIBuild.class);
String buildUrl = ciBuild.getUrl();
String jenkinsUrl = job.getJenkinsUrl() + ":" + job.getJenkinsPort();
// display the jenkins running url in ci
buildUrl = buildUrl.replaceAll("localhost:" + job.getJenkinsPort(), jenkinsUrl);
// list
String response = getJsonResponse(buildUrl + API_JSON);
JsonParser parser = new JsonParser();
JsonElement jsonElement = parser.parse(response);
JsonObject jsonObject = jsonElement.getAsJsonObject();
JsonElement resultJson = jsonObject.get(FrameworkConstants.CI_JOB_BUILD_RESULT);
JsonArray asJsonArray =
jsonObject.getAsJsonArray(FrameworkConstants.CI_JOB_BUILD_ARTIFACTS);
// when build result is not known
if (jsonObject.get(FrameworkConstants.CI_JOB_BUILD_RESULT).toString().equals(STRING_NULL)) {
// it indicates the job is in progress and not yet completed
return -1;
// when build is success and build zip relative path is unknown
} else if (resultJson.getAsString().equals(CI_SUCCESS_FLAG) && asJsonArray.size() < 1) {
return -1;
} else {
return jsonArray.size();
}
} else {
return -1; // When the project is build first time,
}
} catch (ClientHandlerException ex) {
S_LOGGER.error(ex.getLocalizedMessage());
throw new PhrescoException(ex);
}
}
@Override
public DateTime deserialize(JsonElement json, Type type, JsonDeserializationContext context)
throws JsonParseException {
return new DateTime(json.getAsString());
}
