private static void run(Callable c, boolean read, int size) {
// Count all i/o time from here, including all retry overheads
long start_io_ms = System.currentTimeMillis();
while (true) {
try {
long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats
c.call();
TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS);
break;
// Explicitly ignore the following exceptions but
// fail on the rest IOExceptions
} catch (EOFException e) {
ignoreAndWait(e, false);
} catch (SocketTimeoutException e) {
ignoreAndWait(e, false);
} catch (S3Exception e) {
// Preserve S3Exception before IOException
// Since this is tricky code - we are supporting different HDFS version
// New version declares S3Exception as IOException
// But old versions (0.20.xxx) declares it as RuntimeException
// So we have to catch it before IOException !!!
ignoreAndWait(e, false);
} catch (IOException e) {
ignoreAndWait(e, true);
} catch (Exception e) {
throw Log.errRTExcept(e);
}
}
}
private void setSvnCredential(CIJob job) throws JDOMException, IOException {
S_LOGGER.debug("Entering Method CIManagerImpl.setSvnCredential");
try {
String jenkinsTemplateDir = Utility.getJenkinsTemplateDir();
String credentialFilePath = jenkinsTemplateDir + job.getRepoType() + HYPHEN + CREDENTIAL_XML;
if (debugEnabled) {
S_LOGGER.debug("credentialFilePath ... " + credentialFilePath);
}
File credentialFile = new File(credentialFilePath);
SvnProcessor processor = new SvnProcessor(credentialFile);
// DataInputStream in = new DataInputStream(new FileInputStream(credentialFile));
// while (in.available() != 0) {
// System.out.println(in.readLine());
// }
// in.close();
processor.changeNodeValue("credentials/entry//userName", job.getUserName());
processor.changeNodeValue("credentials/entry//password", job.getPassword());
processor.writeStream(new File(Utility.getJenkinsHome() + File.separator + job.getName()));
// jenkins home location
String jenkinsJobHome = System.getenv(JENKINS_HOME);
StringBuilder builder = new StringBuilder(jenkinsJobHome);
builder.append(File.separator);
processor.writeStream(new File(builder.toString() + CI_CREDENTIAL_XML));
} catch (Exception e) {
S_LOGGER.error(
"Entered into the catch block of CIManagerImpl.setSvnCredential "
+ e.getLocalizedMessage());
}
}
private long getStartTime(Query request, String context) throws BeanValidationException {
if (request.getStartAbsolute() != null) {
return request.getStartAbsolute();
} else if (request.getStartRelative() != null) {
return request.getStartRelative().getTimeRelativeTo(System.currentTimeMillis());
} else {
throw new BeanValidationException(
new SimpleConstraintViolation("start_time", "relative or absolute time must be set"),
context);
}
}
private static void run(Callable c, boolean read, int size) {
// Count all i/o time from here, including all retry overheads
long start_io_ms = System.currentTimeMillis();
while (true) {
try {
long start_ns = System.nanoTime(); // Blocking i/o call timing - without counting repeats
c.call();
TimeLine.record_IOclose(start_ns, start_io_ms, read ? 1 : 0, size, Value.HDFS);
break;
// Explicitly ignore the following exceptions but
// fail on the rest IOExceptions
} catch (EOFException e) {
ignoreAndWait(e, false);
} catch (SocketTimeoutException e) {
ignoreAndWait(e, false);
} catch (IOException e) {
ignoreAndWait(e, true);
} catch (Exception e) {
throw Log.errRTExcept(e);
}
}
}
@When("^I read json string from a file$")
public void i_read_json_string_from_a_file() throws Throwable {
FileInputStream fin =
new FileInputStream(
new File(System.getProperty("user.dir") + "//src//test//resources//sample.json"));
InputStreamReader in = new InputStreamReader(fin);
BufferedReader bufferedReader = new BufferedReader(in);
StringBuilder sb = new StringBuilder();
String line;
while ((line = bufferedReader.readLine()) != null) {
sb.append(line);
}
json = sb.toString();
}
@When("^I read the json data file \"(.*?)\"$")
public void i_read_the_json_data_file(String arg1) throws Throwable {
FileInputStream fin =
new FileInputStream(
new File(System.getProperty("user.dir") + "//src//test//resources//" + arg1));
InputStreamReader in = new InputStreamReader(fin);
BufferedReader bufferedReader = new BufferedReader(in);
StringBuilder sb = new StringBuilder();
String line;
while ((line = bufferedReader.readLine()) != null) {
sb.append(line);
}
json = sb.toString();
System.out.println(json);
}
private void setMailCredential(CIJob job) {
if (debugEnabled) {
S_LOGGER.debug("Entering Method CIManagerImpl.setMailCredential");
}
try {
String jenkinsTemplateDir = Utility.getJenkinsTemplateDir();
String mailFilePath = jenkinsTemplateDir + MAIL + HYPHEN + CREDENTIAL_XML;
if (debugEnabled) {
S_LOGGER.debug("configFilePath ... " + mailFilePath);
}
File mailFile = new File(mailFilePath);
SvnProcessor processor = new SvnProcessor(mailFile);
// DataInputStream in = new DataInputStream(new FileInputStream(mailFile));
// while (in.available() != 0) {
// System.out.println(in.readLine());
// }
// in.close();
// Mail have to go with jenkins running email address
InetAddress ownIP = InetAddress.getLocalHost();
processor.changeNodeValue(
CI_HUDSONURL,
HTTP_PROTOCOL
+ PROTOCOL_POSTFIX
+ ownIP.getHostAddress()
+ COLON
+ job.getJenkinsPort()
+ FORWARD_SLASH
+ CI
+ FORWARD_SLASH);
processor.changeNodeValue("smtpAuthUsername", job.getSenderEmailId());
processor.changeNodeValue("smtpAuthPassword", job.getSenderEmailPassword());
processor.changeNodeValue("adminAddress", job.getSenderEmailId());
// jenkins home location
String jenkinsJobHome = System.getenv(JENKINS_HOME);
StringBuilder builder = new StringBuilder(jenkinsJobHome);
builder.append(File.separator);
processor.writeStream(new File(builder.toString() + CI_MAILER_XML));
} catch (Exception e) {
S_LOGGER.error(
"Entered into the catch block of CIManagerImpl.setMailCredential "
+ e.getLocalizedMessage());
}
}
private void updateClusters(
int[] clusters, int count, long chunk, long numrows, int rpc, long updatedRow) {
final int offset = (int) (updatedRow - (rpc * chunk));
final Key chunkKey = ValueArray.getChunkKey(chunk, _job.dest());
final int[] message;
if (count == clusters.length) message = clusters;
else {
message = new int[count];
System.arraycopy(clusters, 0, message, 0, message.length);
}
final int rows = ValueArray.rpc(chunk, rpc, numrows);
new Atomic() {
@Override
public Value atomic(Value val) {
assert val == null || val._key.equals(chunkKey);
AutoBuffer b = new AutoBuffer(rows * ROW_SIZE);
if (val != null) b._bb.put(val.memOrLoad());
for (int i = 0; i < message.length; i++) b.put4((offset + i) * 4, message[i]);
b.position(b.limit());
return new Value(chunkKey, b.buf());
}
}.invoke(chunkKey);
}
/**
* Start this task based on given top-level fork-join task representing job computation.
*
* @param fjtask top-level job computation task.
* @return this job in {@link JobState#RUNNING} state
* @see JobState
* @see H2OCountedCompleter
*/
public
/** FIXME: should be final or at least protected */
Job start(final H2OCountedCompleter fjtask) {
assert state == JobState.CREATED : "Trying to run job which was already run?";
assert fjtask != null : "Starting a job with null working task is not permitted! Fix you API";
_fjtask = fjtask;
start_time = System.currentTimeMillis();
state = JobState.RUNNING;
// Save the full state of the job
UKV.put(self(), this);
// Update job list
new TAtomic<List>() {
@Override
public List atomic(List old) {
if (old == null) old = new List();
Key[] jobs = old._jobs;
old._jobs = Arrays.copyOf(jobs, jobs.length + 1);
old._jobs[jobs.length] = job_key;
return old;
}
}.invoke(LIST);
return this;
}
private long getEndTime(Query request) {
if (request.getEndAbsolute() != null) return request.getEndAbsolute();
else if (request.getEndRelative() != null)
return request.getEndRelative().getTimeRelativeTo(System.currentTimeMillis());
return -1;
}
@Override
@SuppressWarnings("empty-statement")
protected Clan doInBackground() throws Exception {
if (bestMatch == null) { // use fallBack
// URL URL = new
// URL("http://worldoftanks."+gui.getServerRegion()+"/uc/clans/?type=table&offset=0&limit=10&order_by="+this.searchType+"&search="+this.clanTagName+"&echo=2&id=clans_index");
// TODO: does API support orderBy ?!?
URL URL =
new URL(
"http://worldoftanks."
+ gui.getServerRegion()
+ "/uc/clans/api/1.1/?source_token=Intellect_Soft-WoT_Mobile-unofficial_stats&search="
+ this.fallbackInput.replace(" ", "%20")
+ "&offset=0&limit=10");
URLConnection URLConnection = URL.openConnection();
URLConnection.setRequestProperty("Accept", "application/json, text/javascript, */*; q=0.01");
URLConnection.setRequestProperty("Accept-Language", "en-us;q=0.5,en;q=0.3");
URLConnection.setRequestProperty("Accept-Encoding", "paco");
URLConnection.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
URLConnection.setRequestProperty("Connection", "close");
URLConnection.setRequestProperty("X-Requested-With", "XMLHttpRequest");
// timeout after 10 seconds
URLConnection.setConnectTimeout(10000);
BufferedReader clanBufferedReader = null;
StringBuilder data = new StringBuilder(3000);
try {
clanBufferedReader =
new BufferedReader(new InputStreamReader(URLConnection.getInputStream(), "UTF8"));
for (String line; (line = clanBufferedReader.readLine()) != null; data.append(line)) ;
} finally {
if (clanBufferedReader != null) clanBufferedReader.close();
}
JsonParser jsonParser = new JsonParser();
JsonObject json = jsonParser.parse(data.toString()).getAsJsonObject();
if (!"ok".equalsIgnoreCase(json.get("status").getAsString())) {
throw new ClanAPIException(json.get("status_code").getAsString(), this.gui);
}
JsonArray results = json.get("data").getAsJsonObject().get("items").getAsJsonArray();
JsonObject json_clan = null;
for (JsonElement e : results) {
JsonObject o = e.getAsJsonObject();
if (fallbackInput.equalsIgnoreCase(o.get(this.searchType).getAsString())) {
json_clan = o;
break;
}
}
if (json_clan == null) {
throw new ClanNotFoundException(this.gui);
}
String name = json_clan.get("name").getAsString();
String tag = json_clan.get("abbreviation").getAsString();
long ID = json_clan.get("id").getAsLong();
int member_count = json_clan.get("member_count").getAsInt();
ImageIcon emblem =
new ImageIcon(
new URL("http://worldoftanks.eu" + json_clan.get("clan_emblem_url").getAsString()));
bestMatch = new PossibleClan(name, tag, ID, member_count, emblem);
}
// get provinces
GetProvinces provinceWorker = new GetProvinces(bestMatch.getID(), this.gui);
provinceWorker.execute();
// get members
// URL = new
// URL("http://worldoftanks."+gui.getServerRegion()+"/uc/clans/"+clanID+"/members/?type=table&offset=0&limit=100&order_by=name&search=&echo=1&id=clan_members_index");
URL URL =
new URL(
"http://worldoftanks."
+ gui.getServerRegion()
+ "/uc/clans/"
+ bestMatch.getID()
+ "/api/1.1/?source_token=Intellect_Soft-WoT_Mobile-unofficial_stats");
URLConnection URLConnection = URL.openConnection();
URLConnection.setRequestProperty("Accept", "application/json, text/javascript, */*; q=0.01");
URLConnection.setRequestProperty("Accept-Language", "en-us;q=0.5,en;q=0.3");
URLConnection.setRequestProperty("Accept-Encoding", "paco");
URLConnection.setRequestProperty("Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7");
URLConnection.setRequestProperty("Connection", "close");
URLConnection.setRequestProperty("X-Requested-With", "XMLHttpRequest");
// timeout after 15 seconds
URLConnection.setConnectTimeout(15000);
BufferedReader membersBufferedReader = null;
StringBuilder members_data = new StringBuilder(10000);
try {
membersBufferedReader =
new BufferedReader(new InputStreamReader(URLConnection.getInputStream(), "UTF8"));
for (String line;
(line = membersBufferedReader.readLine()) != null;
members_data.append(line)) ;
} finally {
if (membersBufferedReader != null) membersBufferedReader.close();
}
JsonParser jsonParser = new JsonParser();
JsonObject members_json = jsonParser.parse(members_data.toString()).getAsJsonObject();
if (!"ok".equalsIgnoreCase(members_json.get("status").getAsString())) {
throw new ClanAPIException(members_json.get("status_code").getAsString(), this.gui);
}
JsonArray members = members_json.get("data").getAsJsonObject().get("members").getAsJsonArray();
GetPlayerData[] workers = new GetPlayerData[members.size()];
players = new ArrayList<Player>(members.size());
for (int i = 0; i < members.size(); i++) {
JsonObject member = members.get(i).getAsJsonObject();
workers[i] = new GetPlayerData(member.get("account_id").getAsLong(), this.gui);
workers[i].execute();
} // TODO: see if there's some sort of "worker pool" with a getAll() or getFirst()
// In the meantime see if GetProvinces is ready
this.provinces = provinceWorker.get();
ArrayList<Vehicle> vehicles = new ArrayList<Vehicle>(5000);
long start = System.currentTimeMillis();
for (GetPlayerData w : workers) {
Player p = w.get();
players.add(p);
vehicles.addAll(p.getVehicles());
// TODO: update progress bar
// TODO: sort by tier already here ?!? (implementation details)
}
vehicles.trimToSize();
System.out.printf(
"Vs:" + vehicles.size() + "\nOverall time: %dms%n", System.currentTimeMillis() - start);
vehicles = Utils.sortVehiclesByTier(vehicles);
vehicles = Utils.sortVehiclesByClass(vehicles);
vehicles = Utils.sortVehiclesByNation(vehicles);
if (players.size() > 20) { // some small logical optimizations
double avg_top_eff = 0D, avg_eff = 0D, avg_top_wr = 0D, avg_wr = 0D;
players = Utils.sortPlayersByEfficiency(players);
for (int i = 0; i < players.size(); i++) {
if (i == 20) avg_top_eff = avg_eff / 20D;
avg_eff += players.get(i).getEfficiency();
}
avg_eff /= players.size();
players = Utils.sortPlayersByWinrate(players);
for (int i = 0; i < players.size(); i++) {
if (i == 20) avg_top_wr = avg_wr / 20D;
avg_wr += players.get(i).getAvg_wr();
}
avg_wr /= players.size();
return new Clan(
bestMatch.getName(),
bestMatch.getClanTag(),
bestMatch.getID(),
players,
vehicles,
avg_wr,
avg_top_wr,
avg_eff,
avg_top_eff,
bestMatch.getEmblem());
} else {
double avg_eff = 0D, avg_wr = 0D; // top=all
players = Utils.sortPlayersByEfficiency(players);
for (int i = 0; i < players.size(); i++) {
avg_eff += players.get(i).getEfficiency();
}
avg_eff /= players.size();
players = Utils.sortPlayersByWinrate(players);
for (int i = 0; i < players.size(); i++) {
avg_wr += players.get(i).getAvg_wr();
}
avg_wr /= players.size();
return new Clan(
bestMatch.getName(),
bestMatch.getClanTag(),
bestMatch.getID(),
players,
vehicles,
avg_wr,
avg_wr,
avg_eff,
avg_eff,
bestMatch.getEmblem());
}
}
@Override
public JsonElement serialize(
WebSocketMessage src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject root = new JsonObject();
JsonObject jsonNodeData = new JsonObject();
JsonObject jsonRelData = new JsonObject();
JsonArray removedProperties = new JsonArray();
JsonArray modifiedProperties = new JsonArray();
if (src.getCommand() != null) {
root.add("command", new JsonPrimitive(src.getCommand()));
}
if (src.getId() != null) {
root.add("id", new JsonPrimitive(src.getId()));
}
if (src.getPageId() != null) {
root.add("pageId", new JsonPrimitive(src.getPageId()));
}
if (src.getMessage() != null) {
root.add("message", new JsonPrimitive(src.getMessage()));
}
if (src.getCode() != 0) {
root.add("code", new JsonPrimitive(src.getCode()));
}
if (src.getSessionId() != null) {
root.add("sessionId", new JsonPrimitive(src.getSessionId()));
}
if (src.getToken() != null) {
root.add("token", new JsonPrimitive(src.getToken()));
}
if (src.getCallback() != null) {
root.add("callback", new JsonPrimitive(src.getCallback()));
}
if (src.getButton() != null) {
root.add("button", new JsonPrimitive(src.getButton()));
}
if (src.getParent() != null) {
root.add("parent", new JsonPrimitive(src.getParent()));
}
if (src.getView() != null) {
root.add("view", new JsonPrimitive(src.getView()));
}
if (src.getSortKey() != null) {
root.add("sort", new JsonPrimitive(src.getSortKey()));
}
if (src.getSortOrder() != null) {
root.add("order", new JsonPrimitive(src.getSortOrder()));
}
if (src.getPageSize() > 0) {
root.add("pageSize", new JsonPrimitive(src.getPageSize()));
}
if (src.getPage() > 0) {
root.add("page", new JsonPrimitive(src.getPage()));
}
JsonArray nodesWithChildren = new JsonArray();
Set<String> nwc = src.getNodesWithChildren();
if ((nwc != null) && !src.getNodesWithChildren().isEmpty()) {
for (String nodeId : nwc) {
nodesWithChildren.add(new JsonPrimitive(nodeId));
}
root.add("nodesWithChildren", nodesWithChildren);
}
// serialize session valid flag (output only)
root.add("sessionValid", new JsonPrimitive(src.isSessionValid()));
// UPDATE only, serialize only removed and modified properties and use the correct values
if ((src.getGraphObject() != null)) {
GraphObject graphObject = src.getGraphObject();
if (!src.getModifiedProperties().isEmpty()) {
for (PropertyKey modifiedKey : src.getModifiedProperties()) {
modifiedProperties.add(toJsonPrimitive(modifiedKey));
// Object newValue = graphObject.getProperty(modifiedKey);
//
// if (newValue != null) {
//
// if (graphObject instanceof AbstractNode) {
//
// src.getNodeData().put(modifiedKey.jsonName(), newValue);
// } else {
//
// src.getRelData().put(modifiedKey.jsonName(), newValue);
// }
//
// }
}
root.add("modifiedProperties", modifiedProperties);
}
if (!src.getRemovedProperties().isEmpty()) {
for (PropertyKey removedKey : src.getRemovedProperties()) {
removedProperties.add(toJsonPrimitive(removedKey));
}
root.add("removedProperties", removedProperties);
}
}
// serialize node data
if (src.getNodeData() != null) {
for (Entry<String, Object> entry : src.getNodeData().entrySet()) {
Object value = entry.getValue();
String key = entry.getKey();
if (value != null) {
jsonNodeData.add(key, toJsonPrimitive(value));
}
}
root.add("data", jsonNodeData);
}
// serialize relationship data
if (src.getRelData() != null) {
for (Entry<String, Object> entry : src.getRelData().entrySet()) {
Object value = entry.getValue();
String key = entry.getKey();
if (value != null) {
jsonRelData.add(key, toJsonPrimitive(value));
}
}
root.add("relData", jsonRelData);
}
// serialize result list
if (src.getResult() != null) {
if (src.getView() != null) {
try {
propertyView.set(null, src.getView());
} catch (FrameworkException fex) {
logger.log(Level.WARNING, "Unable to set property view", fex);
}
} else {
try {
propertyView.set(null, PropertyView.Ui);
} catch (FrameworkException fex) {
logger.log(Level.WARNING, "Unable to set property view", fex);
}
}
JsonArray result = new JsonArray();
for (GraphObject obj : src.getResult()) {
result.add(graphObjectSerializer.serialize(obj, System.currentTimeMillis()));
}
root.add("result", result);
root.add("rawResultCount", toJsonPrimitive(src.getRawResultCount()));
}
// serialize result tree
// if (src.getResultTree() != null) {
//
// TreeNode node = src.getResultTree();
//
// root.add("root", buildTree(node, context));
//
// }
return root;
}
/**
* Returns job execution time in milliseconds. If job is not running then returns job execution
* time.
*/
public final long runTimeMs() {
long until = end_time != 0 ? end_time : System.currentTimeMillis();
return until - start_time;
}
/** Marks job as finished and records job end time. */
public void remove() {
end_time = System.currentTimeMillis();
if (state == JobState.RUNNING) state = JobState.DONE;
// Overwrite handle - copy end_time, state, msg
replaceByJobHandle();
}