private void send_client_DH_inner_data(long retry_id) {
byte[] b_data = new byte[256];
Common.random.nextBytes(b_data);
BigInteger b = new BigInteger(1, b_data);
BigInteger g_b = g.modPow(b, dh_prime);
Common.logError(
"g_b length: " + g_b.toByteArray().length + " -> " + Common.toBytes(g_b).length);
BigInteger akey = g_a.modPow(b, dh_prime);
Common.logError("auth_key: " + akey.toString());
setAuthKey(Common.toBytes(akey));
// gen data (client_DH_inner_data)
TL.Object data_obj = TL.newObject("client_DH_inner_data", cl_nonce, sv_nonce, retry_id, g_b);
byte[] data = data_obj.serialize();
byte[] hash = Common.getSHA1(data);
byte[] data_with_hash = new byte[(hash.length + data.length + 15) / 16 * 16];
System.arraycopy(hash, 0, data_with_hash, 0, hash.length);
System.arraycopy(data, 0, data_with_hash, hash.length, data.length);
// send set_client_DH_params
TL.Object req_obj =
TL.newObject("set_client_DH_params", cl_nonce, sv_nonce, aes.IGE(data_with_hash, true));
send(req_obj, false, false);
}
@Test
public void testDeleteNonEmptyDirectory() throws CoreException, IOException, SAXException {
String dirPath1 = "sample/directory/path/sample1" + System.currentTimeMillis();
String dirPath2 = "sample/directory/path/sample2" + System.currentTimeMillis();
String fileName = "subfile.txt";
String subDirectory = "subdirectory";
createDirectory(dirPath1);
createFile(dirPath1 + "/" + fileName, "Sample file content");
createDirectory(dirPath2 + "/" + subDirectory);
WebRequest request = getDeleteFilesRequest(dirPath1);
WebResponse response = webConversation.getResponse(request);
assertEquals(
"Could not delete directory with file",
HttpURLConnection.HTTP_OK,
response.getResponseCode());
assertFalse(
"Delete directory with file request returned OK, but the file still exists",
checkDirectoryExists(dirPath1));
request = getDeleteFilesRequest(dirPath2);
response = webConversation.getResponse(request);
assertEquals(
"Could not delete directory with subdirectory",
HttpURLConnection.HTTP_OK,
response.getResponseCode());
assertFalse(
"Delete directory with subdirectory request returned OK, but the file still exists",
checkDirectoryExists(dirPath2));
}
public static void run_single_month(String root, String filename) throws IOException {
long startTime = System.currentTimeMillis();
Final_venezuela_up2 runner = new Final_venezuela_up2();
runner.run_timewindow(root, filename);
System.out.println(runner.temp_words.stream().collect(Collectors.joining(", ")));
long endTime2 = System.currentTimeMillis();
// System.out.println(String.valueOf(endTime2-startTime));
}
@Test
public void testReadFileContents() throws CoreException, IOException, SAXException {
String directoryPath = "sample/directory/path" + System.currentTimeMillis();
createDirectory(directoryPath);
String fileName = "sampleFile" + System.currentTimeMillis() + ".txt";
String fileContent = "Sample File Cotnent " + System.currentTimeMillis();
createFile(directoryPath + "/" + fileName, fileContent);
WebRequest request = getGetFilesRequest(directoryPath + "/" + fileName);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
assertEquals("Invalid file content", fileContent, response.getText());
}
@Test
public void testCreateDirectory() throws CoreException, IOException, SAXException, JSONException {
String directoryPath = "sample/directory/path" + System.currentTimeMillis();
createDirectory(directoryPath);
String dirName = "testdir";
webConversation.setExceptionsThrownOnErrorStatus(false);
WebRequest request =
getPostFilesRequest(directoryPath, getNewDirJSON(dirName).toString(), dirName);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_CREATED, response.getResponseCode());
assertTrue(
"Create directory response was OK, but the directory does not exist",
checkDirectoryExists(directoryPath + "/" + dirName));
assertEquals(
"Response should contain directory metadata in JSON, but was " + response.getText(),
"application/json",
response.getContentType());
JSONObject responseObject = new JSONObject(response.getText());
assertNotNull("No directory information in response", responseObject);
checkDirectoryMetadata(responseObject, dirName, null, null, null, null, null);
// should be able to perform GET on location header to obtain metadata
String location = response.getHeaderField(ProtocolConstants.HEADER_LOCATION);
request = getGetFilesRequest(location);
response = webConversation.getResource(request);
assertNotNull(location);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
responseObject = new JSONObject(response.getText());
assertNotNull("No direcory information in responce", responseObject);
checkDirectoryMetadata(responseObject, dirName, null, null, null, null, null);
}
@Test
public void testCopyFileOverwrite() throws Exception {
String directoryPath = "/testCopyFile/directory/path" + System.currentTimeMillis();
String sourcePath = directoryPath + "/source.txt";
String destName = "destination.txt";
String destPath = directoryPath + "/" + destName;
createDirectory(directoryPath);
createFile(sourcePath, "This is the contents");
createFile(destPath, "Original file");
// with no-overwrite, copy should fail
JSONObject requestObject = new JSONObject();
addSourceLocation(requestObject, sourcePath);
WebRequest request =
getPostFilesRequest(directoryPath, requestObject.toString(), "destination.txt");
request.setHeaderField("X-Create-Options", "copy,no-overwrite");
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, response.getResponseCode());
// now omit no-overwrite and copy should succeed and return 200 instead of 201
request = getPostFilesRequest(directoryPath, requestObject.toString(), "destination.txt");
request.setHeaderField("X-Create-Options", "copy");
response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
JSONObject responseObject = new JSONObject(response.getText());
checkFileMetadata(responseObject, destName, null, null, null, null, null, null, null);
assertTrue(checkFileExists(sourcePath));
assertTrue(checkFileExists(destPath));
}
@Test
public void testCreateTopLevelFile()
throws CoreException, IOException, SAXException, JSONException {
String directoryPath = "sample" + System.currentTimeMillis();
createDirectory(directoryPath);
String fileName = "testfile.txt";
WebRequest request =
getPostFilesRequest(directoryPath, getNewFileJSON(fileName).toString(), fileName);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_CREATED, response.getResponseCode());
assertTrue(
"Create file response was OK, but the file does not exist",
checkFileExists(directoryPath + "/" + fileName));
assertEquals(
"Response should contain file metadata in JSON, but was " + response.getText(),
"application/json",
response.getContentType());
JSONObject responseObject = new JSONObject(response.getText());
assertNotNull("No file information in responce", responseObject);
checkFileMetadata(responseObject, fileName, null, null, null, null, null, null, null);
// should be able to perform GET on location header to obtain metadata
String location = response.getHeaderField("Location");
request = getGetFilesRequest(location + "?parts=meta");
response = webConversation.getResource(request);
assertNotNull(location);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
responseObject = new JSONObject(response.getText());
assertNotNull("No direcory information in responce", responseObject);
checkFileMetadata(responseObject, fileName, null, null, null, null, null, null, null);
}
private long GEN_message_id() {
long time = System.currentTimeMillis() + time_delta * 1000;
long id = (time / 1000L << 32) + (time % 1000) / 4 * 4;
if (last_message_id >= id) id = last_message_id + 4;
last_message_id = id;
return id;
}
@Test
public void testReadDirectoryChildren()
throws CoreException, IOException, SAXException, JSONException {
String dirName = "path" + System.currentTimeMillis();
String directoryPath = "sample/directory/" + dirName;
createDirectory(directoryPath);
String subDirectory = "subdirectory";
createDirectory(directoryPath + "/" + subDirectory);
String subFile = "subfile.txt";
createFile(directoryPath + "/" + subFile, "Sample file");
WebRequest request = getGetFilesRequest(directoryPath + "?depth=1");
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
List<JSONObject> children = getDirectoryChildren(new JSONObject(response.getText()));
assertEquals("Wrong number of directory children", 2, children.size());
for (JSONObject child : children) {
if (child.getBoolean("Directory")) {
checkDirectoryMetadata(child, subDirectory, null, null, null, null, null);
} else {
checkFileMetadata(child, subFile, null, null, null, null, null, null, null);
}
}
}
@Test
public void testDirectoryWithSpaces() throws CoreException, IOException, SAXException {
String basePath = "sampe/dir with spaces/long" + System.currentTimeMillis();
createDirectory(basePath);
WebRequest request = getGetFilesRequest(basePath);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
}
private void TL_Server_DH_inner_data(TL.Object obj) {
g = BigInteger.valueOf(obj.getInt("g"));
dh_prime = new BigInteger(1, obj.getBytes("dh_prime"));
g_a = new BigInteger(1, obj.getBytes("g_a"));
time_delta =
(int) ((long) obj.getInt("server_time") - (long) System.currentTimeMillis() / 1000L);
last_message_id = 0;
send_client_DH_inner_data(0);
}
@Test
public void testDeleteEmptyDir() throws CoreException, IOException, SAXException {
String dirPath = "sample/directory/path/sample" + System.currentTimeMillis();
createDirectory(dirPath);
WebRequest request = getDeleteFilesRequest(dirPath);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
assertFalse(
"Delete request returned OK, but the directory still exists",
checkDirectoryExists(dirPath));
}
private void run_month(String folder_path) throws IOException {
System.out.println(folder_path);
long startTime = System.currentTimeMillis();
File folder = new File(folder_path);
File[] files = folder.listFiles();
List<File> listOfFiles = Arrays.asList(files);
List<File> graphFiles =
listOfFiles
.stream()
.filter(f -> f.getName().contains("graph"))
.collect(Collectors.toList());
this.max_score = 0;
for (int i = 0; i < graphFiles.size(); i++) {
String fname = graphFiles.get(i).getName();
this.run_timewindow(folder_path, fname);
}
this.ignore_words.addAll(this.temp_words);
long endTime = System.currentTimeMillis();
System.out.println(
"***" + folder_path + " done in:" + String.valueOf(endTime - startTime) + "***");
}
@Test
public void testReadDirectory() throws CoreException, IOException, SAXException, JSONException {
String dirName = "path" + System.currentTimeMillis();
String directoryPath = "sample/directory/" + dirName;
createDirectory(directoryPath);
WebRequest request = getGetFilesRequest(directoryPath);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
JSONObject dirObject = new JSONObject(response.getText());
checkDirectoryMetadata(dirObject, dirName, null, null, null, null, null);
}
@Test
public void testCopyFileInvalidSource() throws Exception {
String directoryPath = "/testCopyFile/directory/path" + System.currentTimeMillis();
createDirectory(directoryPath);
JSONObject requestObject = new JSONObject();
requestObject.put("Location", "/this/does/not/exist/at/all");
WebRequest request =
getPostFilesRequest(directoryPath, requestObject.toString(), "destination.txt");
request.setHeaderField("X-Create-Options", "copy");
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_NOT_FOUND, response.getResponseCode());
JSONObject responseObject = new JSONObject(response.getText());
assertEquals("Error", responseObject.get("Severity"));
}
@Test
public void testReadFileMetadata() throws Exception {
String directoryPath = "sample/directory/path" + System.currentTimeMillis();
createDirectory(directoryPath);
String fileName = "sampleFile" + System.currentTimeMillis() + ".txt";
String fileContent = "Sample File Cotnent " + System.currentTimeMillis();
createFile(directoryPath + "/" + fileName, fileContent);
WebRequest request = getGetFilesRequest(directoryPath + "/" + fileName + "?parts=meta");
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
JSONObject result = new JSONObject(response.getText());
assertEquals(fileName, result.optString(ProtocolConstants.KEY_NAME));
JSONArray parents = result.optJSONArray(ProtocolConstants.KEY_PARENTS);
assertNotNull(parents);
assertEquals(3, parents.length());
IPath parentPath = new Path(directoryPath);
// immediate parent
JSONObject parent = parents.getJSONObject(0);
assertEquals(parentPath.segment(2), parent.getString(ProtocolConstants.KEY_NAME));
// grandparent
parent = parents.getJSONObject(1);
assertEquals(parentPath.segment(1), parent.getString(ProtocolConstants.KEY_NAME));
// ensure all parent locations end with trailing slash
for (int i = 0; i < parents.length(); i++) {
parent = parents.getJSONObject(i);
String location = parent.getString(ProtocolConstants.KEY_LOCATION);
assertTrue(location.endsWith("/"));
location = parent.getString(ProtocolConstants.KEY_CHILDREN_LOCATION);
URI childrenLocation = new URI(location);
assertTrue(childrenLocation.getPath().endsWith("/"));
}
}
private void TL_ResPQ(TL.Object obj) {
sv_nonce = obj.getBytes("server_nonce");
BigInteger pq = new BigInteger(1, obj.getBytes("pq"));
TL.Vector v_fp = obj.getVector("server_public_key_fingerprints");
fp = v_fp.getLong(0);
Common.logError("pq: " + pq.toString());
// prime factorization for pq
BigInteger q = Common.rho(pq);
BigInteger p = pq.divide(q);
if (p.compareTo(q) > 0) {
BigInteger t = p;
p = q;
q = t;
}
SecureRandom rnd = new SecureRandom();
new_nonce = new byte[32];
rnd.nextBytes(new_nonce);
// generate encrypted_data
TL.Object data_obj = TL.newObject("p_q_inner_data", pq, p, q, cl_nonce, sv_nonce, new_nonce);
byte[] data = data_obj.serialize();
byte[] hash = Common.getSHA1(data);
byte[] data_with_hash = new byte[255];
System.arraycopy(hash, 0, data_with_hash, 0, hash.length);
System.arraycopy(data, 0, data_with_hash, hash.length, data.length);
GEN_random_bytes(data_with_hash, data.length + hash.length, 255);
byte[] encrypted_data = Common.RSA(RSA_MODULUS, RSA_EXPONENT, data_with_hash);
// req_DH_params
TL.Object req_obj = TL.newObject("req_DH_params", cl_nonce, sv_nonce, p, q, fp, encrypted_data);
send(req_obj, false, false);
}
// El string debe estar em formato json
public boolean updateRegistry(String dataJson) {
try {
// Crea objeto json con string por parametro
JSONObject json = new JSONObject(dataJson);
// Obtiene el array de Registos
JSONArray arr = json.getJSONArray("Registry");
// Recorre el array
for (int i = 0; i < arr.length(); i++) {
// Obtiene los datos idSensor y value
int idSensor = arr.getJSONObject(i).getInt("idSensor");
int value = arr.getJSONObject(i).getInt("value");
// Recorre la configuracion de registro
for (RegistryConf reg : registryConf) {
// Se fija si el registro corresponde a esta configuracion
if (reg.getIdSensor() == idSensor) {
// Checkea el criterio para guardar, o no en la BD
// Checkea tambien si el valor es igual al anterior
if (reg.getSaveTypeString() == "ONCHANGE" && lastRead.get(idSensor) != value) {
// Actualizo la ultima lectura y guardo en la BD
lastRead.put(idSensor, value);
saveRegistry(idSensor, value);
} else if (reg.getSaveTypeString() == "ONTIME") {
// Variables auxiliares, para checkear tiempo
Long auxLong = System.currentTimeMillis() / 1000;
int now = auxLong.intValue();
int timeToSave = lastRead.get(idSensor) + reg.getValue();
// Checkea si ya es tiempo para guerdar un nuevo registro
if (now >= timeToSave) {
// Actualizo el ultimo guardado
lastRead.put(idSensor, now);
saveRegistry(idSensor, value);
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return false;
}
@Test
public void testDeleteFile() throws CoreException, IOException, SAXException {
String dirPath = "sample/directory/path";
String fileName = System.currentTimeMillis() + ".txt";
String filePath = dirPath + "/" + fileName;
createDirectory(dirPath);
createFile(filePath, "Sample file content");
WebRequest request = getDeleteFilesRequest(filePath);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
assertFalse("Delete request returned OK, but the file still exists", checkFileExists(filePath));
assertTrue(
"File was deleted but the above directory was deleted as well",
checkDirectoryExists(dirPath));
}
@Test
public void testCopyFileNoOverwrite() throws Exception {
String directoryPath = "/testCopyFile/directory/path" + System.currentTimeMillis();
String sourcePath = directoryPath + "/source.txt";
String destName = "destination.txt";
String destPath = directoryPath + "/" + destName;
createDirectory(directoryPath);
createFile(sourcePath, "This is the contents");
JSONObject requestObject = new JSONObject();
addSourceLocation(requestObject, sourcePath);
WebRequest request =
getPostFilesRequest(directoryPath, requestObject.toString(), "destination.txt");
request.setHeaderField("X-Create-Options", "copy");
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_CREATED, response.getResponseCode());
JSONObject responseObject = new JSONObject(response.getText());
checkFileMetadata(responseObject, destName, null, null, null, null, null, null, null);
assertTrue(checkFileExists(sourcePath));
assertTrue(checkFileExists(destPath));
}
public static void main(String args[]) {
if (args.length < 1) {
System.err.println(
"Usage: java org.globusonline.transfer.ErrorTest "
+ "username [cafile certfile keyfile [baseurl]]]");
System.exit(1);
}
String username = args[0];
String cafile = null;
if (args.length > 1 && args[1].length() > 0) cafile = args[1];
String certfile = null;
if (args.length > 2 && args[2].length() > 0) certfile = args[2];
String keyfile = null;
if (args.length > 3 && args[3].length() > 0) keyfile = args[3];
String baseUrl = null;
if (args.length > 4 && args[4].length() > 0) baseUrl = args[4];
try {
JSONTransferAPIClient c =
new JSONTransferAPIClient(username, cafile, certfile, keyfile, baseUrl);
System.out.println("base url: " + c.getBaseUrl());
ErrorTest et = new ErrorTest(c);
et.run();
// test auth error
System.out.println("=== 400 Auth Failed ===");
c = new JSONTransferAPIClient(username, cafile, null, null, baseUrl);
try {
c.getResult("/tasksummary");
} catch (APIError e) {
e.printStackTrace();
}
} catch (Exception e) {
e.printStackTrace();
}
}
@Test
public void testCreateFileOverwrite()
throws CoreException, IOException, SAXException, JSONException {
String directoryPath = "sample/directory/path" + System.currentTimeMillis();
createDirectory(directoryPath);
String fileName = "testfile.txt";
WebRequest request =
getPostFilesRequest(directoryPath, getNewFileJSON(fileName).toString(), fileName);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_CREATED, response.getResponseCode());
// creating again at the same location should succeed but return OK rather than CREATED
request = getPostFilesRequest(directoryPath, getNewFileJSON(fileName).toString(), fileName);
response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
// creating with no-overwrite should fail if it already exists
request = getPostFilesRequest(directoryPath, getNewFileJSON(fileName).toString(), fileName);
request.setHeaderField("X-Create-Options", "no-overwrite");
response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_PRECON_FAILED, response.getResponseCode());
}
public class Logger {
private final JSONObject callLog;
private static final String SEPARATOR = System.getProperty("line.separator");
public Logger() {
callLog = new JSONObject();
}
private void logTimestamp() {
callLog.put("timestamp", System.currentTimeMillis());
}
private void logTargetClass(Object target) {
callLog.put("class", target.getClass().getName());
}
private void logMethod(Method method) {
callLog.put("method", method.getName());
}
public void logMethodCall(Method method, Object[] args, Object target) throws Throwable {
Throwable caught = null;
Object returnValue = null;
logTimestamp();
logTargetClass(target);
logMethod(method);
logArguments(args);
}
private void logArguments(Object[] args) {
IdentityHashMap<Iterable, Boolean> identityHashMap = new IdentityHashMap<Iterable, Boolean>();
if (args != null) {
callLog.put("arguments", logIterable(Arrays.asList(args), identityHashMap));
} else {
callLog.put("arguments", new JSONArray());
}
}
private Object logIterable(
Iterable iterable, IdentityHashMap<Iterable, Boolean> identityHashMap) {
if (identityHashMap.containsKey(iterable)) {
return "cyclic";
}
identityHashMap.put(iterable, true);
JSONArray iterableValues = new JSONArray();
if (iterable != null) {
for (Object value : iterable) {
if (value instanceof Iterable) {
iterableValues.put(logIterable((Iterable) value, identityHashMap));
} else if ((value != null) && (value.getClass().isArray())) {
iterableValues.put(value.toString());
} else {
iterableValues.put(value);
}
}
}
return iterableValues;
}
public void logReturnValue(Object returnValue) {
IdentityHashMap<Iterable, Boolean> identityHashMap = new IdentityHashMap<Iterable, Boolean>();
if (returnValue == null) {
callLog.put("returnValue", JSONObject.NULL);
} else if (!(returnValue instanceof Iterable)) {
callLog.put("returnValue", returnValue);
} else {
callLog.put("returnValue", logIterable((Iterable) returnValue, identityHashMap));
}
}
public void logThrown(Throwable thrown) {
callLog.put("thrown", thrown.toString());
}
// for test purposes
public JSONObject getResultObject() {
return callLog;
}
@Override
public String toString() {
return (callLog.toString(2) + SEPARATOR);
}
}
/**
* Parses a text file of JSONs into an array of tweets
*
* @param textFile The textfile containing the JSONs
*/
public parseJSON() {
// get folder paths
String currentDir = System.getProperty("user.dir");
String root = currentDir;
String textFile = root + "/tweet_input/tweets.txt";
String outputFolderF1 = root + "/tweet_output/f1.txt";
String outputFolderF2 = root + "/tweet_output/f2.txt";
int numTweets = 0;
int numGoodTweets = 0;
hashtagEdges hashEdges = new hashtagEdges();
// try objects
try {
// create a reader object for tweet's textfile and read-in first line
BufferedReader reader = new BufferedReader(new FileReader(textFile));
String currentJSON = reader.readLine();
// System.out.println(textFile);
// initate a writer object with the outputFolder name
PrintWriter writerF1 = new PrintWriter(outputFolderF1, "UTF-8");
PrintWriter writerF2 = new PrintWriter(outputFolderF2, "UTF-8");
// create an array list to save parsedTweets and declare problem variables
List<parsedTweet> tweetList = new ArrayList<parsedTweet>();
String tweetText, tweetTime;
JSONObject objJSON;
String[] hashArray;
parsedTweet tweetObj;
float currentAverage;
while (currentJSON != null) {
try {
objJSON = new JSONObject(currentJSON);
// if the JSON has a text and time stamp, process it
numTweets = numTweets + 1;
if (objJSON.has("created_at") && objJSON.has("text")) {
// get timestamp and text from JSON object
tweetTime = objJSON.getString("created_at");
tweetText = objJSON.getString("text");
// create a tweet object and add it to folder
tweetObj = new parsedTweet(tweetTime, tweetText, currentJSON);
tweetList.add(tweetObj);
// update hashObjet
hashArray = tweetObj.hashtags.toArray(new String[tweetObj.hashtags.size()]);
currentAverage = hashEdges.updateEdges(hashArray, tweetTime);
// write correctly-formated cleen-tweet to output folder
writerF1.println(tweetObj.formatTweet());
numGoodTweets = numGoodTweets + 1;
writerF2.format("%.2f%n", currentAverage);
}
} catch (Exception e) {
System.out.println("Error in parseJSON - 1");
e.printStackTrace();
}
// read next line (which has the next JSON object)
currentJSON = reader.readLine();
}
writerF1.close();
writerF2.close();
} catch (Exception e) {
System.out.println("Error in parseJSON - 2");
e.printStackTrace();
}
}
/**
* Layout core logic.
*
* <p>This iterates over all the nodes and determines their velocities based on a force directed
* layout approach, performs collision detection/avoidance and then ultimately their new positions
* as they move.
*/
private void doLayout() {
double maxVelocity = 100;
double numOfIterations = 0;
double step = 100;
double skip = step;
int counter = 1;
boolean dividerChanged = false;
// active = true;
while (maxVelocity > 1 && active && (numOfIterations < MAX_NUM_OF_ITERATIONS)) {
counter++;
maxVelocity = 0;
numOfIterations++;
Iterator<Node> outer = controller.getNodeIterator();
startTime = System.currentTimeMillis();
double systemMovement = 0;
double kineticEnergy = 0;
/* Iterate over all nodes and calculate velocity */
while (outer.hasNext()) {
Node current = outer.next();
current.resetVelocity();
current.resetAcceleration();
/*
* If current node is not pinned or selected,
* perform force directed layout calculations here.
*/
if (!((current == controller.getGraphSelected()) || current.isPinned())) {
calculateNodeAttractiveForce(current);
calculateNodeRepulsiveForces(current);
// applyNodeFriction(current);
// if (IS_COOLING) {
// calculateNodeCooling(current, numOfIterations);
// }
// calculate velocities
current.setVX(current.getVX() + current.accelx * TIMESTEP * DAMPING);
current.setVY(current.getVY() + current.accely * TIMESTEP * DAMPING);
double speedSquared =
current.getVX() * current.getVX() + current.getVY() * current.getVY();
kineticEnergy += 0.5 * current.weight * speedSquared;
// cap the velocity
if (current.getVX() >= 0) {
current.setVX(Math.min(current.getVX(), MAX_DIST_PER_MOVE));
} else {
current.setVX(Math.max(current.getVX(), -MAX_DIST_PER_MOVE));
}
if (current.getVY() >= 0) {
current.setVY(Math.min(current.getVY(), MAX_DIST_PER_MOVE));
} else {
current.setVY(Math.max(current.getVY(), -MAX_DIST_PER_MOVE));
}
// System.out.println("vy "+ current.getVY() + ", vx " + current.getVX());
}
}
/* Iterate over all nodes move them after all the velocities are set */
outer = controller.getNodeIterator();
while (outer.hasNext()) {
Node current = outer.next();
int xStart = current.getX();
int yStart = current.getY();
/* update node position */
int xEnd = current.getX() + (int) current.getVX();
int yEnd = current.getY() + (int) current.getVY();
int dX = Math.abs(xStart - xEnd);
int dY = Math.abs(yStart - yEnd);
// systemMovement += dX;
// systemMovement += dY;
// if(dX + dY > 6) {
controller.moveNode(current, xEnd, yEnd);
// }
// controller.moveNode(current, Math.min(current.getX() + (int)
// current.getVX(),MAX_DIST_PER_MOVE), Math.min(current.getY() + (int) current.getVY(),
// MAX_DIST_PER_MOVE));
/* compute maxVelocity for layout iteration */
maxVelocity =
Math.max(
maxVelocity,
Math.sqrt(Math.pow(current.getVX(), 2) + Math.pow(current.getVY(), 2)));
}
if (kineticEnergy < ENERGY_THRESHOLD) {
numOfIterations = MAX_NUM_OF_ITERATIONS;
}
/* Make animation slower */
try {
Thread.sleep(sleepInterval);
} catch (InterruptedException ex) {
}
/* Calculate how long the iteration took */
stopTime = System.currentTimeMillis();
long duration = stopTime - startTime;
if ((duration > MAX_ITERATION_TIME) && (sleepInterval > 5)) {
sleepInterval -= 5;
} else if (duration < MIN_ITERATION_TIME) {
sleepInterval += 5;
}
if (numOfIterations > skip) {
skip += step;
System.out.print('.');
}
}
/* We've reached a stable layout, hold on for now */
stable = true;
// System.out.println("ran in " + numOfIterations + " iterations.");
}
public static ArrayList<ContentValues> parsePosts(
TagNode aThread,
int aThreadId,
int unreadIndex,
int opId,
AwfulPreferences prefs,
int startIndex) {
ArrayList<ContentValues> result = new ArrayList<ContentValues>();
boolean lastReadFound = false;
int index = startIndex;
String update_time = new Timestamp(System.currentTimeMillis()).toString();
Log.v(TAG, "Update time: " + update_time);
try {
if (!Constants.isICS()
|| !prefs.inlineYoutube) { // skipping youtube support for now, it kinda sucks.
aThread = convertVideos(aThread);
}
TagNode[] postNodes = aThread.getElementsByAttValue("class", "post", true, true);
for (TagNode node : postNodes) {
// fyad status, to prevent processing postbody twice if we are in fyad
ContentValues post = new ContentValues();
post.put(THREAD_ID, aThreadId);
// We'll just reuse the array of objects rather than create
// a ton of them
int id = Integer.parseInt(node.getAttributeByName("id").replaceAll("post", ""));
post.put(ID, id);
post.put(AwfulProvider.UPDATED_TIMESTAMP, update_time);
post.put(POST_INDEX, index);
if (index > unreadIndex) {
post.put(PREVIOUSLY_READ, 0);
lastReadFound = true;
} else {
post.put(PREVIOUSLY_READ, 1);
}
index++;
post.put(IS_MOD, 0);
post.put(IS_ADMIN, 0);
TagNode[] postContent = node.getElementsHavingAttribute("class", true);
for (TagNode pc : postContent) {
if (pc.getAttributeByName("class").contains("author")) {
post.put(USERNAME, pc.getText().toString().trim());
}
if (pc.getAttributeByName("class").contains("role-mod")) {
post.put(IS_MOD, 1);
}
if (pc.getAttributeByName("class").contains("role-admin")) {
post.put(IS_ADMIN, 1);
}
if (pc.getAttributeByName("class").equalsIgnoreCase("title")
&& pc.getChildTags().length > 0) {
TagNode[] avatar = pc.getElementsByName("img", true);
if (avatar.length > 0) {
post.put(AVATAR, avatar[0].getAttributeByName("src"));
}
post.put(AVATAR_TEXT, pc.getText().toString().trim());
}
if (pc.getAttributeByName("class").equalsIgnoreCase("postbody")
|| pc.getAttributeByName("class").contains("complete_shit")) {
TagNode[] images = pc.getElementsByName("img", true);
for (TagNode img : images) {
// don't alter video mock buttons
if ((img.hasAttribute("class")
&& img.getAttributeByName("class").contains("videoPlayButton"))) {
continue;
}
boolean dontLink = false;
TagNode parent = img.getParent();
String src = img.getAttributeByName("src");
if ((parent != null && parent.getName().equals("a"))
|| (img.hasAttribute("class")
&& img.getAttributeByName("class")
.contains("nolink"))) { // image is linked, don't override
dontLink = true;
}
if (src.contains(".gif")) {
img.setAttribute(
"class",
(img.hasAttribute("class") ? img.getAttributeByName("class") + " " : "")
+ "gif");
}
if (img.hasAttribute("title")) {
if (!prefs.showSmilies) { // kill all emotes
String name = img.getAttributeByName("title");
img.setName("p");
img.addChild(new ContentNode(name));
}
} else {
if (!lastReadFound && prefs.hideOldImages || !prefs.imagesEnabled) {
if (!dontLink) {
img.setName("a");
img.setAttribute("href", src);
img.addChild(new ContentNode(src));
} else {
img.setName("p");
img.addChild(new ContentNode(src));
}
} else {
if (!dontLink) {
img.setName("a");
img.setAttribute("href", src);
TagNode newimg = new TagNode("img");
if (!prefs.imgurThumbnails.equals("d") && src.contains("i.imgur.com")) {
int lastSlash = src.lastIndexOf('/');
if (src.length() - lastSlash <= 9) {
int pos = src.length() - 4;
src = src.substring(0, pos) + prefs.imgurThumbnails + src.substring(pos);
}
}
newimg.setAttribute("src", src);
img.addChild(newimg);
}
}
}
}
StringBuffer fixedContent = new StringBuffer();
Matcher fixCharMatch = fixCharacters_regex.matcher(NetworkUtils.getAsString(pc));
while (fixCharMatch.find()) {
fixCharMatch.appendReplacement(fixedContent, "");
}
fixCharMatch.appendTail(fixedContent);
post.put(CONTENT, fixedContent.toString());
}
if (pc.getAttributeByName("class").equalsIgnoreCase("postdate")) {
post.put(
DATE,
NetworkUtils.unencodeHtml(pc.getText().toString())
.replaceAll("[^\\w\\s:,]", "")
.trim());
}
if (pc.getAttributeByName("class").equalsIgnoreCase("profilelinks")) {
TagNode[] links = pc.getElementsHavingAttribute("href", true);
if (links.length > 0) {
String href = links[0].getAttributeByName("href").trim();
String userId = href.substring(href.lastIndexOf("rid=") + 4);
post.put(USER_ID, userId);
if (Integer.toString(opId).equals(userId)) { // ugh
post.put(IS_OP, 1);
} else {
post.put(IS_OP, 0);
}
}
}
if (pc.getAttributeByName("class").equalsIgnoreCase("editedby")
&& pc.getChildTags().length > 0) {
post.put(EDITED, "<i>" + pc.getChildTags()[0].getText().toString() + "</i>");
}
}
TagNode[] editImgs = node.getElementsByAttValue("alt", "Edit", true, true);
if (editImgs.length > 0) {
post.put(EDITABLE, 1);
} else {
post.put(EDITABLE, 0);
}
result.add(post);
}
Log.i(
TAG,
Integer.toString(postNodes.length) + " posts found, " + result.size() + " posts parsed.");
} catch (Exception e) {
e.printStackTrace();
}
return result;
}
public void run_timewindow(String rootFolder, String filename) throws IOException {
System.out.println(filename);
long startTime = System.currentTimeMillis();
// R
// String[] elements = filename.split("_", -1);
ArrayList<Integer> R_list = new ArrayList<Integer>();
int day = Integer.parseInt(filename.substring(8, 10));
int length = Integer.parseInt(filename.split("_", -1)[1]);
IntStream.range(day, day + length).forEach(R_list::add);
int[] R = ArrayUtils.toPrimitive(R_list.toArray(new Integer[0]));
// lambdas
BufferedReader br =
new BufferedReader(new FileReader(rootFolder + filename.replace("graph", "lambda")));
String line = br.readLine();
JSONObject obj = new JSONObject(line.trim());
ArrayList<Double> lambdas_list = new ArrayList<Double>();
Iterator<?> keys = obj.keys();
while (keys.hasNext()) {
String key = (String) keys.next();
lambdas_list.add(obj.getDouble(key));
}
double[] lambdas = lambdas_list.stream().mapToDouble(d -> d).toArray();
br.close();
// word dictionary
BufferedReader br2 =
new BufferedReader(new FileReader(rootFolder + filename.replace("graph", "index")));
String line2 = br2.readLine();
JSONObject word_dict = new JSONObject(line2.trim());
br2.close();
// edges and c
APDMInputFormat apdm = new APDMInputFormat(new File(rootFolder + filename));
ArrayList<Integer[]> edges = new ArrayList<Integer[]>(); // 1. graph G, input parameter G
for (int[] edge : apdm.inputData.edges.keySet()) {
edges.add(new Integer[] {edge[0], edge[1]});
}
double[] c = apdm.getPValue();
double[] c2 = new double[c.length];
for (int i = 0; i < c.length; i++) {
c2[i] = c[i];
}
// detect only up or down
if (this.up_down.equals("down")) {
for (int i = 0; i < c.length; i++) {
if (c2[i] > lambdas[i]) {
c2[i] = lambdas[i];
}
}
} else if (this.up_down.equals("up")) {
for (int i = 0; i < c.length; i++) {
if (c2[i] < lambdas[i]) {
c2[i] = lambdas[i];
}
}
}
if (!this.is_first_time) {
for (String iw : this.ignore_words) {
for (int i = 0; i < apdm.numNodes; i++) {
if (word_dict.getString(String.valueOf(i)).equals(iw)) {
c2[i] = lambdas[i];
}
}
}
}
// sparsity s
// int s = apdm.trueSubGraphNodes.length ;
int s = 8;
GraphModelIHT graphModelIHT =
new GraphModelIHT(
edges, null, c2, null, s / 2, 1, s - 1 + 0.0D, 10, false, null, null, null, R, lambdas);
// post-process
List<Integer> index_list =
IntStream.of(graphModelIHT.resultNodes_Tail).boxed().collect(Collectors.toList());
List<String> words = new ArrayList<String>();
List<Double> weights = new ArrayList<Double>();
for (Integer a : index_list) {
words.add(word_dict.getString(String.valueOf(a)));
weights.add(c[a]);
}
ArrayList<String[]> sub_edges = new ArrayList<String[]>();
for (Integer[] e : edges) {
if (index_list.contains(e[0])
&& index_list.contains(e[1])
&& !word_dict
.getString(String.valueOf(e[0]))
.equals(word_dict.getString(String.valueOf(e[1])))) {
sub_edges.add(
new String[] {
word_dict.getString(String.valueOf(e[0])), word_dict.getString(String.valueOf(e[1]))
});
}
}
if (graphModelIHT.funcValue > this.max_score) {
this.max_score = graphModelIHT.funcValue;
this.best_subgraph = graphModelIHT.resultNodes_Tail;
this.temp_words = words;
this.bestWindow = filename;
this.subgraph = sub_edges;
this.weights = weights;
}
long endTime2 = System.currentTimeMillis();
System.out.println(String.valueOf(endTime2 - startTime));
}
private void logTimestamp() {
callLog.put("timestamp", System.currentTimeMillis());
}
private void exit_btActionPerformed(
java.awt.event.ActionEvent evt) { // GEN-FIRST:event_exit_btActionPerformed
// TODO add your handling code here:
System.exit(0);
} // GEN-LAST:event_exit_btActionPerformed
@Test
public void testDirectoryDepth() throws CoreException, IOException, SAXException, JSONException {
String basePath = "sampe/directory/long" + System.currentTimeMillis();
String longPath = basePath + "/dir1/dir2/dir3/dir4";
createDirectory(longPath);
WebRequest request = getGetFilesRequest(basePath);
WebResponse response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
assertEquals(
"Directory information with depth = 0 return children",
0,
getDirectoryChildren(new JSONObject(response.getText())).size());
request = getGetFilesRequest(basePath + "?depth=1");
response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
List<JSONObject> depthChildren = getDirectoryChildren(new JSONObject(response.getText()));
assertEquals(
"Directory information with depth = 1 returned too shallow", 1, depthChildren.size());
checkDirectoryMetadata(depthChildren.get(0), "dir1", null, null, null, null, null);
assertEquals(
"Directory information with depth = 1 returned too deep",
0,
getDirectoryChildren(depthChildren.get(0)).size());
request = getGetFilesRequest(basePath + "?depth=2");
response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
depthChildren =
getDirectoryChildren(getDirectoryChildren(new JSONObject(response.getText())).get(0));
assertEquals(
"Directory information with depth = 2 returned too shallow", 1, depthChildren.size());
checkDirectoryMetadata(depthChildren.get(0), "dir2", null, null, null, null, null);
assertEquals(
"Directory information with depth = 2 returned too deep",
0,
getDirectoryChildren(depthChildren.get(0)).size());
request = getGetFilesRequest(basePath + "?depth=3");
response = webConversation.getResponse(request);
assertEquals(HttpURLConnection.HTTP_OK, response.getResponseCode());
depthChildren =
getDirectoryChildren(
getDirectoryChildren(getDirectoryChildren(new JSONObject(response.getText())).get(0))
.get(0));
assertEquals(
"Directory information with depth = 3 returned too shallow", 1, depthChildren.size());
checkDirectoryMetadata(depthChildren.get(0), "dir3", null, null, null, null, null);
assertEquals(
"Directory information with depth = 3 returned too deep",
0,
getDirectoryChildren(depthChildren.get(0)).size());
}