private String[] store(String names[], long size, int i, NameValueRecord nvr) {
// store file
boolean error1 = false;
while (true) {
boolean error = false;
try {
names = createFileAndStore(names, size, binary, metadata, nvr, true, times, 0);
// nvr = retrieveMetadata(names[1], times, 0);
} catch (HoneycombTestException e) {
flex("store size=" + size + " name=" + (names == null ? "null" : names[0]), e);
log.log("store problem: " + e.getMessage());
// e.printStackTrace();
error = true;
} catch (Throwable t) {
flex("store size=" + size + " name=" + (names == null ? "null" : names[0]), t);
log.log("store: got throwable");
t.printStackTrace();
error = true;
}
if (error) {
error1 = true;
store_retries++;
sleep(SLEEP);
continue;
}
break;
}
if (error1) flog("#E ok\n");
// save info
total_files++;
total_bytes += size;
store_bytes += size;
time_store += times[0];
total_store_time += times[0];
oids[i] = names[1];
shas[i] = names[2];
if (fo != null) {
String s =
"" + size + '\t' + names[0] + '\t' + // filename
names[1] + '\t' + // oid
names[2] + '\t' + // sha1
names[3] + '\t' + // store date
binary + '\t' + metadata + '\t' + times[0] + '\n';
flog(s);
}
return names;
}
private void readwrite(long s1[], long s2[], int pass) {
// oids[] should correspond to s1[] going in, s2[] going out
String[] names = null;
time_retrieve = 0;
store_bytes = 0;
time_store = 0;
String algorithm_id = algorithms[randIndex(algorithms.length)];
String content_type = content_types[randIndex(content_types.length)];
StringBuffer sb = new StringBuffer();
for (int i = 0; i < s1.length; i++) {
String retrieved = null;
boolean error1 = false;
long t1 = System.currentTimeMillis();
while (true) {
boolean error = false;
try {
retrieved = retrieve(oids[i], times, 0);
time_retrieve += times[0];
total_retrieve_time += times[0];
} catch (HoneycombTestException e) {
flex("retrv " + oids[i], e);
String s = "#e retrv " + oids[i] + ": " + e.getMessage();
log.log(s);
// flog(s + "\n");
// e.printStackTrace();
error = true;
} catch (Throwable t) {
flex("retrv " + oids[i], t);
log.log("retrieve got throwable");
t.printStackTrace();
error = true;
}
if (error) {
error1 = true;
retrieve_retries++;
if (System.currentTimeMillis() - t1 > GIVEUP) {
retrieve_failures++;
retrieved = null;
break;
}
sleep(SLEEP);
continue;
}
break;
}
if (retrieved == null) {
failed_rtrv.add(oids[i]);
} else {
if (error1) flog("#E ok\n");
dot("r");
// compare retrieved w/ original
File f = new File(retrieved);
if (f.length() != s1[i]) {
bad_file_errors++;
String s =
"#e size mismatch ["
+ oids[i]
+ "] store/retrieve: "
+ s1[i]
+ " / "
+ f.length()
+ "\n";
flog(s);
} else {
String sha1sum = null;
try {
sha1sum = shell.sha1sum(retrieved);
} catch (Exception e) {
e.printStackTrace();
}
if (sha1sum != null && !sha1sum.equals(shas[i])) {
bad_file_errors++;
String s =
"#e sha1 mismatch ["
+ oids[i]
+ "] store/retrieve: "
+ shas[i]
+ " / "
+ sha1sum
+ "\n";
flog(s);
}
}
deleteFile(retrieved);
}
// compare mdata
t1 = System.currentTimeMillis();
NameValueRecord nvr = null;
error1 = false;
while (true) {
boolean error = false;
try {
nvr = retrieveMetadata(oids[i], times, 0);
md_retrieve_time += times[0];
md_retrieves++;
} catch (HoneycombTestException e) {
flex("md retrv " + oids[i], e);
String s = "#e md retrv " + oids[i] + ": " + e.getMessage();
log.log(s);
// flog(s + "\n");
// e.printStackTrace();
error = true;
} catch (Throwable t) {
flex("md retrv " + oids[i], t);
log.log("md retrieve got throwable");
t.printStackTrace();
error = true;
}
if (error) {
error1 = true;
md_retrieve_retries++;
if (System.currentTimeMillis() - t1 > GIVEUP) {
md_retrieve_failures++;
nvr = null;
flog("#E " + oids[i] + ": giving up\n");
break;
}
sleep(SLEEP);
continue;
}
break;
}
if (nvr == null) {
failed_md_rtrv.add(oids[i]);
} else {
if (error1) flog("#E ok\n");
SystemRecord sr = nvr.getSystemRecord();
String stored_oid = sr.getObjectIdentifier().toString();
if (!oids[i].equals(stored_oid)) {
// one oid looked up another
sb.append("#e oid mismatch! ").append(oids[i]);
sb.append(" / ").append(stored_oid).append("\n");
}
String sha2 = nvr.getString("sha1");
if (sha2 == null) {
sb.append("#e no sha1 in metadata\n");
} else if (!sha2.equals(shas[i])) {
sb.append("#e original/md sha: ").append(shas[i]);
sb.append(" / ").append(sha2).append("\n");
}
checkMD(sb, nvr, "cedars_subject_id", subject_id);
checkMD(sb, nvr, "cedars_study_id", study_id);
checkMD(sb, nvr, "cedars_species_id", species_id);
checkMD(sb, nvr, "cedars_sample_id", sample_id);
checkMD(sb, nvr, "cedars_collection_date", collection_date);
checkMD(sb, nvr, "cedars_processing_date", processing_date);
checkMD(sb, nvr, "cedars_collected_by", collected_by);
checkMD(sb, nvr, "cedars_processed_by", processed_by);
checkMD(sb, nvr, "cedars_sample_type", sample_type);
checkMD(sb, nvr, "cedars_experiment_date", experiment_date);
checkMD(sb, nvr, "cedars_experiment_number", experiment_number);
checkMD(sb, nvr, "cedars_scan_number", Integer.toString(i));
if (pass > 2) {
checkMD(sb, nvr, "cedars_algorithm_id", prev_algorithm_id);
checkMD(sb, nvr, "cedars_content_type", prev_content_type);
}
if (sb.length() > 0) {
bad_md_errors++;
flog("#e md error " + oids[i] + "\n");
flog(sb.toString());
sb.setLength(0);
}
}
if (s2 != null) {
// store next smaller file and save oid
nvr = createRecord();
nvr.put("cedars_subject_id", subject_id);
nvr.put("cedars_study_id", study_id);
nvr.put("cedars_species_id", species_id);
nvr.put("cedars_sample_id", sample_id);
nvr.put("cedars_collection_date", collection_date);
nvr.put("cedars_processing_date", processing_date);
nvr.put("cedars_collected_by", collected_by);
nvr.put("cedars_processed_by", processed_by);
nvr.put("cedars_sample_type", sample_type);
nvr.put("cedars_experiment_date", experiment_date);
nvr.put("cedars_experiment_number", experiment_number);
nvr.put("cedars_pass", Long.toString(pass));
nvr.put("cedars_scan_number", Long.toString(i));
nvr.put("cedars_algorithm_id", algorithm_id);
nvr.put("cedars_content_type", content_type);
names = store(names, s2[i], i, nvr);
oids[i] = names[1];
dot("s");
}
}
prev_algorithm_id = algorithm_id;
prev_content_type = content_type;
if (names != null && names[0] != null) deleteFile(names[0]);
}
