private void readwrite(long s1[], long s2[], int pass) {
// oids[] should correspond to s1[] going in, s2[] going out
String[] names = null;
time_retrieve = 0;
store_bytes = 0;
time_store = 0;
String algorithm_id = algorithms[randIndex(algorithms.length)];
String content_type = content_types[randIndex(content_types.length)];
StringBuffer sb = new StringBuffer();
for (int i = 0; i < s1.length; i++) {
String retrieved = null;
boolean error1 = false;
long t1 = System.currentTimeMillis();
while (true) {
boolean error = false;
try {
retrieved = retrieve(oids[i], times, 0);
time_retrieve += times[0];
total_retrieve_time += times[0];
} catch (HoneycombTestException e) {
flex("retrv " + oids[i], e);
String s = "#e retrv " + oids[i] + ": " + e.getMessage();
log.log(s);
// flog(s + "\n");
// e.printStackTrace();
error = true;
} catch (Throwable t) {
flex("retrv " + oids[i], t);
log.log("retrieve got throwable");
t.printStackTrace();
error = true;
}
if (error) {
error1 = true;
retrieve_retries++;
if (System.currentTimeMillis() - t1 > GIVEUP) {
retrieve_failures++;
retrieved = null;
break;
}
sleep(SLEEP);
continue;
}
break;
}
if (retrieved == null) {
failed_rtrv.add(oids[i]);
} else {
if (error1) flog("#E ok\n");
dot("r");
// compare retrieved w/ original
File f = new File(retrieved);
if (f.length() != s1[i]) {
bad_file_errors++;
String s =
"#e size mismatch ["
+ oids[i]
+ "] store/retrieve: "
+ s1[i]
+ " / "
+ f.length()
+ "\n";
flog(s);
} else {
String sha1sum = null;
try {
sha1sum = shell.sha1sum(retrieved);
} catch (Exception e) {
e.printStackTrace();
}
if (sha1sum != null && !sha1sum.equals(shas[i])) {
bad_file_errors++;
String s =
"#e sha1 mismatch ["
+ oids[i]
+ "] store/retrieve: "
+ shas[i]
+ " / "
+ sha1sum
+ "\n";
flog(s);
}
}
deleteFile(retrieved);
}
// compare mdata
t1 = System.currentTimeMillis();
NameValueRecord nvr = null;
error1 = false;
while (true) {
boolean error = false;
try {
nvr = retrieveMetadata(oids[i], times, 0);
md_retrieve_time += times[0];
md_retrieves++;
} catch (HoneycombTestException e) {
flex("md retrv " + oids[i], e);
String s = "#e md retrv " + oids[i] + ": " + e.getMessage();
log.log(s);
// flog(s + "\n");
// e.printStackTrace();
error = true;
} catch (Throwable t) {
flex("md retrv " + oids[i], t);
log.log("md retrieve got throwable");
t.printStackTrace();
error = true;
}
if (error) {
error1 = true;
md_retrieve_retries++;
if (System.currentTimeMillis() - t1 > GIVEUP) {
md_retrieve_failures++;
nvr = null;
flog("#E " + oids[i] + ": giving up\n");
break;
}
sleep(SLEEP);
continue;
}
break;
}
if (nvr == null) {
failed_md_rtrv.add(oids[i]);
} else {
if (error1) flog("#E ok\n");
SystemRecord sr = nvr.getSystemRecord();
String stored_oid = sr.getObjectIdentifier().toString();
if (!oids[i].equals(stored_oid)) {
// one oid looked up another
sb.append("#e oid mismatch! ").append(oids[i]);
sb.append(" / ").append(stored_oid).append("\n");
}
String sha2 = nvr.getString("sha1");
if (sha2 == null) {
sb.append("#e no sha1 in metadata\n");
} else if (!sha2.equals(shas[i])) {
sb.append("#e original/md sha: ").append(shas[i]);
sb.append(" / ").append(sha2).append("\n");
}
checkMD(sb, nvr, "cedars_subject_id", subject_id);
checkMD(sb, nvr, "cedars_study_id", study_id);
checkMD(sb, nvr, "cedars_species_id", species_id);
checkMD(sb, nvr, "cedars_sample_id", sample_id);
checkMD(sb, nvr, "cedars_collection_date", collection_date);
checkMD(sb, nvr, "cedars_processing_date", processing_date);
checkMD(sb, nvr, "cedars_collected_by", collected_by);
checkMD(sb, nvr, "cedars_processed_by", processed_by);
checkMD(sb, nvr, "cedars_sample_type", sample_type);
checkMD(sb, nvr, "cedars_experiment_date", experiment_date);
checkMD(sb, nvr, "cedars_experiment_number", experiment_number);
checkMD(sb, nvr, "cedars_scan_number", Integer.toString(i));
if (pass > 2) {
checkMD(sb, nvr, "cedars_algorithm_id", prev_algorithm_id);
checkMD(sb, nvr, "cedars_content_type", prev_content_type);
}
if (sb.length() > 0) {
bad_md_errors++;
flog("#e md error " + oids[i] + "\n");
flog(sb.toString());
sb.setLength(0);
}
}
if (s2 != null) {
// store next smaller file and save oid
nvr = createRecord();
nvr.put("cedars_subject_id", subject_id);
nvr.put("cedars_study_id", study_id);
nvr.put("cedars_species_id", species_id);
nvr.put("cedars_sample_id", sample_id);
nvr.put("cedars_collection_date", collection_date);
nvr.put("cedars_processing_date", processing_date);
nvr.put("cedars_collected_by", collected_by);
nvr.put("cedars_processed_by", processed_by);
nvr.put("cedars_sample_type", sample_type);
nvr.put("cedars_experiment_date", experiment_date);
nvr.put("cedars_experiment_number", experiment_number);
nvr.put("cedars_pass", Long.toString(pass));
nvr.put("cedars_scan_number", Long.toString(i));
nvr.put("cedars_algorithm_id", algorithm_id);
nvr.put("cedars_content_type", content_type);
names = store(names, s2[i], i, nvr);
oids[i] = names[1];
dot("s");
}
}
prev_algorithm_id = algorithm_id;
prev_content_type = content_type;
if (names != null && names[0] != null) deleteFile(names[0]);
}
private void doIt() throws ArchiveException, IOException {
// create id for the run/sample series
try {
subject_id = new String(getRandomChars(32));
study_id = new String(getRandomChars(32));
species_id = species[randIndex(species.length)];
sample_id = new String(getRandomChars(32));
collection_date = new Date(System.currentTimeMillis() - 336000).toString();
processing_date = new Date().toString();
collected_by = collectors[randIndex(collectors.length)];
processed_by = processors[randIndex(processors.length)];
sample_type = types[randIndex(types.length)];
experiment_date = new Date(System.currentTimeMillis() - 636000).toString();
experiment_number = Integer.toString(randIndex(9999999));
} catch (Exception e) {
e.printStackTrace();
System.exit(1);
}
metadata_s =
"#M\tN\t"
+ n_files
+ "\tcedars_subject_id\t"
+ subject_id
+ "\tcedars_study_id\t"
+ study_id
+ "\tcedars_species_id\t"
+ species_id
+ "\tcedars_sample_id\t"
+ sample_id
+ "\tcedars_collection_date\t"
+ collection_date
+ "\tcedars_processing_date\t"
+ processing_date
+ "\tcedars_collected_by\t"
+ collected_by
+ "\tcedars_processed_by\t"
+ processed_by
+ "\tcedars_sample_type\t"
+ sample_type
+ "\tcedars_experiment_date\t"
+ experiment_date
+ "\tcedars_experiment_number\t"
+ experiment_number;
flog(metadata_s + "\n");
// pass 1: create/store n_files of ~30 MB each
System.err.println("Storing 30M (N=" + n_files + "):");
String[] names = null;
for (int i = 0; i < sizes.length; i++) {
// store file
NameValueRecord nvr = createRecord();
nvr.put("cedars_subject_id", subject_id);
nvr.put("cedars_study_id", study_id);
nvr.put("cedars_species_id", species_id);
nvr.put("cedars_sample_id", sample_id);
nvr.put("cedars_collection_date", collection_date);
nvr.put("cedars_processing_date", processing_date);
nvr.put("cedars_collected_by", collected_by);
nvr.put("cedars_processed_by", processed_by);
nvr.put("cedars_sample_type", sample_type);
nvr.put("cedars_experiment_date", experiment_date);
nvr.put("cedars_experiment_number", experiment_number);
nvr.put("cedars_pass", Long.toString(1));
nvr.put("cedars_scan_number", Long.toString(i));
names = store(names, sizes[i], i, nvr);
dot("s");
}
deleteFile(names[0]);
closeDots();
bytes_30M = store_bytes;
time_store_30M = time_store;
// pass 2: read each ~30 MB file & write a 3 MB file
System.err.println("Retrieving 30M / storing 3M (N=" + n_files + "):");
readwrite(sizes, sizes2, 2);
closeDots();
time_retrv_30M = time_retrieve;
bytes_3M = store_bytes;
time_store_3M = time_store;
// pass 3: read each ~3 MB file & write a 3K file
System.err.println("Retrieving 3M / storing 300K (N=" + n_files + "):");
readwrite(sizes2, sizes3, 3);
closeDots();
time_retrv_3M = time_retrieve;
bytes_300K = store_bytes;
time_store_300K = time_store;
// pass 4: verify each 3K file
System.err.println("Retrieving 300K (N=" + n_files + "):");
readwrite(sizes3, null, 4);
closeDots();
time_retrv_300K = time_retrieve;
if (!update) return;
// pass 5: update all files with blood type
String blood_type = blood_types[randIndex(blood_types.length)];
System.err.println(
"Adding cedars_blood_type=" + blood_type + " to all files (3xN=" + (n_files * 3) + "):");
flog("#m\tcedars_blood_type\t" + blood_type + "\n");
if (out_file != null) {
out_file += ".final";
flog("# updated oid file is " + out_file + "\n");
System.err.println(" ( updated oid file is " + out_file + ")");
try {
String host = clnthost;
fo = new FileWriter(out_file, true); // append=true
flog("#S Cedars/final [" + host + "] " + new Date() + "\n");
flog(metadata_s + "\tcedars_blood_type\t" + blood_type + "\n");
} catch (Exception e) {
System.err.println("Opening " + out_file);
e.printStackTrace();
System.exit(1);
}
}
updating = true;
ObjectIdentifier o = null;
QueryResultSet qr = null;
while (true) {
boolean error = false;
try {
qr = query("\"cedars_sample_id\" = \"" + sample_id + "\"", times, 0);
} catch (Throwable t) {
flex("query getting oid's", t);
String s = "#e getting oid's: " + t.getMessage();
log.log(s);
// flog(s + "\n");
// e.printStackTrace();
error = true;
}
if (error) {
query_retries++;
sleep(SLEEP);
continue;
}
break;
}
oid_query_time = times[0];
NameValueRecord nvr = createRecord();
nvr.put("cedars_blood_type", blood_type);
// XXX PORT Bill, please review
try {
while (qr.next()) {
o = qr.getObjectIdentifier();
Object[] oo = null;
while (true) {
boolean error = false;
try {
oo = addMetadata(o, nvr, times, 0);
} catch (Throwable t) {
flex("adding md to " + o.toString(), t);
String s = "#e adding md to " + o.toString() + ": " + t.getMessage();
log.log(s);
error = true;
}
if (error) {
update_retries++;
sleep(SLEEP);
continue;
}
break;
}
update_time += times[0];
updates++;
// write to new log - trust mdata for now
SystemRecord sr = (SystemRecord) oo[0];
NameValueRecord new_nvr = (NameValueRecord) oo[1];
String oid = sr.getObjectIdentifier().toString();
String s =
new_nvr.getString("filesize")
+ '\t'
+ new_nvr.getString("filename")
+ '\t'
+ oid
+ '\t'
+ new_nvr.getString("sha1")
+ '\t'
+ new_nvr.getString("storedate")
+ '\t'
+ binary
+ '\t'
+ metadata
+ '\t'
+ 0
+ '\n';
flog(s);
dot("u");
}
} catch (Throwable t) {
// XXX PORT Bill review
flex("getting next result ", t);
String s = "#e getting next result: " + t.getMessage();
log.log(s);
/*
error = true;
if (error) {
update_retries++;
sleep(SLEEP);
continue;
}
*/
}
closeDots();
}