private String[] store(String names[], long size, int i, NameValueRecord nvr) { // store file boolean error1 = false; while (true) { boolean error = false; try { names = createFileAndStore(names, size, binary, metadata, nvr, true, times, 0); // nvr = retrieveMetadata(names[1], times, 0); } catch (HoneycombTestException e) { flex("store size=" + size + " name=" + (names == null ? "null" : names[0]), e); log.log("store problem: " + e.getMessage()); // e.printStackTrace(); error = true; } catch (Throwable t) { flex("store size=" + size + " name=" + (names == null ? "null" : names[0]), t); log.log("store: got throwable"); t.printStackTrace(); error = true; } if (error) { error1 = true; store_retries++; sleep(SLEEP); continue; } break; } if (error1) flog("#E ok\n"); // save info total_files++; total_bytes += size; store_bytes += size; time_store += times[0]; total_store_time += times[0]; oids[i] = names[1]; shas[i] = names[2]; if (fo != null) { String s = "" + size + '\t' + names[0] + '\t' + // filename names[1] + '\t' + // oid names[2] + '\t' + // sha1 names[3] + '\t' + // store date binary + '\t' + metadata + '\t' + times[0] + '\n'; flog(s); } return names; }
private void readwrite(long s1[], long s2[], int pass) { // oids[] should correspond to s1[] going in, s2[] going out String[] names = null; time_retrieve = 0; store_bytes = 0; time_store = 0; String algorithm_id = algorithms[randIndex(algorithms.length)]; String content_type = content_types[randIndex(content_types.length)]; StringBuffer sb = new StringBuffer(); for (int i = 0; i < s1.length; i++) { String retrieved = null; boolean error1 = false; long t1 = System.currentTimeMillis(); while (true) { boolean error = false; try { retrieved = retrieve(oids[i], times, 0); time_retrieve += times[0]; total_retrieve_time += times[0]; } catch (HoneycombTestException e) { flex("retrv " + oids[i], e); String s = "#e retrv " + oids[i] + ": " + e.getMessage(); log.log(s); // flog(s + "\n"); // e.printStackTrace(); error = true; } catch (Throwable t) { flex("retrv " + oids[i], t); log.log("retrieve got throwable"); t.printStackTrace(); error = true; } if (error) { error1 = true; retrieve_retries++; if (System.currentTimeMillis() - t1 > GIVEUP) { retrieve_failures++; retrieved = null; break; } sleep(SLEEP); continue; } break; } if (retrieved == null) { failed_rtrv.add(oids[i]); } else { if (error1) flog("#E ok\n"); dot("r"); // compare retrieved w/ original File f = new File(retrieved); if (f.length() != s1[i]) { bad_file_errors++; String s = "#e size mismatch [" + oids[i] + "] store/retrieve: " + s1[i] + " / " + f.length() + "\n"; flog(s); } else { String sha1sum = null; try { sha1sum = shell.sha1sum(retrieved); } catch (Exception e) { e.printStackTrace(); } if (sha1sum != null && !sha1sum.equals(shas[i])) { bad_file_errors++; String s = "#e sha1 mismatch [" + oids[i] + "] store/retrieve: " + shas[i] + " / " + sha1sum + "\n"; flog(s); } } deleteFile(retrieved); } // compare mdata t1 = System.currentTimeMillis(); NameValueRecord nvr = null; error1 = false; while (true) { boolean error = false; try { nvr = retrieveMetadata(oids[i], times, 0); md_retrieve_time += times[0]; md_retrieves++; } catch (HoneycombTestException e) { flex("md retrv " + oids[i], e); String s = "#e md retrv " + oids[i] + ": " + e.getMessage(); log.log(s); // flog(s + "\n"); // e.printStackTrace(); error = true; } catch (Throwable t) { flex("md retrv " + oids[i], t); log.log("md retrieve got throwable"); t.printStackTrace(); error = true; } if (error) { error1 = true; md_retrieve_retries++; if (System.currentTimeMillis() - t1 > GIVEUP) { md_retrieve_failures++; nvr = null; flog("#E " + oids[i] + ": giving up\n"); break; } sleep(SLEEP); continue; } break; } if (nvr == null) { failed_md_rtrv.add(oids[i]); } else { if (error1) flog("#E ok\n"); SystemRecord sr = nvr.getSystemRecord(); String stored_oid = sr.getObjectIdentifier().toString(); if (!oids[i].equals(stored_oid)) { // one oid looked up another sb.append("#e oid mismatch! ").append(oids[i]); sb.append(" / ").append(stored_oid).append("\n"); } String sha2 = nvr.getString("sha1"); if (sha2 == null) { sb.append("#e no sha1 in metadata\n"); } else if (!sha2.equals(shas[i])) { sb.append("#e original/md sha: ").append(shas[i]); sb.append(" / ").append(sha2).append("\n"); } checkMD(sb, nvr, "cedars_subject_id", subject_id); checkMD(sb, nvr, "cedars_study_id", study_id); checkMD(sb, nvr, "cedars_species_id", species_id); checkMD(sb, nvr, "cedars_sample_id", sample_id); checkMD(sb, nvr, "cedars_collection_date", collection_date); checkMD(sb, nvr, "cedars_processing_date", processing_date); checkMD(sb, nvr, "cedars_collected_by", collected_by); checkMD(sb, nvr, "cedars_processed_by", processed_by); checkMD(sb, nvr, "cedars_sample_type", sample_type); checkMD(sb, nvr, "cedars_experiment_date", experiment_date); checkMD(sb, nvr, "cedars_experiment_number", experiment_number); checkMD(sb, nvr, "cedars_scan_number", Integer.toString(i)); if (pass > 2) { checkMD(sb, nvr, "cedars_algorithm_id", prev_algorithm_id); checkMD(sb, nvr, "cedars_content_type", prev_content_type); } if (sb.length() > 0) { bad_md_errors++; flog("#e md error " + oids[i] + "\n"); flog(sb.toString()); sb.setLength(0); } } if (s2 != null) { // store next smaller file and save oid nvr = createRecord(); nvr.put("cedars_subject_id", subject_id); nvr.put("cedars_study_id", study_id); nvr.put("cedars_species_id", species_id); nvr.put("cedars_sample_id", sample_id); nvr.put("cedars_collection_date", collection_date); nvr.put("cedars_processing_date", processing_date); nvr.put("cedars_collected_by", collected_by); nvr.put("cedars_processed_by", processed_by); nvr.put("cedars_sample_type", sample_type); nvr.put("cedars_experiment_date", experiment_date); nvr.put("cedars_experiment_number", experiment_number); nvr.put("cedars_pass", Long.toString(pass)); nvr.put("cedars_scan_number", Long.toString(i)); nvr.put("cedars_algorithm_id", algorithm_id); nvr.put("cedars_content_type", content_type); names = store(names, s2[i], i, nvr); oids[i] = names[1]; dot("s"); } } prev_algorithm_id = algorithm_id; prev_content_type = content_type; if (names != null && names[0] != null) deleteFile(names[0]); }