Example #1
0
  private String[] store(String names[], long size, int i, NameValueRecord nvr) {
    // store file
    boolean error1 = false;
    while (true) {
      boolean error = false;
      try {
        names = createFileAndStore(names, size, binary, metadata, nvr, true, times, 0);
        // nvr = retrieveMetadata(names[1], times, 0);
      } catch (HoneycombTestException e) {
        flex("store size=" + size + " name=" + (names == null ? "null" : names[0]), e);
        log.log("store problem: " + e.getMessage());
        // e.printStackTrace();
        error = true;
      } catch (Throwable t) {
        flex("store size=" + size + " name=" + (names == null ? "null" : names[0]), t);
        log.log("store: got throwable");
        t.printStackTrace();
        error = true;
      }
      if (error) {
        error1 = true;
        store_retries++;
        sleep(SLEEP);
        continue;
      }
      break;
    }
    if (error1) flog("#E ok\n");

    // save info
    total_files++;
    total_bytes += size;
    store_bytes += size;
    time_store += times[0];
    total_store_time += times[0];

    oids[i] = names[1];
    shas[i] = names[2];
    if (fo != null) {
      String s =
          "" + size + '\t' + names[0] + '\t' + // filename
              names[1] + '\t' + // oid
              names[2] + '\t' + // sha1
              names[3] + '\t' + // store date
              binary + '\t' + metadata + '\t' + times[0] + '\n';
      flog(s);
    }
    return names;
  }
Example #2
0
  private void readwrite(long s1[], long s2[], int pass) {
    // oids[] should correspond to s1[] going in, s2[] going out
    String[] names = null;
    time_retrieve = 0;
    store_bytes = 0;
    time_store = 0;

    String algorithm_id = algorithms[randIndex(algorithms.length)];
    String content_type = content_types[randIndex(content_types.length)];
    StringBuffer sb = new StringBuffer();
    for (int i = 0; i < s1.length; i++) {
      String retrieved = null;
      boolean error1 = false;
      long t1 = System.currentTimeMillis();
      while (true) {
        boolean error = false;
        try {
          retrieved = retrieve(oids[i], times, 0);
          time_retrieve += times[0];
          total_retrieve_time += times[0];
        } catch (HoneycombTestException e) {
          flex("retrv " + oids[i], e);
          String s = "#e retrv " + oids[i] + ": " + e.getMessage();
          log.log(s);
          // flog(s + "\n");
          // e.printStackTrace();
          error = true;
        } catch (Throwable t) {
          flex("retrv " + oids[i], t);
          log.log("retrieve got throwable");
          t.printStackTrace();
          error = true;
        }
        if (error) {
          error1 = true;
          retrieve_retries++;
          if (System.currentTimeMillis() - t1 > GIVEUP) {
            retrieve_failures++;
            retrieved = null;
            break;
          }
          sleep(SLEEP);
          continue;
        }
        break;
      }
      if (retrieved == null) {
        failed_rtrv.add(oids[i]);
      } else {
        if (error1) flog("#E ok\n");
        dot("r");
        // compare retrieved w/ original
        File f = new File(retrieved);
        if (f.length() != s1[i]) {
          bad_file_errors++;
          String s =
              "#e size mismatch ["
                  + oids[i]
                  + "] store/retrieve: "
                  + s1[i]
                  + " / "
                  + f.length()
                  + "\n";
          flog(s);
        } else {
          String sha1sum = null;
          try {
            sha1sum = shell.sha1sum(retrieved);
          } catch (Exception e) {
            e.printStackTrace();
          }
          if (sha1sum != null && !sha1sum.equals(shas[i])) {
            bad_file_errors++;
            String s =
                "#e sha1 mismatch ["
                    + oids[i]
                    + "] store/retrieve: "
                    + shas[i]
                    + " / "
                    + sha1sum
                    + "\n";
            flog(s);
          }
        }
        deleteFile(retrieved);
      }
      // compare mdata
      t1 = System.currentTimeMillis();
      NameValueRecord nvr = null;
      error1 = false;
      while (true) {
        boolean error = false;
        try {
          nvr = retrieveMetadata(oids[i], times, 0);
          md_retrieve_time += times[0];
          md_retrieves++;
        } catch (HoneycombTestException e) {
          flex("md retrv " + oids[i], e);
          String s = "#e md retrv " + oids[i] + ": " + e.getMessage();
          log.log(s);
          // flog(s + "\n");
          // e.printStackTrace();
          error = true;
        } catch (Throwable t) {
          flex("md retrv " + oids[i], t);
          log.log("md retrieve got throwable");
          t.printStackTrace();
          error = true;
        }
        if (error) {
          error1 = true;
          md_retrieve_retries++;
          if (System.currentTimeMillis() - t1 > GIVEUP) {
            md_retrieve_failures++;
            nvr = null;
            flog("#E " + oids[i] + ": giving up\n");
            break;
          }
          sleep(SLEEP);
          continue;
        }
        break;
      }
      if (nvr == null) {
        failed_md_rtrv.add(oids[i]);
      } else {
        if (error1) flog("#E ok\n");
        SystemRecord sr = nvr.getSystemRecord();
        String stored_oid = sr.getObjectIdentifier().toString();
        if (!oids[i].equals(stored_oid)) {
          // one oid looked up another
          sb.append("#e  oid mismatch! ").append(oids[i]);
          sb.append(" / ").append(stored_oid).append("\n");
        }
        String sha2 = nvr.getString("sha1");
        if (sha2 == null) {
          sb.append("#e  no sha1 in metadata\n");
        } else if (!sha2.equals(shas[i])) {
          sb.append("#e  original/md sha:  ").append(shas[i]);
          sb.append(" / ").append(sha2).append("\n");
        }

        checkMD(sb, nvr, "cedars_subject_id", subject_id);
        checkMD(sb, nvr, "cedars_study_id", study_id);
        checkMD(sb, nvr, "cedars_species_id", species_id);
        checkMD(sb, nvr, "cedars_sample_id", sample_id);
        checkMD(sb, nvr, "cedars_collection_date", collection_date);
        checkMD(sb, nvr, "cedars_processing_date", processing_date);
        checkMD(sb, nvr, "cedars_collected_by", collected_by);
        checkMD(sb, nvr, "cedars_processed_by", processed_by);
        checkMD(sb, nvr, "cedars_sample_type", sample_type);
        checkMD(sb, nvr, "cedars_experiment_date", experiment_date);
        checkMD(sb, nvr, "cedars_experiment_number", experiment_number);

        checkMD(sb, nvr, "cedars_scan_number", Integer.toString(i));

        if (pass > 2) {
          checkMD(sb, nvr, "cedars_algorithm_id", prev_algorithm_id);
          checkMD(sb, nvr, "cedars_content_type", prev_content_type);
        }

        if (sb.length() > 0) {
          bad_md_errors++;
          flog("#e md error " + oids[i] + "\n");
          flog(sb.toString());
          sb.setLength(0);
        }
      }

      if (s2 != null) {
        // store next smaller file and save oid
        nvr = createRecord();
        nvr.put("cedars_subject_id", subject_id);
        nvr.put("cedars_study_id", study_id);
        nvr.put("cedars_species_id", species_id);

        nvr.put("cedars_sample_id", sample_id);
        nvr.put("cedars_collection_date", collection_date);
        nvr.put("cedars_processing_date", processing_date);
        nvr.put("cedars_collected_by", collected_by);
        nvr.put("cedars_processed_by", processed_by);
        nvr.put("cedars_sample_type", sample_type);

        nvr.put("cedars_experiment_date", experiment_date);
        nvr.put("cedars_experiment_number", experiment_number);

        nvr.put("cedars_pass", Long.toString(pass));
        nvr.put("cedars_scan_number", Long.toString(i));

        nvr.put("cedars_algorithm_id", algorithm_id);
        nvr.put("cedars_content_type", content_type);

        names = store(names, s2[i], i, nvr);
        oids[i] = names[1];
        dot("s");
      }
    }
    prev_algorithm_id = algorithm_id;
    prev_content_type = content_type;

    if (names != null && names[0] != null) deleteFile(names[0]);
  }