Exemple #1
0
  private void readwrite(long s1[], long s2[], int pass) {
    // oids[] should correspond to s1[] going in, s2[] going out
    String[] names = null;
    time_retrieve = 0;
    store_bytes = 0;
    time_store = 0;

    String algorithm_id = algorithms[randIndex(algorithms.length)];
    String content_type = content_types[randIndex(content_types.length)];
    StringBuffer sb = new StringBuffer();
    for (int i = 0; i < s1.length; i++) {
      String retrieved = null;
      boolean error1 = false;
      long t1 = System.currentTimeMillis();
      while (true) {
        boolean error = false;
        try {
          retrieved = retrieve(oids[i], times, 0);
          time_retrieve += times[0];
          total_retrieve_time += times[0];
        } catch (HoneycombTestException e) {
          flex("retrv " + oids[i], e);
          String s = "#e retrv " + oids[i] + ": " + e.getMessage();
          log.log(s);
          // flog(s + "\n");
          // e.printStackTrace();
          error = true;
        } catch (Throwable t) {
          flex("retrv " + oids[i], t);
          log.log("retrieve got throwable");
          t.printStackTrace();
          error = true;
        }
        if (error) {
          error1 = true;
          retrieve_retries++;
          if (System.currentTimeMillis() - t1 > GIVEUP) {
            retrieve_failures++;
            retrieved = null;
            break;
          }
          sleep(SLEEP);
          continue;
        }
        break;
      }
      if (retrieved == null) {
        failed_rtrv.add(oids[i]);
      } else {
        if (error1) flog("#E ok\n");
        dot("r");
        // compare retrieved w/ original
        File f = new File(retrieved);
        if (f.length() != s1[i]) {
          bad_file_errors++;
          String s =
              "#e size mismatch ["
                  + oids[i]
                  + "] store/retrieve: "
                  + s1[i]
                  + " / "
                  + f.length()
                  + "\n";
          flog(s);
        } else {
          String sha1sum = null;
          try {
            sha1sum = shell.sha1sum(retrieved);
          } catch (Exception e) {
            e.printStackTrace();
          }
          if (sha1sum != null && !sha1sum.equals(shas[i])) {
            bad_file_errors++;
            String s =
                "#e sha1 mismatch ["
                    + oids[i]
                    + "] store/retrieve: "
                    + shas[i]
                    + " / "
                    + sha1sum
                    + "\n";
            flog(s);
          }
        }
        deleteFile(retrieved);
      }
      // compare mdata
      t1 = System.currentTimeMillis();
      NameValueRecord nvr = null;
      error1 = false;
      while (true) {
        boolean error = false;
        try {
          nvr = retrieveMetadata(oids[i], times, 0);
          md_retrieve_time += times[0];
          md_retrieves++;
        } catch (HoneycombTestException e) {
          flex("md retrv " + oids[i], e);
          String s = "#e md retrv " + oids[i] + ": " + e.getMessage();
          log.log(s);
          // flog(s + "\n");
          // e.printStackTrace();
          error = true;
        } catch (Throwable t) {
          flex("md retrv " + oids[i], t);
          log.log("md retrieve got throwable");
          t.printStackTrace();
          error = true;
        }
        if (error) {
          error1 = true;
          md_retrieve_retries++;
          if (System.currentTimeMillis() - t1 > GIVEUP) {
            md_retrieve_failures++;
            nvr = null;
            flog("#E " + oids[i] + ": giving up\n");
            break;
          }
          sleep(SLEEP);
          continue;
        }
        break;
      }
      if (nvr == null) {
        failed_md_rtrv.add(oids[i]);
      } else {
        if (error1) flog("#E ok\n");
        SystemRecord sr = nvr.getSystemRecord();
        String stored_oid = sr.getObjectIdentifier().toString();
        if (!oids[i].equals(stored_oid)) {
          // one oid looked up another
          sb.append("#e  oid mismatch! ").append(oids[i]);
          sb.append(" / ").append(stored_oid).append("\n");
        }
        String sha2 = nvr.getString("sha1");
        if (sha2 == null) {
          sb.append("#e  no sha1 in metadata\n");
        } else if (!sha2.equals(shas[i])) {
          sb.append("#e  original/md sha:  ").append(shas[i]);
          sb.append(" / ").append(sha2).append("\n");
        }

        checkMD(sb, nvr, "cedars_subject_id", subject_id);
        checkMD(sb, nvr, "cedars_study_id", study_id);
        checkMD(sb, nvr, "cedars_species_id", species_id);
        checkMD(sb, nvr, "cedars_sample_id", sample_id);
        checkMD(sb, nvr, "cedars_collection_date", collection_date);
        checkMD(sb, nvr, "cedars_processing_date", processing_date);
        checkMD(sb, nvr, "cedars_collected_by", collected_by);
        checkMD(sb, nvr, "cedars_processed_by", processed_by);
        checkMD(sb, nvr, "cedars_sample_type", sample_type);
        checkMD(sb, nvr, "cedars_experiment_date", experiment_date);
        checkMD(sb, nvr, "cedars_experiment_number", experiment_number);

        checkMD(sb, nvr, "cedars_scan_number", Integer.toString(i));

        if (pass > 2) {
          checkMD(sb, nvr, "cedars_algorithm_id", prev_algorithm_id);
          checkMD(sb, nvr, "cedars_content_type", prev_content_type);
        }

        if (sb.length() > 0) {
          bad_md_errors++;
          flog("#e md error " + oids[i] + "\n");
          flog(sb.toString());
          sb.setLength(0);
        }
      }

      if (s2 != null) {
        // store next smaller file and save oid
        nvr = createRecord();
        nvr.put("cedars_subject_id", subject_id);
        nvr.put("cedars_study_id", study_id);
        nvr.put("cedars_species_id", species_id);

        nvr.put("cedars_sample_id", sample_id);
        nvr.put("cedars_collection_date", collection_date);
        nvr.put("cedars_processing_date", processing_date);
        nvr.put("cedars_collected_by", collected_by);
        nvr.put("cedars_processed_by", processed_by);
        nvr.put("cedars_sample_type", sample_type);

        nvr.put("cedars_experiment_date", experiment_date);
        nvr.put("cedars_experiment_number", experiment_number);

        nvr.put("cedars_pass", Long.toString(pass));
        nvr.put("cedars_scan_number", Long.toString(i));

        nvr.put("cedars_algorithm_id", algorithm_id);
        nvr.put("cedars_content_type", content_type);

        names = store(names, s2[i], i, nvr);
        oids[i] = names[1];
        dot("s");
      }
    }
    prev_algorithm_id = algorithm_id;
    prev_content_type = content_type;

    if (names != null && names[0] != null) deleteFile(names[0]);
  }
Exemple #2
0
  private void doIt() throws ArchiveException, IOException {

    // create id for the run/sample series
    try {
      subject_id = new String(getRandomChars(32));
      study_id = new String(getRandomChars(32));
      species_id = species[randIndex(species.length)];
      sample_id = new String(getRandomChars(32));
      collection_date = new Date(System.currentTimeMillis() - 336000).toString();
      processing_date = new Date().toString();
      collected_by = collectors[randIndex(collectors.length)];
      processed_by = processors[randIndex(processors.length)];
      sample_type = types[randIndex(types.length)];
      experiment_date = new Date(System.currentTimeMillis() - 636000).toString();
      experiment_number = Integer.toString(randIndex(9999999));
    } catch (Exception e) {
      e.printStackTrace();
      System.exit(1);
    }

    metadata_s =
        "#M\tN\t"
            + n_files
            + "\tcedars_subject_id\t"
            + subject_id
            + "\tcedars_study_id\t"
            + study_id
            + "\tcedars_species_id\t"
            + species_id
            + "\tcedars_sample_id\t"
            + sample_id
            + "\tcedars_collection_date\t"
            + collection_date
            + "\tcedars_processing_date\t"
            + processing_date
            + "\tcedars_collected_by\t"
            + collected_by
            + "\tcedars_processed_by\t"
            + processed_by
            + "\tcedars_sample_type\t"
            + sample_type
            + "\tcedars_experiment_date\t"
            + experiment_date
            + "\tcedars_experiment_number\t"
            + experiment_number;
    flog(metadata_s + "\n");

    // pass 1: create/store n_files of ~30 MB each
    System.err.println("Storing 30M (N=" + n_files + "):");
    String[] names = null;
    for (int i = 0; i < sizes.length; i++) {
      // store file
      NameValueRecord nvr = createRecord();

      nvr.put("cedars_subject_id", subject_id);
      nvr.put("cedars_study_id", study_id);
      nvr.put("cedars_species_id", species_id);

      nvr.put("cedars_sample_id", sample_id);
      nvr.put("cedars_collection_date", collection_date);
      nvr.put("cedars_processing_date", processing_date);
      nvr.put("cedars_collected_by", collected_by);
      nvr.put("cedars_processed_by", processed_by);
      nvr.put("cedars_sample_type", sample_type);

      nvr.put("cedars_experiment_date", experiment_date);
      nvr.put("cedars_experiment_number", experiment_number);

      nvr.put("cedars_pass", Long.toString(1));
      nvr.put("cedars_scan_number", Long.toString(i));

      names = store(names, sizes[i], i, nvr);
      dot("s");
    }
    deleteFile(names[0]);
    closeDots();
    bytes_30M = store_bytes;
    time_store_30M = time_store;

    // pass 2: read each ~30 MB file & write a 3 MB file
    System.err.println("Retrieving 30M / storing 3M (N=" + n_files + "):");
    readwrite(sizes, sizes2, 2);
    closeDots();
    time_retrv_30M = time_retrieve;
    bytes_3M = store_bytes;
    time_store_3M = time_store;

    // pass 3: read each ~3 MB file & write a 3K file
    System.err.println("Retrieving 3M / storing 300K (N=" + n_files + "):");
    readwrite(sizes2, sizes3, 3);
    closeDots();
    time_retrv_3M = time_retrieve;
    bytes_300K = store_bytes;
    time_store_300K = time_store;

    // pass 4: verify each 3K file
    System.err.println("Retrieving 300K (N=" + n_files + "):");
    readwrite(sizes3, null, 4);
    closeDots();
    time_retrv_300K = time_retrieve;

    if (!update) return;

    // pass 5: update all files with blood type
    String blood_type = blood_types[randIndex(blood_types.length)];
    System.err.println(
        "Adding cedars_blood_type=" + blood_type + " to all files (3xN=" + (n_files * 3) + "):");
    flog("#m\tcedars_blood_type\t" + blood_type + "\n");
    if (out_file != null) {
      out_file += ".final";
      flog("# updated oid file is " + out_file + "\n");
      System.err.println("    ( updated oid file is " + out_file + ")");
      try {
        String host = clnthost;
        fo = new FileWriter(out_file, true); // append=true
        flog("#S Cedars/final [" + host + "] " + new Date() + "\n");
        flog(metadata_s + "\tcedars_blood_type\t" + blood_type + "\n");
      } catch (Exception e) {
        System.err.println("Opening " + out_file);
        e.printStackTrace();
        System.exit(1);
      }
    }

    updating = true;
    ObjectIdentifier o = null;
    QueryResultSet qr = null;
    while (true) {
      boolean error = false;
      try {
        qr = query("\"cedars_sample_id\" = \"" + sample_id + "\"", times, 0);
      } catch (Throwable t) {
        flex("query getting oid's", t);
        String s = "#e getting oid's: " + t.getMessage();
        log.log(s);
        // flog(s + "\n");
        // e.printStackTrace();
        error = true;
      }
      if (error) {
        query_retries++;
        sleep(SLEEP);
        continue;
      }
      break;
    }
    oid_query_time = times[0];
    NameValueRecord nvr = createRecord();
    nvr.put("cedars_blood_type", blood_type);

    // XXX PORT Bill, please review
    try {
      while (qr.next()) {
        o = qr.getObjectIdentifier();
        Object[] oo = null;
        while (true) {
          boolean error = false;
          try {
            oo = addMetadata(o, nvr, times, 0);
          } catch (Throwable t) {
            flex("adding md to " + o.toString(), t);
            String s = "#e adding md to " + o.toString() + ": " + t.getMessage();
            log.log(s);
            error = true;
          }
          if (error) {
            update_retries++;
            sleep(SLEEP);
            continue;
          }
          break;
        }
        update_time += times[0];
        updates++;

        // write to new log - trust mdata for now
        SystemRecord sr = (SystemRecord) oo[0];
        NameValueRecord new_nvr = (NameValueRecord) oo[1];
        String oid = sr.getObjectIdentifier().toString();
        String s =
            new_nvr.getString("filesize")
                + '\t'
                + new_nvr.getString("filename")
                + '\t'
                + oid
                + '\t'
                + new_nvr.getString("sha1")
                + '\t'
                + new_nvr.getString("storedate")
                + '\t'
                + binary
                + '\t'
                + metadata
                + '\t'
                + 0
                + '\n';
        flog(s);
        dot("u");
      }
    } catch (Throwable t) {
      // XXX PORT Bill review
      flex("getting next result ", t);
      String s = "#e getting next result: " + t.getMessage();
      log.log(s);
      /*
          error = true;
      if (error) {
          update_retries++;
          sleep(SLEEP);
          continue;
      }
      */
    }
    closeDots();
  }