Esempio n. 1
0
  @Override
  public void interpretArguments(
      final JSAP jsap, final JSAPResult result, final DAVOptions options) {
    timeService = new TimeLoggingService("execute-splits");
    timeService.start();

    super.interpretArguments(jsap, result, options);
    optionalModelIds = parseOptionalModelIdProperties(configurationProperties);

    evaluateStatistics = result.getBoolean("evaluate-statistics");
    if (!evaluateStatistics) {
      System.out.println(
          "Will not evaluate statistics as run proceed. Run restat on results directory to obtain statistics.");
    } else {
      System.out.println("Will evaluate statistics as run proceed.");
    }
    try {
      final String filename = result.getString("splits");
      splitPlanFilename = filename;
      splitPlan.load(filename);
    } catch (Exception e) {
      LOGGER.error("An error occurred reading splits file. " + result.getString("splits"), e);
    }
    // collect keys for parameters that are not required (have default values)
    paramKeysNotRequired = new ObjectArrayList<String>();
    if (!result.userSpecified("seed")) {
      paramKeysNotRequired.add("--seed");
      paramKeysNotRequired.add(Integer.toString(options.randomSeed));
    }

    modelId = ShortHash.shortHash(getOriginalArgs());
    options.modelId = modelId;
    timeService.setModelId(options.modelId);

    final Map<String, String> additionalConditionsMap = new HashMap<String, String>();
    additionalConditionsMap.put("model-id", modelId);

    for (final OptionalModelId optionalModelId : optionalModelIds) {
      final String[] originalArgs1 = expandShortArgs(getOriginalArgs(), jsap);
      final String[] filteredArgs = filterArgs(originalArgs1, optionalModelId);
      final String optionalModelIdValue = ShortHash.shortHash(filteredArgs);

      additionalConditionsMap.put(optionalModelId.columnIdentifier, optionalModelIdValue);
    }

    final String modelConditionsFilename = "model-conditions.txt";
    final Set<String> skipJsapConditions = new HashSet<String>();
    skipJsapConditions.add("model-id");
    skipJsapConditions.add("mode");
    try {
      writeConditions(
          modelConditionsFilename, jsap, result, additionalConditionsMap, skipJsapConditions);
    } catch (IOException e) {
      LOGGER.error("Error writing " + modelConditionsFilename + " file", e);
    }
  }
Esempio n. 2
0
  /**
   * Parse properties to extract optional model id definitions. The format is as follow:
   *
   * <p>
   *
   * <PRE>
   * define.model-id.column-id=modelid-noScaler
   * define.model-id.modelid-noScaler.exclude=a,b
   * define.model-id.modelid-noScaler.exclude.a.argument=scaler-name
   * define.model-id.modelid-noScaler.exclude.a.skip=1
   * define.model-id.modelid-noScaler.exclude.b.argument=normalizer-name
   * define.model-id.modelid-noScaler.exclude.b.skip=1
   * </PRE>
   *
   * These properties would define one new model-id called, to be written in a column called
   * modelid-noScaler, which excludes two arguments and one value each from the hashcode modelId
   * calculation.
   */
  public static OptionalModelId[] parseOptionalModelIdProperties(
      final Properties configurationProperties) {
    final ObjectList<OptionalModelId> result = new ObjectArrayList<OptionalModelId>();
    if (configurationProperties != null) {
      // inspect properties to figure out which optional model ids to create:
      final ObjectSet<String> optionalModelIdColumnNames = new ObjectArraySet<String>();

      for (final String propertyName : configurationProperties.stringPropertyNames()) {
        if (propertyName.startsWith("define.model-id.column-id")) {
          final String columnIdNames = configurationProperties.getProperty(propertyName);
          final String[] names = columnIdNames.split(",");
          for (final String name : names) {
            optionalModelIdColumnNames.add(name);
          }
        }
      }

      for (final String optionalColumnId : optionalModelIdColumnNames) {
        final String defineModelIdExcludePropertyName =
            "define.model-id." + optionalColumnId + ".exclude";
        final String argumentKeys =
            configurationProperties.getProperty(defineModelIdExcludePropertyName);
        final String[] keys;
        if (argumentKeys == null) {
          System.err.println(
              "Error parsing properties. Cannot find key=" + defineModelIdExcludePropertyName);
          keys = ArrayUtils.EMPTY_STRING_ARRAY;
        } else {
          keys = argumentKeys.split(",");
        }

        final OptionalModelId newOne = new OptionalModelId(optionalColumnId);
        for (String key : keys) {
          key = key.trim();
          final String excludeArgumentName =
              configurationProperties.getProperty(
                  defineModelIdExcludePropertyName + "." + key + ".argument");
          final String excludeArgumentSkip =
              configurationProperties.getProperty(
                  defineModelIdExcludePropertyName + "." + key + ".skip");
          newOne.addExcludeArgument(excludeArgumentName, Integer.parseInt(excludeArgumentSkip));
        }
        result.add(newOne);
        LOGGER.debug("Defined  modelId: " + newOne);
      }
    }
    return result.toArray(new OptionalModelId[result.size()]);
  }
Esempio n. 3
0
  private String[] filterArgs(final String[] originalArgs, final OptionalModelId optionalModelId) {
    final ObjectList<String> filteredArgs = new ObjectArrayList<String>();
    for (int i = 0; i < originalArgs.length; i++) {
      final String argumentName = originalArgs[i].replaceAll("--", "");
      if (optionalModelId.columnIdentifier.equalsIgnoreCase(
          argumentName)) { // skip optional modelIds arguments as well.
        final int skipNumber = 1;
        LOGGER.info(
            "For optional modelId: "
                + optionalModelId.columnIdentifier
                + " Filtering out argument "
                + argumentName
                + " total args skipped: "
                + skipNumber);

        i += skipNumber; // skip argument name and 'skip' number of arguments.
      }
      if (optionalModelId.excludeArgumentNames.contains(argumentName)) {
        final int skipNumber = optionalModelId.skipValue(argumentName);
        LOGGER.info(
            "For optional modelId: "
                + optionalModelId.columnIdentifier
                + " Filtering out argument "
                + argumentName
                + " total args skipped: "
                + skipNumber);

        i += skipNumber; // skip argument name and 'skip' number of arguments.
      } else {
        LOGGER.debug("Adding argument to hashCode: " + originalArgs[i]);
        filteredArgs.add(originalArgs[i]);
      }

      if (i >= originalArgs.length) {
        break;
      }
    }
    // Hashcode will depend on argument order, so we sort them after filtering:
    Collections.sort(filteredArgs);
    return filteredArgs.toArray(new String[filteredArgs.size()]);
  }
Esempio n. 4
0
  private String[] adjustArguments(
      final String[] args, final int splitIndex, final ObjectList<String> explicitOptions) {
    final ObjectList<String> filteredArgs = new ObjectArrayList<String>();

    int ignoreCount = 0;

    for (final String arg : args) {
      if (arg.equals("--sequence-file")) {
        ignoreCount = 1;
        filteredArgs.add("--sequence-file");
      }
      if (arg.equals("--mode") || arg.equals("-m")) {
        ignoreCount = 2;
        filteredArgs.add("--mode");
        filteredArgs.add("sequence");
      }
      if (arg.equals("--splits")) {
        ignoreCount = 2;
      }
      if (arg.equals("--evaluate-statistics")) {
        ignoreCount = 2;
      }
      if (ignoreCount > 0) {
        ignoreCount--;
      } else {
        filteredArgs.add(arg);
      }
    }
    filteredArgs.add("--process-split-id");
    filteredArgs.add(Integer.toString(splitIndex));
    filteredArgs.add("--split-id");
    filteredArgs.add(Integer.toString(splitIndex));

    filteredArgs.add("--model-id");
    filteredArgs.add(modelId);

    filteredArgs.add("--split-plan");
    filteredArgs.add(splitPlanFilename);
    filteredArgs.add("--other-options");
    final StringBuilder argOtherOptions = new StringBuilder();

    for (final String option : explicitOptions) {
      argOtherOptions.append(option);
      argOtherOptions.append(' ');
    }

    filteredArgs.add(argOtherOptions.toString());
    return filteredArgs.toArray(new String[filteredArgs.size()]);
  }
Esempio n. 5
0
 public void addRemovalListener(RemovalListener removalListener) {
   removalListeners.add(removalListener);
 }