private double[] getSparkModelInfoFromHDFS(Path location, Configuration conf) throws Exception {

    FileSystem fileSystem = FileSystem.get(location.toUri(), conf);
    FileStatus[] files = fileSystem.listStatus(location);

    if (files == null) throw new Exception("Couldn't find Spark Truck ML weights at: " + location);

    ArrayList<Double> modelInfo = new ArrayList<Double>();
    for (FileStatus file : files) {

      if (file.getPath().getName().startsWith("_")) {
        continue;
      }

      InputStream stream = fileSystem.open(file.getPath());

      StringWriter writer = new StringWriter();
      IOUtils.copy(stream, writer, "UTF-8");
      String raw = writer.toString();
      for (String str : raw.split("\n")) {
        modelInfo.add(Double.valueOf(str));
      }
    }

    return Doubles.toArray(modelInfo);
  }
  @Override
  public boolean apply(@Nullable Scenario scenario) {
    final List<Double> loads =
        newArrayList(
            relative ? Metrics.measureRelativeLoad(scenario) : Metrics.measureLoad(scenario));
    final int toAdd = desiredLoadList.size() - loads.size();
    for (int j = 0; j < toAdd; j++) {
      loads.add(0d);
    }

    final double[] deviations =
        abs(subtract(Doubles.toArray(desiredLoadList), Doubles.toArray(loads)));
    final double mean = StatUtils.mean(deviations);
    final double max = Doubles.max(deviations);
    return max <= maxMax && mean <= maxMean;
  }
    @SuppressWarnings("synthetic-access")
    @Override
    public CurrencyLabelledMatrix1D buildObject(
        final FudgeDeserializer deserializer, final FudgeMsg message) {
      final FudgeMsg msg = message.getMessage(MATRIX_FIELD_NAME);

      final Queue<String> labelTypes = new LinkedList<String>();
      final Queue<FudgeField> labelValues = new LinkedList<FudgeField>();

      final List<Currency> keys = new LinkedList<Currency>();
      final List<Object> labels = new LinkedList<Object>();
      final List<Double> values = new LinkedList<Double>();

      for (final FudgeField field : msg) {
        switch (field.getOrdinal()) {
          case LABEL_TYPE_ORDINAL:
            labelTypes.add((String) field.getValue());
            break;
          case KEY_ORDINAL:
            keys.add(Currency.of((String) field.getValue()));
            break;
          case LABEL_ORDINAL:
            labelValues.add(field);
            break;
          case VALUE_ORDINAL:
            values.add((Double) field.getValue());
            break;
        }

        if (!labelTypes.isEmpty() && !labelValues.isEmpty()) {
          // Have a type and a value, which can be consumed
          final String labelType = labelTypes.remove();
          Class<?> labelClass;
          try {
            labelClass = LabelledMatrix1DBuilder.getLabelClass(labelType, _loadedClasses);
          } catch (final ClassNotFoundException ex) {
            throw new OpenGammaRuntimeException(
                "Could not deserialize label of type " + labelType, ex);
          }
          final FudgeField labelValue = labelValues.remove();
          final Object label = deserializer.fieldValueToObject(labelClass, labelValue);
          //          labels.add(Currency.of((String) label));
          labels.add(label);
        }
      }

      final int matrixSize = keys.size();
      final Currency[] keysArray = new Currency[matrixSize];
      keys.toArray(keysArray);
      final Object[] labelsArray = new Object[matrixSize];
      labels.toArray(labelsArray);
      final double[] valuesArray = Doubles.toArray(values);
      return new CurrencyLabelledMatrix1D(keysArray, labelsArray, valuesArray);
    }
 private double addScores(final IDType target, final Iterable<Integer> ids) {
   Collection<Double> scores = new ArrayList<>();
   for (Integer id : ids) {
     Optional<Integer> my = mapping.getUnchecked(Pair.make(target, id));
     if (!my.isPresent()) continue;
     Double s = this.scores.get(my.get());
     if (s == null) continue;
     scores.add(s);
   }
   if (scores.isEmpty()) return Double.NaN;
   if (scores.size() == 1) return scores.iterator().next().doubleValue();
   return operator.combine(Doubles.toArray(scores));
 }
    @Override
    public LocalDateLabelledMatrix1D buildObject(
        final FudgeDeserializer deserializer, final FudgeMsg message) {
      final FudgeMsg msg = message.getMessage(MATRIX_FIELD_NAME);

      final Queue<String> labelTypes = new LinkedList<String>();
      final Queue<FudgeField> labelValues = new LinkedList<FudgeField>();

      final List<LocalDate> keys = new LinkedList<LocalDate>();
      final List<Object> labels = new LinkedList<Object>();
      final List<Double> values = new LinkedList<Double>();

      for (final FudgeField field : msg) {
        switch (field.getOrdinal()) {
          case LABEL_TYPE_ORDINAL:
            labelTypes.add((String) field.getValue());
            break;
          case KEY_ORDINAL:
            keys.add(((FudgeDate) field.getValue()).toLocalDate());
            break;
          case LABEL_ORDINAL:
            labelValues.add(field);
            break;
          case VALUE_ORDINAL:
            values.add((Double) field.getValue());
            break;
        }

        if (!labelTypes.isEmpty() && !labelValues.isEmpty()) {
          // Have a type and a value, which can be consumed
          final String labelType = labelTypes.remove();
          Class<?> labelClass = getClass(labelType);
          final FudgeField labelValue = labelValues.remove();
          final Object label = deserializer.fieldValueToObject(labelClass, labelValue);
          labels.add(label);
        }
      }

      final int matrixSize = keys.size();
      final LocalDate[] keysArray = new LocalDate[matrixSize];
      keys.toArray(keysArray);
      final Object[] labelsArray = new Object[matrixSize];
      labels.toArray(labelsArray);
      final double[] valuesArray = Doubles.toArray(values);
      return new LocalDateLabelledMatrix1D(keysArray, labelsArray, valuesArray);
    }
 @Override
 public SurfaceCurrencyParameterSensitivity surfaceCurrencyParameterSensitivity(
     IborCapletFloorletSensitivity point) {
   ArgChecker.isTrue(
       point.getIndex().equals(index),
       "Ibor index of provider must be the same as Ibor index of point sensitivity");
   double expiry = relativeTime(point.getExpiry());
   double strike = point.getStrike();
   // copy to ImmutableMap to lock order (keySet and values used separately but must match)
   Map<DoublesPair, Double> result =
       ImmutableMap.copyOf(surface.zValueParameterSensitivity(expiry, strike));
   SurfaceCurrencyParameterSensitivity parameterSensi =
       SurfaceCurrencyParameterSensitivity.of(
           updateSurfaceMetadata(result.keySet()),
           point.getCurrency(),
           DoubleArray.copyOf(Doubles.toArray(result.values())));
   return parameterSensi.multipliedBy(point.getSensitivity());
 }
  private static Double evaluate(Collection<?> values, int quantile) {
    List<Double> doubleValues = new ArrayList<>();

    for (Object value : values) {
      Double doubleValue = (Double) TypeUtil.parseOrCast(DataType.DOUBLE, value);

      doubleValues.add(doubleValue);
    }

    double[] data = Doubles.toArray(doubleValues);

    // The data must be (at least partially) ordered
    Arrays.sort(data);

    Percentile percentile = new Percentile();
    percentile.setData(data);

    return percentile.evaluate(quantile);
  }
 @Override
 public StringLabelledMatrix1D buildObject(
     final FudgeDeserializer deserializer, final FudgeMsg message) {
   final FudgeMsg msg = message.getMessage(MATRIX_FIELD_NAME);
   final List<String> keys = new LinkedList<String>();
   final List<Double> values = new LinkedList<Double>();
   for (final FudgeField field : msg) {
     switch (field.getOrdinal()) {
       case KEY_ORDINAL:
         keys.add((String) field.getValue());
         break;
       case VALUE_ORDINAL:
         values.add((Double) field.getValue());
         break;
     }
   }
   String[] keysArray = keys.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
   final double[] valuesArray = Doubles.toArray(values);
   return new StringLabelledMatrix1D(keysArray, valuesArray);
 }
  private static Map<String, double[]> parseVectorDictionary(
      SupportVectorMachineModel supportVectorMachineModel) {
    VectorDictionary vectorDictionary = supportVectorMachineModel.getVectorDictionary();

    VectorFields vectorFields = vectorDictionary.getVectorFields();

    List<FieldRef> fieldRefs = vectorFields.getFieldRefs();

    Map<String, double[]> result = new LinkedHashMap<>();

    List<VectorInstance> vectorInstances = vectorDictionary.getVectorInstances();
    for (VectorInstance vectorInstance : vectorInstances) {
      String id = vectorInstance.getId();
      if (id == null) {
        throw new InvalidFeatureException(vectorInstance);
      }

      Array array = vectorInstance.getArray();
      RealSparseArray sparseArray = vectorInstance.getREALSparseArray();

      List<? extends Number> values;

      if (array != null && sparseArray == null) {
        values = ArrayUtil.asNumberList(array);
      } else if (array == null && sparseArray != null) {
        values = SparseArrayUtil.asNumberList(sparseArray);
      } else {
        throw new InvalidFeatureException(vectorInstance);
      } // End if

      if (fieldRefs.size() != values.size()) {
        throw new InvalidFeatureException(vectorInstance);
      }

      double[] vector = Doubles.toArray(values);

      result.put(id, vector);
    }

    return result;
  }
  @Override
  public void execute(Tuple input) {
    String line = input.getString(0);

    List<Double> list = new ArrayList<Double>();

    int firstIndex = line.indexOf("[");
    int lastIndex = line.indexOf("]");

    String subLine = line.substring(firstIndex + 1, lastIndex);
    //  System.out.println(subLine);
    String[] subStrings = subLine.split(",");

    String author = subStrings[0];
    for (int i = 1; i < subStrings.length; i++) {
      list.add(Double.valueOf(subStrings[i]));
    }

    vector = Doubles.toArray(list);
    //    System.out.println(author+" "+list+" "+vector);
    collector.emit(input, new Values(author, list, vector));
    collector.ack(input);
  }
 @Override
 public PersistedData serializeCollection(Collection<Double> value, SerializationContext context) {
   return context.create(Doubles.toArray(value));
 }