private double[] getSparkModelInfoFromHDFS(Path location, Configuration conf) throws Exception {
FileSystem fileSystem = FileSystem.get(location.toUri(), conf);
FileStatus[] files = fileSystem.listStatus(location);
if (files == null) throw new Exception("Couldn't find Spark Truck ML weights at: " + location);
ArrayList<Double> modelInfo = new ArrayList<Double>();
for (FileStatus file : files) {
if (file.getPath().getName().startsWith("_")) {
continue;
}
InputStream stream = fileSystem.open(file.getPath());
StringWriter writer = new StringWriter();
IOUtils.copy(stream, writer, "UTF-8");
String raw = writer.toString();
for (String str : raw.split("\n")) {
modelInfo.add(Double.valueOf(str));
}
}
return Doubles.toArray(modelInfo);
}
@Override
public boolean apply(@Nullable Scenario scenario) {
final List<Double> loads =
newArrayList(
relative ? Metrics.measureRelativeLoad(scenario) : Metrics.measureLoad(scenario));
final int toAdd = desiredLoadList.size() - loads.size();
for (int j = 0; j < toAdd; j++) {
loads.add(0d);
}
final double[] deviations =
abs(subtract(Doubles.toArray(desiredLoadList), Doubles.toArray(loads)));
final double mean = StatUtils.mean(deviations);
final double max = Doubles.max(deviations);
return max <= maxMax && mean <= maxMean;
}
@SuppressWarnings("synthetic-access")
@Override
public CurrencyLabelledMatrix1D buildObject(
final FudgeDeserializer deserializer, final FudgeMsg message) {
final FudgeMsg msg = message.getMessage(MATRIX_FIELD_NAME);
final Queue<String> labelTypes = new LinkedList<String>();
final Queue<FudgeField> labelValues = new LinkedList<FudgeField>();
final List<Currency> keys = new LinkedList<Currency>();
final List<Object> labels = new LinkedList<Object>();
final List<Double> values = new LinkedList<Double>();
for (final FudgeField field : msg) {
switch (field.getOrdinal()) {
case LABEL_TYPE_ORDINAL:
labelTypes.add((String) field.getValue());
break;
case KEY_ORDINAL:
keys.add(Currency.of((String) field.getValue()));
break;
case LABEL_ORDINAL:
labelValues.add(field);
break;
case VALUE_ORDINAL:
values.add((Double) field.getValue());
break;
}
if (!labelTypes.isEmpty() && !labelValues.isEmpty()) {
// Have a type and a value, which can be consumed
final String labelType = labelTypes.remove();
Class<?> labelClass;
try {
labelClass = LabelledMatrix1DBuilder.getLabelClass(labelType, _loadedClasses);
} catch (final ClassNotFoundException ex) {
throw new OpenGammaRuntimeException(
"Could not deserialize label of type " + labelType, ex);
}
final FudgeField labelValue = labelValues.remove();
final Object label = deserializer.fieldValueToObject(labelClass, labelValue);
// labels.add(Currency.of((String) label));
labels.add(label);
}
}
final int matrixSize = keys.size();
final Currency[] keysArray = new Currency[matrixSize];
keys.toArray(keysArray);
final Object[] labelsArray = new Object[matrixSize];
labels.toArray(labelsArray);
final double[] valuesArray = Doubles.toArray(values);
return new CurrencyLabelledMatrix1D(keysArray, labelsArray, valuesArray);
}
private double addScores(final IDType target, final Iterable<Integer> ids) {
Collection<Double> scores = new ArrayList<>();
for (Integer id : ids) {
Optional<Integer> my = mapping.getUnchecked(Pair.make(target, id));
if (!my.isPresent()) continue;
Double s = this.scores.get(my.get());
if (s == null) continue;
scores.add(s);
}
if (scores.isEmpty()) return Double.NaN;
if (scores.size() == 1) return scores.iterator().next().doubleValue();
return operator.combine(Doubles.toArray(scores));
}
@Override
public LocalDateLabelledMatrix1D buildObject(
final FudgeDeserializer deserializer, final FudgeMsg message) {
final FudgeMsg msg = message.getMessage(MATRIX_FIELD_NAME);
final Queue<String> labelTypes = new LinkedList<String>();
final Queue<FudgeField> labelValues = new LinkedList<FudgeField>();
final List<LocalDate> keys = new LinkedList<LocalDate>();
final List<Object> labels = new LinkedList<Object>();
final List<Double> values = new LinkedList<Double>();
for (final FudgeField field : msg) {
switch (field.getOrdinal()) {
case LABEL_TYPE_ORDINAL:
labelTypes.add((String) field.getValue());
break;
case KEY_ORDINAL:
keys.add(((FudgeDate) field.getValue()).toLocalDate());
break;
case LABEL_ORDINAL:
labelValues.add(field);
break;
case VALUE_ORDINAL:
values.add((Double) field.getValue());
break;
}
if (!labelTypes.isEmpty() && !labelValues.isEmpty()) {
// Have a type and a value, which can be consumed
final String labelType = labelTypes.remove();
Class<?> labelClass = getClass(labelType);
final FudgeField labelValue = labelValues.remove();
final Object label = deserializer.fieldValueToObject(labelClass, labelValue);
labels.add(label);
}
}
final int matrixSize = keys.size();
final LocalDate[] keysArray = new LocalDate[matrixSize];
keys.toArray(keysArray);
final Object[] labelsArray = new Object[matrixSize];
labels.toArray(labelsArray);
final double[] valuesArray = Doubles.toArray(values);
return new LocalDateLabelledMatrix1D(keysArray, labelsArray, valuesArray);
}
@Override
public SurfaceCurrencyParameterSensitivity surfaceCurrencyParameterSensitivity(
IborCapletFloorletSensitivity point) {
ArgChecker.isTrue(
point.getIndex().equals(index),
"Ibor index of provider must be the same as Ibor index of point sensitivity");
double expiry = relativeTime(point.getExpiry());
double strike = point.getStrike();
// copy to ImmutableMap to lock order (keySet and values used separately but must match)
Map<DoublesPair, Double> result =
ImmutableMap.copyOf(surface.zValueParameterSensitivity(expiry, strike));
SurfaceCurrencyParameterSensitivity parameterSensi =
SurfaceCurrencyParameterSensitivity.of(
updateSurfaceMetadata(result.keySet()),
point.getCurrency(),
DoubleArray.copyOf(Doubles.toArray(result.values())));
return parameterSensi.multipliedBy(point.getSensitivity());
}
private static Double evaluate(Collection<?> values, int quantile) {
List<Double> doubleValues = new ArrayList<>();
for (Object value : values) {
Double doubleValue = (Double) TypeUtil.parseOrCast(DataType.DOUBLE, value);
doubleValues.add(doubleValue);
}
double[] data = Doubles.toArray(doubleValues);
// The data must be (at least partially) ordered
Arrays.sort(data);
Percentile percentile = new Percentile();
percentile.setData(data);
return percentile.evaluate(quantile);
}
@Override
public StringLabelledMatrix1D buildObject(
final FudgeDeserializer deserializer, final FudgeMsg message) {
final FudgeMsg msg = message.getMessage(MATRIX_FIELD_NAME);
final List<String> keys = new LinkedList<String>();
final List<Double> values = new LinkedList<Double>();
for (final FudgeField field : msg) {
switch (field.getOrdinal()) {
case KEY_ORDINAL:
keys.add((String) field.getValue());
break;
case VALUE_ORDINAL:
values.add((Double) field.getValue());
break;
}
}
String[] keysArray = keys.toArray(ArrayUtils.EMPTY_STRING_ARRAY);
final double[] valuesArray = Doubles.toArray(values);
return new StringLabelledMatrix1D(keysArray, valuesArray);
}
private static Map<String, double[]> parseVectorDictionary(
SupportVectorMachineModel supportVectorMachineModel) {
VectorDictionary vectorDictionary = supportVectorMachineModel.getVectorDictionary();
VectorFields vectorFields = vectorDictionary.getVectorFields();
List<FieldRef> fieldRefs = vectorFields.getFieldRefs();
Map<String, double[]> result = new LinkedHashMap<>();
List<VectorInstance> vectorInstances = vectorDictionary.getVectorInstances();
for (VectorInstance vectorInstance : vectorInstances) {
String id = vectorInstance.getId();
if (id == null) {
throw new InvalidFeatureException(vectorInstance);
}
Array array = vectorInstance.getArray();
RealSparseArray sparseArray = vectorInstance.getREALSparseArray();
List<? extends Number> values;
if (array != null && sparseArray == null) {
values = ArrayUtil.asNumberList(array);
} else if (array == null && sparseArray != null) {
values = SparseArrayUtil.asNumberList(sparseArray);
} else {
throw new InvalidFeatureException(vectorInstance);
} // End if
if (fieldRefs.size() != values.size()) {
throw new InvalidFeatureException(vectorInstance);
}
double[] vector = Doubles.toArray(values);
result.put(id, vector);
}
return result;
}
@Override
public void execute(Tuple input) {
String line = input.getString(0);
List<Double> list = new ArrayList<Double>();
int firstIndex = line.indexOf("[");
int lastIndex = line.indexOf("]");
String subLine = line.substring(firstIndex + 1, lastIndex);
// System.out.println(subLine);
String[] subStrings = subLine.split(",");
String author = subStrings[0];
for (int i = 1; i < subStrings.length; i++) {
list.add(Double.valueOf(subStrings[i]));
}
vector = Doubles.toArray(list);
// System.out.println(author+" "+list+" "+vector);
collector.emit(input, new Values(author, list, vector));
collector.ack(input);
}
@Override
public PersistedData serializeCollection(Collection<Double> value, SerializationContext context) {
return context.create(Doubles.toArray(value));
}