/** Corresponds to <element name="javaExecutable"> */
private Element createJavaExecutableElement(Document doc, JavaTask t) {
Element executableE =
doc.createElementNS(Schemas.SCHEMA_LATEST.namespace, XMLTags.JAVA_EXECUTABLE.getXMLName());
setAttribute(executableE, XMLAttributes.TASK_CLASS_NAME, t.getExecutableClassName(), true);
// <ref name="javaParameters"/>
try {
Map<String, Serializable> args = t.getArguments();
if ((args != null) && (args.size() > 0)) {
// <element name="parameter">
Element paramsE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_PARAMETERS.getXMLName());
for (String name : args.keySet()) {
Serializable val = args.get(name);
Element paramE =
doc.createElementNS(
Schemas.SCHEMA_LATEST.namespace, XMLTags.TASK_PARAMETER.getXMLName());
setAttribute(paramE, XMLAttributes.COMMON_NAME, name, true);
setAttribute(paramE, XMLAttributes.COMMON_VALUE, val.toString(), true);
paramsE.appendChild(paramE);
}
executableE.appendChild(paramsE);
}
} catch (Exception e) {
logger.error("Could not add arguments for Java Executable element of task " + t.getName(), e);
}
return executableE;
}
/**
* @param test_name Name used to refer to the serialized instance. If a test output dir is
* specified (see constructor), a file will be created with this name.
*/
private boolean serializesCorrectly(Serializable obj, String test_name) {
assert obj instanceof Externalizable : obj + " is not Externalizable";
ObjectOutputStream oout = null;
ObjectInputStream oin = null;
try {
ByteArrayOutputStream bout = new ByteArrayOutputStream();
oout = new ObjectOutputStream(bout);
oout.writeObject(obj);
oout.close();
ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray());
oin = new ObjectInputStream(bin);
Object new_obj = oin.readObject();
if (!obj.equals(new_obj)) {
System.err.println("Non-matching serialized objects (in-VM test):");
System.err.println(" Original: " + obj);
System.err.println(" Deserialized: " + new_obj);
return false;
}
if (obj instanceof THash) {
assertEquals(
((THash) obj).getAutoCompactionFactor(),
((THash) new_obj).getAutoCompactionFactor(),
0.0);
}
oin.close();
if (serialized_object_output_dir != null) {
try {
File file = new File(serialized_object_output_dir, test_name + ".obj");
ObjectOutputStream test_out = new ObjectOutputStream(new FileOutputStream(file));
test_out.writeObject(obj);
test_out.close();
} catch (IOException ex) {
ex.printStackTrace();
fail("Unable to write to test file: " + test_name);
}
}
// Look for previous versions
int version = 0;
for (; ; version++) {
InputStream stream =
SerializationTest.class.getResourceAsStream(
"old_serialized_versions/" + version + "/" + test_name + ".obj");
if (stream == null) break;
System.out.println("Testing " + test_name + " against version " + version + "...");
oin = new ObjectInputStream(stream);
new_obj = oin.readObject();
if (!obj.equals(new_obj)) {
System.err.println("Non-matching serialized objects (version " + version + "):");
System.err.println(" Original: " + obj);
System.err.println(" Deserialized: " + new_obj);
return false;
}
}
return true;
} catch (Exception ex) {
ex.printStackTrace();
return false;
} finally {
if (oout != null) {
try {
oout.close();
} catch (IOException ex) {
// ignore
}
}
if (oin != null) {
try {
oin.close();
} catch (IOException ex) {
// ignore
}
}
}
}
/**
* @param type May be null if the type is unknown.
* @param elementType May be null if the type is unknown.
* @return May be null.
*/
public <T> T readValue(Class<T> type, Class elementType, Object jsonData) {
if (jsonData == null) return null;
if (jsonData instanceof OrderedMap) {
OrderedMap<String, Object> jsonMap = (OrderedMap) jsonData;
String className = typeName == null ? null : (String) jsonMap.remove(typeName);
if (className != null) {
try {
type = (Class<T>) Class.forName(className);
} catch (ClassNotFoundException ex) {
type = tagToClass.get(className);
if (type == null) throw new SerializationException(ex);
}
}
Object object;
if (type != null) {
Serializer serializer = classToSerializer.get(type);
if (serializer != null) return (T) serializer.read(this, jsonMap, type);
object = newInstance(type);
if (object instanceof Serializable) {
((Serializable) object).read(this, jsonMap);
return (T) object;
}
if (object instanceof HashMap) {
HashMap result = (HashMap) object;
for (Entry entry : jsonMap.entries())
result.put(entry.key, readValue(elementType, null, entry.value));
return (T) result;
}
} else object = new OrderedMap();
if (object instanceof ObjectMap) {
ObjectMap result = (ObjectMap) object;
for (String key : jsonMap.orderedKeys())
result.put(key, readValue(elementType, null, jsonMap.get(key)));
return (T) result;
}
readFields(object, jsonMap);
return (T) object;
}
if (type != null) {
Serializer serializer = classToSerializer.get(type);
if (serializer != null) return (T) serializer.read(this, jsonData, type);
}
if (jsonData instanceof Array) {
Array array = (Array) jsonData;
if (type == null || Array.class.isAssignableFrom(type)) {
Array newArray = type == null ? new Array() : (Array) newInstance(type);
newArray.ensureCapacity(array.size);
for (int i = 0, n = array.size; i < n; i++)
newArray.add(readValue(elementType, null, array.get(i)));
return (T) newArray;
}
if (ArrayList.class.isAssignableFrom(type)) {
ArrayList newArray = type == null ? new ArrayList() : (ArrayList) newInstance(type);
newArray.ensureCapacity(array.size);
for (int i = 0, n = array.size; i < n; i++)
newArray.add(readValue(elementType, null, array.get(i)));
return (T) newArray;
}
if (type.isArray()) {
Class componentType = type.getComponentType();
if (elementType == null) elementType = componentType;
Object newArray = java.lang.reflect.Array.newInstance(componentType, array.size);
for (int i = 0, n = array.size; i < n; i++)
java.lang.reflect.Array.set(newArray, i, readValue(elementType, null, array.get(i)));
return (T) newArray;
}
throw new SerializationException(
"Unable to convert value to required type: " + jsonData + " (" + type.getName() + ")");
}
if (jsonData instanceof Float) {
Float floatValue = (Float) jsonData;
try {
if (type == null || type == float.class || type == Float.class)
return (T) (Float) floatValue;
if (type == int.class || type == Integer.class) return (T) (Integer) floatValue.intValue();
if (type == long.class || type == Long.class) return (T) (Long) floatValue.longValue();
if (type == double.class || type == Double.class)
return (T) (Double) floatValue.doubleValue();
if (type == short.class || type == Short.class) return (T) (Short) floatValue.shortValue();
if (type == byte.class || type == Byte.class) return (T) (Byte) floatValue.byteValue();
} catch (NumberFormatException ignored) {
}
jsonData = String.valueOf(jsonData);
}
if (jsonData instanceof Boolean) jsonData = String.valueOf(jsonData);
if (jsonData instanceof String) {
String string = (String) jsonData;
if (type == null || type == String.class) return (T) jsonData;
try {
if (type == int.class || type == Integer.class) return (T) Integer.valueOf(string);
if (type == float.class || type == Float.class) return (T) Float.valueOf(string);
if (type == long.class || type == Long.class) return (T) Long.valueOf(string);
if (type == double.class || type == Double.class) return (T) Double.valueOf(string);
if (type == short.class || type == Short.class) return (T) Short.valueOf(string);
if (type == byte.class || type == Byte.class) return (T) Byte.valueOf(string);
} catch (NumberFormatException ignored) {
}
if (type == boolean.class || type == Boolean.class) return (T) Boolean.valueOf(string);
if (type == char.class || type == Character.class) return (T) (Character) string.charAt(0);
if (type.isEnum()) {
Object[] constants = type.getEnumConstants();
for (int i = 0, n = constants.length; i < n; i++)
if (string.equals(constants[i].toString())) return (T) constants[i];
}
if (type == CharSequence.class) return (T) string;
throw new SerializationException(
"Unable to convert value to required type: " + jsonData + " (" + type.getName() + ")");
}
return null;
}
/**
* @param value May be null.
* @param knownType May be null if the type is unknown.
* @param elementType May be null if the type is unknown.
*/
public void writeValue(Object value, Class knownType, Class elementType) {
try {
if (value == null) {
writer.value(null);
return;
}
Class actualType = value.getClass();
if (actualType.isPrimitive()
|| actualType == String.class
|| actualType == Integer.class
|| actualType == Boolean.class
|| actualType == Float.class
|| actualType == Long.class
|| actualType == Double.class
|| actualType == Short.class
|| actualType == Byte.class
|| actualType == Character.class) {
writer.value(value);
return;
}
if (value instanceof Serializable) {
writeObjectStart(actualType, knownType);
((Serializable) value).write(this);
writeObjectEnd();
return;
}
Serializer serializer = classToSerializer.get(actualType);
if (serializer != null) {
serializer.write(this, value, knownType);
return;
}
if (value instanceof Array) {
if (knownType != null && actualType != knownType)
throw new SerializationException(
"Serialization of an Array other than the known type is not supported.\n"
+ "Known type: "
+ knownType
+ "\nActual type: "
+ actualType);
writeArrayStart();
Array array = (Array) value;
for (int i = 0, n = array.size; i < n; i++) writeValue(array.get(i), elementType, null);
writeArrayEnd();
return;
}
if (value instanceof Collection) {
if (knownType != null && actualType != knownType && actualType != ArrayList.class)
throw new SerializationException(
"Serialization of a Collection other than the known type is not supported.\n"
+ "Known type: "
+ knownType
+ "\nActual type: "
+ actualType);
writeArrayStart();
for (Object item : (Collection) value) writeValue(item, elementType, null);
writeArrayEnd();
return;
}
if (actualType.isArray()) {
if (elementType == null) elementType = actualType.getComponentType();
int length = java.lang.reflect.Array.getLength(value);
writeArrayStart();
for (int i = 0; i < length; i++)
writeValue(java.lang.reflect.Array.get(value, i), elementType, null);
writeArrayEnd();
return;
}
if (value instanceof OrderedMap) {
if (knownType == null) knownType = OrderedMap.class;
writeObjectStart(actualType, knownType);
OrderedMap map = (OrderedMap) value;
for (Object key : map.orderedKeys()) {
writer.name(convertToString(key));
writeValue(map.get(key), elementType, null);
}
writeObjectEnd();
return;
}
if (value instanceof ArrayMap) {
if (knownType == null) knownType = ArrayMap.class;
writeObjectStart(actualType, knownType);
ArrayMap map = (ArrayMap) value;
for (int i = 0, n = map.size; i < n; i++) {
writer.name(convertToString(map.keys[i]));
writeValue(map.values[i], elementType, null);
}
writeObjectEnd();
return;
}
if (value instanceof ObjectMap) {
if (knownType == null) knownType = OrderedMap.class;
writeObjectStart(actualType, knownType);
for (Entry entry : ((ObjectMap<?, ?>) value).entries()) {
writer.name(convertToString(entry.key));
writeValue(entry.value, elementType, null);
}
writeObjectEnd();
return;
}
if (value instanceof Map) {
if (knownType == null) knownType = OrderedMap.class;
writeObjectStart(actualType, knownType);
for (Map.Entry entry : ((Map<?, ?>) value).entrySet()) {
writer.name(convertToString(entry.getKey()));
writeValue(entry.getValue(), elementType, null);
}
writeObjectEnd();
return;
}
if (actualType.isEnum()) {
writer.value(value);
return;
}
writeObjectStart(actualType, knownType);
writeFields(value);
writeObjectEnd();
} catch (IOException ex) {
throw new SerializationException(ex);
}
}
/**
* @param clazz May be null if the type is unknown.
* @param elementType May be null if the type is unknown.
* @return May be null.
*/
public <T> T readValue(Class<T> clazz, Class elementType, JsonValue jsonData) {
if (jsonData == null) return null;
Type type = ReflectionCache.getType(clazz);
if (jsonData.isObject()) {
String className = typeName == null ? null : jsonData.getString(typeName, null);
if (className != null) {
jsonData.remove(typeName);
try {
type = ReflectionCache.forName(className);
} catch (ClassNotFoundException ex) {
type = tagToClass.get(className);
if (type == null) throw new SerializationException(ex);
}
}
Object object;
if (type != null) {
Serializer serializer = classToSerializer.get(type);
if (serializer != null) return (T) serializer.read(this, jsonData, type.getClassOfType());
object = newInstance(type);
if (object instanceof Serializable) {
((Serializable) object).read(this, jsonData);
return (T) object;
}
if (object instanceof HashMap) {
HashMap result = (HashMap) object;
for (JsonValue child = jsonData.child(); child != null; child = child.next())
result.put(child.name(), readValue(elementType, null, child));
return (T) result;
}
} else object = new OrderedMap();
if (object instanceof ObjectMap) {
ObjectMap result = (ObjectMap) object;
for (JsonValue child = jsonData.child(); child != null; child = child.next())
result.put(child.name(), readValue(elementType, null, child));
return (T) result;
}
readFields(object, jsonData);
return (T) object;
}
if (type != null) {
Serializer serializer = classToSerializer.get(type);
if (serializer != null) return (T) serializer.read(this, jsonData, type.getClassOfType());
}
if (jsonData.isArray()) {
if (type == null || type.isAssignableFrom(ReflectionCache.getType(Array.class))) {
Array newArray = new Array();
for (JsonValue child = jsonData.child(); child != null; child = child.next())
newArray.add(readValue(elementType, null, child));
return (T) newArray;
}
if (type.isAssignableFrom(ReflectionCache.getType(ArrayList.class))) {
ArrayList newArray = new ArrayList();
for (JsonValue child = jsonData.child(); child != null; child = child.next())
newArray.add(readValue(elementType, null, child));
return (T) newArray;
}
if (type.isArray()) {
Class componentType = type.getComponentType();
if (elementType == null) elementType = componentType;
Object newArray = ReflectionCache.newArray(componentType, jsonData.size());
Type arrayType = ReflectionCache.getType(newArray.getClass());
int i = 0;
for (JsonValue child = jsonData.child(); child != null; child = child.next())
arrayType.setArrayElement(newArray, i++, readValue(elementType, null, child));
return (T) newArray;
}
throw new SerializationException(
"Unable to convert value to required type: " + jsonData + " (" + type.getName() + ")");
}
Class t = type == null ? null : type.getClassOfType();
if (jsonData.isNumber()) {
try {
if (type == null || t == float.class || t == Float.class)
return (T) (Float) jsonData.asFloat();
if (t == int.class || t == Integer.class) return (T) (Integer) jsonData.asInt();
if (t == long.class || t == Long.class) return (T) (Long) jsonData.asLong();
if (t == double.class || t == Double.class) return (T) (Double) (double) jsonData.asFloat();
if (t == String.class) return (T) Float.toString(jsonData.asFloat());
if (t == short.class || t == Short.class) return (T) (Short) (short) jsonData.asInt();
if (t == byte.class || t == Byte.class) return (T) (Byte) (byte) jsonData.asInt();
} catch (NumberFormatException ignored) {
}
jsonData = new JsonValue(jsonData.asString());
}
if (jsonData.isBoolean()) {
try {
if (type == null || t == boolean.class || t == Boolean.class)
return (T) (Boolean) jsonData.asBoolean();
} catch (NumberFormatException ignored) {
}
jsonData = new JsonValue(jsonData.asString());
}
if (jsonData.isString()) {
String string = jsonData.asString();
if (type == null || t == String.class) return (T) string;
try {
if (t == int.class || t == Integer.class) return (T) Integer.valueOf(string);
if (t == float.class || t == Float.class) return (T) Float.valueOf(string);
if (t == long.class || t == Long.class) return (T) Long.valueOf(string);
if (t == double.class || t == Double.class) return (T) Double.valueOf(string);
if (t == short.class || t == Short.class) return (T) Short.valueOf(string);
if (t == byte.class || t == Byte.class) return (T) Byte.valueOf(string);
} catch (NumberFormatException ignored) {
}
if (t == boolean.class || t == Boolean.class) return (T) Boolean.valueOf(string);
if (t == char.class || t == Character.class) return (T) (Character) string.charAt(0);
if (type.isEnum()) {
Object[] constants = type.getEnumConstants();
for (int i = 0, n = constants.length; i < n; i++)
if (string.equals(constants[i].toString())) return (T) constants[i];
}
if (t == CharSequence.class) return (T) string;
throw new SerializationException(
"Unable to convert value to required type: " + jsonData + " (" + type.getName() + ")");
}
return null;
}
/**
* Searches the registered Serializers for one, that is responsible for the given class.
*
* @param clazz class of objects, that can be de/serialized with returned Serializer
* @return a Serializer that can handle objects of given class
*/
private Serializable findResponsible(Class<?> clazz) {
for (Serializable serializer : serializers)
if (serializer.isResponsible(clazz)) return serializer;
return null;
}
/**
* Deserializes binary data to an object of given class by delegating the request to the
* responsible Serializer.
*
* @param data the binary data
* @param clazz the class of the resulting object
* @return the deserialized object
*/
@Override
public <T> T deserialize(byte[] data, Class<T> clazz) {
Serializable serializer = findResponsible(clazz);
return serializer.deserialize(data, clazz);
}
/**
* Serializes an given object to a binary representation by delegating the request to the
* responsible Serializer.
*
* @param o an object
* @return a binary representation
*/
@Override
public byte[] serialize(Object o) {
Serializable serializer = findResponsible(o.getClass());
return serializer.serialize(o);
}
/** @param args */
public static void main(final String[] args) throws Exception {
final BufferedReader br =
new BufferedReader(
new InputStreamReader(
(new GZIPInputStream(
new FileInputStream(
new File(
new File(System.getProperty("user.dir")), "testdata/mobo1.txt.gz"))))));
final List<Instance> instances = Lists.newLinkedList();
int count = 0;
while (true) {
count++;
final String line = br.readLine();
if (line == null) {
break;
}
final JSONObject jo = (JSONObject) JSONValue.parse(line);
final HashMapAttributes a = new HashMapAttributes();
a.putAll((JSONObject) jo.get("attributes"));
Instance instance = new Instance(a, (String) jo.get("output"));
instances.add(instance);
}
final List<Instance> train = instances.subList(0, instances.size() / 2);
final List<Instance> test = instances.subList(instances.size() / 2 + 1, instances.size() - 1);
System.out.println("Read " + instances.size() + " instances");
System.out.println("Testing scorers with single decision node");
for (final Scorer scorer : Sets.newHashSet(new Scorer1())) {
final TreeBuilder tb = new TreeBuilder(scorer);
final long startTime = System.currentTimeMillis();
final Node tree = tb.buildPredictiveModel(train).node;
System.out.println(
scorer.getClass().getSimpleName()
+ " build time "
+ (System.currentTimeMillis() - startTime)
+ ", size: "
+ tree.size()
+ " mean depth: "
+ tree.meanDepth());
int correctlyClassified = 0;
for (Instance testInstance : test) {
String result = (String) tree.getLeaf(testInstance.getAttributes()).getBestClassification();
if (result.equals(testInstance.getClassification())) {
correctlyClassified++;
}
}
System.out.println(", accuracy: " + (double) correctlyClassified / test.size());
System.out.println("Testing random forest");
for (int i = 2; i <= 20; i++) {
RandomForestBuilder rfBuilder = new RandomForestBuilder(new TreeBuilder());
RandomForest randomForest = rfBuilder.buildPredictiveModel(train);
correctlyClassified = 0;
for (Instance testInstance : test) {
Serializable result =
randomForest.getClassificationByMaxProb(testInstance.getAttributes());
if (result.equals(testInstance.getClassification())) {
correctlyClassified++;
}
}
System.out.println(
"accuracy with " + i + " trees: " + (double) correctlyClassified / test.size());
// ObjectOutputStream out = new ObjectOutputStream(new FileOutputStream(new
// File("baggedTree.ser")));
// out.writeObject(baggedTree);
}
}
}