/**
* Set the output format. Takes the current average class values and m_InputFormat and calls
* setOutputFormat(Instances) appropriately.
*/
private void setOutputFormat() {
Instances newData;
FastVector newAtts, newVals;
// Compute new attributes
newAtts = new FastVector(getInputFormat().numAttributes());
for (int j = 0; j < getInputFormat().numAttributes(); j++) {
Attribute att = getInputFormat().attribute(j);
if (!m_AttIndices.isInRange(j) || !att.isString()) {
// We don't have to copy the attribute because the
// attribute index remains unchanged.
newAtts.addElement(att);
} else {
// Compute list of attribute values
newVals = new FastVector(att.numValues());
for (int i = 0; i < att.numValues(); i++) {
newVals.addElement(att.value(i));
}
newAtts.addElement(new Attribute(att.name(), newVals));
}
}
// Construct new header
newData = new Instances(getInputFormat().relationName(), newAtts, 0);
newData.setClassIndex(getInputFormat().classIndex());
setOutputFormat(newData);
}
@Override
public int compare(InstanceHolder o1, InstanceHolder o2) {
// both missing is equal
if (o1.m_instance.isMissing(m_attribute) && o2.m_instance.isMissing(m_attribute)) {
return 0;
}
// one missing - missing instances should all be at the end
// regardless of whether order is ascending or descending
if (o1.m_instance.isMissing(m_attribute)) {
return 1;
}
if (o2.m_instance.isMissing(m_attribute)) {
return -1;
}
int cmp = 0;
if (!m_attribute.isString() && !m_attribute.isRelationValued()) {
double val1 = o1.m_instance.value(m_attribute);
double val2 = o2.m_instance.value(m_attribute);
cmp = Double.compare(val1, val2);
} else if (m_attribute.isString()) {
String val1 = o1.m_stringVals.get(m_attribute.name());
String val2 = o2.m_stringVals.get(m_attribute.name());
/*
* String val1 = o1.stringValue(m_attribute); String val2 =
* o2.stringValue(m_attribute);
*/
// TODO case insensitive?
cmp = val1.compareTo(val2);
} else {
throw new IllegalArgumentException(
"Can't sort according to " + "relation-valued attribute values!");
}
if (m_descending) {
return -cmp;
}
return cmp;
}
/** Find the fold attribute within a dataset. */
private Attribute getAttribute(Instances data) {
SingleIndex index = new SingleIndex(super.getAttributeIndex());
index.setUpper(data.numAttributes() - 1);
Attribute att = data.attribute(index.getIndex());
if (att == null)
throw new NoSuchElementException(
"attribute #" + super.getAttributeIndex() + " does not exist");
if (!att.isNominal() && !att.isString())
throw new IllegalArgumentException("Attribute '" + att + "' is not nominal");
return att;
}
/**
* Constructs an instance suitable for passing to the model for scoring
*
* @param incoming the incoming instance
* @return an instance with values mapped to be consistent with what the model is expecting
*/
protected Instance mapIncomingFieldsToModelFields(Instance incoming) {
Instances modelHeader = m_model.getHeader();
double[] vals = new double[modelHeader.numAttributes()];
for (int i = 0; i < modelHeader.numAttributes(); i++) {
if (m_attributeMap[i] < 0) {
// missing or type mismatch
vals[i] = Utils.missingValue();
continue;
}
Attribute modelAtt = modelHeader.attribute(i);
Attribute incomingAtt = incoming.dataset().attribute(m_attributeMap[i]);
if (incoming.isMissing(incomingAtt.index())) {
vals[i] = Utils.missingValue();
continue;
}
if (modelAtt.isNumeric()) {
vals[i] = incoming.value(m_attributeMap[i]);
} else if (modelAtt.isNominal()) {
String incomingVal = incoming.stringValue(m_attributeMap[i]);
int modelIndex = modelAtt.indexOfValue(incomingVal);
if (modelIndex < 0) {
vals[i] = Utils.missingValue();
} else {
vals[i] = modelIndex;
}
} else if (modelAtt.isString()) {
vals[i] = 0;
modelAtt.setStringValue(incoming.stringValue(m_attributeMap[i]));
}
}
if (modelHeader.classIndex() >= 0) {
// set class to missing value
vals[modelHeader.classIndex()] = Utils.missingValue();
}
Instance newInst = null;
if (incoming instanceof SparseInstance) {
newInst = new SparseInstance(incoming.weight(), vals);
} else {
newInst = new DenseInstance(incoming.weight(), vals);
}
newInst.setDataset(modelHeader);
return newInst;
}
public MappingInfo(Instances dataSet, MiningSchema miningSchema, Logger log) throws Exception {
m_log = log;
// miningSchema.convertStringAttsToNominal();
Instances fieldsI = miningSchema.getMiningSchemaAsInstances();
m_fieldsMap = new int[fieldsI.numAttributes()];
m_nominalValueMaps = new int[fieldsI.numAttributes()][];
for (int i = 0; i < fieldsI.numAttributes(); i++) {
String schemaAttName = fieldsI.attribute(i).name();
boolean found = false;
for (int j = 0; j < dataSet.numAttributes(); j++) {
if (dataSet.attribute(j).name().equals(schemaAttName)) {
Attribute miningSchemaAtt = fieldsI.attribute(i);
Attribute incomingAtt = dataSet.attribute(j);
// check type match
if (miningSchemaAtt.type() != incomingAtt.type()) {
throw new Exception(
"[MappingInfo] type mismatch for field "
+ schemaAttName
+ ". Mining schema type "
+ miningSchemaAtt.toString()
+ ". Incoming type "
+ incomingAtt.toString()
+ ".");
}
// check nominal values (number, names...)
if (miningSchemaAtt.numValues() != incomingAtt.numValues()) {
String warningString =
"[MappingInfo] WARNING: incoming nominal attribute "
+ incomingAtt.name()
+ " does not have the same "
+ "number of values as the corresponding mining "
+ "schema attribute.";
if (m_log != null) {
m_log.logMessage(warningString);
} else {
System.err.println(warningString);
}
}
if (miningSchemaAtt.isNominal() || miningSchemaAtt.isString()) {
int[] valuesMap = new int[incomingAtt.numValues()];
for (int k = 0; k < incomingAtt.numValues(); k++) {
String incomingNomVal = incomingAtt.value(k);
int indexInSchema = miningSchemaAtt.indexOfValue(incomingNomVal);
if (indexInSchema < 0) {
String warningString =
"[MappingInfo] WARNING: incoming nominal attribute "
+ incomingAtt.name()
+ " has value "
+ incomingNomVal
+ " that doesn't occur in the mining schema.";
if (m_log != null) {
m_log.logMessage(warningString);
} else {
System.err.println(warningString);
}
valuesMap[k] = UNKNOWN_NOMINAL_VALUE;
} else {
valuesMap[k] = indexInSchema;
}
}
m_nominalValueMaps[i] = valuesMap;
}
/*if (miningSchemaAtt.isNominal()) {
for (int k = 0; k < miningSchemaAtt.numValues(); k++) {
if (!miningSchemaAtt.value(k).equals(incomingAtt.value(k))) {
throw new Exception("[PMMLUtils] value " + k + " (" +
miningSchemaAtt.value(k) + ") does not match " +
"incoming value (" + incomingAtt.value(k) +
") for attribute " + miningSchemaAtt.name() +
".");
}
}
}*/
found = true;
m_fieldsMap[i] = j;
}
}
if (!found) {
throw new Exception(
"[MappingInfo] Unable to find a match for mining schema "
+ "attribute "
+ schemaAttName
+ " in the "
+ "incoming instances!");
}
}
// check class attribute (if set)
if (fieldsI.classIndex() >= 0) {
if (dataSet.classIndex() < 0) {
// first see if we can find a matching class
String className = fieldsI.classAttribute().name();
Attribute classMatch = dataSet.attribute(className);
if (classMatch == null) {
throw new Exception(
"[MappingInfo] Can't find match for target field "
+ className
+ "in incoming instances!");
}
dataSet.setClass(classMatch);
} else if (!fieldsI.classAttribute().name().equals(dataSet.classAttribute().name())) {
throw new Exception(
"[MappingInfo] class attribute in mining schema does not match "
+ "class attribute in incoming instances!");
}
}
// Set up the textual description of the mapping
fieldsMappingString(fieldsI, dataSet);
}
/**
* The procedure implementing the SMOTE algorithm. The output instances are pushed onto the output
* queue for collection.
*
* @throws Exception if provided options cannot be executed on input instances
*/
protected void doSMOTE() throws Exception {
int minIndex = 0;
int min = Integer.MAX_VALUE;
if (m_DetectMinorityClass) {
// find minority class
int[] classCounts =
getInputFormat().attributeStats(getInputFormat().classIndex()).nominalCounts;
for (int i = 0; i < classCounts.length; i++) {
if (classCounts[i] != 0 && classCounts[i] < min) {
min = classCounts[i];
minIndex = i;
}
}
} else {
String classVal = getClassValue();
if (classVal.equalsIgnoreCase("first")) {
minIndex = 1;
} else if (classVal.equalsIgnoreCase("last")) {
minIndex = getInputFormat().numClasses();
} else {
minIndex = Integer.parseInt(classVal);
}
if (minIndex > getInputFormat().numClasses()) {
throw new Exception("value index must be <= the number of classes");
}
minIndex--; // make it an index
}
int nearestNeighbors;
if (min <= getNearestNeighbors()) {
nearestNeighbors = min - 1;
} else {
nearestNeighbors = getNearestNeighbors();
}
if (nearestNeighbors < 1) throw new Exception("Cannot use 0 neighbors!");
// compose minority class dataset
// also push all dataset instances
Instances sample = getInputFormat().stringFreeStructure();
Enumeration instanceEnum = getInputFormat().enumerateInstances();
while (instanceEnum.hasMoreElements()) {
Instance instance = (Instance) instanceEnum.nextElement();
push((Instance) instance.copy());
if ((int) instance.classValue() == minIndex) {
sample.add(instance);
}
}
// compute Value Distance Metric matrices for nominal features
Map vdmMap = new HashMap();
Enumeration attrEnum = getInputFormat().enumerateAttributes();
while (attrEnum.hasMoreElements()) {
Attribute attr = (Attribute) attrEnum.nextElement();
if (!attr.equals(getInputFormat().classAttribute())) {
if (attr.isNominal() || attr.isString()) {
double[][] vdm = new double[attr.numValues()][attr.numValues()];
vdmMap.put(attr, vdm);
int[] featureValueCounts = new int[attr.numValues()];
int[][] featureValueCountsByClass =
new int[getInputFormat().classAttribute().numValues()][attr.numValues()];
instanceEnum = getInputFormat().enumerateInstances();
while (instanceEnum.hasMoreElements()) {
Instance instance = (Instance) instanceEnum.nextElement();
int value = (int) instance.value(attr);
int classValue = (int) instance.classValue();
featureValueCounts[value]++;
featureValueCountsByClass[classValue][value]++;
}
for (int valueIndex1 = 0; valueIndex1 < attr.numValues(); valueIndex1++) {
for (int valueIndex2 = 0; valueIndex2 < attr.numValues(); valueIndex2++) {
double sum = 0;
for (int classValueIndex = 0;
classValueIndex < getInputFormat().numClasses();
classValueIndex++) {
double c1i = featureValueCountsByClass[classValueIndex][valueIndex1];
double c2i = featureValueCountsByClass[classValueIndex][valueIndex2];
double c1 = featureValueCounts[valueIndex1];
double c2 = featureValueCounts[valueIndex2];
double term1 = c1i / c1;
double term2 = c2i / c2;
sum += Math.abs(term1 - term2);
}
vdm[valueIndex1][valueIndex2] = sum;
}
}
}
}
}
// use this random source for all required randomness
Random rand = new Random(getRandomSeed());
// find the set of extra indices to use if the percentage is not evenly
// divisible by 100
List extraIndices = new LinkedList();
double percentageRemainder = (getPercentage() / 100) - Math.floor(getPercentage() / 100.0);
int extraIndicesCount = (int) (percentageRemainder * sample.numInstances());
if (extraIndicesCount >= 1) {
for (int i = 0; i < sample.numInstances(); i++) {
extraIndices.add(i);
}
}
Collections.shuffle(extraIndices, rand);
extraIndices = extraIndices.subList(0, extraIndicesCount);
Set extraIndexSet = new HashSet(extraIndices);
// the main loop to handle computing nearest neighbors and generating SMOTE
// examples from each instance in the original minority class data
Instance[] nnArray = new Instance[nearestNeighbors];
for (int i = 0; i < sample.numInstances(); i++) {
Instance instanceI = sample.instance(i);
// find k nearest neighbors for each instance
List distanceToInstance = new LinkedList();
for (int j = 0; j < sample.numInstances(); j++) {
Instance instanceJ = sample.instance(j);
if (i != j) {
double distance = 0;
attrEnum = getInputFormat().enumerateAttributes();
while (attrEnum.hasMoreElements()) {
Attribute attr = (Attribute) attrEnum.nextElement();
if (!attr.equals(getInputFormat().classAttribute())) {
double iVal = instanceI.value(attr);
double jVal = instanceJ.value(attr);
if (attr.isNumeric()) {
distance += Math.pow(iVal - jVal, 2);
} else {
distance += ((double[][]) vdmMap.get(attr))[(int) iVal][(int) jVal];
}
}
}
distance = Math.pow(distance, .5);
distanceToInstance.add(new Object[] {distance, instanceJ});
}
}
// sort the neighbors according to distance
Collections.sort(
distanceToInstance,
new Comparator() {
public int compare(Object o1, Object o2) {
double distance1 = (Double) ((Object[]) o1)[0];
double distance2 = (Double) ((Object[]) o2)[0];
return Double.compare(distance1, distance2);
}
});
// populate the actual nearest neighbor instance array
Iterator entryIterator = distanceToInstance.iterator();
int j = 0;
while (entryIterator.hasNext() && j < nearestNeighbors) {
nnArray[j] = (Instance) ((Object[]) entryIterator.next())[1];
j++;
}
// create synthetic examples
int n = (int) Math.floor(getPercentage() / 100);
while (n > 0 || extraIndexSet.remove(i)) {
double[] values = new double[sample.numAttributes()];
int nn = rand.nextInt(nearestNeighbors);
attrEnum = getInputFormat().enumerateAttributes();
while (attrEnum.hasMoreElements()) {
Attribute attr = (Attribute) attrEnum.nextElement();
if (!attr.equals(getInputFormat().classAttribute())) {
if (attr.isNumeric()) {
double dif = nnArray[nn].value(attr) - instanceI.value(attr);
double gap = rand.nextDouble();
values[attr.index()] = (instanceI.value(attr) + gap * dif);
} else if (attr.isDate()) {
double dif = nnArray[nn].value(attr) - instanceI.value(attr);
double gap = rand.nextDouble();
values[attr.index()] = (long) (instanceI.value(attr) + gap * dif);
} else {
int[] valueCounts = new int[attr.numValues()];
int iVal = (int) instanceI.value(attr);
valueCounts[iVal]++;
for (int nnEx = 0; nnEx < nearestNeighbors; nnEx++) {
int val = (int) nnArray[nnEx].value(attr);
valueCounts[val]++;
}
int maxIndex = 0;
int max = Integer.MIN_VALUE;
for (int index = 0; index < attr.numValues(); index++) {
if (valueCounts[index] > max) {
max = valueCounts[index];
maxIndex = index;
}
}
values[attr.index()] = maxIndex;
}
}
}
values[sample.classIndex()] = minIndex;
Instance synthetic = new Instance(1.0, values);
push(synthetic);
n--;
}
}
}
