/**
* Creates attribute meta data that represents the attribute that will be generated for the
* provided arguments.
*
* @return the {@link AttributeMetaData} for the provided arguments
*/
public static AttributeMetaData generateAttributeMetaData(
ExampleSet exampleSet, String name, ExpressionType expressionType) {
AttributeMetaData newAttribute = null;
Attribute existingAtt = exampleSet.getAttributes().get(name);
int ontology = expressionType.getAttributeType();
if (ontology == Ontology.BINOMINAL) {
newAttribute = new AttributeMetaData(name, Ontology.BINOMINAL);
HashSet<String> values = new HashSet<>();
values.add("false");
values.add("true");
newAttribute.setValueSet(values, SetRelation.EQUAL);
} else {
newAttribute = new AttributeMetaData(name, ontology);
}
// restore role if attribute existed already
if (existingAtt != null) {
newAttribute.setRole(exampleSet.getAttributes().getRole(existingAtt).getSpecialName());
}
return newAttribute;
}
@Override
protected MetaData modifyMetaData(ExampleSetMetaData metaData) {
AttributeMetaData weightAttribute =
new AttributeMetaData(Attributes.WEIGHT_NAME, Ontology.REAL, Attributes.WEIGHT_NAME);
weightAttribute.setValueRange(getWeightAttributeRange(), getWeightAttributeValueRelation());
metaData.addAttribute(weightAttribute);
return metaData;
}
@Override
protected ExampleSetMetaData modifyMetaData(ExampleSetMetaData exampleSetMetaData)
throws UndefinedParameterError {
AttributeMetaData label = exampleSetMetaData.getLabelMetaData();
if (label != null) {
if (label.isNumerical() && getParameterAsDouble(PARAMETER_LABEL_NOISE) > 0) {
label.setValueSetRelation(SetRelation.SUPERSET);
}
}
double defaultNoise = getParameterAsDouble(PARAMETER_DEFAULT_ATTRIBUTE_NOISE);
if (defaultNoise > 0) {
for (AttributeMetaData amd : exampleSetMetaData.getAllAttributes()) {
if (!amd.isSpecial()) {
if (amd.isNumerical()) {
amd.setValueSetRelation(SetRelation.SUPERSET);
}
}
}
}
int numberOfRandomAttributes = getParameterAsInt(PARAMETER_RANDOM_ATTRIBUTES);
for (int i = 0; i < numberOfRandomAttributes; i++) {
AttributeMetaData amd =
new AttributeMetaData("random" + ((i == 0) ? "" : i + ""), Ontology.REAL);
amd.setValueRange(
new Range(Double.NEGATIVE_INFINITY, Double.POSITIVE_INFINITY), SetRelation.SUBSET);
exampleSetMetaData.addAttribute(amd);
}
return exampleSetMetaData;
}
@Override
public MetaData getGeneratedMetaData() throws OperatorException {
ExampleSetMetaData emd = new ExampleSetMetaData();
AttributeMetaData amd = new AttributeMetaData("label", Ontology.NOMINAL, Attributes.LABEL_NAME);
emd.addAttribute(amd);
int desirendNumberOfAttributes = getParameterAsInt(PARAMETER_NUMBER_ATTRIBUTES);
double mean = getParameterAsDouble(PARAMETER_SPARSE_FRACTION);
if (desirendNumberOfAttributes > 20) {
emd.attributesAreSuperset();
// first ten
for (int i = 1; i < 11; i++) {
AttributeMetaData newAMD = new AttributeMetaData("att" + i, Ontology.REAL);
newAMD.setValueRange(new Range(0, 1), SetRelation.EQUAL);
newAMD.setMean(new MDReal(mean));
emd.addAttribute(newAMD);
}
// last ten
for (int i = desirendNumberOfAttributes - 10; i <= desirendNumberOfAttributes; i++) {
AttributeMetaData newAMD = new AttributeMetaData("att" + i, Ontology.REAL);
newAMD.setValueRange(new Range(0, 1), SetRelation.EQUAL);
newAMD.setMean(new MDReal(mean));
emd.addAttribute(newAMD);
}
} else {
for (int i = 0; i < desirendNumberOfAttributes; i++) {
AttributeMetaData newAMD = new AttributeMetaData("att" + (i + 1), Ontology.REAL);
newAMD.setValueRange(new Range(0, 1), SetRelation.EQUAL);
newAMD.setMean(new MDReal(mean));
emd.addAttribute(newAMD);
}
}
return emd;
}
@Override
protected MetaData modifyMetaData(ExampleSetMetaData metaData) {
ExampleSetMetaData subset = attributeSelector.getMetaDataSubset(metaData, true);
Iterator<AttributeMetaData> amdIterator = metaData.getAllAttributes().iterator();
while (amdIterator.hasNext()) {
AttributeMetaData amd = amdIterator.next();
AttributeMetaData subsetAMD = subset.getAttributeByName(amd.getName());
if (subsetAMD == null) {
amdIterator.remove();
}
}
return metaData;
}
@Override
protected Collection<AttributeMetaData> modifyAttributeMetaData(
ExampleSetMetaData emd, AttributeMetaData amd) throws UndefinedParameterError {
AttributeMetaData newAMD =
new AttributeMetaData(amd.getName(), Ontology.NOMINAL, amd.getRole());
Set<String> valueSet = new TreeSet<String>();
if (getParameterAsInt(PARAMETER_RANGE_NAME_TYPE) == DiscretizationModel.RANGE_NAME_SHORT) {
for (int i = 0; i < getParameterAsInt(PARAMETER_NUMBER_OF_BINS); i++) {
valueSet.add("range" + (i + 1));
}
newAMD.setValueSet(valueSet, SetRelation.EQUAL);
} else {
newAMD.setValueSet(valueSet, SetRelation.SUPERSET);
}
return Collections.singletonList(newAMD);
}
private void setMetaData(
String value, AttributeMetaData targetAttribute, String i18nCode, Object[] i18nArguments) {
if (targetAttribute != null) {
if (targetAttribute.isNominal()) {
targetAttribute.getValueSet().add(value);
} else {
try {
targetAttribute.getValueRange().add(Double.parseDouble(value));
} catch (NumberFormatException e) {
// add warning
addError(
new SimpleProcessSetupError(
Severity.WARNING, getPortOwner(), i18nCode, i18nArguments));
}
}
}
}
/**
* Subclasses might override this method to define the meta data transformation performed by this
* operator. The default implementation takes all attributes specified by the {@link
* AttributeSubsetSelector} and passes them to {@link #modifyAttributeMetaData(ExampleSetMetaData,
* AttributeMetaData)} and replaces them accordingly.
*
* @throws UndefinedParameterError
*/
@Override
protected ExampleSetMetaData modifyMetaData(ExampleSetMetaData exampleSetMetaData)
throws UndefinedParameterError {
ExampleSetMetaData subsetMetaData =
attributeSelector.getMetaDataSubset(exampleSetMetaData, isSupportingAttributeRoles());
checkSelectedSubsetMetaData(subsetMetaData);
for (AttributeMetaData amd : subsetMetaData.getAllAttributes()) {
Collection<AttributeMetaData> replacement = null;
replacement = modifyAttributeMetaData(exampleSetMetaData, amd);
if (replacement != null) {
if (replacement.size() == 1) {
AttributeMetaData replacementAttribute = replacement.iterator().next();
replacementAttribute.setRole(
exampleSetMetaData.getAttributeByName(amd.getName()).getRole());
}
exampleSetMetaData.removeAttribute(amd);
exampleSetMetaData.addAllAttributes(replacement);
}
}
return exampleSetMetaData;
}
public Vector<String> getAttributeNames() {
Vector<String> names = new Vector<>();
Vector<String> regularNames = new Vector<>();
MetaData metaData = getMetaData();
if (metaData != null) {
if (metaData instanceof ExampleSetMetaData) {
ExampleSetMetaData emd = (ExampleSetMetaData) metaData;
for (AttributeMetaData amd : emd.getAllAttributes()) {
if (!isFilteredOut(amd) && isOfAllowedType(amd.getValueType())) {
if (amd.isSpecial()) {
names.add(amd.getName());
} else {
regularNames.add(amd.getName());
}
}
}
} else if (metaData instanceof ModelMetaData) {
ModelMetaData mmd = (ModelMetaData) metaData;
ExampleSetMetaData emd = mmd.getTrainingSetMetaData();
if (emd != null) {
for (AttributeMetaData amd : emd.getAllAttributes()) {
if (!isFilteredOut(amd) && isOfAllowedType(amd.getValueType())) {
if (amd.isSpecial()) {
names.add(amd.getName());
} else {
regularNames.add(amd.getName());
}
}
}
}
}
}
Collections.sort(names);
Collections.sort(regularNames);
names.addAll(regularNames);
return names;
}
private ExampleSetMetaData applyRulesOnMetaData(
List<String> rules, MetaData metaData, FilterConditon condition)
throws UndefinedParameterError {
if (metaData == null || !(metaData instanceof ExampleSetMetaData) || condition == null) {
return new ExampleSetMetaData();
}
ExampleSetMetaData sortedMetaData = new ExampleSetMetaData();
ExampleSetMetaData originalMetaData = (ExampleSetMetaData) metaData;
Collection<AttributeMetaData> allAttributes = originalMetaData.getAllAttributes();
// iterate over all rules
for (String currentRule : rules) {
// iterate over all original attributes and check if rule applies
Iterator<AttributeMetaData> iterator = allAttributes.iterator();
while (iterator.hasNext()) {
AttributeMetaData attrMD = iterator.next();
// skip special attributes
if (attrMD.isSpecial()) {
continue;
}
// if rule applies, remove attribute from unmachted list and add it to rules matched
// list
if (condition.match(currentRule, attrMD.getName())) {
iterator.remove();
sortedMetaData.addAttribute(attrMD);
}
}
}
if (!getParameterAsString(PARAMETER_HANDLE_UNMATCHED_ATTRIBUTES)
.equals(REMOVE_UNMATCHED_MODE)) {
sortedMetaData.addAllAttributes(allAttributes);
}
return sortedMetaData;
}
public MetaDataInfo isFilteredOutMetaData(AttributeMetaData attribute, ParameterHandler handler)
throws ConditionCreationException {
if ((attributeNames == null) || (attributeNames.length() == 0))
throw new ConditionCreationException(
"The condition for a single attribute needs a non-empty attribute parameter string.");
boolean found = false;
for (String attributeName : attributeNames.split("\\|")) {
if (attribute.getName().equals(attributeName)) {
found = true;
}
}
return found ? MetaDataInfo.NO : MetaDataInfo.YES;
}
@Override
protected MetaData modifyMetaData(ExampleSetMetaData metaData) throws UndefinedParameterError {
AttributeMetaData amd =
metaData.getAttributeByName(getParameterAsString(PARAMETER_ATTRIBUTE_NAME));
if (amd != null) {
AttributeMetaData newAttribute = amd.clone();
newAttribute.setType(Ontology.DATE_TIME);
newAttribute.getMean().setUnkown();
newAttribute.setValueSetRelation(SetRelation.UNKNOWN);
if (!getParameterAsBoolean(PARAMETER_KEEP_OLD_ATTRIBUTE)) metaData.removeAttribute(amd);
else newAttribute.setName(newAttribute.getName() + "_AS_DATE");
metaData.addAttribute(newAttribute);
}
return metaData;
}
@Override
public ExampleSetMetaData applyOnFilteredMetaData(ExampleSetMetaData metaData) {
for (AttributeMetaData amd : metaData.getAllAttributes()) {
if (amd.isNumerical() && !amd.isSpecial()) {
Range range = amd.getValueRange();
amd.setValueRange(
new Range(0, Math.max(Math.abs(range.getLower()), Math.abs(range.getUpper()))),
amd.getValueSetRelation());
amd.getMean().setUnkown();
}
}
return metaData;
}
@Override
public ExampleSetMetaData applyOnFilteredMetaData(ExampleSetMetaData emd) {
boolean round = getParameterAsBoolean(PARAMETER_ROUND);
for (AttributeMetaData amd : emd.getAllAttributes()) {
if ((Ontology.ATTRIBUTE_VALUE_TYPE.isA(amd.getValueType(), Ontology.NUMERICAL))
&& (!Ontology.ATTRIBUTE_VALUE_TYPE.isA(amd.getValueType(), Ontology.INTEGER))) {
amd.setType(Ontology.INTEGER);
}
if (round) {
amd.setValueRange(
new Range(
Math.round(amd.getValueRange().getLower()),
Math.round(amd.getValueRange().getUpper())),
SetRelation.EQUAL);
} else {
amd.setValueRange(
new Range((long) amd.getValueRange().getLower(), (long) amd.getValueRange().getUpper()),
SetRelation.EQUAL);
}
}
return emd;
}
@Override
protected MetaData modifyMetaData(ExampleSetMetaData metaData) {
try {
AttributeMetaData amd1 =
metaData.getAttributeByName(getParameterAsString(PARAMETER_FIRST_ATTRIBUTE));
AttributeMetaData amd2 =
metaData.getAttributeByName(getParameterAsString(PARAMETER_SECOND_ATTRIBUTE));
if (amd1 != null && amd2 != null) {
String role1 = amd1.getRole();
amd1.setRole(amd2.getRole());
amd2.setRole(role1);
}
} catch (UndefinedParameterError e) {
}
return metaData;
}
@Override
protected Collection<AttributeMetaData> modifyAttributeMetaData(
ExampleSetMetaData emd, AttributeMetaData amd) throws UndefinedParameterError {
amd.setValueSetRelation(SetRelation.SUBSET);
return Collections.singleton(amd);
}
/** Standard behavior is to return true for all numerical attributes */
@Override
public boolean supportsAttribute(AttributeMetaData amd) {
return amd.isNumerical();
}
@Override
protected MetaData modifyMetaData(ExampleSetMetaData metaData) throws UndefinedParameterError {
ExampleSetMetaData resultMD = metaData.clone();
resultMD.clear();
// add group by attributes
if (isParameterSet(PARAMETER_GROUP_BY_ATTRIBUTES)
&& !getParameterAsString(PARAMETER_GROUP_BY_ATTRIBUTES).isEmpty()) {
String attributeRegex = getParameterAsString(PARAMETER_GROUP_BY_ATTRIBUTES);
Pattern pattern = Pattern.compile(attributeRegex);
for (AttributeMetaData amd : metaData.getAllAttributes()) {
if (pattern.matcher(amd.getName()).matches()) {
if (amd.isNumerical()
&& getCompatibilityLevel().isAtMost(VERSION_5_1_6)) { // converting type to mimic
// NumericalToPolynomial used below
amd.setType(Ontology.NOMINAL);
amd.setValueSet(Collections.<String>emptySet(), SetRelation.SUPERSET);
}
resultMD.addAttribute(amd);
}
}
resultMD.getNumberOfExamples().reduceByUnknownAmount();
}
if (resultMD.getAllAttributes().isEmpty() && getCompatibilityLevel().isAtMost(VERSION_5_1_6)) {
AttributeMetaData allGroup = new AttributeMetaData(GENERIC_GROUP_NAME, Ontology.NOMINAL);
Set<String> values = new TreeSet<String>();
values.add(GENERIC_ALL_NAME);
allGroup.setValueSet(values, SetRelation.EQUAL);
resultMD.addAttribute(allGroup);
resultMD.setNumberOfExamples(new MDInteger(1));
}
// add aggregated attributes of default aggregation: They will apply only to those attribute not
// mentioned explicitly
List<String[]> parameterList = this.getParameterList(PARAMETER_AGGREGATION_ATTRIBUTES);
HashSet<String> explicitDefinedAttributes = new HashSet<String>();
for (String[] function : parameterList) {
explicitDefinedAttributes.add(function[0]);
}
if (getParameterAsBoolean(PARAMETER_USE_DEFAULT_AGGREGATION)) {
String defaultFunction = getParameterAsString(PARAMETER_DEFAULT_AGGREGATION_FUNCTION);
ExampleSetMetaData metaDataSubset = attributeSelector.getMetaDataSubset(metaData, false);
for (AttributeMetaData amd : metaDataSubset.getAllAttributes()) {
if (!explicitDefinedAttributes.contains(amd.getName())) {
AttributeMetaData newAMD =
AggregationFunction.getAttributeMetaData(
defaultFunction, amd, getExampleSetInputPort());
if (newAMD != null) resultMD.addAttribute(newAMD);
}
}
}
// add explicitly defined attributes of list
for (String[] function : parameterList) {
AttributeMetaData amd = metaData.getAttributeByName(function[0]);
if (amd != null) {
AttributeMetaData newMD =
AggregationFunction.getAttributeMetaData(function[1], amd, getExampleSetInputPort());
if (newMD != null) resultMD.addAttribute(newMD);
} else {
// in this case we should register a warning, but continue anyway in cases we don't have the
// correct set available
getExampleSetInputPort()
.addError(
new SimpleMetaDataError(
Severity.WARNING,
getExampleSetInputPort(),
"aggregation.attribute_unknown",
function[0]));
AttributeMetaData newAMD =
AggregationFunction.getAttributeMetaData(
function[1],
new AttributeMetaData(function[0], Ontology.ATTRIBUTE_VALUE),
getExampleSetInputPort());
if (newAMD != null) resultMD.addAttribute(newAMD);
}
}
return resultMD;
}