public void count() {
TIntIntHashMap docCounts = new TIntIntHashMap();
int index = 0;
if (instances.size() == 0) {
logger.info("Instance list is empty");
return;
}
if (instances.get(0).getData() instanceof FeatureSequence) {
for (Instance instance : instances) {
FeatureSequence features = (FeatureSequence) instance.getData();
for (int i = 0; i < features.getLength(); i++) {
docCounts.adjustOrPutValue(features.getIndexAtPosition(i), 1, 1);
}
int[] keys = docCounts.keys();
for (int i = 0; i < keys.length - 1; i++) {
int feature = keys[i];
featureCounts[feature] += docCounts.get(feature);
documentFrequencies[feature]++;
}
docCounts = new TIntIntHashMap();
index++;
if (index % 1000 == 0) {
System.err.println(index);
}
}
} else if (instances.get(0).getData() instanceof FeatureVector) {
for (Instance instance : instances) {
FeatureVector features = (FeatureVector) instance.getData();
for (int location = 0; location < features.numLocations(); location++) {
int feature = features.indexAtLocation(location);
double value = features.valueAtLocation(location);
documentFrequencies[feature]++;
featureCounts[feature] += value;
}
index++;
if (index % 1000 == 0) {
System.err.println(index);
}
}
} else {
logger.info("Unsupported data class: " + instances.get(0).getData().getClass().getName());
}
}
public Instance pipe(Instance carrier) {
TokenSequence ts = (TokenSequence) carrier.getData();
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
String s = t.getText();
if (distinguishBorders) s = startBorderChar + s + endBorderChar;
int slen = s.length();
for (int j = 0; j < gramSizes.length; j++) {
int size = gramSizes[j];
for (int k = 0; k < (slen - size) + 1; k++)
t.setFeatureValue((prefix + s.substring(k, k + size)).intern(), 1.0);
}
}
return carrier;
}
public Instance pipe(Instance carrier) {
TokenSequence ts = (TokenSequence) carrier.getData();
for (int i = 0; i < ts.size(); i++) {
Token t = ts.get(i);
String s = t.getText();
if (distinguishBorders) s = startBorderChar + s + endBorderChar;
int slen = s.length();
for (int j = 0; j < gramSizes.length; j++) {
int size = gramSizes[j];
for (int k = 0; k < slen - size; k++)
t.setFeatureValue(
s.substring(k, k + size), 1.0); // original was substring(k, size), changed by Fuchun
}
}
return carrier;
}
public Instance pipe(Instance carrier) {
TokenSequence ts = (TokenSequence) carrier.getData();
TokenSequence targets =
carrier.getTarget() instanceof TokenSequence ? (TokenSequence) carrier.getTarget() : null;
TokenSequence source =
carrier.getSource() instanceof TokenSequence ? (TokenSequence) carrier.getSource() : null;
StringBuffer sb = new StringBuffer();
if (prefix != null) sb.append(prefix);
sb.append("name: " + carrier.getName() + "\n");
for (int i = 0; i < ts.size(); i++) {
if (source != null) {
sb.append(source.get(i).getText());
sb.append(' ');
}
if (carrier.getTarget() instanceof TokenSequence) {
sb.append(((TokenSequence) carrier.getTarget()).get(i).getText());
sb.append(' ');
}
if (carrier.getTarget() instanceof FeatureSequence) {
sb.append(((FeatureSequence) carrier.getTarget()).getObjectAtPosition(i).toString());
sb.append(' ');
}
PropertyList pl = ts.get(i).getFeatures();
if (pl != null) {
PropertyList.Iterator iter = pl.iterator();
while (iter.hasNext()) {
iter.next();
double v = iter.getNumericValue();
if (v == 1.0) sb.append(iter.getKey());
else sb.append(iter.getKey() + '=' + v);
sb.append(' ');
}
}
sb.append('\n');
}
System.out.print(sb.toString());
return carrier;
}
public Instance pipe(Instance carrier) {
carrier.setData(new TokenIterator((TokenSequence) carrier.getData()));
return carrier;
}