public ArrayList<Pair<String, Counter<String>>> compileUsageForAllColumns(
int minLength, int maxLength, int maxLines) {
if (maxLines < 0) maxLines = Integer.MAX_VALUE;
int nTillReport = 100000;
int nObjsRead = 0;
int nObjsReadTotal = 0;
ArrayList<Pair<String, Counter<String>>> returnCounts =
new ArrayList<Pair<String, Counter<String>>>();
try {
int objNumCol = DumpColumnConfigInfo.getMuseumIdColumnIndex();
DumpColumnConfigInfo[] colDumpInfo = null;
try {
colDumpInfo = DumpColumnConfigInfo.getAllColumnInfo(columnNames);
} catch (IllegalArgumentException iae) {
String msg =
"Mismatched columns between metadata configuration and metadata file.\n"
+ iae.getLocalizedMessage();
debug(0, msg);
throw new RuntimeException(msg);
}
int nCols = columnNames.length;
boolean[] skipCol = new boolean[nCols];
int[] countIndexes = new int[nCols];
for (int iCol = 0; iCol < nCols; iCol++) {
if (!(skipCol[iCol] = !colDumpInfo[iCol].columnMinedForAnyFacet())) {
countIndexes[iCol] = returnCounts.size();
returnCounts.add(
new Pair<String, Counter<String>>(columnNames[iCol], new Counter<String>()));
} else {
countIndexes[iCol] = -1;
}
}
resetToLine1();
Pair<Integer, ArrayList<String>> objInfo = null;
while ((objInfo = getNextObjectAsColumns()) != null) {
int currID = objInfo.getFirst();
ArrayList<String> objStrings = objInfo.getSecond();
assert (objStrings.size() == nCols) : "Bad parse for obj:" + currID;
// Have a complete line. Check for validity
String objNumStr = objStrings.get(objNumCol);
if (DumpColumnConfigInfo.objNumIsValid(objNumStr)) {
nObjsReadTotal++;
if (++nObjsRead >= nTillReport) {
nObjsRead = 0;
debug(1, "MDR.compileUsageForAllColumns: read: " + nObjsReadTotal + " objects");
}
for (int iCol = 1; iCol < nCols; iCol++) {
if (skipCol[iCol]) // Do not bother with misc cols
continue;
String source = objStrings.get(iCol);
if ((source.length() >= minLength) && (source.length() < maxLength)) {
ArrayList<Pair<String, ArrayList<String>>> tokenList =
ConfigStringUtils.prepareSourceTokens(source, colDumpInfo[iCol]);
for (Pair<String, ArrayList<String>> pair : tokenList) {
String subtoken = pair.getFirst().trim();
if (subtoken.length() > 1)
returnCounts.get(countIndexes[iCol]).getSecond().incrementCount(subtoken, 1);
}
}
}
}
}
} catch (RuntimeException e) {
e.printStackTrace();
}
return returnCounts;
}
public Counter<String> compileUsageForColumn(
int iCol, int minLength, int maxLength, int maxLines, StringBuilder statusSB) {
Counter<String> vocabCounts = null;
int nLinesEmpty = 0;
int nLinesTooLong = 0;
int reportFreq = 10000;
vocabCounts = new Counter<String>();
// noiseTokens.add("each");
if (maxLines < 0) maxLines = Integer.MAX_VALUE;
if (iCol < 1) throw new RuntimeException("Illegal column index: " + iCol);
try {
DumpColumnConfigInfo colDumpInfo = null;
try {
colDumpInfo = DumpColumnConfigInfo.GetColInfo(columnNames[iCol], "Compile Usage");
} catch (IllegalArgumentException iae) {
String msg = "Problem with metadata configuration.\n" + iae.getLocalizedMessage();
debug(0, msg);
throw new RuntimeException(msg);
}
resetToLine1();
String token = null;
while (((token = getNextLineColumn(iCol)) != null) && (currLine < maxLines)) {
if (currLine % reportFreq == 0) {
debug(
1,
"MDR.compileUsageForColumn("
+ iCol
+ ") read "
+ currLine
+ " lines so far..."
+ "\n Skipped "
+ nLinesEmpty
+ " empty lines and "
+ nLinesTooLong
+ " too long lines"
+ "\n Found "
+ vocabCounts.size()
+ " distinct tokens (a total of: "
+ vocabCounts.totalCount()
+ " tokens)");
}
if (token.length() < minLength) {
nLinesEmpty++;
continue;
}
if (token.length() > maxLength) {
nLinesTooLong++;
continue;
}
ArrayList<Pair<String, ArrayList<String>>> tokenList =
ConfigStringUtils.prepareSourceTokens(token, colDumpInfo);
for (Pair<String, ArrayList<String>> pair : tokenList) {
String subtoken = pair.getFirst();
vocabCounts.incrementCount(subtoken, 1);
}
}
} catch (RuntimeException e) {
e.printStackTrace();
}
debug(
1,
"MDR.compileUsageForColumn("
+ iCol
+ ") read "
+ currLine
+ " lines."
+ "\n Skipped "
+ nLinesEmpty
+ " empty lines and "
+ nLinesTooLong
+ " too long lines"
+ "\n Found "
+ vocabCounts.size()
+ " distinct tokens (a total of: "
+ vocabCounts.totalCount()
+ " tokens)");
if (statusSB != null) {
statusSB.append("Read ");
statusSB.append(currLine);
statusSB.append(" lines. Found ");
statusSB.append(vocabCounts.size());
statusSB.append(" distinct tokens");
}
return vocabCounts;
}