/**
* Determines and returns (if possible) the structure (internally the header) of the data set as
* an empty set of instances.
*
* @return the structure of the data set as an empty set of Instances
* @exception IOException if an error occurs
*/
@Override
public Instances getStructure() throws IOException {
if ((m_sourceFile == null) && (m_sourceReader == null)) {
throw new IOException("No source has been specified");
}
if (m_structure == null) {
try {
m_st = new StreamTokenizer(m_sourceReader);
initTokenizer(m_st);
readStructure(m_st);
} catch (FileNotFoundException ex) {
}
}
return m_structure;
}
/**
* Return the full data set. If the structure hasn't yet been determined by a call to getStructure
* then method should do so before processing the rest of the data set.
*
* @return the structure of the data set as an empty set of Instances
* @exception IOException if there is no source or parsing fails
*/
@Override
public Instances getDataSet() throws IOException {
if ((m_sourceFile == null) && (m_sourceReader == null)) {
throw new IOException("No source has been specified");
}
if (m_structure == null) {
getStructure();
}
if (m_st == null) {
m_st = new StreamTokenizer(m_sourceReader);
initTokenizer(m_st);
}
m_st.ordinaryChar(m_FieldSeparator.charAt(0));
m_cumulativeStructure = new ArrayList<Hashtable<Object, Integer>>(m_structure.numAttributes());
for (int i = 0; i < m_structure.numAttributes(); i++) {
m_cumulativeStructure.add(new Hashtable<Object, Integer>());
}
m_cumulativeInstances = new ArrayList<ArrayList<Object>>();
ArrayList<Object> current;
while ((current = getInstance(m_st)) != null) {
m_cumulativeInstances.add(current);
}
ArrayList<Attribute> atts = new ArrayList<Attribute>(m_structure.numAttributes());
for (int i = 0; i < m_structure.numAttributes(); i++) {
String attname = m_structure.attribute(i).name();
Hashtable<Object, Integer> tempHash = m_cumulativeStructure.get(i);
if (tempHash.size() == 0) {
atts.add(new Attribute(attname));
} else {
if (m_StringAttributes.isInRange(i)) {
atts.add(new Attribute(attname, (ArrayList<String>) null));
} else {
ArrayList<String> values = new ArrayList<String>(tempHash.size());
// add dummy objects in order to make the ArrayList's size == capacity
for (int z = 0; z < tempHash.size(); z++) {
values.add("dummy");
}
Enumeration e = tempHash.keys();
while (e.hasMoreElements()) {
Object ob = e.nextElement();
// if (ob instanceof Double) {
int index = ((Integer) tempHash.get(ob)).intValue();
String s = ob.toString();
if (s.startsWith("'") || s.startsWith("\"")) s = s.substring(1, s.length() - 1);
values.set(index, new String(s));
// }
}
atts.add(new Attribute(attname, values));
}
}
}
// make the instances
String relationName;
if (m_sourceFile != null)
relationName = (m_sourceFile.getName()).replaceAll("\\.[cC][sS][vV]$", "");
else relationName = "stream";
Instances dataSet = new Instances(relationName, atts, m_cumulativeInstances.size());
for (int i = 0; i < m_cumulativeInstances.size(); i++) {
current = m_cumulativeInstances.get(i);
double[] vals = new double[dataSet.numAttributes()];
for (int j = 0; j < current.size(); j++) {
Object cval = current.get(j);
if (cval instanceof String) {
if (((String) cval).compareTo(m_MissingValue) == 0) {
vals[j] = Utils.missingValue();
} else {
if (dataSet.attribute(j).isString()) {
vals[j] = dataSet.attribute(j).addStringValue((String) cval);
} else if (dataSet.attribute(j).isNominal()) {
// find correct index
Hashtable<Object, Integer> lookup = m_cumulativeStructure.get(j);
int index = ((Integer) lookup.get(cval)).intValue();
vals[j] = index;
} else {
throw new IllegalStateException(
"Wrong attribute type at position " + (i + 1) + "!!!");
}
}
} else if (dataSet.attribute(j).isNominal()) {
// find correct index
Hashtable<Object, Integer> lookup = m_cumulativeStructure.get(j);
int index = ((Integer) lookup.get(cval)).intValue();
vals[j] = index;
} else if (dataSet.attribute(j).isString()) {
vals[j] = dataSet.attribute(j).addStringValue("" + cval);
} else {
vals[j] = ((Double) cval).doubleValue();
}
}
dataSet.add(new DenseInstance(1.0, vals));
}
m_structure = new Instances(dataSet, 0);
setRetrieval(BATCH);
m_cumulativeStructure = null; // conserve memory
// close the stream
m_sourceReader.close();
return dataSet;
}
@Override
public Instance getNextInstance(Instances structure) throws IOException {
m_structure = structure;
if (getRetrieval() == BATCH) {
throw new IOException("Cannot mix getting instances in both incremental and batch modes");
}
setRetrieval(INCREMENTAL);
if (m_dataDumper != null) {
// close the uneeded temp files (if necessary)
m_dataDumper.close();
m_dataDumper = null;
}
if (m_rowBuffer.size() > 0 && m_incrementalReader == null) {
StringBuilder tempB = new StringBuilder();
for (String r : m_rowBuffer) {
tempB.append(r).append("\n");
}
m_numBufferedRows = m_rowBuffer.size();
Reader batchReader = new BufferedReader(new StringReader(tempB.toString()));
m_incrementalReader =
new ArffReader(batchReader, m_structure, 0, 0, m_fieldSeparatorAndEnclosures);
m_rowBuffer.clear();
}
if (m_numBufferedRows == 0) {
// m_incrementalReader = new ArffReader(m_sourceReader, m_structure, 0,
// 0);
m_numBufferedRows = -1;
m_st = new StreamTokenizer(m_sourceReader);
initTokenizer(m_st);
m_st.ordinaryChar(m_FieldSeparator.charAt(0));
//
m_incrementalReader = null;
}
Instance current = null;
if (m_sourceReader != null) {
if (m_incrementalReader != null) {
current = m_incrementalReader.readInstance(m_structure);
} else {
if (getInstance(m_st) != null) {
current = makeInstance();
}
}
if (current == null) {}
if (m_numBufferedRows > 0) {
m_numBufferedRows--;
}
}
if ((m_sourceReader != null) && (current == null)) {
try {
// close the stream
m_sourceReader.close();
m_sourceReader = null;
// reset();
} catch (Exception ex) {
ex.printStackTrace();
}
}
return current;
}
private void readHeader() throws IOException {
m_rowCount = 1;
m_incrementalReader = null;
m_current = new ArrayList<Object>();
openTempFiles();
m_rowBuffer = new ArrayList<String>();
String firstRow = m_sourceReader.readLine();
if (firstRow == null) {
throw new IOException("No data in the file!");
}
if (m_noHeaderRow) {
m_rowBuffer.add(firstRow);
}
ArrayList<Attribute> attribNames = new ArrayList<Attribute>();
// now tokenize to determine attribute names (or create att names if
// no header row
StringReader sr = new StringReader(firstRow + "\n");
// System.out.print(firstRow + "\n");
m_st = new StreamTokenizer(sr);
initTokenizer(m_st);
m_st.ordinaryChar(m_FieldSeparator.charAt(0));
int attNum = 1;
StreamTokenizerUtils.getFirstToken(m_st);
if (m_st.ttype == StreamTokenizer.TT_EOF) {
StreamTokenizerUtils.errms(m_st, "premature end of file");
}
boolean first = true;
boolean wasSep;
while (m_st.ttype != StreamTokenizer.TT_EOL && m_st.ttype != StreamTokenizer.TT_EOF) {
// Get next token
if (!first) {
StreamTokenizerUtils.getToken(m_st);
}
if (m_st.ttype == m_FieldSeparator.charAt(0) || m_st.ttype == StreamTokenizer.TT_EOL) {
wasSep = true;
} else {
wasSep = false;
String attName = null;
if (m_noHeaderRow) {
attName = "att" + attNum;
attNum++;
} else {
attName = m_st.sval;
}
attribNames.add(new Attribute(attName, (java.util.List<String>) null));
}
if (!wasSep) {
StreamTokenizerUtils.getToken(m_st);
}
first = false;
}
String relationName;
if (m_sourceFile != null) {
relationName = (m_sourceFile.getName()).replaceAll("\\.[cC][sS][vV]$", "");
} else {
relationName = "stream";
}
m_structure = new Instances(relationName, attribNames, 0);
m_NominalAttributes.setUpper(m_structure.numAttributes() - 1);
m_StringAttributes.setUpper(m_structure.numAttributes() - 1);
m_dateAttributes.setUpper(m_structure.numAttributes() - 1);
m_numericAttributes.setUpper(m_structure.numAttributes() - 1);
m_nominalVals = new HashMap<Integer, LinkedHashSet<String>>();
m_types = new TYPE[m_structure.numAttributes()];
for (int i = 0; i < m_structure.numAttributes(); i++) {
if (m_NominalAttributes.isInRange(i)) {
m_types[i] = TYPE.NOMINAL;
LinkedHashSet<String> ts = new LinkedHashSet<String>();
m_nominalVals.put(i, ts);
} else if (m_StringAttributes.isInRange(i)) {
m_types[i] = TYPE.STRING;
} else if (m_dateAttributes.isInRange(i)) {
m_types[i] = TYPE.DATE;
} else if (m_numericAttributes.isInRange(i)) {
m_types[i] = TYPE.NUMERIC;
} else {
m_types[i] = TYPE.UNDETERMINED;
}
}
if (m_nominalLabelSpecs.size() > 0) {
for (String spec : m_nominalLabelSpecs) {
String[] attsAndLabels = spec.split(":");
if (attsAndLabels.length == 2) {
String[] labels = attsAndLabels[1].split(",");
try {
// try as a range string first
Range tempR = new Range();
tempR.setRanges(attsAndLabels[0].trim());
tempR.setUpper(m_structure.numAttributes() - 1);
int[] rangeIndexes = tempR.getSelection();
for (int i = 0; i < rangeIndexes.length; i++) {
m_types[rangeIndexes[i]] = TYPE.NOMINAL;
LinkedHashSet<String> ts = new LinkedHashSet<String>();
for (String lab : labels) {
ts.add(lab);
}
m_nominalVals.put(rangeIndexes[i], ts);
}
} catch (IllegalArgumentException e) {
// one or more named attributes?
String[] attNames = attsAndLabels[0].split(",");
for (String attN : attNames) {
Attribute a = m_structure.attribute(attN.trim());
if (a != null) {
int attIndex = a.index();
m_types[attIndex] = TYPE.NOMINAL;
LinkedHashSet<String> ts = new LinkedHashSet<String>();
for (String lab : labels) {
ts.add(lab);
}
m_nominalVals.put(attIndex, ts);
}
}
}
}
}
}
// Prevents the first row from getting lost in the
// case where there is no header row and we're
// running in batch mode
if (m_noHeaderRow && getRetrieval() == BATCH) {
StreamTokenizer tempT = new StreamTokenizer(new StringReader(firstRow));
initTokenizer(tempT);
tempT.ordinaryChar(m_FieldSeparator.charAt(0));
String checked = getInstance(tempT);
dumpRow(checked);
}
m_st = new StreamTokenizer(m_sourceReader);
initTokenizer(m_st);
m_st.ordinaryChar(m_FieldSeparator.charAt(0));
// try and determine a more accurate structure from the first batch
readData(false || getRetrieval() == BATCH);
makeStructure();
}