/**
* Creates a new ValueArray with classes. New ValueArray is not aligned with source one
* unfortunately so have to send results to each chunk owner using Atomic.
*/
@Override
public void map(Key key) {
assert key.home();
if (Job.isRunning(_job.self())) {
ValueArray va = DKV.get(_arykey).get();
AutoBuffer bits = va.getChunk(key);
long startRow = va.startRow(ValueArray.getChunkIndex(key));
int rows = va.rpc(ValueArray.getChunkIndex(key));
int rpc = (int) (ValueArray.CHUNK_SZ / ROW_SIZE);
long chunk = ValueArray.chknum(startRow, va.numRows(), ROW_SIZE);
long updatedChk = chunk;
long updatedRow = startRow;
double[] values = new double[_cols.length - 1];
ClusterDist cd = new ClusterDist();
int[] clusters = new int[rows];
int count = 0;
for (int row = 0; row < rows; row++) {
KMeans.datad(va, bits, row, _cols, _normalized, values);
KMeans.closest(_clusters, values, cd);
chunk = ValueArray.chknum(startRow + row, va.numRows(), ROW_SIZE);
if (chunk != updatedChk) {
updateClusters(clusters, count, updatedChk, va.numRows(), rpc, updatedRow);
updatedChk = chunk;
updatedRow = startRow + row;
count = 0;
}
clusters[count++] = cd._cluster;
}
if (count > 0) updateClusters(clusters, count, chunk, va.numRows(), rpc, updatedRow);
_job.updateProgress(1);
}
_job = null;
_arykey = null;
_cols = null;
_clusters = null;
}
@Override
public void map(Key key) {
_rows = new long[_clusters.length];
_dist = new double[_clusters.length];
assert key.home();
ValueArray va = DKV.get(_arykey).get();
AutoBuffer bits = va.getChunk(key);
int rows = va.rpc(ValueArray.getChunkIndex(key));
double[] values = new double[_cols.length - 1];
ClusterDist cd = new ClusterDist();
for (int row = 0; row < rows; row++) {
KMeans.datad(va, bits, row, _cols, _normalized, values);
KMeans.closest(_clusters, values, cd);
_rows[cd._cluster]++;
_dist[cd._cluster] += cd._dist;
}
_arykey = null;
_cols = null;
_clusters = null;
}
/**
* Map function for distributed parsing of the CSV files.
*
* <p>In first phase it calculates the min, max, means, encodings and other statistics about the
* dataset, determines the number of columns.
*
* <p>The second pass then encodes the parsed dataset to the result key, splitting it into equal
* sized chunks.
*/
@Override
public void map(Key key) {
try {
Key aryKey = null;
boolean arraylet = key._kb[0] == Key.ARRAYLET_CHUNK;
boolean skipFirstLine = _skipFirstLine;
if (arraylet) {
aryKey = ValueArray.getArrayKey(key);
_chunkId = ValueArray.getChunkIndex(key);
skipFirstLine = skipFirstLine || (ValueArray.getChunkIndex(key) != 0);
}
switch (_phase) {
case ONE:
assert (_ncolumns != 0);
// initialize the column statistics
phaseOneInitialize();
// perform the parse
CsvParser p = new CsvParser(aryKey, _ncolumns, _sep, _decSep, this, skipFirstLine);
p.parse(key);
if (arraylet) {
long idx = ValueArray.getChunkIndex(key);
int idx2 = (int) idx;
assert idx2 == idx;
assert (_nrows[idx2] == 0)
: idx
+ ": "
+ Arrays.toString(_nrows)
+ " ("
+ _nrows[idx2]
+ " -- "
+ _myrows
+ ")";
_nrows[idx2] = _myrows;
}
break;
case TWO:
assert (_ncolumns != 0);
// initialize statistics - invalid rows, sigma and row size
phaseTwoInitialize();
// calculate the first row and the number of rows to parse
int firstRow = 0;
int lastRow = _myrows;
_myrows = 0;
if (arraylet) {
long origChunkIdx = ValueArray.getChunkIndex(key);
firstRow = (origChunkIdx == 0) ? 0 : _nrows[(int) origChunkIdx - 1];
lastRow = _nrows[(int) origChunkIdx];
}
int rowsToParse = lastRow - firstRow;
// create the output streams
_outputStreams2 = createRecords(firstRow, rowsToParse);
assert (_outputStreams2.length > 0);
_ab = _outputStreams2[0].initialize();
// perform the second parse pass
CsvParser p2 = new CsvParser(aryKey, _ncolumns, _sep, _decSep, this, skipFirstLine);
p2.parse(key);
// store the last stream if not stored during the parse
if (_ab != null) _outputStreams2[_outputIdx].store();
break;
default:
assert (false);
}
ParseStatus.update(_resultKey, DKV.get(key).length(), _phase);
} catch (Exception e) {
e.printStackTrace();
_error = e.getMessage();
}
}
