/**
* @param outer execution mode of outer parfor loop
* @param inner execution mode of inner parfor loop
* @param instType execution mode of instructions
*/
private void runParForDataPartitioningTest(
PDataPartitioner partitioner,
PExecMode mode,
boolean small,
boolean sparse,
boolean multiParts) {
RUNTIME_PLATFORM oldRT = rtplatform;
boolean oldUseSparkConfig = DMLScript.USE_LOCAL_SPARK_CONFIG;
if (partitioner == PDataPartitioner.REMOTE_SPARK || mode == PExecMode.REMOTE_SPARK) {
DMLScript.USE_LOCAL_SPARK_CONFIG = true;
rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK;
}
try {
// inst exec type, influenced via rows
int rows = small ? rows1 : rows2;
int cols = small ? cols1 : cols2;
// script
int scriptNum = -1;
switch (partitioner) {
case NONE:
scriptNum = 1;
break;
case LOCAL:
if (mode == PExecMode.LOCAL) scriptNum = 2;
else scriptNum = 3;
break;
case REMOTE_MR:
if (mode == PExecMode.LOCAL) {
if (!multiParts) scriptNum = 4;
else scriptNum = 6;
} else scriptNum = 5;
break;
case REMOTE_SPARK:
if (mode == PExecMode.LOCAL) {
if (!multiParts) scriptNum = 7;
else scriptNum = 9;
} else scriptNum = 8;
break;
default:
// do nothing
}
TestConfiguration config = getTestConfiguration(TEST_NAME);
config.addVariable("rows", rows);
config.addVariable("cols", cols);
loadTestConfiguration(config);
/* This is for running the junit test the new way, i.e., construct the arguments directly */
String HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml";
programArgs =
new String[] {
"-args", input("V"), Integer.toString(rows), Integer.toString(cols), output("R")
};
fullRScriptName = HOME + TEST_NAME + (multiParts ? "6" : "") + ".R";
rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
long seed = System.nanoTime();
double sparsity = -1;
if (sparse) sparsity = sparsity2;
else sparsity = sparsity1;
double[][] V = getRandomMatrix(rows, cols, 0, 1, sparsity, seed);
writeInputMatrix("V", V, true);
boolean exceptionExpected = false;
runTest(true, exceptionExpected, null, -1);
runRScript(true);
// compare matrices
HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R");
HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout");
TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R");
} finally {
rtplatform = oldRT;
DMLScript.USE_LOCAL_SPARK_CONFIG = oldUseSparkConfig;
}
}
/**
* Shared test driver for tests of univariate statistics over continuous, scaled, and weighted
* data
*
* @param sz size of primary input vector
* @param rng range of randomly-generated values to use
* @param sp sparsity of generated data
* @param rt backend platform to test
*/
protected void testWeightedScaleWithR(SIZE sz, RANGE rng, SPARSITY sp, RUNTIME_PLATFORM rt) {
RUNTIME_PLATFORM oldrt = rtplatform;
rtplatform = rt;
try {
TestConfiguration config = getTestConfiguration("WeightedScaleTest");
config.addVariable("rows1", sz.size);
config.addVariable("rows2", rows2);
loadTestConfiguration(config);
// This is for running the junit test the new way, i.e., construct
// the arguments directly
String S_HOME = SCRIPT_DIR + TEST_DIR;
fullDMLScriptName = S_HOME + "WeightedScaleTest" + ".dml";
programArgs =
new String[] {
"-args",
input("vector"),
Integer.toString(sz.size),
input("weight"),
input("prob"),
Integer.toString(rows2),
output("mean"),
output("std"),
output("se"),
output("var"),
output("cv"),
output("min"),
output("max"),
output("rng"),
output("g1"),
output("se_g1"),
output("g2"),
output("se_g2"),
output("median"),
output("iqm"),
output("out_minus"),
output("out_plus"),
output("quantile")
};
fullRScriptName = S_HOME + "WeightedScaleTest" + ".R";
rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir();
createHelperMatrix();
double[][] vector =
getRandomMatrix(sz.size, 1, rng.min, rng.max, sp.sparsity, System.currentTimeMillis());
double[][] weight = getRandomMatrix(sz.size, 1, 1, 10, 1, System.currentTimeMillis());
OrderStatisticsTest.round(weight);
double[][] prob = getRandomMatrix(rows2, 1, 0, 1, 1, System.currentTimeMillis());
writeInputMatrix("vector", vector, true);
writeInputMatrix("weight", weight, true);
writeInputMatrix("prob", prob, true);
//
// Expected number of jobs:
// Reblock - 1 job
// While loop iteration - 10 jobs
// Final output write - 1 job
runTest(true, false, null, -1);
runRScript(true);
// disableOutAndExpectedDeletion();
for (String file : config.getOutputFiles()) {
// NOte that some files do not contain matrix, but just a single
// scalar value inside
HashMap<CellIndex, Double> dmlfile;
HashMap<CellIndex, Double> rfile;
if (file.endsWith(".scalar")) {
file = file.replace(".scalar", "");
dmlfile = readDMLScalarFromHDFS(file);
rfile = readRScalarFromFS(file);
} else {
dmlfile = readDMLMatrixFromHDFS(file);
rfile = readRMatrixFromFS(file);
}
TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R");
}
} finally {
// reset runtime platform
rtplatform = oldrt;
}
}