/** * @param outer execution mode of outer parfor loop * @param inner execution mode of inner parfor loop * @param instType execution mode of instructions */ private void runParForDataPartitioningTest( PDataPartitioner partitioner, PExecMode mode, boolean small, boolean sparse, boolean multiParts) { RUNTIME_PLATFORM oldRT = rtplatform; boolean oldUseSparkConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; if (partitioner == PDataPartitioner.REMOTE_SPARK || mode == PExecMode.REMOTE_SPARK) { DMLScript.USE_LOCAL_SPARK_CONFIG = true; rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; } try { // inst exec type, influenced via rows int rows = small ? rows1 : rows2; int cols = small ? cols1 : cols2; // script int scriptNum = -1; switch (partitioner) { case NONE: scriptNum = 1; break; case LOCAL: if (mode == PExecMode.LOCAL) scriptNum = 2; else scriptNum = 3; break; case REMOTE_MR: if (mode == PExecMode.LOCAL) { if (!multiParts) scriptNum = 4; else scriptNum = 6; } else scriptNum = 5; break; case REMOTE_SPARK: if (mode == PExecMode.LOCAL) { if (!multiParts) scriptNum = 7; else scriptNum = 9; } else scriptNum = 8; break; default: // do nothing } TestConfiguration config = getTestConfiguration(TEST_NAME); config.addVariable("rows", rows); config.addVariable("cols", cols); loadTestConfiguration(config); /* This is for running the junit test the new way, i.e., construct the arguments directly */ String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml"; programArgs = new String[] { "-args", input("V"), Integer.toString(rows), Integer.toString(cols), output("R") }; fullRScriptName = HOME + TEST_NAME + (multiParts ? "6" : "") + ".R"; rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir(); long seed = System.nanoTime(); double sparsity = -1; if (sparse) sparsity = sparsity2; else sparsity = sparsity1; double[][] V = getRandomMatrix(rows, cols, 0, 1, sparsity, seed); writeInputMatrix("V", V, true); boolean exceptionExpected = false; runTest(true, exceptionExpected, null, -1); runRScript(true); // compare matrices HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R"); HashMap<CellIndex, Double> rfile = readRMatrixFromFS("Rout"); TestUtils.compareMatrices(dmlfile, rfile, eps, "DML", "R"); } finally { rtplatform = oldRT; DMLScript.USE_LOCAL_SPARK_CONFIG = oldUseSparkConfig; } }
/** * Shared test driver for tests of univariate statistics over continuous, scaled, and weighted * data * * @param sz size of primary input vector * @param rng range of randomly-generated values to use * @param sp sparsity of generated data * @param rt backend platform to test */ protected void testWeightedScaleWithR(SIZE sz, RANGE rng, SPARSITY sp, RUNTIME_PLATFORM rt) { RUNTIME_PLATFORM oldrt = rtplatform; rtplatform = rt; try { TestConfiguration config = getTestConfiguration("WeightedScaleTest"); config.addVariable("rows1", sz.size); config.addVariable("rows2", rows2); loadTestConfiguration(config); // This is for running the junit test the new way, i.e., construct // the arguments directly String S_HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = S_HOME + "WeightedScaleTest" + ".dml"; programArgs = new String[] { "-args", input("vector"), Integer.toString(sz.size), input("weight"), input("prob"), Integer.toString(rows2), output("mean"), output("std"), output("se"), output("var"), output("cv"), output("min"), output("max"), output("rng"), output("g1"), output("se_g1"), output("g2"), output("se_g2"), output("median"), output("iqm"), output("out_minus"), output("out_plus"), output("quantile") }; fullRScriptName = S_HOME + "WeightedScaleTest" + ".R"; rCmd = "Rscript" + " " + fullRScriptName + " " + inputDir() + " " + expectedDir(); createHelperMatrix(); double[][] vector = getRandomMatrix(sz.size, 1, rng.min, rng.max, sp.sparsity, System.currentTimeMillis()); double[][] weight = getRandomMatrix(sz.size, 1, 1, 10, 1, System.currentTimeMillis()); OrderStatisticsTest.round(weight); double[][] prob = getRandomMatrix(rows2, 1, 0, 1, 1, System.currentTimeMillis()); writeInputMatrix("vector", vector, true); writeInputMatrix("weight", weight, true); writeInputMatrix("prob", prob, true); // // Expected number of jobs: // Reblock - 1 job // While loop iteration - 10 jobs // Final output write - 1 job runTest(true, false, null, -1); runRScript(true); // disableOutAndExpectedDeletion(); for (String file : config.getOutputFiles()) { // NOte that some files do not contain matrix, but just a single // scalar value inside HashMap<CellIndex, Double> dmlfile; HashMap<CellIndex, Double> rfile; if (file.endsWith(".scalar")) { file = file.replace(".scalar", ""); dmlfile = readDMLScalarFromHDFS(file); rfile = readRScalarFromFS(file); } else { dmlfile = readDMLMatrixFromHDFS(file); rfile = readRMatrixFromFS(file); } TestUtils.compareMatrices(dmlfile, rfile, epsilon, file + "-DML", file + "-R"); } } finally { // reset runtime platform rtplatform = oldrt; } }