// This function is replicated in Dag.java
@SuppressWarnings("unused")
private Format getChildFormat(Lop node) throws LopsException {
if (node.getOutputParameters().getFile_name() != null
|| node.getOutputParameters().getLabel() != null) {
return node.getOutputParameters().getFormat();
} else {
// Reblock lop should always have a single child
if (node.getInputs().size() > 1)
throw new LopsException(this.printErrorLocation() + "Should only have one child! \n");
/*
* Return the format of the child node (i.e., input lop)
* No need of recursion here.. because
* 1) Reblock lop's input can either be DataLop or some intermediate computation
* If it is Data then we just take its format (TEXT or BINARY)
* If it is intermediate lop then it is always BINARY
* since we assume that all intermediate computations will be in Binary format
* 2) Note that Reblock job will never have any instructions in the mapper
* => the input lop (if it is other than Data) is always executed in a different job
*/
// return getChildFormat(node.getInputs().get(0));
return node.getInputs().get(0).getOutputParameters().getFormat();
}
}
/**
* Constructor to setup read or write LOP In case of write: <code>input</code> must be provided.
* This will always be added as the first element in <code>input</code> array. For literals: this
* function is invoked through the static method <code>createLiteralLop</code>.
*
* @param op
* @param input
* @param inputParametersLops
* @param name
* @param literal
* @param dt
* @param vt
* @param isTransient
* @throws LopsException
*/
public Data(
Data.OperationTypes op,
Lop input,
HashMap<String, Lop> inputParametersLops,
String name,
String literal,
DataType dt,
ValueType vt,
boolean isTransient,
FileFormatTypes fmt)
throws LopsException {
super(Lop.Type.Data, dt, vt);
operation = op;
transient_var = isTransient;
// Either <code>name</code> or <code>literal</code> can be non-null.
if (literal != null) {
if (transient_var)
throw new LopsException(
"Invalid parameter values while setting up a Data LOP -- transient flag is invalid for a literal.");
literal_var = true;
this.getOutputParameters().setLabel(literal);
} else if (name != null) {
if (transient_var) this.getOutputParameters().setLabel(name); // tvar+name
else this.getOutputParameters().setLabel("p" + op + name);
} else {
throw new LopsException(
"Invalid parameter values while setting up a Data LOP -- the lop must have either literal value or a name.");
}
// WRITE operation must have an input Lops, we always put this
// input Lops as the first element of WRITE input. The parameters of
// WRITE operation are then put as the following input elements.
if (input != null && operation == OperationTypes.WRITE) {
this.addInput(input);
input.addOutput(this);
}
_inputParams = inputParametersLops;
if (_inputParams != null) {
for (Lop lop : inputParametersLops.values()) {
this.addInput(lop);
lop.addOutput(this);
}
if (inputParametersLops.get(DataExpression.IO_FILENAME) != null
&& inputParametersLops.get(DataExpression.IO_FILENAME) instanceof Data) {
OutputParameters outParams =
((Data) inputParametersLops.get(DataExpression.IO_FILENAME)).getOutputParameters();
String fName = outParams.getLabel();
this.getOutputParameters().setFile_name(fName);
}
}
setFileFormatAndProperties(fmt);
}
public ReBlock(
Lop input,
Long rows_per_block,
Long cols_per_block,
DataType dt,
ValueType vt,
boolean outputEmptyBlocks,
ExecType et)
throws LopsException {
super(Lop.Type.ReBlock, dt, vt);
this.addInput(input);
input.addOutput(this);
_rows_per_block = rows_per_block;
_cols_per_block = cols_per_block;
_outputEmptyBlocks = outputEmptyBlocks;
boolean breaksAlignment = false;
boolean aligner = false;
boolean definesMRJob = true;
lps.addCompatibility(JobType.REBLOCK);
if (et == ExecType.MR)
lps.setProperties(
inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob);
else if (et == ExecType.SPARK)
lps.setProperties(
inputs,
ExecType.SPARK,
ExecLocation.ControlProgram,
breaksAlignment,
aligner,
definesMRJob);
else throw new LopsException("Incorrect execution type for Reblock:" + et);
}
