// This function is replicated in Dag.java @SuppressWarnings("unused") private Format getChildFormat(Lop node) throws LopsException { if (node.getOutputParameters().getFile_name() != null || node.getOutputParameters().getLabel() != null) { return node.getOutputParameters().getFormat(); } else { // Reblock lop should always have a single child if (node.getInputs().size() > 1) throw new LopsException(this.printErrorLocation() + "Should only have one child! \n"); /* * Return the format of the child node (i.e., input lop) * No need of recursion here.. because * 1) Reblock lop's input can either be DataLop or some intermediate computation * If it is Data then we just take its format (TEXT or BINARY) * If it is intermediate lop then it is always BINARY * since we assume that all intermediate computations will be in Binary format * 2) Note that Reblock job will never have any instructions in the mapper * => the input lop (if it is other than Data) is always executed in a different job */ // return getChildFormat(node.getInputs().get(0)); return node.getInputs().get(0).getOutputParameters().getFormat(); } }
/** * Constructor to setup read or write LOP In case of write: <code>input</code> must be provided. * This will always be added as the first element in <code>input</code> array. For literals: this * function is invoked through the static method <code>createLiteralLop</code>. * * @param op * @param input * @param inputParametersLops * @param name * @param literal * @param dt * @param vt * @param isTransient * @throws LopsException */ public Data( Data.OperationTypes op, Lop input, HashMap<String, Lop> inputParametersLops, String name, String literal, DataType dt, ValueType vt, boolean isTransient, FileFormatTypes fmt) throws LopsException { super(Lop.Type.Data, dt, vt); operation = op; transient_var = isTransient; // Either <code>name</code> or <code>literal</code> can be non-null. if (literal != null) { if (transient_var) throw new LopsException( "Invalid parameter values while setting up a Data LOP -- transient flag is invalid for a literal."); literal_var = true; this.getOutputParameters().setLabel(literal); } else if (name != null) { if (transient_var) this.getOutputParameters().setLabel(name); // tvar+name else this.getOutputParameters().setLabel("p" + op + name); } else { throw new LopsException( "Invalid parameter values while setting up a Data LOP -- the lop must have either literal value or a name."); } // WRITE operation must have an input Lops, we always put this // input Lops as the first element of WRITE input. The parameters of // WRITE operation are then put as the following input elements. if (input != null && operation == OperationTypes.WRITE) { this.addInput(input); input.addOutput(this); } _inputParams = inputParametersLops; if (_inputParams != null) { for (Lop lop : inputParametersLops.values()) { this.addInput(lop); lop.addOutput(this); } if (inputParametersLops.get(DataExpression.IO_FILENAME) != null && inputParametersLops.get(DataExpression.IO_FILENAME) instanceof Data) { OutputParameters outParams = ((Data) inputParametersLops.get(DataExpression.IO_FILENAME)).getOutputParameters(); String fName = outParams.getLabel(); this.getOutputParameters().setFile_name(fName); } } setFileFormatAndProperties(fmt); }
public ReBlock( Lop input, Long rows_per_block, Long cols_per_block, DataType dt, ValueType vt, boolean outputEmptyBlocks, ExecType et) throws LopsException { super(Lop.Type.ReBlock, dt, vt); this.addInput(input); input.addOutput(this); _rows_per_block = rows_per_block; _cols_per_block = cols_per_block; _outputEmptyBlocks = outputEmptyBlocks; boolean breaksAlignment = false; boolean aligner = false; boolean definesMRJob = true; lps.addCompatibility(JobType.REBLOCK); if (et == ExecType.MR) lps.setProperties( inputs, ExecType.MR, ExecLocation.MapAndReduce, breaksAlignment, aligner, definesMRJob); else if (et == ExecType.SPARK) lps.setProperties( inputs, ExecType.SPARK, ExecLocation.ControlProgram, breaksAlignment, aligner, definesMRJob); else throw new LopsException("Incorrect execution type for Reblock:" + et); }