/**
* Constructor to setup read or write LOP In case of write: <code>input</code> must be provided.
* This will always be added as the first element in <code>input</code> array. For literals: this
* function is invoked through the static method <code>createLiteralLop</code>.
*
* @param op
* @param input
* @param inputParametersLops
* @param name
* @param literal
* @param dt
* @param vt
* @param isTransient
* @throws LopsException
*/
public Data(
Data.OperationTypes op,
Lop input,
HashMap<String, Lop> inputParametersLops,
String name,
String literal,
DataType dt,
ValueType vt,
boolean isTransient,
FileFormatTypes fmt)
throws LopsException {
super(Lop.Type.Data, dt, vt);
operation = op;
transient_var = isTransient;
// Either <code>name</code> or <code>literal</code> can be non-null.
if (literal != null) {
if (transient_var)
throw new LopsException(
"Invalid parameter values while setting up a Data LOP -- transient flag is invalid for a literal.");
literal_var = true;
this.getOutputParameters().setLabel(literal);
} else if (name != null) {
if (transient_var) this.getOutputParameters().setLabel(name); // tvar+name
else this.getOutputParameters().setLabel("p" + op + name);
} else {
throw new LopsException(
"Invalid parameter values while setting up a Data LOP -- the lop must have either literal value or a name.");
}
// WRITE operation must have an input Lops, we always put this
// input Lops as the first element of WRITE input. The parameters of
// WRITE operation are then put as the following input elements.
if (input != null && operation == OperationTypes.WRITE) {
this.addInput(input);
input.addOutput(this);
}
_inputParams = inputParametersLops;
if (_inputParams != null) {
for (Lop lop : inputParametersLops.values()) {
this.addInput(lop);
lop.addOutput(this);
}
if (inputParametersLops.get(DataExpression.IO_FILENAME) != null
&& inputParametersLops.get(DataExpression.IO_FILENAME) instanceof Data) {
OutputParameters outParams =
((Data) inputParametersLops.get(DataExpression.IO_FILENAME)).getOutputParameters();
String fName = outParams.getLabel();
this.getOutputParameters().setFile_name(fName);
}
}
setFileFormatAndProperties(fmt);
}
/**
* Fills link_data_total table
*
* @param link
* @param param output parameters
* @return the stored row
* @throws Exception
*/
private LinkDataTotal fill_total(Link link, OutputParameters params) throws Exception {
LinkDataTotal db_ldt = new LinkDataTotal();
db_ldt.setLinkId(str2id(link.getId()));
db_ldt.setNetworkId(str2id(link.myNetwork.getId()));
db_ldt.setDataSources(db_simulation_run.getDataSources());
db_ldt.setTs(ts.getTime());
db_ldt.setAggregation("raw");
db_ldt.setType("mean");
db_ldt.setCellNumber(Integer.valueOf(0));
// mean density, vehicles
double density = SiriusMath.sum(link.cumulative_density[0]) / params.getNsteps();
db_ldt.setDensity(BigDecimal.valueOf(density));
FundamentalDiagram fd = link.currentFD(0);
if (null != fd) {
if (params.doExportFlows()) {
// input flow, vehicles
db_ldt.setInFlow(BigDecimal.valueOf(SiriusMath.sum(link.cumulative_inflow[0])));
// output flow, vehicles
double outflow = SiriusMath.sum(link.cumulative_outflow[0]);
db_ldt.setOutFlow(BigDecimal.valueOf(outflow));
// free flow speed, m/s
BigDecimal ffspeed = fd.getFreeFlowSpeed();
// speed, m/s
if (density <= 0) db_ldt.setSpeed(ffspeed);
else {
double speed =
outflow * link.getLength().doubleValue() / (params.getOutputPeriod() * density);
if (null != ffspeed && speed > ffspeed.doubleValue()) db_ldt.setSpeed(ffspeed);
else if (!Double.isNaN(speed)) db_ldt.setSpeed(BigDecimal.valueOf(speed));
}
}
// free flow speed, m/s
db_ldt.setFreeFlowSpeed(fd.getFreeFlowSpeed());
// critical speed, m/s
db_ldt.setCriticalSpeed(fd.getCriticalSpeed());
// congestion wave speed, m/s
db_ldt.setCongestionWaveSpeed(fd.getCongestionSpeed());
// maximum flow, vehicles per second per lane
db_ldt.setCapacity(fd.getCapacity());
// jam density, vehicles per meter per lane
db_ldt.setJamDensity(fd.getJamDensity());
// capacity drop, vehicle per second per lane
db_ldt.setCapacityDrop(fd.getCapacityDrop());
}
db_ldt.save();
return db_ldt;
}
/**
* Fills link_data_detailed table
*
* @param link
* @param params output parameters
* @param total_speed speed for the cell as a whole, m/s
* @throws Exception
*/
private void fill_detailed(Link link, OutputParameters params, BigDecimal total_speed)
throws Exception {
for (int vt_ind = 0; vt_ind < db_vehicle_type.length; ++vt_ind) {
LinkDataDetailed db_ldd = new LinkDataDetailed();
db_ldd.setLinkId(str2id(link.getId()));
db_ldd.setNetworkId(str2id(link.myNetwork.getId()));
db_ldd.setDataSources(db_simulation_run.getDataSources());
db_ldd.setVehicleTypes(db_vehicle_type[vt_ind]);
db_ldd.setTs(ts.getTime());
db_ldd.setAggregation("raw");
db_ldd.setType("mean");
db_ldd.setCellNumber(Integer.valueOf(0));
// mean density, vehicles
double density = link.cumulative_density[0][vt_ind] / params.getNsteps();
db_ldd.setDensity(new BigDecimal(density));
if (params.doExportFlows()) {
// input flow, vehicles
db_ldd.setInFlow(new BigDecimal(link.cumulative_inflow[0][vt_ind]));
// output flow, vehicles
double outflow = link.cumulative_outflow[0][vt_ind];
db_ldd.setOutFlow(new BigDecimal(outflow));
if (density <= 0) db_ldd.setSpeed(total_speed);
else {
// speed, m/s
double speed =
outflow * link.getLength().doubleValue() / (params.getOutputPeriod() * density);
FundamentalDiagram fd = link.currentFD(0);
// free flow speed, m/s
BigDecimal ffspeed = null == fd ? null : fd.getFreeFlowSpeed();
if (null != ffspeed && speed > ffspeed.doubleValue()) db_ldd.setSpeed(ffspeed);
else if (!Double.isNaN(speed)) db_ldd.setSpeed(new BigDecimal(speed));
}
}
db_ldd.save();
}
}
public String getCreateVarInstructions(String outputFileName, String outputLabel)
throws LopsException {
if (getDataType() == DataType.MATRIX || getDataType() == DataType.FRAME) {
if (isTransient())
throw new LopsException("getInstructions() should not be called for transient nodes.");
OutputParameters oparams = getOutputParameters();
String fmt = "";
// TODO: following logic should change once we LOPs encode key-value-class information.
if (oparams.getFormat() == Format.TEXT) fmt = "textcell";
else if (oparams.getFormat() == Format.MM) fmt = "matrixmarket";
else if (oparams.getFormat() == Format.CSV) fmt = "csv";
else {
if (oparams.getRowsInBlock() > 0 || oparams.getColsInBlock() > 0) fmt = "binaryblock";
else fmt = "binarycell";
}
StringBuilder sb = new StringBuilder();
sb.append("CP");
sb.append(OPERAND_DELIMITOR);
sb.append("createvar");
sb.append(OPERAND_DELIMITOR);
sb.append(outputLabel);
sb.append(OPERAND_DELIMITOR);
sb.append(outputFileName);
sb.append(OPERAND_DELIMITOR);
sb.append(false);
sb.append(OPERAND_DELIMITOR); // only persistent reads come here!
sb.append(fmt);
sb.append(OPERAND_DELIMITOR);
sb.append(oparams.getNumRows());
sb.append(OPERAND_DELIMITOR);
sb.append(oparams.getNumCols());
sb.append(OPERAND_DELIMITOR);
sb.append(oparams.getRowsInBlock());
sb.append(OPERAND_DELIMITOR);
sb.append(oparams.getColsInBlock());
sb.append(OPERAND_DELIMITOR);
sb.append(oparams.getNnz());
/* Format-specific properties */
if (oparams.getFormat() == Format.CSV) {
sb.append(OPERAND_DELIMITOR);
sb.append(createVarCSVHelper());
}
return sb.toString();
} else {
throw new LopsException(
this.printErrorLocation() + "In Data Lop, Unexpected data type " + getDataType());
}
}
/**
* Method to get CP instructions for reading/writing scalars and matrices from/to HDFS. This
* method generates CP read/write instructions.
*/
@Override
public String getInstructions(String input1, String input2) throws LopsException {
if (getOutputParameters().getFile_name() == null && operation == OperationTypes.READ)
throw new LopsException(
this.printErrorLocation()
+ "Data.getInstructions(): Exepecting a SCALAR data type, encountered "
+ getDataType());
StringBuilder sb = new StringBuilder();
if (this.getExecType() == ExecType.SPARK) sb.append("SPARK");
else sb.append("CP");
sb.append(OPERAND_DELIMITOR);
if (operation == OperationTypes.READ) {
sb.append("read");
sb.append(OPERAND_DELIMITOR);
sb.append(this.prepInputOperand(input1));
} else if (operation == OperationTypes.WRITE) {
sb.append("write");
sb.append(OPERAND_DELIMITOR);
sb.append(getInputs().get(0).prepInputOperand(input1));
} else
throw new LopsException(
this.printErrorLocation() + "In Data Lop, Unknown operation: " + operation);
sb.append(OPERAND_DELIMITOR);
Lop fnameLop = _inputParams.get(DataExpression.IO_FILENAME);
boolean literal = (fnameLop instanceof Data && ((Data) fnameLop).isLiteral());
sb.append(prepOperand(input2, DataType.SCALAR, ValueType.STRING, literal));
// attach outputInfo in case of matrices
OutputParameters oparams = getOutputParameters();
if (operation == OperationTypes.WRITE) {
sb.append(OPERAND_DELIMITOR);
String fmt = "";
if (getDataType() == DataType.MATRIX) {
if (oparams.getFormat() == Format.MM) fmt = "matrixmarket";
else if (oparams.getFormat() == Format.TEXT) fmt = "textcell";
else if (oparams.getFormat() == Format.CSV) fmt = "csv";
else if (oparams.getFormat() == Format.BINARY) {
if (oparams.getRowsInBlock() > 0 || oparams.getColsInBlock() > 0) fmt = "binaryblock";
else fmt = "binarycell";
} else {
throw new LopsException("Unexpected format: " + oparams.getFormat());
}
} else {
// scalars will always be written in text format
fmt = "textcell";
}
sb.append(prepOperand(fmt, DataType.SCALAR, ValueType.STRING, true));
if (oparams.getFormat() == Format.CSV) {
Data headerLop = (Data) getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
Data delimLop = (Data) getNamedInputLop(DataExpression.DELIM_DELIMITER);
Data sparseLop = (Data) getNamedInputLop(DataExpression.DELIM_SPARSE);
if (headerLop.isVariable())
throw new LopsException(
this.printErrorLocation()
+ "Parameter "
+ DataExpression.DELIM_HAS_HEADER_ROW
+ " must be a literal for a seq operation.");
if (delimLop.isVariable())
throw new LopsException(
this.printErrorLocation()
+ "Parameter "
+ DataExpression.DELIM_DELIMITER
+ " must be a literal for a seq operation.");
if (sparseLop.isVariable())
throw new LopsException(
this.printErrorLocation()
+ "Parameter "
+ DataExpression.DELIM_SPARSE
+ " must be a literal for a seq operation.");
sb.append(OPERAND_DELIMITOR);
sb.append(headerLop.getBooleanValue());
sb.append(OPERAND_DELIMITOR);
sb.append(delimLop.getStringValue());
sb.append(OPERAND_DELIMITOR);
sb.append(sparseLop.getBooleanValue());
if (this.getExecType() == ExecType.SPARK) {
boolean isInputMatrixBlock = true;
Lop input = getInputs().get(0);
if (input instanceof ParameterizedBuiltin
&& ((ParameterizedBuiltin) input).getOp()
== ParameterizedBuiltin.OperationTypes.TRANSFORM) {
// in the case of transform input, the input will be Text strings insteadof MatrixBlocks
// This information is used to have correct class information while accessing RDDs from
// the symbol table
isInputMatrixBlock = false;
}
sb.append(OPERAND_DELIMITOR);
sb.append(isInputMatrixBlock);
}
}
}
return sb.toString();
}
/**
* Method to generate appropriate MR write instructions. Explicit write instructions are generated
* only in case of external file formats (e.g., CSV) except for MatrixMarket. MM format is
* overridden by TextCell, instead.
*/
@Override
public String getInstructions(int input_index, int output_index) throws LopsException {
OutputParameters oparams = getOutputParameters();
if (operation != OperationTypes.WRITE)
throw new LopsException(
"This method should only be executed for generating MR Write instructions.");
if (oparams.getFormat() != Format.CSV)
throw new LopsException(
"MR Write instructions can not be generated for the output format: "
+ oparams.getFormat());
StringBuilder sb = new StringBuilder();
sb.append("MR");
sb.append(OPERAND_DELIMITOR);
// Generate opcode based on the output format
if (oparams.getFormat() == Format.CSV) sb.append("csvwrite");
else
throw new LopsException(
"MR Write instructions can not be generated for the output format: "
+ oparams.getFormat());
sb.append(OPERAND_DELIMITOR);
sb.append(getInputs().get(0).prepInputOperand(input_index));
sb.append(OPERAND_DELIMITOR);
sb.append(this.prepOutputOperand(output_index));
// Attach format-specific properties
if (oparams.getFormat() == Format.CSV) {
Data headerLop = (Data) getNamedInputLop(DataExpression.DELIM_HAS_HEADER_ROW);
Data delimLop = (Data) getNamedInputLop(DataExpression.DELIM_DELIMITER);
Data sparseLop = (Data) getNamedInputLop(DataExpression.DELIM_SPARSE);
if (headerLop.isVariable())
throw new LopsException(
this.printErrorLocation()
+ "Parameter "
+ DataExpression.DELIM_HAS_HEADER_ROW
+ " must be a literal.");
if (delimLop.isVariable())
throw new LopsException(
this.printErrorLocation()
+ "Parameter "
+ DataExpression.DELIM_DELIMITER
+ " must be a literal.");
if (sparseLop.isVariable())
throw new LopsException(
this.printErrorLocation()
+ "Parameter "
+ DataExpression.DELIM_SPARSE
+ " must be a literal.");
sb.append(OPERAND_DELIMITOR);
sb.append(headerLop.getBooleanValue());
sb.append(OPERAND_DELIMITOR);
sb.append(delimLop.getStringValue());
sb.append(OPERAND_DELIMITOR);
sb.append(sparseLop.getBooleanValue());
} else {
throw new LopsException(
"MR Write instructions can not be generated for the output format: "
+ oparams.getFormat());
}
return sb.toString();
}