public boolean imp() {
boolean res = false;
DbVariable dbVariable = null;
String fullFileName = "";
String errMessage = null;
DbImportFile dbInFile = new DbImportFile();
try {
dbVariable = new DbVariable();
// Store the file on server filesystem
fullFileName = dbInFile.storeImportFileBLOB(conn_viss, ifid);
FileParser fileParser = new FileParser(fullFileName);
fileParser.Parse(
FileTypeDefinitionList.matchingDefinitions(
FileTypeDefinition.VARIABLE, FileTypeDefinition.LIST));
dbVariable.CreateVariables(fileParser, connection, sampleUnitId, Integer.parseInt(userId));
errMessage = dbVariable.getErrorMessage();
Assertion.assertMsg(errMessage == null || errMessage.trim().equals(""), errMessage);
dbInFile.setStatus(conn_viss, ifid, "IMPORTED");
// dbInFile.UpdateImportFile(connection,null,null,"Done",Integer.parseInt(ifid),Integer.parseInt(userId));
// Add a message to the log
dbInFile.addErrMsg(
conn_viss,
ifid,
"File imported to sampling unit "
+ DbSamplingUnit.getSUName(conn_viss, Integer.toString(sampleUnitId))
+ "Note: Markers is always imported in Create mode.");
res = true;
} catch (Exception e) {
Errors.logError("ImportVariables.imp(...)" + e.getMessage());
dbInFile.setStatus(conn_viss, ifid, "ERROR");
// dbInFile.UpdateImportFile(connection,null,null,e.getMessage(),Integer.parseInt(ifid),Integer.parseInt(userId));
// Add a message to the log
dbInFile.addErrMsg(conn_viss, ifid, e.getMessage());
e.printStackTrace(System.err);
if (errMessage == null) {
errMessage = e.getMessage();
}
} finally {
try {
/*
* Delete files uploaded
*/
File tmp = new File(fullFileName);
tmp.delete();
} catch (Exception ignore) {
}
}
return res;
}
public boolean check() {
Errors.logDebug("CheckVariables started");
boolean res = false;
DbImportFile dbInFile = new DbImportFile();
String fullFileName = "";
String checkFileName = "";
String errMessage = null;
try {
// Create the variable
DbVariable dbVariable = new DbVariable();
dbInFile.setStatus(conn_viss, ifid, "0%");
// Store the file on server filesystem
fullFileName = dbInFile.storeImportFileBLOB(conn_viss, ifid);
checkFileName = fullFileName + "_checked";
FileHeader header = FileParser.scanFileHeader(fullFileName);
String type = header.formatTypeName().toUpperCase();
char delimiter = header.delimiter().charValue();
FileParser fileParser = new FileParser(fullFileName);
fileParser.Parse(
FileTypeDefinitionList.matchingDefinitions(
FileTypeDefinition.VARIABLE, FileTypeDefinition.LIST));
// Write out the result to a new file
FileWriter fileOut = new FileWriter(checkFileName);
fileOut.write(
header.objectTypeName()
+ "/"
+ header.formatTypeName()
+ "/"
+ header.version()
+ "/"
+ header.delimiter()
+ "\n");
String titles[] = fileParser.columnTitles();
for (int j = 0; j < titles.length; j++) {
fileOut.write(titles[j] + delimiter);
}
fileOut.write("\n");
String errorMsg = "";
errorMsg = checkList(fileParser, fileOut, delimiter);
fileOut.close();
/*
* Save the file to database
*/
dbInFile.saveCheckedFile(conn_viss, ifid, checkFileName);
if (errorMsg.startsWith("ERROR:")) {
res = false;
dbInFile.setStatus(conn_viss, ifid, "ERROR");
// Add a message to the log
dbInFile.addErrMsg(
conn_viss,
ifid,
"File checked failed for sampling unit "
+ DbSamplingUnit.getSUName(conn_viss, Integer.toString(sampleUnitId))
+ ". <br>"
+ errorMsg);
Errors.logDebug("ErrorMsg=" + errorMsg);
} else if (errorMsg.startsWith("WARNING:")) {
dbInFile.setStatus(conn_viss, ifid, "WARNING");
res = true;
// Add a message to the log
dbInFile.addErrMsg(conn_viss, ifid, "Warnings exists: " + errorMsg);
} else {
dbInFile.setStatus(conn_viss, ifid, "CHECKED");
res = true;
// Add a message to the log
dbInFile.addErrMsg(
conn_viss,
ifid,
"File checked for sampling unit "
+ DbSamplingUnit.getSUName(conn_viss, Integer.toString(sampleUnitId))
+ ".<br>"
+ errorMsg);
}
/*
if (errorMsg.length()>0)
{
res = false;
dbInFile.setStatus(conn_viss,ifid,"ERROR");
// Add a message to the log
dbInFile.addErrMsg(conn_viss,ifid,"File failed the check for sampling unit "+DbSamplingUnit.getSUName(conn_viss,Integer.toString(sampleUnitId)) +"<br>"+errorMsg);
}
else
{
res = true;
dbInFile.setStatus(conn_viss,ifid,"CHECKED");
// Add a message to the log
dbInFile.addErrMsg(conn_viss,ifid,"File checked for sampling unit "+DbSamplingUnit.getSUName(conn_viss,Integer.toString(sampleUnitId)) +"<br>"+errorMsg);
}*/
} catch (Exception e) {
dbInFile.setStatus(conn_viss, ifid, "ERROR");
// Add a message to the log
dbInFile.addErrMsg(conn_viss, ifid, e.getMessage());
e.printStackTrace(System.err);
if (errMessage == null) {
errMessage = e.getMessage();
}
} finally {
try {
/*
* Delete files uploaded
*/
File tmp = new File(checkFileName);
tmp.delete();
tmp = new File(fullFileName);
tmp.delete();
} catch (Exception ignore) {
}
}
Errors.logDebug("CheckVariables completed");
return res;
}
public boolean imp() {
boolean res = false;
String errMessage = null;
DbImportFile dbInFile = new DbImportFile();
DbPhenotype dbp = new DbPhenotype();
String fullFileName = null;
try {
Errors.logInfo("CheckPhenotype started");
// connection.setAutoCommit(false);
dbInFile.setStatus(conn_viss, ifid, "0%");
fullFileName = dbInFile.storeImportFileBLOB(conn_viss, ifid);
FileHeader header = FileParser.scanFileHeader(fullFileName);
FileParser fileParser = new FileParser(fullFileName);
// Set status
dbInFile.setStatus(conn_viss, ifid, "10%");
// Ensure file format is list or matrix
Assertion.assertMsg(
header.formatTypeName().equalsIgnoreCase(FileTypeDefinition.LIST)
|| header.formatTypeName().equalsIgnoreCase(FileTypeDefinition.MATRIX),
"Format type name should be list or matrix "
+ "but found found "
+ header.formatTypeName());
// If file is a list
if (header.formatTypeName().equalsIgnoreCase(FileTypeDefinition.LIST)) {
fileParser.Parse(
FileTypeDefinitionList.matchingDefinitions(
FileTypeDefinition.PHENOTYPE, FileTypeDefinition.LIST));
dbInFile.setStatus(conn_viss, ifid, "20%");
if (updateMethod.equals("CREATE")) {
dbp.CreatePhenotypesList(
fileParser, connection, sampleUnitId, Integer.valueOf(userId).intValue());
} else if (updateMethod.equals("UPDATE")) {
dbp.UpdatePhenotypesList(
fileParser, connection, sampleUnitId, Integer.valueOf(userId).intValue());
} else if (updateMethod.equals("CREATE_OR_UPDATE")) {
dbp.CreateOrUpdatePhenotypesList(
fileParser, connection, sampleUnitId, Integer.valueOf(userId).intValue());
}
}
// If file is a matrix
else if (header.formatTypeName().equalsIgnoreCase(FileTypeDefinition.MATRIX)) {
fileParser.Parse(
FileTypeDefinitionList.matchingDefinitions(
FileTypeDefinition.PHENOTYPE, FileTypeDefinition.MATRIX));
dbInFile.setStatus(conn_viss, ifid, "20%");
if (updateMethod.equals("CREATE")) {
dbp.CreatePhenotypesMatrix(
fileParser, connection, sampleUnitId, Integer.valueOf(userId).intValue());
} else if (updateMethod.equals("UPDATE")) {
dbp.UpdatePhenotypesMatrix(
fileParser, connection, sampleUnitId, Integer.valueOf(userId).intValue());
} else if (updateMethod.equals("CREATE_OR_UPDATE")) {
dbp.CreateOrUpdatePhenotypesMatrix(
fileParser, connection, sampleUnitId, Integer.valueOf(userId).intValue());
}
}
errMessage = dbp.getErrorMessage();
Assertion.assertMsg(errMessage == null || errMessage.trim().equals(""), errMessage);
dbInFile.setStatus(conn_viss, ifid, "IMPORTED");
// Add a message to the log
dbInFile.addErrMsg(
conn_viss,
ifid,
"File imported for sampling unit "
+ DbSamplingUnit.getSUName(conn_viss, Integer.toString(sampleUnitId)));
res = true;
Errors.logInfo("Check Phenotype ended");
} catch (Exception e) {
// Flag for error and set the errMessage if it has not been set
// isOk = false;
dbInFile.setStatus(conn_viss, ifid, "ERROR");
// Add a message to the log
dbInFile.addErrMsg(conn_viss, ifid, e.getMessage());
e.printStackTrace(System.err);
if (errMessage == null) {
errMessage = e.getMessage();
}
}
return res;
}
public boolean check() {
Errors.logDebug("CheckPhenotype started");
boolean res = false;
String errMessage = null;
FileWriter fileOut = null;
DbImportFile dbInFile = new DbImportFile();
String fullFileName = "";
String checkFileName = "";
try {
dbInFile.setStatus(conn_viss, ifid, "0%");
fullFileName = dbInFile.storeImportFileBLOB(conn_viss, ifid);
checkFileName = fullFileName + "_checked";
// Create the Phenotype
DbPhenotype dbPhenotype = new DbPhenotype();
FileHeader header = FileParser.scanFileHeader(fullFileName);
String type = header.formatTypeName().toUpperCase();
char delimiter = header.delimiter().charValue();
// AbstractValueFileParser fp = null;
FileParser fp = null;
if (type.equals("LIST")) {
fp = new FileParser(fullFileName);
fp.Parse(
FileTypeDefinitionList.matchingDefinitions(
FileTypeDefinition.PHENOTYPE, FileTypeDefinition.LIST));
} else if (type.equals("MATRIX")) {
fp = new FileParser(fullFileName);
fp.Parse(
FileTypeDefinitionList.matchingDefinitions(
FileTypeDefinition.PHENOTYPE, FileTypeDefinition.MATRIX));
}
// Write out the result to a new file
fileOut = new FileWriter(checkFileName);
fileOut.write(
header.objectTypeName()
+ "/"
+ header.formatTypeName()
+ "/"
+ header.version()
+ "/"
+ header.delimiter()
+ "\n");
String titles[] = fp.columnTitles();
for (int j = 0; j < titles.length; j++) {
fileOut.write(titles[j] + delimiter);
}
fileOut.write("\n");
// Garbage collect the unused variables
header = null;
// fullFileName = null;
// checkFileName = null;
// Fix to upper case
updateMethod = updateMethod.toUpperCase();
Vector fatalErrors = new Vector();
if (type.equals("LIST")) checkListTitles(titles, fatalErrors);
else checkMatrixTitles(titles, fatalErrors);
writeTitleErrors(fileOut, fatalErrors);
String indId;
if (titles[0].equals("IDENTITY")) indId = "IDENTITY";
else indId = "Alias";
String errMsg = "";
if (type.equals("LIST")) errMsg = checkList((FileParser) fp, fileOut, delimiter, indId);
// errMsg = checkList((FileParser)fp, fatalErrors,fileOut,delimiter,indId);
else if (type.equals("MATRIX"))
errMsg = checkMatrix((FileParser) fp, fatalErrors, fileOut, delimiter, indId);
// Close the file
fileOut.close();
/*
* Save the file to database
*/
dbInFile.saveCheckedFile(conn_viss, ifid, checkFileName);
// Get the error message from the database object. If it is set an
// error occured during the operation so an error is thrown.
// errMessage = dbIndividual.getErrorMessage();
// Assertion.assertMsg(errMessage == null ||
// errMessage.trim().equals(""), errMessage);
if (errMsg.startsWith("ERROR:")) {
dbInFile.setStatus(conn_viss, ifid, "ERROR");
res = false;
} else if (errMsg.startsWith("WARNING:")) {
dbInFile.setStatus(conn_viss, ifid, "WARNING");
res = true;
} else {
dbInFile.setStatus(conn_viss, ifid, "CHECKED");
res = true;
}
// Add a message to the log
dbInFile.addErrMsg(
conn_viss,
ifid,
"File checked for sampling unit "
+ DbSamplingUnit.getSUName(conn_viss, Integer.toString(sampleUnitId))
+ "<br>\n"
+ errMsg);
} catch (Exception e) {
// Flag for error and set the errMessage if it has not been set
// isOk = false;
dbInFile.setStatus(conn_viss, ifid, "ERROR");
// dbInFile.UpdateImportFile(connection,null,null,e.getMessage(),Integer.parseInt(ifid),Integer.parseInt(userId));
// Add a message to the log
dbInFile.addErrMsg(conn_viss, ifid, e.getMessage());
e.printStackTrace(System.err);
if (errMessage == null) {
errMessage = e.getMessage();
}
} finally {
try {
/*
* Delete temporary file
*/
File tmp = new File(fullFileName);
tmp.delete();
tmp = null;
tmp = new File(checkFileName);
tmp.delete();
tmp = null;
} catch (Exception ignore) {
}
}
Errors.logDebug("CheckPhenotype completed");
return res;
}
public String checkMatrix(
FileParser fp, Vector fatalErrors, FileWriter fileOut, char delimiter, String indId) {
String errMsg = "";
// String ind, marker = "", allele1 = "", allele2 = ""; //, raw1, raw2; //, //ref; //, comm;
String ind = "", variable = "", value = "";
// String alleles[];
int nrErrors = 0;
int nrWarnings = 0;
int nrDeviations = 0;
/*
Vector errorMessages = new Vector();
Vector warningMessages = new Vector();
Vector deviationMessages = new Vector();
Vector databaseValues = new Vector();
*/
DbImportFile dbInFile = new DbImportFile();
String statusStr;
double status;
double status_last = 0.0;
int dataRows = fp.dataRows();
String titles[] = fp.columnTitles();
String variables[] = new String[titles.length - 1];
for (int i = 0; i < variables.length; i++) variables[i] = titles[i + 1];
Vector deviationMessages = null;
Vector databaseValues = null;
Vector newAlleles = null;
Vector values = null;
warningList = new ArrayList();
errorList = new ArrayList();
for (int row = 0; row < fp.dataRows(); row++) {
deviationMessages = new Vector();
databaseValues = new Vector();
values = new Vector();
ind = fp.getValue(indId, row);
// newAlleles = new Vector();
// check the whole row
for (int mNum = 0; mNum < variables.length; mNum++) {
// String old_alleles[]=null;
variable = variables[mNum];
value = fp.getValue(variable, row);
// Add the values for error writing
values.add(value);
// check that values exist, have correct length etc
checkValues(ind, variable, value, null, null, null);
if (updateMethod.equals("CREATE"))
checkCreate(titles[0], ind, variable, value, null, null, null, sampleUnitId);
else if (updateMethod.equals("UPDATE"))
checkUpdate(
titles[0],
ind,
variable,
value,
null,
null,
null,
sampleUnitId,
deviationMessages,
databaseValues,
delimiter);
else if (updateMethod.equals("CREATE_OR_UPDATE"))
checkCreateOrUpdate(
titles[0],
ind,
variable,
value,
null,
null,
null,
sampleUnitId,
deviationMessages,
databaseValues,
delimiter);
} // for markers
nrErrors += errorList.size();
nrDeviations += deviationMessages.size();
nrWarnings += warningList.size();
writeMatrixErrors(
fileOut, deviationMessages, databaseValues, values, ind, delimiter, variable, value);
/*
//newAlleles= new Vector();
databaseValues = new Vector();
errorMessages=new Vector();
warningMessages=new Vector();
deviationMessages=new Vector();
*/
/*
* Set the status of the import, visible to the user
*/
status = (new Double(row * 100 / (1.0 * dataRows))).doubleValue();
if (status_last + 5 < status) {
status_last = status;
statusStr = Integer.toString((new Double(status)).intValue()) + "%";
dbInFile.setStatus(conn_viss, ifid, statusStr);
}
errorList.clear();
warningList.clear();
} // for rows
if (nrErrors > 0) errMsg = "ERROR: Import of the genotypes failed.";
else if (nrWarnings > 0) errMsg = "WARNING: Some warnings exist in the import file";
else errMsg = "Genotype file is correct";
errMsg += "\nDeviations:" + nrDeviations + "\nWarnings:" + nrWarnings + "\nErrors:" + nrErrors;
return errMsg;
}
// public String checkList(FileParser fp, Vector errorMessages, FileWriter fileOut, char
// delimiter, String indId)
public String checkList(FileParser fp, FileWriter fileOut, char delimiter, String indId) {
// String ind, marker, allele1, allele2, raw1, raw2, ref, comm;
String ind, variable, value, date, ref, comm;
String errMsg = "";
int nrErrors = 0;
int nrWarnings = 0;
int nrDeviations = 0;
int dataRows = fp.dataRows();
String titles[] = fp.columnTitles();
DbImportFile dbInFile = new DbImportFile();
String statusStr;
double status;
double status_last = 0.0;
warningList = new ArrayList();
errorList = new ArrayList();
for (int i = 0; i < dataRows; i++) {
// Vector errorMessages = new Vector();
// Vector warningMessages = new Vector();
Vector deviationMessages = new Vector();
Vector databaseValues = new Vector();
ind = ((FileParser) fp).getValue(indId, i).trim();
variable = ((FileParser) fp).getValue("VARIABLE", i).trim();
value = ((FileParser) fp).getValue("VALUE", i).trim();
date = ((FileParser) fp).getValue("DATE", i).trim();
ref = ((FileParser) fp).getValue("REF", i).trim();
comm = ((FileParser) fp).getValue("COMMENT", i).trim();
// Check for valid data values.
// Check for length, remove null and so on.
// Syntax check.
checkValues(ind, variable, value, date, ref, comm);
// checkValues(ind, variable, value, date, ref, comm, fatalErrors);
// If create updateMethod
if (updateMethod == null || updateMethod.equals("CREATE"))
checkCreate(titles[0], ind, variable, value, date, ref, comm, sampleUnitId);
// If update updateMethod
else if (updateMethod.equals("UPDATE"))
checkUpdate(
titles[0],
ind,
variable,
value,
date,
ref,
comm,
sampleUnitId,
deviationMessages,
databaseValues,
delimiter);
// if both update and add
else if (updateMethod.equals("CREATE_OR_UPDATE"))
checkCreateOrUpdate(
titles[0],
ind,
variable,
value,
date,
ref,
comm,
sampleUnitId,
deviationMessages,
databaseValues,
delimiter);
nrErrors += errorList.size();
nrDeviations += deviationMessages.size();
nrWarnings += warningList.size();
// write row + all errors encountered to file
writeListErrors(
fileOut,
deviationMessages,
databaseValues,
ind,
delimiter,
variable,
value,
date,
ref,
comm);
/*
* Set the status of the import, visible to the user
*/
status = (new Double(i * 100 / (1.0 * dataRows))).doubleValue();
if (status_last + 5 < status) {
status_last = status;
statusStr = Integer.toString((new Double(status)).intValue()) + "%";
dbInFile.setStatus(conn_viss, ifid, statusStr);
}
errorList.clear();
warningList.clear();
}
if (nrErrors > 0) errMsg = "ERROR: Import of the Phenotypes failed.";
else if (nrWarnings > 0) errMsg = "WARNING: Some warnings exist in the import file";
else errMsg = "Phenotype file is correct";
errMsg += "\nDeviations:" + nrDeviations + "\nWarnings:" + nrWarnings + "\nErrors:" + nrErrors;
return errMsg;
}
