private List<Status> fetchPreviousThreadNextStatus() throws TechnicalException {
outln("waiting for previous thread to either provide data and/or be done");
List<Status> previousStatusList = this.previousQueryThread.getChangedStatusList();
List<Status> tmp = null;
Status firstStatus = null;
// Obtain lock on previous status
synchronized (previousStatusList) {
while (previousStatusList.isEmpty()) {
try {
previousStatusList.wait();
} catch (InterruptedException e) {
throw new TechnicalException(e);
}
}
if (!previousStatusList.isEmpty()) {
firstStatus =
previousStatusList.remove(0); // remove status from list, it will then be processed
} else {
MyUtils.errorProgram("SPONTANEOUS NOTIFICATION!!!!!!!!"); // shouldn't happen really
}
if (QueryRunnerPrototypeConstants.CHECK) tmp = new ArrayList<Status>(previousStatusList);
}
outln("previous status to process: " + firstStatus);
return tmp;
}
public boolean checkDatabase(String templateName, DatabaseCheck databaseCheck, String rangeName)
throws TechnicalException, FunctionalException {
String datasetNameFromTableName = this.nctm.getDatasetName();
if (null != datasetNameFromTableName
&& !MyUtils.isEmpty(datasetNameFromTableName)
&& !datasetNameFromTableName.equals(rangeName)) {
throw new FunctionalException(
"Mismatching dataset identifiers: datasetNameFromTableName = "
+ datasetNameFromTableName
+ ", rangeName = "
+ rangeName);
}
if (this.getTableConstraint() == null || this.getField() == null) {
return false;
}
String tableName = null;
if (this.main) {
return true;
} else {
tableName =
datasetNameFromTableName == null
? rangeName + "__" + this.getTableConstraint()
: this.getTableConstraint();
MyUtils.checkStatusProgram(
tableName.split("__").length == 3, "tableName = " + tableName, true);
}
return databaseCheck.checkDatabase(templateName, tableName, this.getField())
&& databaseCheck.checkDatabase(templateName, tableName, this.getKey());
}
public void process() throws Exception {
/*createFiles(0, 2, 1000);
createFiles(10, 2, 1000);
createFiles(50, 2, 1000);
createFiles(100, 2, 1000);*/
/*createFiles(0, 1, 1000000);
createFiles(10, 1, 1000000);
createFiles(50, 1, 1000000);
createFiles(100, 1, 1000000);*/
// createFiles(10, 1, 1000000, "bottom");
// createFiles(50, 1, 1000000, "bottom");
/*changeRdbs(Rdbs.MYSQL);
createFiles(10, 1, 1000000, "bottom");
createFiles(50, 1, 1000000, "bottom");
changeRdbs(Rdbs.POSTGRESQL);
createFiles(10, 1, 1000000, "bottom");
createFiles(50, 1, 1000000, "bottom");*/
changeRdbs(Rdbs.ORACLE);
createFiles(10, 1, 1000000, BOTTOM2);
createFiles(50, 1, 1000000, BOTTOM2);
// createFiles(50, 1, 100000, "bottom");
// createFiles(100, 1, 10000000);
String scriptFileName = "script_" + this.rdbs + SCRIPT_EXTENSION;
String scriptFilePathAndName =
GENERAL_OUPUT_FOLDER_PATH_AND_NAME + MyUtils.FILE_SEPARATOR + scriptFileName;
MyUtils.writeFile(scriptFilePathAndName, stringBuffer.toString());
System.out.println("msl < " + scriptFilePathAndName);
}
private boolean checkDatabaseForTable(String templateName, String tableName) {
TemplateDatabaseDescription templateDatabaseDescription =
templateNameToDatabaseDescriptionMap.get(templateName);
MyUtils.checkStatusProgram(templateDatabaseDescription != null);
List<String> tableList = templateDatabaseDescription.getTableList();
return MartConfiguratorUtils.containsIgnoreCase(tableList, tableName);
}
public void fetchTableList(String templateName) throws TechnicalException {
String databaseName = getDatabaseName(templateName);
MyUtils.checkStatusProgram(null != databaseName);
try {
List<String> tableList = null;
String tableListSerialFilePathAndName =
this.transformationsGeneralSerialFolderPathAndName
+ TABLE_LIST_SERIAL_FILE_PREFIX
+ databaseName;
if (new File(tableListSerialFilePathAndName).exists()) {
tableList = (List<String>) MyUtils.readSerializedObject(tableListSerialFilePathAndName);
System.out.println("Using serial, quick check: " + tableList.size());
} else {
System.out.println("No serial at " + tableListSerialFilePathAndName + " , connecting...");
connect(databaseName);
tableList = new ArrayList<String>();
// Prepare list of tables
this.preparedStatementTable = this.connection.prepareStatement("show tables");
ResultSet resultSet = this.preparedStatementTable.executeQuery();
tableList = new ArrayList<String>();
resultSet.beforeFirst();
while (resultSet.next()) {
tableList.add(resultSet.getString(1));
}
this.preparedStatementTable.close();
resultSet.close();
disconnect();
MyUtils.writeSerializedObject(tableList, tableListSerialFilePathAndName);
}
MyUtils.checkStatusProgram(templateNameToDatabaseDescriptionMap.get(templateName) == null);
templateNameToDatabaseDescriptionMap.put(
templateName, new TemplateDatabaseDescription(tableList));
} catch (SQLException e) {
throw new TechnicalException(e);
}
}
public Document fetchTemplateXml(String virtualSchema, String templateName)
throws TechnicalException {
// Find what database the dataset/template belongs to
String databaseName = getDatabaseName(templateName);
MyUtils.checkStatusProgram(null != databaseName, templateName);
return fetchTemplateXml(virtualSchema, templateName, databaseName);
}
@Deprecated
public boolean isTemplate(String name) throws FunctionalException {
Boolean matchesDatasetName = false;
Boolean matchesTemplateName = false;
String databaseNameMatch = null;
for (Iterator<String> it = databaseNameToDatasetInformationMap.keySet().iterator();
it.hasNext(); ) {
String databaseName = it.next();
HashMap<String, List<String>> map = databaseNameToDatasetInformationMap.get(databaseName);
for (Iterator<String> it2 = map.keySet().iterator(); it2.hasNext(); ) {
String templateName = it2.next();
List<String> datasetNames = map.get(templateName);
for (String datasetName : datasetNames) {
if (name.equals(datasetName)) {
MyUtils.checkStatusProgram(
!matchesDatasetName && !matchesTemplateName && null == databaseNameMatch);
matchesDatasetName = true;
databaseNameMatch = databaseName;
}
if (name.equals(templateName)) {
MyUtils.checkStatusProgram(
!matchesDatasetName
&& (!matchesTemplateName
|| (null != databaseNameMatch && databaseNameMatch.equals(databaseName))));
matchesTemplateName = true;
databaseNameMatch = databaseName;
}
}
}
}
if (!matchesDatasetName && !matchesTemplateName) {
throw new FunctionalException("Unknown dataset/template name: " + name);
} else if (matchesDatasetName && matchesTemplateName) {
throw new FunctionalException("Ambiguous dataset/template name: " + name);
}
return matchesTemplateName;
}
private boolean checkDatabaseForField(String templateName, String tableName, String columnName)
throws FunctionalException {
TemplateDatabaseDescription templateDatabaseDescription =
templateNameToDatabaseDescriptionMap.get(templateName);
MyUtils.checkStatusProgram(templateDatabaseDescription != null);
Map<String, List<String>> tableColumnMap = templateDatabaseDescription.getTableColumnMap();
List<String> columnList = tableColumnMap.get(tableName);
if (null == columnList) {
throw new FunctionalException("No table called " + tableName + " found in map");
}
return MartConfiguratorUtils.containsIgnoreCase(columnList, columnName);
}
public void updateTransformedDatasetNameInPlainDatasetNameToTransformedDatasetName(
String datasetOrTemplateName, String transformedDatasetName, boolean isWebService) {
boolean atLeastOne = false;
for (Iterator<String> it = this.plainDatasetNameToTransformedDatasetName.keySet().iterator();
it.hasNext(); ) {
String pointedDatasetPlainName = it.next();
String datasetOrTemplateNameTmp =
this.plainDatasetNameToTransformedDatasetName.get(pointedDatasetPlainName);
if (datasetOrTemplateNameTmp.equals(datasetOrTemplateName)) {
atLeastOne = true;
/*MyUtils.checkStatusProgram(datasetOrTemplateNameTmp!=null &&
(isWebService || (!datasetOrTemplateNameTmp.equals(transformedDatasetName))),
(datasetOrTemplateNameTmp!=null) + ", " + isWebService + ", " + (!datasetOrTemplateNameTmp.equals(transformedDatasetName)) + ", " +
datasetOrTemplateNameTmp + ", " + transformedDatasetName);*/
this.plainDatasetNameToTransformedDatasetName.put(
pointedDatasetPlainName, transformedDatasetName);
}
}
MyUtils.checkStatusProgram(atLeastOne);
}
private void process() throws TechnicalException, FunctionalException {
try {
Connection connection = connect();
while (true) { // broken when done
this.checkCancelIndex = 0;
if (checkCancelThread()) { // regularily check for cancel request and exit when encountered
break;
}
QueryRunnerIntermediaryResult previousThreadResults = null;
if (!first) {
List<Status> tmp = fetchPreviousThreadNextStatus();
if (checkCancelThread()) { // see above similar line
break;
}
previousThreadResults = processPreviousThreadDoneStatus(tmp);
if (previousThreadResults == null) { // means previous thread is done
break;
}
}
if (checkCancelThread()) { // see above similar line
break;
}
StringBuffer currentQuery = new StringBuffer(this.queryTemplate);
boolean potentialResults = true;
if (first) {
previousThreadResults = new QueryRunnerIntermediaryResult(this.queryThreadIndex);
outln("first: no input needed");
} else {
outln("Processing input");
potentialResults = customizeQuery(previousThreadResults, currentQuery);
}
this.shortenedQuery =
currentQuery.length() < QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH
? currentQuery.toString()
: (currentQuery
.substring(0, QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2)
.toString()
+ " ... "
+ currentQuery
.substring(
currentQuery.length()
- QueryRunnerPrototypeConstants.MAX_DISPLAY_QUERY_LENGTH2,
currentQuery.length())
.toString());
outln(potentialResults + ", " + "shortenedQuery = " + shortenedQuery);
writelnQueryFile(shortenedQuery);
if (checkCancelThread()) { // see above similar line
break;
}
outln("preparing statement");
synchronized (
this
.activeQuery) { // always get the lock on that object to handle the query
// related-object: 2 threads can use them (main one and current one)
lockln("lock acquired on activeQuery");
if (this.preparedStatement == null) {
check(!this.activeQuery.getValue());
check(this.resultSet == null); // not sure
prepareStreamingStatement(connection, currentQuery);
} else {
check(this.activeQuery.getValue());
check(this.resultSet != null); // not sure
this.resultSet.close();
this.preparedStatement.close();
prepareStreamingStatement(connection, currentQuery);
}
this.activeQuery
.setTrue(); // set it to true: meaning cancel would have to actually cancel the query
// and close as much as it can
lockln("lock released on activeQuery");
}
if (checkCancelThread()) { // see above similar line
break;
}
Timer queryTimer = new Timer();
queryTimer.startTimer();
Timer queryLaunchTimer = new Timer();
queryLaunchTimer.startTimer();
Integer columnCount = null;
if (potentialResults) {
outln("executing query");
synchronized (this.activeQuery) { // see above similar line
lockln("lock acquired on activeQuery");
this.resultSet =
this.preparedStatement.executeQuery(
// currentQuery.toString()); // for Statement
); // for PreparedStatement
outln("query prepared");
columnCount = this.resultSet.getMetaData().getColumnCount();
// activeQuery is already set to true here: it means we must always check whether
// resultSet is null or not before trying to close it
lockln("lock released on activeQuery");
}
}
queryLaunchTimer.stopTimer();
this.launchQueryAverageTime.addToAverage(queryLaunchTimer.getTimeEllapsedMs());
if (checkCancelThread()) { // see above similar line
break;
}
int batchNumber = 0; // For the current query
queryRow = 0; // For the current query
Boolean next = null;
boolean queryCancellation = false; // so we can break the 2 embedded query while loops
while ( // 1st query while loop: query one
(potentialResults && ((next = resultSetNext(resultSet)) || batchNumber == 0))
|| (!potentialResults
&& batchNumber == 0)) { // We want to go inside the first time even if no results
this.currentBatchResults = new ArrayList<List<String>>();
int batchRow = 0; // For the current batch for the current query
outln("Fetching batch");
Timer batchTimer = new Timer();
batchTimer.startTimer();
while ( // 2st query while loop: batch one
batchRow < this.batchSize
&& potentialResults
&& ((batchRow == 0 && next) || (batchRow > 0 && resultSetNext(resultSet)))) {
List<String> list = new ArrayList<String>();
// we also lock on that object when retrieving a row
synchronized (this.activeQuery) { // see above similar line
lockln("lock acquired on activeQuery");
if (this.activeQuery.getValue()) {
for (int i = 0; i < columnCount; i++) {
String fieldValue = resultSet.getString(i + 1);
list.add(fieldValue); // accepting null values
}
} else {
queryCancellation = true;
}
lockln("lock released on activeQuery");
if (queryCancellation) { // break 2nd query while loop: batch one (not the main one)
break;
}
}
currentBatchResults.add(list);
batchRow++;
queryRow++;
} // end of the 2nd query while loop (batch)
batchTimer.stopTimer();
this.fetchingBatchAverageTime.addToAverage(batchTimer.getTimeEllapsedMs());
writelnQueryFile(
MyUtils.TAB_SEPARATOR
+ MyUtils.TAB_SEPARATOR
+ batchRow
+ " rows in "
+ batchTimer.getTimeEllapsedMs()
+ " ms");
StringBuffer stringBuffer =
new StringBuffer(); /*stringBuffer.append("{");for (int i = 0; i < currentBatchResults.size(); i++) {stringBuffer.append(currentBatchResults.get(i).get(0) + ",");}stringBuffer.append("}");stringBuffer.toString();*/
outln("currentBatchResults.size() = " + currentBatchResults.size() + ", " + stringBuffer);
if (queryCancellation) { // break 1st query while loop: query one (not the main one)
break;
}
// if query produced results, process them
if (!currentBatchResults.isEmpty()) {
if (first) outln(MyUtils.DASH_LINE + currentBatchResults.get(0));
boolean threadCancellation =
processBatchResults(
queryThreadIndex, previousThreadResults); // also check for cancelation
if (threadCancellation) { // break 1st query while loop (not the main one)
break;
}
}
batchNumber++;
} // end of the 1st query while loop (query)s
queryTimer.stopTimer();
this.fetchingQueryAverageTime.addToAverage(queryTimer.getTimeEllapsedMs());
MyUtils.checkStatusProgram(potentialResults || queryRow == 0);
writelnQueryFile(
MyUtils.TAB_SEPARATOR
+ queryRow
+ " rows (sum) in "
+ queryTimer.getTimeEllapsedMs()
+ " ms");
// First thread only has 1 query and is finished
if (first) {
outln("Setting thread to done");
synchronized (this.done) {
this.done.setTrue();
}
synchronized (this.statusList) {
this.statusList.add(Status.FIRST_DONE);
this.statusList.notify(); // To notify second thread
}
break;
}
} // end of the main while loop
// close resources
outln("trying to acquire query lock");
synchronized (this.activeQuery) { // see above similar line
lockln("lock acquired on activeQuery");
if (this.activeQuery.getValue()) {
if (null != this.resultSet) {
this.resultSet.close();
}
check(this.preparedStatement != null);
this.preparedStatement.close();
this.activeQuery.setFalse();
}
lockln("lock released on activeQuery");
}
connection.close();
// Update average times
this.launchQueryAverageTime.update();
this.fetchingQueryAverageTime.update();
this.fetchingBatchAverageTime.update();
this.intermediaryResultsProcessingAverageTime.update();
} catch (SQLException e) {
throw new TechnicalException(e);
} catch (IOException e) {
throw new TechnicalException(e);
}
}
private void check(boolean test, String message) {
if (QueryRunnerPrototypeConstants.CHECK) MyUtils.checkStatusProgram(test, checklog(message));
}
private OldElement(
boolean isAttribute,
Element jdomElement,
String value,
List<Element> childrenList,
String internalName,
String hidden,
String hideDisplay,
String description,
String displayName,
String tableConstraint,
String key,
String field,
String pointerDataset,
String pointerInterface,
String pointerElement,
String checkForNulls)
throws FunctionalException {
super(jdomElement, internalName);
if (this.hidden) { // assigned in OldNode
this.valid = false;
return;
}
this.isAttribute = isAttribute;
this.hideDisplay = TransformationUtils.getBooleanValueFromString(hideDisplay, "hideDisplay");
this.description = description;
this.displayName = displayName;
this.tableConstraint = tableConstraint;
this.key = key;
this.field = field;
this.pointerDataset = pointerDataset;
this.pointerInterface = pointerInterface;
this.pointerElement = pointerElement;
this.checkForNulls =
TransformationUtils.getBooleanValueFromString(checkForNulls, "checkForNulls");
this.hasChildren = childrenList != null && childrenList.size() != 0;
this.hasSubChildren = false;
this.tree = false;
if (this.hasChildren) {
Element firstChild = childrenList.get(0);
this.firstChildrenType = ElementChildrenType.getElementChildrenType(firstChild);
// Will check for the 1st child that has subChildren (if any) and check the type of the
// subchild
for (Element child : childrenList) {
List<Element> subChildrenList = child.getChildren();
this.hasSubChildren = subChildrenList != null && subChildrenList.size() != 0;
if (this.hasSubChildren) {
ElementChildrenType subChildrenType =
ElementChildrenType.getElementChildrenType(subChildrenList.get(0));
this.tree =
ElementChildrenType.OPTION_VALUE.equals(this.firstChildrenType)
&& ElementChildrenType.OPTION_VALUE.equals(subChildrenType);
break;
}
}
}
this.pointer =
this.pointerElement != null && this.pointerInterface != null && this.pointerDataset != null;
this.main =
!this.pointer && tableConstraint != null && TransformationUtils.isMain(tableConstraint);
this.dimension = !this.pointer && tableConstraint != null && !this.main;
this.option = this instanceof OldOptionFilter;
if (this.main || this.dimension) {
this.nctm = new NamingConventionTableName(this.tableConstraint);
} else {
this.filterGroup = !pointer && !this.option;
MyUtils.checkStatusProgram(
!this.filterGroup || hasChildren, XmlUtils.displayJdomElement(jdomElement));
}
this.dimensionPartition = new DimensionPartition(this.tableConstraint, this.key);
if (this.dimension) {
this.dimensionPartition.searchForPotentialDimensionTablePartition();
}
if (!finalChecks()) {
// System.out.println(this.internalName + "
// ##########@@@@@@@@@@@@@######################*****************####################################");
System.err.println(
"######################### element has failed final checks" + this.internalName);
this.valid = false;
}
}
public Document fetchTemplateXml(String virtualSchema, String templateName, String databaseName)
throws TechnicalException {
Document document = null;
try {
String templateXmlSerialFilePathAndName =
this.transformationsGeneralSerialFolderPathAndName
+ TEMPLATE_XML_CONTENT_SERIAL_FILE_PREFIX
+ virtualSchema
+ MyUtils.INFO_SEPARATOR
+ MyUtils.INFO_SEPARATOR
+ templateName;
String templateXmlFilePathAndName =
this.transformationsGeneralSerialFolderPathAndName
+ virtualSchema
+ MyUtils.INFO_SEPARATOR
+ MyUtils.INFO_SEPARATOR
+ templateName;
if (new File(templateXmlSerialFilePathAndName).exists()) {
System.out.println("Using serial at " + templateXmlSerialFilePathAndName);
document = (Document) MyUtils.readSerializedObject(templateXmlSerialFilePathAndName);
System.out.println("quick check: " + document.getDocType());
} else {
System.out.println("No serial at " + templateXmlSerialFilePathAndName + " , connecting...");
connect(databaseName);
this.preparedStatementTemplateXml =
this.connection.prepareStatement(
"select meta_template__xml__dm.compressed_xml "
+ "from meta_template__xml__dm "
+ "where meta_template__xml__dm.template = '"
+ templateName
+ "';");
ResultSet resultSet = this.preparedStatementTemplateXml.executeQuery();
resultSet.first();
byte[] bytes = resultSet.getBytes(1);
MyUtils.checkStatusProgram(!resultSet.next()); // Check only 1 row
this.preparedStatementTemplateXml.close();
resultSet.close();
disconnect();
/* Transform into uncompressed XML document:
Compression of XML in MartEditor, see:
DatabaseDatasetConfigUtils.storeTemplateXML (compress)
MartRegistryXMLUtils.DataSourceToRegistryDocument (uncompress)
*/
InputStream rstream = null;
rstream = new GZIPInputStream(new ByteArrayInputStream(bytes));
SAXBuilder builder = new SAXBuilder();
InputSource is = new InputSource(rstream);
document = builder.build(is);
MyUtils.writeSerializedObject(document, templateXmlSerialFilePathAndName);
XmlUtils.writeXmlFile(document, templateXmlFilePathAndName);
}
} catch (SQLException e) {
throw new TechnicalException(e);
} catch (JDOMException e) {
throw new TechnicalException(e);
} catch (IOException e) {
throw new TechnicalException(e);
}
return document;
}
public TemplateDatabaseDescription getTemplateDatabaseDescription(String templateName) {
TemplateDatabaseDescription templateDatabaseDescription =
this.templateNameToDatabaseDescriptionMap.get(templateName);
MyUtils.checkStatusProgram(null != templateDatabaseDescription);
return templateDatabaseDescription;
}
public void fetchTableColumnMap(String templateName) throws TechnicalException {
String databaseName = getDatabaseName(templateName);
MyUtils.checkStatusProgram(null != databaseName);
try {
Map<String, List<String>> tableColumnMap = null;
TemplateDatabaseDescription templateDatabaseDescription =
templateNameToDatabaseDescriptionMap.get(templateName);
MyUtils.checkStatusProgram(templateDatabaseDescription != null);
String tableColumnMapContentSerialFilePathAndName =
this.transformationsGeneralSerialFolderPathAndName
+ TABLE_COLUMN_LIST_SERIAL_FILE_PREFIX
+ databaseName;
if (new File(tableColumnMapContentSerialFilePathAndName).exists()) {
tableColumnMap =
(Map<String, List<String>>)
MyUtils.readSerializedObject(tableColumnMapContentSerialFilePathAndName);
System.out.println("Using serial, quick check: " + tableColumnMap.size());
} else {
System.out.println(
"No serial at " + tableColumnMapContentSerialFilePathAndName + " , connecting...");
connect(databaseName);
tableColumnMap = new HashMap<String, List<String>>();
List<String> tableList = templateDatabaseDescription.getTableList();
int table = 0;
for (String tableName : tableList) {
System.out.print(
"tableName = " + tableName + ", " + (table + 1) + "/" + tableList.size());
// Prepare list of descriptions
this.preparedStatementField = this.connection.prepareStatement("desc " + tableName);
ResultSet resultSet = this.preparedStatementField.executeQuery();
List<String> columnList = new ArrayList<String>();
resultSet.beforeFirst();
while (resultSet.next()) {
columnList.add(resultSet.getString(1));
}
tableColumnMap.put(tableName, columnList);
this.preparedStatementField.close();
resultSet.close();
System.out.println(", " + columnList.size());
table++;
}
disconnect();
MyUtils.writeSerializedObject(tableColumnMap, tableColumnMapContentSerialFilePathAndName);
}
templateDatabaseDescription.setTableColumnMap(tableColumnMap);
} catch (SQLException e) {
throw new TechnicalException(e);
}
}
private boolean processBatchResults(
int currentQueryThreadIndex, QueryRunnerIntermediaryResult previousThreadResults)
throws TechnicalException, FunctionalException {
if (!first) {
outln("Merging results");
}
Timer intermediaryResultsProcessingTimer = null;
if (!this.first && !this.last) {
intermediaryResultsProcessingTimer = new Timer();
intermediaryResultsProcessingTimer.startTimer();
}
// Prepare results for join with the next query
this.batchResult = prepareResults(previousThreadResults, currentBatchResults);
if (!last) {
// wait till next thread is start processing the latest results
List<Status> nextReadyList = nextQueryThread.getReadyList();
List<Status> tmp = null;
Status firstStatus = null;
synchronized (nextReadyList) {
outln("wait for next thread to be ready");
while (nextReadyList.isEmpty()) {
try {
nextReadyList.wait();
} catch (InterruptedException e) {
throw new TechnicalException(e);
}
}
if (!nextReadyList.isEmpty()) {
firstStatus = nextReadyList.remove(0);
} else {
MyUtils.errorProgram("SPONTANEOUS NOTIFICATION!!!!!!!!"); // shouldn't happen really
}
tmp = new ArrayList<Status>(nextReadyList);
}
outln("next status to process: " + firstStatus);
if (checkCancelThread()) { // check here in case we sent a signal because of cancelling
return true;
}
check(tmp.isEmpty(), checklog("nextReadyList = " + nextReadyList));
check(
null
== this
.currentResult); // Must have been freed by next thread (call to
// resetCurrentResult())
// Add the newest results for the next thread to process as soon as it's ready
outln("Add input for next thread");
this.currentResult = batchResult;
synchronized (this.statusList) {
this.statusList.add(Status.RESULTS);
this.statusList.notify(); // To notify next thread
}
} else {
// Add final results
outln("Add final results");
List<Integer> ignoreList = ThreadCommunication.getIgnoreList();
boolean cancelationInProcess =
this.rows != null && ThreadCommunication.getTotalRows() >= this.rows;
if (!cancelationInProcess) {
try {
for (List<String> list : batchResult.getValues()) {
boolean first = true;
for (int i = 0; i < list.size(); i++) {
if (!ignoreList.contains(i)) {
ThreadCommunication.bufferedWriter.write(
(first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
if (!QueryRunnerPrototypeConstants.LOG) {
System.out.print((first ? "" : MyUtils.TAB_SEPARATOR) + list.get(i));
}
first = false;
}
}
ThreadCommunication.bufferedWriter.write(MyUtils.LINE_SEPARATOR);
if (!QueryRunnerPrototypeConstants.LOG) {
System.out.println();
}
int totalRows = ThreadCommunication.addRowToTotalRows();
if (!this.unsynchronizedFirstFinalResults && totalRows == 1) { // if first row
outln("notify first results have arrived");
synchronized (this.firstFinalResults) {
this.firstFinalResults.setTrue();
this.firstFinalResults.notify(); // notify main thread
this.unsynchronizedFirstFinalResults = true;
}
}
if (!this.unsynchronizedSomeFinalResults
&& this.rows != null
&& totalRows == this.rows.intValue()) {
outln("notify some results have arrived");
synchronized (this.someFinalResults) {
this.unsynchronizedSomeFinalResults = true;
this.someFinalResults.setTrue();
this.someFinalResults.notify(); // notify main thread
}
break;
}
}
if (QueryRunnerPrototypeConstants.LOG) {
ThreadCommunication.bufferedWriter.flush();
}
} catch (IOException e) {
throw new TechnicalException(e);
}
}
}
this.batchResult = null;
if (!this.first && !this.last) {
intermediaryResultsProcessingTimer.stopTimer();
this.intermediaryResultsProcessingAverageTime.addToAverage(
intermediaryResultsProcessingTimer.getTimeEllapsedMs());
}
return false;
}
public void fetchDatasetInformation() throws TechnicalException {
try {
String databaseMapSerialFilePathAndName =
this.serialFolderPathAndName + DATABASE_MAP_SERIAL_FILE_NAME;
String templateMapSerialFilePathAndName =
this.serialFolderPathAndName + TEMPLATE_MAP_SERIAL_FILE_NAME;
String datasetMapSerialFilePathAndName =
this.serialFolderPathAndName + DATASET_MAP_SERIAL_FILE_NAME;
String datasetMap2SerialFilePathAndName =
this.serialFolderPathAndName + DATASET_MAP2_SERIAL_FILE_NAME;
if (new File(databaseMapSerialFilePathAndName).exists()
&& new File(templateMapSerialFilePathAndName).exists()
&& new File(datasetMapSerialFilePathAndName).exists()
&& new File(datasetMap2SerialFilePathAndName).exists()) {
databaseNameToDatasetInformationMap =
(HashMap<String, HashMap<String, List<String>>>)
MyUtils.readSerializedObject(databaseMapSerialFilePathAndName);
System.out.println(
"Using serial, quick check: " + this.databaseNameToDatasetInformationMap.size());
/*datasetNameToDatabaseNameMap = (HashMap<String, String>)MyUtils.readSerializedObject(
datasetMapSerialFilePathAndName);
System.out.println("Using serial, quick check: " + this.datasetNameToDatabaseNameMap.size());*/
templateNameToDatabaseNameMap =
(HashMap<String, String>)
MyUtils.readSerializedObject(templateMapSerialFilePathAndName);
System.out.println(
"Using serial, quick check: " + this.templateNameToDatabaseNameMap.size());
datasetNameToTemplateName =
(HashMap<String, String>) MyUtils.readSerializedObject(datasetMapSerialFilePathAndName);
System.out.println("Using serial, quick check: " + this.datasetNameToTemplateName.size());
datasetNameToDatasetType =
(HashMap<String, String>)
MyUtils.readSerializedObject(datasetMap2SerialFilePathAndName);
System.out.println("Using serial, quick check: " + this.datasetNameToDatasetType.size());
} else {
System.out.println("No serial at " + databaseMapSerialFilePathAndName + " , connecting...");
this.databaseNameToDatasetInformationMap =
new HashMap<String, HashMap<String, List<String>>>();
// Check every database for the dataset or the template name
for (String databaseName : this.databaseNames) {
connect(databaseName);
// Prepare list of tables
this.preparedStatementDatabase =
this.connection.prepareStatement(
"select meta_conf__dataset__main.dataset, meta_conf__dataset__main.type, meta_template__template__main.template "
+ "from meta_conf__dataset__main, meta_template__template__main "
+ "where meta_conf__dataset__main.dataset_id_key = meta_template__template__main.dataset_id_key;");
ResultSet resultSet = this.preparedStatementDatabase.executeQuery();
HashMap<String, List<String>> map = new HashMap<String, List<String>>();
resultSet.beforeFirst();
while (resultSet.next()) {
String datasetName = resultSet.getString(1);
String datasetType = resultSet.getString(2);
String templateName = resultSet.getString(3);
List<String> list = map.get(templateName);
if (null == list) {
list = new ArrayList<String>();
}
list.add(datasetName);
map.put(templateName, list);
MyUtils.checkStatusProgram(
TransformationUtils.isGenomicSequence(datasetType)
|| TransformationUtils.isTableSet(datasetType));
/*MyUtils.checkStatusProgram(datasetNameToDatabaseNameMap.get(datasetName)==null);
datasetNameToDatabaseNameMap.put(datasetName, databaseName);*/
MyUtils.checkStatusProgram(datasetNameToTemplateName.get(datasetName) == null);
datasetNameToTemplateName.put(datasetName, templateName);
MyUtils.checkStatusProgram(datasetNameToDatasetType.get(datasetName) == null);
datasetNameToDatasetType.put(datasetName, datasetType);
String databaseNameTmp = templateNameToDatabaseNameMap.get(templateName);
if (null == databaseNameTmp) {
templateNameToDatabaseNameMap.put(templateName, databaseName);
} else {
MyUtils.checkStatusProgram(databaseNameTmp.equals(databaseName));
}
}
this.preparedStatementDatabase.close();
resultSet.close();
disconnect();
MyUtils.checkStatusProgram(
this.databaseNameToDatasetInformationMap.get(databaseName) == null);
this.databaseNameToDatasetInformationMap.put(databaseName, map);
MyUtils.writeSerializedObject(
this.databaseNameToDatasetInformationMap, databaseMapSerialFilePathAndName);
// MyUtils.writeSerializedObject(this.datasetNameToDatabaseNameMap,
// datasetMapSerialFilePathAndName);
MyUtils.writeSerializedObject(
this.templateNameToDatabaseNameMap, templateMapSerialFilePathAndName);
MyUtils.writeSerializedObject(
this.datasetNameToTemplateName, datasetMapSerialFilePathAndName);
MyUtils.writeSerializedObject(
this.datasetNameToDatasetType, datasetMap2SerialFilePathAndName);
}
}
System.out.println(databaseNameToDatasetInformationMap);
} catch (SQLException e) {
throw new TechnicalException(e);
}
}