/**
* Reads in all VehicleEvents from the database that were between the beginTime and endTime.
*
* @param agencyId Which project getting data for
* @param beginTime Specifies time range for query
* @param endTime Specifies time range for query
* @param sqlClause Optional. Can specify an SQL clause to winnow down the data, such as "AND
* routeId='71'".
* @return
*/
public static List<VehicleEvent> getVehicleEvents(
String agencyId, Date beginTime, Date endTime, String sqlClause) {
IntervalTimer timer = new IntervalTimer();
// Get the database session. This is supposed to be pretty light weight
Session session = HibernateUtils.getSession(agencyId);
// Create the query. Table name is case sensitive and needs to be the
// class name instead of the name of the db table.
String hql =
"FROM VehicleEvent " + " WHERE time >= :beginDate " + " AND time < :endDate";
if (sqlClause != null) hql += " " + sqlClause;
Query query = session.createQuery(hql);
// Set the parameters
query.setTimestamp("beginDate", beginTime);
query.setTimestamp("endDate", endTime);
try {
@SuppressWarnings("unchecked")
List<VehicleEvent> vehicleEvents = query.list();
logger.debug("Getting VehicleEvents from database took {} msec", timer.elapsedMsec());
return vehicleEvents;
} catch (HibernateException e) {
logger.error(e.getMessage(), e);
return null;
} finally {
// Clean things up. Not sure if this absolutely needed nor if
// it might actually be detrimental and slow things down.
session.close();
}
}
/**
* Returns the first (there can be multiple) GTFS agency object for the specified agencyId. The
* GTFS agency object is cached. First time it is accessed it is read from server via RMI. If
* can't connect via RMI then will timeout after a few seconds. This means that any call to this
* method can take several seconds if there is a problem.
*
* @return The Agency object, or null if can't access the agency via RMI
*/
public Agency getAgency() {
// If agency hasn't been accessed yet do so now...
if (agency == null) {
ConfigInterface inter = ConfigInterfaceFactory.get(agencyId);
if (inter == null) {
logger.error(
"Could not access via RMI agencyId={}. The "
+ "ConfigInterfaceFactory returned null for the "
+ "agency ID.",
agencyId);
} else {
IntervalTimer timer = new IntervalTimer();
try {
// Get the agencies via RMI
List<Agency> agencies = inter.getAgencies();
// Use the first agency if there are multiple ones
agency = agencies.get(0);
} catch (RemoteException e) {
logger.error(
"Could not get Agency object for agencyId={}. "
+ "Exception occurred after {} msec. {}",
agencyId,
timer.elapsedMsec(),
e.getMessage());
}
}
}
// Return the RMI based agency object
return agency;
}
/**
* Reads in WebAgency objects from the database and returns them as a map keyed on agencyId.
*
* @return
* @throws HibernateException
*/
private static Map<String, WebAgency> getMapFromDb() throws HibernateException {
String webAgencyDbName = getWebAgencyDbName();
logger.info("Reading WebAgencies data from database \"{}\"...", webAgencyDbName);
IntervalTimer timer = new IntervalTimer();
Session session = HibernateUtils.getSession(webAgencyDbName);
try {
String hql = "FROM WebAgency";
Query query = session.createQuery(hql);
@SuppressWarnings("unchecked")
List<WebAgency> list = query.list();
Map<String, WebAgency> map = new HashMap<String, WebAgency>();
for (WebAgency webAgency : list) {
map.put(webAgency.getAgencyId(), webAgency);
}
logger.info(
"Done reading WebAgencies from database. Took {} msec. " + "They are {}",
timer.elapsedMsec(),
map.values());
return map;
} catch (Exception e) {
throw e;
} finally {
// Make sure that the session always gets closed, even if
// exception occurs
session.close();
}
}
/**
* Parse the GTFS file. Reads in the header info and then each line. Calls the abstract
* handleRecord() method for each record. Adds each resulting GTFS object to the _gtfsObjecgts
* array.
*/
private void parse() {
CSVRecord record = null;
try {
IntervalTimer timer = new IntervalTimer();
logger.debug("Parsing CSV file {} ...", fileName);
// Open the file for reading. Use UTF-8 format since that will work
// for both regular ASCII format and UTF-8 extended format files
// since UTF-8 was designed to be backwards compatible with ASCII.
// This way will work for Chinese and other character sets. Use
// InputStreamReader so can specify that using UTF-8 format. Use
// BufferedReader so that can determine if first character is an
// optional BOM (Byte Order Mark) character used to indicate that
// file is in UTF-8 format. BufferedReader allows us to read in
// first character and then discard if it is a BOM character or
// reset the reader to back to the beginning if it is not. This
// way the CSV parser will process the file starting with the first
// true character.
Reader in = new BufferedReader(new InputStreamReader(new FileInputStream(fileName), "UTF-8"));
// Deal with the possible BOM character at the beginning of the file
in.mark(1);
int firstRead = in.read();
final int BOM_CHARACTER = 0xFEFF;
if (firstRead != BOM_CHARACTER) in.reset();
// Get ready to parse the CSV file.
// Allow lines to be comments if they start with "-" so that can
// easily comment out problems and also test what happens when
// certain data is missing. Using the '-' character so can
// comment out line that starts with "--", which is what is
// used for SQL.
CSVFormat formatter = CSVFormat.DEFAULT.withHeader().withCommentStart('-');
// Parse the file
Iterable<CSVRecord> records = formatter.parse(in);
logger.debug("Finished CSV parsing of file {}. Took {} msec.", fileName, timer.elapsedMsec());
int lineNumberWhenLogged = 0;
timer = new IntervalTimer();
IntervalTimer loggingTimer = new IntervalTimer();
Iterator<CSVRecord> iterator = records.iterator();
while (iterator.hasNext()) {
// Determine the record to process
record = iterator.next();
// Process the record using appropriate handler
// and create the corresponding GTFS object
T gtfsObject;
try {
gtfsObject = handleRecord(record, supplemental);
} catch (ParseException e) {
logger.error(
"ParseException occurred on line {} for filename {} . {}",
record.getRecordNumber(),
fileName,
e.getMessage());
// Continue even though there was an error so that all errors
// logged at once.
continue;
} catch (NumberFormatException e) {
logger.error(
"NumberFormatException occurred on line {} for filename {} . {}",
record.getRecordNumber(),
fileName,
e.getMessage());
// Continue even though there was an error so that all errors
// logged at once.
continue;
}
// Add the newly created GTFS object to the object list
gtfsObjects.add(gtfsObject);
// Log info if it has been a while. Check only every 20,000 lines
// to see if the 10 seconds has gone by. If so, then log number
// of lines. By only looking at timer every 20,000 lines not slowing
// things down by for every line doing system call for to get current time.
final int LINES_TO_PROCESS_BEFORE_CHECKING_IF_SHOULD_LOG = 20000;
final long SECONDS_ELSAPSED_UNTIL_SHOULD_LOG = 5;
if (record.getRecordNumber()
>= lineNumberWhenLogged + LINES_TO_PROCESS_BEFORE_CHECKING_IF_SHOULD_LOG) {
lineNumberWhenLogged = (int) record.getRecordNumber();
if (loggingTimer.elapsedMsec() > SECONDS_ELSAPSED_UNTIL_SHOULD_LOG * Time.MS_PER_SEC) {
logger.info(
" Processed {} lines. Took {} msec...", lineNumberWhenLogged, timer.elapsedMsec());
loggingTimer = new IntervalTimer();
}
}
} // End of while iterating over records
// Close up the file reader
in.close();
logger.debug("Finished parsing file {} . Took {} msec.", fileName, timer.elapsedMsec());
} catch (FileNotFoundException e) {
if (required) logger.error("Required GTFS file {} not found.", fileName);
else logger.info("GTFS file {} not found but OK because this file not required.", fileName);
} catch (IOException e) {
logger.error("IOException occurred when reading in filename {}.", fileName, e);
}
}
