private static void read(Path file, String... alternativeName) throws IOException {
try (CSVReader csvReader =
new CSVReader(
new BufferedReader(
new InputStreamReader(Files.newInputStream(file), Charset.forName("CP1252"))))) {
String name;
if (alternativeName.length > 0) {
name = alternativeName[0];
} else {
name = FilenameUtils.getBaseName(file.getFileName().toString());
}
Set<Timeseries> dataset = Data.oldDataset(name);
if (dataset == null) {
dataset = Data.addOldDataset(name);
}
// Check all the CDIDs in the header row:
// int duplicates = 0;
String[] header = csvReader.readNext();
for (int i = 1; i < header.length; i++) {
Timeseries timeseries = Data.timeseries(header[i]);
if (timeseries == null) {
timeseries = Data.addTimeseries(header[i]);
}
dataset.add(timeseries);
timeseries.sourceDatasets.add(name);
}
// Now read the data - each row *may* contain one additional value
// for each timeseries:
String[] row;
rows:
while ((row = csvReader.readNext()) != null) {
// There is a blank line between the data and the
// additional information below, so stop reading there:
if (row.length == 0 || StringUtils.isBlank(row[0])) {
break rows;
}
// Add data to timeseries:
String date = row[0];
Data.addDateOption(date);
for (int i = 1; i < Math.min(header.length, row.length); i++) {
if (StringUtils.isNotBlank(header[i]) && StringUtils.isNotBlank(row[i])) {
Timeseries timeseries = Data.timeseries(header[i]);
String cdid = header[i];
if (cdid == null) {
// This one was marked as a duplicate
continue;
}
String value = row[i];
TimeseriesValue timeseriesValue = new TimeseriesValue();
timeseriesValue.date = StringUtils.trim(date);
timeseriesValue.value = StringUtils.trim(value);
timeseriesValue.sourceDataset = name;
timeseries.add(timeseriesValue);
// if ("404".equals(value)) {
// System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 404: "
// + cdid + ": " + timeseriesValue + " (" +
// timeseries.name + ")");
// }
// Scale values if necessary:
if (timeseries.cdid().equalsIgnoreCase("abmi")) {
// System.out.println("ABMI: " +
// timeseries.multiply());
}
scale(timeseriesValue, timeseries);
}
}
}
// Print out some sanity-check information:
for (int i = 1; i < header.length; i++) {
if (StringUtils.isNotBlank(header[i])) {
Timeseries timeseries = Data.timeseries(header[i]);
if (timeseries.years.size() == 0
&& timeseries.quarters.size() == 0
&& timeseries.months.size() == 0) {
System.out.println(timeseries + " has no data.");
}
}
}
// if (duplicates > 0) {
// System.out.println(name + " contains " + dataset.size() +
// " timeseries (" + duplicates + " duplicates)");
// } else {
System.out.println(name + " contains " + dataset.size() + " timeseries");
// }
}
}
