private static void read(Path file, String... alternativeName) throws IOException { try (CSVReader csvReader = new CSVReader( new BufferedReader( new InputStreamReader(Files.newInputStream(file), Charset.forName("CP1252"))))) { String name; if (alternativeName.length > 0) { name = alternativeName[0]; } else { name = FilenameUtils.getBaseName(file.getFileName().toString()); } Set<Timeseries> dataset = Data.oldDataset(name); if (dataset == null) { dataset = Data.addOldDataset(name); } // Check all the CDIDs in the header row: // int duplicates = 0; String[] header = csvReader.readNext(); for (int i = 1; i < header.length; i++) { Timeseries timeseries = Data.timeseries(header[i]); if (timeseries == null) { timeseries = Data.addTimeseries(header[i]); } dataset.add(timeseries); timeseries.sourceDatasets.add(name); } // Now read the data - each row *may* contain one additional value // for each timeseries: String[] row; rows: while ((row = csvReader.readNext()) != null) { // There is a blank line between the data and the // additional information below, so stop reading there: if (row.length == 0 || StringUtils.isBlank(row[0])) { break rows; } // Add data to timeseries: String date = row[0]; Data.addDateOption(date); for (int i = 1; i < Math.min(header.length, row.length); i++) { if (StringUtils.isNotBlank(header[i]) && StringUtils.isNotBlank(row[i])) { Timeseries timeseries = Data.timeseries(header[i]); String cdid = header[i]; if (cdid == null) { // This one was marked as a duplicate continue; } String value = row[i]; TimeseriesValue timeseriesValue = new TimeseriesValue(); timeseriesValue.date = StringUtils.trim(date); timeseriesValue.value = StringUtils.trim(value); timeseriesValue.sourceDataset = name; timeseries.add(timeseriesValue); // if ("404".equals(value)) { // System.out.println("!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! 404: " // + cdid + ": " + timeseriesValue + " (" + // timeseries.name + ")"); // } // Scale values if necessary: if (timeseries.cdid().equalsIgnoreCase("abmi")) { // System.out.println("ABMI: " + // timeseries.multiply()); } scale(timeseriesValue, timeseries); } } } // Print out some sanity-check information: for (int i = 1; i < header.length; i++) { if (StringUtils.isNotBlank(header[i])) { Timeseries timeseries = Data.timeseries(header[i]); if (timeseries.years.size() == 0 && timeseries.quarters.size() == 0 && timeseries.months.size() == 0) { System.out.println(timeseries + " has no data."); } } } // if (duplicates > 0) { // System.out.println(name + " contains " + dataset.size() + // " timeseries (" + duplicates + " duplicates)"); // } else { System.out.println(name + " contains " + dataset.size() + " timeseries"); // } } }