public void runSP() {
// main method to run the synthetic population generator
if (!ResourceUtil.getBooleanProperty(rb, PROPERTIES_RUN_SP)) return;
logger.info("Generating synthetic populations of household/persons, dwellings and jobs");
identifyUniquePUMAzones();
readControlTotals();
createJobs();
processPums();
generateAutoOwnership();
// summarizeData.summarizeAutoOwnershipByCounty();
addVacantDwellings();
if (ResourceUtil.getBooleanProperty(rb, PROPERTIES_VALIDATE_SYNTH_POP)) validateHHandDD();
logger.info(" Total number of households created " + Household.getHouseholdCount());
logger.info(" Total number of persons created " + Person.getPersonCount());
logger.info(" Total number of dwellings created " + Dwelling.getDwellingCount());
logger.info(" Total number of jobs created " + Job.getJobCount());
calculateVacancyRate();
if (!jobErrorCounter.isEmpty()) {
logger.warn(
" Could not find sufficient number of jobs in these PUMA zones (note that most of these "
+ "zones are outside the MSTM area):");
Set<Integer> list = jobErrorCounter.keySet();
for (Integer puma : list)
logger.warn(" -> " + puma + " is missing " + jobErrorCounter.get(puma) + " jobs.");
} else {
logger.info(" Succeeded in assigning job to every worker.");
}
// summarizeVacantJobsByRegion();
// summarizeByPersonRelationship();
summarizeData.writeOutSyntheticPopulation(rb, SiloUtil.getBaseYear());
// writeSyntheticPopulation();
logger.info(" Completed generation of synthetic population");
}
private void readControlTotals() {
// read control totals of households by size and dwellings
logger.info(" Reading control total data for households and dwellings");
TableDataSet pop =
SiloUtil.readCSVfile(
SiloUtil.baseDirectory
+ ResourceUtil.getProperty(rb, PROPERTIES_HOUSEHOLD_CONTROL_TOTAL));
householdTarget = new HashMap<>();
for (int row = 1; row <= pop.getRowCount(); row++) {
String fips = String.valueOf(pop.getValueAt(row, "Fips"));
// note: doesn't make much sense to store these data in a HashMap. It's legacy code.
householdTarget.put(fips, (int) pop.getValueAt(row, "TotalHouseholds"));
}
hhDistribution =
SiloUtil.readCSVfile(
SiloUtil.baseDirectory
+ ResourceUtil.getProperty(rb, PROPERTIES_HOUSEHOLD_DISTRIBUTION));
hhDistribution.buildIndex(hhDistribution.getColumnPosition(";SMZ_N"));
}
private void createJobs() {
// method to generate synthetic jobs
logger.info(" Generating base year jobs");
TableDataSet jobs =
SiloUtil.readCSVfile(
ResourceUtil.getProperty(rb, JobDataManager.PROPERTIES_JOB_CONTROL_TOTAL));
new JobType(rb);
// jobInventory by [industry][taz]
float[][] jobInventory =
new float[JobType.getNumberOfJobTypes()][geoData.getHighestZonalId() + 1];
tazByWorkZonePuma =
new HashMap<>(); // this HashMap has same content as "HashMap tazByPuma", though is kept
// separately in case external workzones will be defined
// read employment data
// For reasons that are not explained in the documentation, some of the PUMA work zones were
// aggregated to the
// next higher level. Keep this information.
for (int row = 1; row <= jobs.getRowCount(); row++) {
int taz = (int) jobs.getValueAt(row, "SMZ");
int pumaOfWorkZone = geoData.getSimplifiedPUMAofZone(taz);
if (tazByWorkZonePuma.containsKey(pumaOfWorkZone)) {
int[] list = tazByWorkZonePuma.get(pumaOfWorkZone);
int[] newList = SiloUtil.expandArrayByOneElement(list, taz);
tazByWorkZonePuma.put(pumaOfWorkZone, newList);
} else {
tazByWorkZonePuma.put(pumaOfWorkZone, new int[] {taz});
}
for (int jobTp = 0; jobTp < JobType.getNumberOfJobTypes(); jobTp++) {
jobInventory[jobTp][taz] = jobs.getValueAt(row, JobType.getJobType(jobTp) + "00");
}
}
// create base year employment
for (int zone : geoData.getZones()) {
for (int jobTp = 0; jobTp < JobType.getNumberOfJobTypes(); jobTp++) {
if (jobInventory[jobTp][zone] > 0) {
for (int i = 1; i <= jobInventory[jobTp][zone]; i++) {
int id = JobDataManager.getNextJobId();
new Job(id, zone, -1, JobType.getJobType(jobTp));
if (id == SiloUtil.trackJj) {
SiloUtil.trackWriter.println("Generated job with following attributes:");
Job.getJobFromId(id).logAttributes(SiloUtil.trackWriter);
}
}
}
}
}
identifyVacantJobsByZone();
}
public static void main(String... args) {
org.apache.log4j.BasicConfigurator.configure();
LogManager.getRootLogger().setLevel(Level.INFO);
logger.info("usage: org.sandag.abm.ctramp.BikeLogsum properties origin_mgra dest_mgra");
Map<String, String> properties = ResourceUtil.getResourceBundleAsHashMap(args[0]);
int originMgra = Integer.parseInt(args[1]);
int destMgra = Integer.parseInt(args[2]);
BikeLogsum bls = BikeLogsum.getBikeLogsum(properties);
BikeLogsumSegment defaultSegment = new BikeLogsumSegment(true, true, true);
double logsum = bls.getLogsum(new BikeLogsumSegment(true, true, true), originMgra, destMgra);
double time = bls.getTime(new BikeLogsumSegment(true, true, true), originMgra, destMgra);
logger.info(
String.format(
"omgra: %s, dmgra: %s, logsum: %s, time: %s", originMgra, destMgra, logsum, time));
}
private void addVacantDwellings() {
// PUMS generates too few vacant dwellings, add vacant dwellings to match vacancy rate
logger.info(" Adding empty dwellings to match vacancy rate");
HashMap<String, ArrayList<Integer>> ddPointer = new HashMap<>();
// summarize vacancy
int[][][] ddCount = new int[geoData.getHighestZonalId() + 1][DwellingType.values().length][2];
for (Dwelling dd : Dwelling.getDwellingArray()) {
int taz = dd.getZone();
int occ = dd.getResidentId();
ddCount[taz][dd.getType().ordinal()][0]++;
if (occ > 0) ddCount[taz][dd.getType().ordinal()][1]++;
// set pointer to this dwelling
String code = taz + "_" + dd.getType();
if (ddPointer.containsKey(code)) {
ArrayList<Integer> dList = ddPointer.get(code);
dList.add(dd.getId());
ddPointer.put(code, dList);
} else {
ArrayList<Integer> dList = new ArrayList<>();
dList.add(dd.getId());
ddPointer.put(code, dList);
}
}
TableDataSet countyLevelVacancies =
SiloUtil.readCSVfile(rb.getString(PROPERTIES_COUNTY_VACANCY_RATES));
countyLevelVacancies.buildIndex(countyLevelVacancies.getColumnPosition("Fips"));
double[] expectedVacancies = ResourceUtil.getDoubleArray(rb, PROPERTIES_VACANCY_RATES);
for (int taz : geoData.getZones()) {
float vacRateCountyTarget;
try {
vacRateCountyTarget =
countyLevelVacancies.getIndexedValueAt(geoData.getCountyOfZone(taz), "VacancyRate");
} catch (Exception e) {
vacRateCountyTarget =
countyLevelVacancies.getIndexedValueAt(99999, "VacancyRate"); // use average value
}
int ddInThisTaz = 0;
for (DwellingType dt : DwellingType.values()) {
String code = taz + "_" + dt;
if (!ddPointer.containsKey(code)) continue;
ddInThisTaz += ddPointer.get(code).size();
}
int targetVacantDdThisZone = (int) (ddInThisTaz * vacRateCountyTarget + 0.5);
for (DwellingType dt : DwellingType.values()) {
String code = taz + "_" + dt;
if (!ddPointer.containsKey(code)) continue;
ArrayList<Integer> dList = ddPointer.get(code);
if (ddCount[taz][dt.ordinal()][0] == 0)
continue; // no values for this zone and dwelling type in modeled data
float vacRateTargetThisDwellingType = (float) expectedVacancies[dt.ordinal()];
float targetThisTypeThisZoneAbs =
(float)
(vacRateTargetThisDwellingType
/ SiloUtil.getSum(expectedVacancies)
* targetVacantDdThisZone);
float vacDwellingsModel =
((float) (ddCount[taz][dt.ordinal()][0] - ddCount[taz][dt.ordinal()][1]));
Integer[] ids = dList.toArray(new Integer[dList.size()]);
while (vacDwellingsModel < SiloUtil.rounder(targetThisTypeThisZoneAbs, 0)) {
int selected = SiloUtil.select(ids.length) - 1;
Dwelling dd = Dwelling.getDwellingFromId(ids[selected]);
int newDdId = RealEstateDataManager.getNextDwellingId();
new Dwelling(
newDdId,
dd.getZone(),
-1,
dd.getType(),
dd.getBedrooms(),
dd.getQuality(),
dd.getPrice(),
0f,
dd.getYearBuilt());
ddCount[taz][dt.ordinal()][0]++;
vacDwellingsModel++;
if (newDdId == SiloUtil.trackDd) {
SiloUtil.trackWriter.println("Generated vacant dwelling with following attributes:");
Dwelling.getDwellingFromId(newDdId).logAttributes(SiloUtil.trackWriter);
}
}
}
}
}
private void processPums() {
// read PUMS data
logger.info(" Reading PUMS data");
String partlyCovered =
SiloUtil.baseDirectory + ResourceUtil.getProperty(rb, PROPERTIES_PARTLY_COVERED_PUMAS);
TableDataSet partlyCoveredPumas = SiloUtil.readCSVfile(partlyCovered);
int highestPUMA = 5500000;
float[] pumaScaler = SiloUtil.createArrayWithValue((highestPUMA), 1f);
for (int row = 1; row <= partlyCoveredPumas.getRowCount(); row++) {
pumaScaler[(int) partlyCoveredPumas.getValueAt(row, "fullPumaCode")] =
partlyCoveredPumas.getValueAt(row, "mstmPop2000")
/ partlyCoveredPumas.getValueAt(row, "fullPop2000");
}
String age90plusFile =
SiloUtil.baseDirectory + ResourceUtil.getProperty(rb, PROPERTIES_AGE_DISTRIBUTION_90PLUS);
TableDataSet age90plus = SiloUtil.readCSVfile(age90plusFile);
float[] probAge90plusMale = age90plus.getColumnAsFloat("male");
float[] probAge90plusFemale = age90plus.getColumnAsFloat("female");
String[] states = {"MD", "DC", "DE", "PA", "VA", "WV"};
int[] stateNumber = {24, 11, 10, 42, 51, 54}; // FIPS code of String states[]
jobErrorCounter = new HashMap<>();
new Accessibility(
rb, SiloUtil.getBaseYear()); // read in travel times and trip length frequency distribution
for (int st = 0; st < states.length; st++) {
String pumsFileName =
SiloUtil.baseDirectory
+ ResourceUtil.getProperty(rb, PROPERTIES_PUMS_FILES)
+ states[st]
+ "/REVISEDPUMS5_"
+ stateNumber[st]
+ ".TXT";
logger.info(" Creating synthetic population for " + states[st]);
String recString = "";
int recCount = 0;
int hhCount = 0;
int recInStudyAreaCount = 0;
try {
BufferedReader in = new BufferedReader(new FileReader(pumsFileName));
int hhSize = 0;
int personCounter = 0;
// define variables
int pumaZone = 0;
int weight = 0;
int ddType = 0;
int bedRooms = 0;
int autos = 0;
int rent = 0;
int mortgage = 0;
int quality = 0;
int yearBuilt = 0;
int[] relShp = new int[100];
int[] gender = new int[100];
int[] age = new int[100];
Race[] race = new Race[100];
int[] occupation = new int[100];
int[] workPumaZone = new int[100];
int[] workState = new int[100];
int[] income = new int[100];
// boolean[] fullTime = new boolean[100];
while ((recString = in.readLine()) != null) {
recCount++;
String recType = recString.substring(0, 1);
switch (recType) {
case "H":
if (hhSize != personCounter)
logger.error(
"Inconsistent PUMS data: Found "
+ personCounter
+ " person(s) in dwelling with "
+ hhSize
+ " residents (Record "
+ (recCount - 1)
+ ").");
hhCount++;
hhSize = convertToInteger(recString.substring(105, 107));
int vacancy = convertToInteger(recString.substring(110, 111));
if ((hhSize != 0 && vacancy != 0) || (hhSize == 0 && vacancy == 0))
logger.error(
"Inconsistent PUMS "
+ "data: Found hhSize "
+ hhSize
+ " in dwelling with vacancy code "
+ vacancy
+ " (rec "
+ recCount
+ ")");
pumaZone = convertToInteger(recString.substring(9, 11) + recString.substring(13, 18));
weight = convertToInteger(recString.substring(101, 105));
// some PUMA zones are only partly covered by MSTM study area. Therefore, weight needs
// to be reduced by the share of population in this PUMA that is covered by MSTM
weight = (int) ((weight * 1f) * pumaScaler[pumaZone] + 0.5);
ddType = convertToInteger(recString.substring(114, 116));
bedRooms = convertToInteger(recString.substring(123, 124));
autos = convertToInteger(recString.substring(133, 134));
rent = convertToInteger(recString.substring(161, 165));
mortgage = convertToInteger(recString.substring(170, 175));
yearBuilt = convertToInteger(recString.substring(117, 118));
int completePlumbing = convertToInteger(recString.substring(126, 127));
int completeKitchen = convertToInteger(recString.substring(127, 128));
quality = guessQuality(completePlumbing, completeKitchen, yearBuilt);
personCounter = 0;
for (int i = 0; i < gender[i]; i++)
gender[i] =
0; // set gender variable to zero which practically erases previous household
break;
case "P":
relShp[personCounter] = convertToInteger(recString.substring(16, 18));
gender[personCounter] = convertToInteger(recString.substring(22, 23));
age[personCounter] = convertToInteger(recString.substring(24, 26));
if (age[personCounter] >= 90) {
if (gender[personCounter] == 1)
age[personCounter] = 90 + SiloUtil.select(probAge90plusMale);
else age[personCounter] = 90 + SiloUtil.select(probAge90plusFemale);
}
int hispanic = convertToInteger(recString.substring(27, 29));
int singleRace = convertToInteger(recString.substring(37, 38));
race[personCounter] = defineRace(hispanic, singleRace);
// int school = convertToInteger(recString.substring(48,
// 49));
occupation[personCounter] = convertToInteger(recString.substring(153, 154));
workPumaZone[personCounter] = convertToInteger(recString.substring(160, 165));
workState[personCounter] = convertToInteger(recString.substring(156, 159));
// fullTime[personCounter] = false;
// int hoursWorked =
// convertToInteger(recString.substring(240, 242));
// if (hoursWorked > 34) fullTime[personCounter] = true;
income[personCounter] =
Math.max(
convertToInteger(recString.substring(296, 303)),
0); // PUMS reports negative income for loss, which cannot be long-term income
personCounter++;
break;
default:
logger.error("Wrong record type in PUMS data in line " + recCount);
break;
}
// "personCounter == hhSize" after all person records for this household have been read
if (personCounter == hhSize && checkIfPumaInStudyArea(pumaZone)) {
recInStudyAreaCount++;
savePumsRecord(
pumaZone,
weight,
hhSize,
ddType,
bedRooms,
autos,
rent,
mortgage,
quality,
yearBuilt,
gender,
age,
race,
relShp,
occupation,
workPumaZone,
workState,
income);
}
}
logger.info(" Read " + hhCount + " PUMS household records from file: " + pumsFileName);
logger.info(" " + recInStudyAreaCount + " thereof located in study area");
} catch (IOException e) {
logger.fatal("IO Exception caught reading synpop household file: " + pumsFileName);
logger.fatal("recCount = " + recCount + ", recString = <" + recString + ">");
}
}
}