private SocialPerson createPerson(VertexRecord record, SQLDumpReader sqlData) {
Person matsimPerson = PersonImpl.createPerson(Id.create(record.id, Person.class));
SocialPerson person = new SocialPerson(matsimPerson);
int age;
if (record.isEgo) age = sqlData.getEgoAge(record.egoSQLId);
else age = sqlData.getAlterAge(record.alterKeys);
if (age < 0) errLogger.logNoAge(record.isEgo);
else PersonUtils.setAge(matsimPerson, age);
String sex = sqlData.getSex(record);
if (sex != null) PersonUtils.setSex(matsimPerson, sex);
else errLogger.logNoSex(record.isEgo);
if (record.isEgo) PersonUtils.setLicence(matsimPerson, sqlData.getLicense(record));
if (record.isEgo) PersonUtils.setCarAvail(matsimPerson, sqlData.getCarAvail(record));
person.setCitizenship(sqlData.getCitizenship(record));
person.setEducation(sqlData.getEducation(record));
person.setIncome(sqlData.getIncome(record));
person.setCivilStatus(sqlData.getCivilStatus(record));
return person;
}
public SampledGraphProjection<SocialSparseGraph, SocialSparseVertex, SocialSparseEdge> buildGraph(
List<String> alterTables, List<String> egoTables, List<String> sqlDumps) throws IOException {
errLogger = new ErrorLogger();
/*
* Load raw data.
*/
AlterTableReader alterReader = new AlterTableReader(alterTables);
EgoTableReader egoReader = new EgoTableReader(egoTables);
SQLDumpReader sqlReader = new SQLDumpReader(sqlDumps);
/*
* Build the raw graph and a sampled projection.
*/
graph = builder.createGraph();
proj = projBuilder.createGraph(graph);
/*
* Create the vertices.
*/
projMap = new HashMap<SocialSparseVertex, SampledVertexDecorator<SocialSparseVertex>>();
idMap = new HashMap<String, SocialSparseVertex>();
for (Entry<String, VertexRecord> entry : alterReader.getVertices().entrySet()) {
VertexRecord vRecord = entry.getValue();
/*
* Extract the home location.
*/
Point point;
if (vRecord.isEgo) {
point = sqlReader.getEgoLocation(vRecord.egoSQLId);
if (point == null) {
/*
* try getting coordinates via google
*/
logger.info("Requesting google server for coordinates.");
point = egoReader.getEgoLocation(vRecord.id);
}
} else {
point = sqlReader.getAlterLocation(vRecord.alterKeys);
}
if (point == null) {
errLogger.logNoCoordinate(vRecord.isEgo);
// point = geoFacotry.createPoint(new Coordinate(0, 0));
}
/*
* Create a vertex and its projection.
*/
SocialSparseVertex vertex = builder.addVertex(graph, createPerson(vRecord, sqlReader), point);
SampledVertexDecorator<SocialSparseVertex> vProj = projBuilder.addVertex(proj, vertex);
/*
* If it is an ego set the snowball attributes.
*/
if (vRecord.isEgo) {
vProj.sample(infereIterationSampled(new Integer(vRecord.id)));
vProj.detect(vProj.getIterationSampled() - 1);
}
projMap.put(vertex, vProj);
idMap.put(vRecord.id, vertex);
// recordMap.put(vRecord.id, vRecord);
}
/*
* Create the edges.
*/
for (Tuple<VertexRecord, VertexRecord> edge : alterReader.getEdges()) {
SocialSparseVertex v1 = idMap.get(edge.getFirst().id);
SocialSparseVertex v2 = idMap.get(edge.getSecond().id);
SocialSparseEdge socialEdge = builder.addEdge(graph, v1, v2);
/*
* Check if we have double edges.
*/
if (socialEdge != null) {
SampledVertexDecorator<SocialSparseVertex> vProj1 = projMap.get(v1);
SampledVertexDecorator<SocialSparseVertex> vProj2 = projMap.get(v2);
projBuilder.addEdge(proj, vProj1, vProj2, socialEdge);
/*
* Set the snowball attributes if it is not an ego.
*/
if (!vProj1.isSampled()) {
if (vProj1.isDetected())
/*
* If this vertex is already detected check if the adjacent vertex has been sampled earlier.
*/
vProj1.detect(Math.min(vProj1.getIterationDetected(), vProj2.getIterationSampled()));
else vProj1.detect(vProj2.getIterationSampled());
}
if (!vProj2.isSampled()) {
if (vProj2.isDetected())
/*
* If this vertex is already detected check if the adjacent vertex has been sampled earlier.
*/
vProj2.detect(Math.min(vProj2.getIterationDetected(), vProj1.getIterationSampled()));
else vProj2.detect(vProj1.getIterationSampled());
}
/*
* add edge attributes
*/
VertexRecord rec1 = edge.getFirst();
VertexRecord rec2 = edge.getSecond();
double freq = 0;
if (rec1.isEgo) {
freq = sqlReader.getF2FFrequencey(rec1.egoSQLId, rec2.alterKeys.get(rec1.egoSQLId));
} else {
freq = sqlReader.getF2FFrequencey(rec2.egoSQLId, rec1.alterKeys.get(rec2.egoSQLId));
}
socialEdge.setFrequency(freq);
socialEdge.setType(sqlReader.getEdgeType(rec1, rec2));
} else {
errLogger.logDoubleEdge();
}
}
/*
* Sociogram
*/
loadSociogramData(alterReader.getVertices().values(), sqlReader);
logger.info(errLogger.toString());
return proj;
}
private void loadSociogramData(Collection<VertexRecord> records, SQLDumpReader sqlData) {
logger.info("Loading sociogram data...");
Map<String, VertexRecord> map = sqlData.getFullAlterKeyMappping(records);
TObjectIntHashMap<Vertex> rawDegrees = new TObjectIntHashMap<Vertex>();
for (Vertex v : proj.getVertices()) {
rawDegrees.put(v, v.getNeighbours().size());
}
int edgecnt = 0;
int doublecnt = 0;
int egoEdge = 0;
Set<Vertex> notOkVertices = new HashSet<Vertex>();
Set<Vertex> okVertices = new HashSet<Vertex>();
DescriptiveStatistics notOkStats = new DescriptiveStatistics();
DescriptiveStatistics okStats = new DescriptiveStatistics();
DescriptiveStatistics numDistr = new DescriptiveStatistics();
DescriptiveStatistics numDistrNoZero = new DescriptiveStatistics();
DescriptiveStatistics sizeDistr = new DescriptiveStatistics();
TDoubleArrayList sizeValues = new TDoubleArrayList();
TDoubleArrayList kSizeValues = new TDoubleArrayList();
TDoubleArrayList numValues = new TDoubleArrayList();
TDoubleArrayList numValues2 = new TDoubleArrayList();
TDoubleArrayList kNumValues = new TDoubleArrayList();
for (VertexRecord record : records) {
if (record.isEgo) {
List<Set<String>> cliques = sqlData.getCliques(record);
numDistr.addValue(cliques.size());
Vertex v = idMap.get(record.id);
numValues.add(cliques.size());
kNumValues.add(v.getNeighbours().size());
if (!cliques.isEmpty()) numDistrNoZero.addValue(cliques.size());
for (Set<String> clique : cliques) {
sizeDistr.addValue(clique.size());
sizeValues.add(clique.size());
kSizeValues.add(rawDegrees.get(projMap.get(v)));
numValues2.add(cliques.size());
List<SocialSparseVertex> vertices = new ArrayList<SocialSparseVertex>(clique.size());
for (String alter : clique) {
VertexRecord r = map.get(record.egoSQLId + alter);
if (r != null) {
SocialSparseVertex vertex = idMap.get(r.id);
if (vertex != null) {
vertices.add(vertex);
} else {
logger.warn("Vertex not found.");
}
} else {
logger.warn("Record not found.");
}
}
for (int i = 0; i < vertices.size(); i++) {
for (int j = i + 1; j < vertices.size(); j++) {
SampledVertexDecorator<SocialSparseVertex> vProj1 = projMap.get(vertices.get(i));
SampledVertexDecorator<SocialSparseVertex> vProj2 = projMap.get(vertices.get(j));
if (!vProj1.isSampled() && !vProj2.isSampled()) {
if (Math.random() < 0.62) {
SocialSparseEdge socialEdge =
builder.addEdge(graph, vertices.get(i), vertices.get(j));
if (socialEdge != null) {
projBuilder.addEdge(proj, vProj1, vProj2, socialEdge);
edgecnt++;
if (vProj1.isSampled() || vProj2.isSampled()) {
egoEdge++;
if (vProj1.isSampled()) notOkVertices.add(vProj1);
else notOkVertices.add(vProj2);
}
} else {
doublecnt++;
if (vProj1.isSampled()) okVertices.add(vProj1);
else if (vProj2.isSampled()) okVertices.add(vProj2);
}
}
}
}
}
}
}
}
for (Vertex v : okVertices) okStats.addValue(rawDegrees.get(v));
for (Vertex v : notOkVertices) notOkStats.addValue(rawDegrees.get(v));
try {
TDoubleDoubleHashMap hist =
Histogram.createHistogram(okStats, new LinearDiscretizer(1), false);
StatsWriter.writeHistogram(
hist,
"k",
"n",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/k_ok.txt");
TDoubleDoubleHashMap hist2 =
Histogram.createHistogram(notOkStats, new LinearDiscretizer(1), false);
StatsWriter.writeHistogram(
hist2,
"k",
"n",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/k_notok.txt");
TDoubleDoubleHashMap ratio = new TDoubleDoubleHashMap();
double[] keys = hist.keys();
for (double k : keys) {
double val1 = hist2.get(k);
double val2 = hist.get(k);
ratio.put(k, val1 / (val2 + val1));
}
StatsWriter.writeHistogram(
ratio,
"k",
"p",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/k_ratio.txt");
logger.info("Mean num of cliques: " + numDistrNoZero.getMean());
logger.info("Mean size: " + sizeDistr.getMean());
logger.info("Median num of cliques: " + StatUtils.percentile(numDistrNoZero.getValues(), 50));
logger.info("Median size: " + StatUtils.percentile(sizeDistr.getValues(), 50));
TDoubleDoubleHashMap histNum =
Histogram.createHistogram(
numDistrNoZero,
FixedSampleSizeDiscretizer.create(numDistrNoZero.getValues(), 2, 20),
true);
Histogram.normalize(histNum);
StatsWriter.writeHistogram(
histNum,
"num",
"freq",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/numCliques.txt");
TDoubleDoubleHashMap histSize =
Histogram.createHistogram(
sizeDistr, FixedSampleSizeDiscretizer.create(sizeDistr.getValues(), 2, 20), true);
Histogram.normalize(histSize);
StatsWriter.writeHistogram(
histSize,
"size",
"freq",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/numPersons.txt");
Discretizer discretizer =
FixedSampleSizeDiscretizer.create(kSizeValues.toNativeArray(), 20, 20);
TDoubleArrayList valuesX = new TDoubleArrayList();
for (int i = 0; i < kSizeValues.size(); i++) {
valuesX.add(discretizer.discretize(kSizeValues.get(i)));
}
Correlations.writeToFile(
Correlations.mean(valuesX.toNativeArray(), sizeValues.toNativeArray()),
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/size_k.txt",
"k",
"size");
discretizer = FixedSampleSizeDiscretizer.create(kNumValues.toNativeArray(), 20, 20);
valuesX = new TDoubleArrayList();
for (int i = 0; i < kNumValues.size(); i++) {
valuesX.add(discretizer.discretize(kNumValues.get(i)));
}
Correlations.writeToFile(
Correlations.mean(valuesX.toNativeArray(), numValues.toNativeArray()),
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/num_k.txt",
"k",
"n");
Correlations.writeToFile(
Correlations.mean(numValues2.toNativeArray(), sizeValues.toNativeArray()),
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/size_num.txt",
"num",
"size");
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
logger.info(
String.format("Inserted %1$s edges, %2$s edges already present.", edgecnt, doublecnt));
logger.info(String.format("Inserted %1$s edges between at least one ego.", egoEdge));
}
