/**
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
BufferedReader reader =
new BufferedReader(
new FileReader(
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/analysis/plain/ch/obs/spatial/p_accept.raw.txt"));
String line = null;
TDoubleArrayList data = new TDoubleArrayList();
while ((line = reader.readLine()) != null) {
data.add(Double.parseDouble(line));
}
double alpha = estimAlpha(data.toNativeArray(), 2000, 250000);
System.out.println("alpha = " + alpha);
// reader = new BufferedReader(new
// FileReader("/Users/jillenberger/Desktop/powerlaw/data.txt"));
// line = null;
// data = new TDoubleArrayList();
// while((line = reader.readLine()) != null) {
// data.add(Double.parseDouble(line));
// }
//
// alpha = estimAlpha(data.toNativeArray(), 1000, 500000);
// System.out.println("alpha = " + alpha);
}
public static double estimAlpha(double[] data, double xmin, double xmax) {
TDoubleArrayList subdata = new TDoubleArrayList(data.length);
for (double x : data) {
if (x >= xmin && x <= xmax) subdata.add(x);
}
double sum = 0;
for (int i = 0; i < subdata.size(); i++) {
sum += Math.log(subdata.get(i) / xmin);
}
return 1 + subdata.size() / sum;
}
private void analyzeIntern(
Set<Trajectory> trajectories, String purpose, TObjectDoubleHashMap<Trajectory> distancesMap) {
TDoubleArrayList arrivals = new TDoubleArrayList(trajectories.size());
TDoubleArrayList distances = new TDoubleArrayList(trajectories.size());
for (Trajectory t : trajectories) {
for (int i = 0; i < t.getElements().size(); i += 2) {
if (purpose == null || ((Activity) t.getElements().get(i)).getType().equals(purpose)) {
double start = t.getTransitions().get(i);
arrivals.add(start);
distances.add(distancesMap.get(t));
}
}
}
TDoubleDoubleHashMap map =
Correlations.mean(arrivals.toNativeArray(), distances.toNativeArray(), 3600);
// Discretizer d = FixedSampleSizeDiscretizer.create(arrivals.toNativeArray(), 100, 100);
// TDoubleDoubleHashMap map = Correlations.mean(arrivals.toNativeArray(),
// distances.toNativeArray(), d);
try {
if (purpose == null)
StatsWriter.writeHistogram(
map, "arr", "dist", String.format("%1$s/dist_arr.txt", getOutputDirectory()));
else
StatsWriter.writeHistogram(
map,
"arr",
"dist",
String.format("%1$s/dist_arr.%2$s.txt", getOutputDirectory(), purpose));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
/* (non-Javadoc)
* @see playground.johannes.gsv.synPop.analysis.AnalyzerTask#analyze(java.util.Collection, java.util.Map)
*/
@Override
public void analyze(Collection<PlainPerson> persons, Map<String, DescriptiveStatistics> results) {
TDoubleArrayList ages = new TDoubleArrayList();
TDoubleArrayList incomes = new TDoubleArrayList();
for (PlainPerson person : persons) {
String aStr = person.getAttribute(CommonKeys.PERSON_AGE);
String iStr = person.getAttribute(CommonKeys.HH_INCOME);
// String mStr = person.getAttribute(CommonKeys.HH_MEMBERS);
// if(aStr != null && iStr != null && mStr != null) {
if (aStr != null && iStr != null) {
double age = Double.parseDouble(aStr);
double income = Double.parseDouble(iStr);
// double members = Double.parseDouble(mStr);
ages.add(age);
// incomes.add(income/members);
incomes.add(income);
}
}
try {
// TDoubleDoubleHashMap hist = Histogram.createHistogram(ages.toNativeArray(), new
// LinearDiscretizer(5), false);
TDoubleDoubleHashMap hist =
Histogram.createHistogram(ages.toNativeArray(), new DummyDiscretizer(), false);
TXTWriter.writeMap(hist, "age", "n", getOutputDirectory() + "/age.txt");
hist = Histogram.createHistogram(incomes.toNativeArray(), new LinearDiscretizer(500), false);
TXTWriter.writeMap(hist, "income", "n", getOutputDirectory() + "/income.txt");
TXTWriter.writeScatterPlot(
ages, incomes, "age", "income", getOutputDirectory() + "/age.income.txt");
TXTWriter.writeMap(
Correlations.mean(ages.toNativeArray(), incomes.toNativeArray()),
"age",
"income",
getOutputDirectory() + "/age.income.mean.txt");
} catch (IOException e) {
e.printStackTrace();
}
}
private void loadSociogramData(Collection<VertexRecord> records, SQLDumpReader sqlData) {
logger.info("Loading sociogram data...");
Map<String, VertexRecord> map = sqlData.getFullAlterKeyMappping(records);
TObjectIntHashMap<Vertex> rawDegrees = new TObjectIntHashMap<Vertex>();
for (Vertex v : proj.getVertices()) {
rawDegrees.put(v, v.getNeighbours().size());
}
int edgecnt = 0;
int doublecnt = 0;
int egoEdge = 0;
Set<Vertex> notOkVertices = new HashSet<Vertex>();
Set<Vertex> okVertices = new HashSet<Vertex>();
DescriptiveStatistics notOkStats = new DescriptiveStatistics();
DescriptiveStatistics okStats = new DescriptiveStatistics();
DescriptiveStatistics numDistr = new DescriptiveStatistics();
DescriptiveStatistics numDistrNoZero = new DescriptiveStatistics();
DescriptiveStatistics sizeDistr = new DescriptiveStatistics();
TDoubleArrayList sizeValues = new TDoubleArrayList();
TDoubleArrayList kSizeValues = new TDoubleArrayList();
TDoubleArrayList numValues = new TDoubleArrayList();
TDoubleArrayList numValues2 = new TDoubleArrayList();
TDoubleArrayList kNumValues = new TDoubleArrayList();
for (VertexRecord record : records) {
if (record.isEgo) {
List<Set<String>> cliques = sqlData.getCliques(record);
numDistr.addValue(cliques.size());
Vertex v = idMap.get(record.id);
numValues.add(cliques.size());
kNumValues.add(v.getNeighbours().size());
if (!cliques.isEmpty()) numDistrNoZero.addValue(cliques.size());
for (Set<String> clique : cliques) {
sizeDistr.addValue(clique.size());
sizeValues.add(clique.size());
kSizeValues.add(rawDegrees.get(projMap.get(v)));
numValues2.add(cliques.size());
List<SocialSparseVertex> vertices = new ArrayList<SocialSparseVertex>(clique.size());
for (String alter : clique) {
VertexRecord r = map.get(record.egoSQLId + alter);
if (r != null) {
SocialSparseVertex vertex = idMap.get(r.id);
if (vertex != null) {
vertices.add(vertex);
} else {
logger.warn("Vertex not found.");
}
} else {
logger.warn("Record not found.");
}
}
for (int i = 0; i < vertices.size(); i++) {
for (int j = i + 1; j < vertices.size(); j++) {
SampledVertexDecorator<SocialSparseVertex> vProj1 = projMap.get(vertices.get(i));
SampledVertexDecorator<SocialSparseVertex> vProj2 = projMap.get(vertices.get(j));
if (!vProj1.isSampled() && !vProj2.isSampled()) {
if (Math.random() < 0.62) {
SocialSparseEdge socialEdge =
builder.addEdge(graph, vertices.get(i), vertices.get(j));
if (socialEdge != null) {
projBuilder.addEdge(proj, vProj1, vProj2, socialEdge);
edgecnt++;
if (vProj1.isSampled() || vProj2.isSampled()) {
egoEdge++;
if (vProj1.isSampled()) notOkVertices.add(vProj1);
else notOkVertices.add(vProj2);
}
} else {
doublecnt++;
if (vProj1.isSampled()) okVertices.add(vProj1);
else if (vProj2.isSampled()) okVertices.add(vProj2);
}
}
}
}
}
}
}
}
for (Vertex v : okVertices) okStats.addValue(rawDegrees.get(v));
for (Vertex v : notOkVertices) notOkStats.addValue(rawDegrees.get(v));
try {
TDoubleDoubleHashMap hist =
Histogram.createHistogram(okStats, new LinearDiscretizer(1), false);
StatsWriter.writeHistogram(
hist,
"k",
"n",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/k_ok.txt");
TDoubleDoubleHashMap hist2 =
Histogram.createHistogram(notOkStats, new LinearDiscretizer(1), false);
StatsWriter.writeHistogram(
hist2,
"k",
"n",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/k_notok.txt");
TDoubleDoubleHashMap ratio = new TDoubleDoubleHashMap();
double[] keys = hist.keys();
for (double k : keys) {
double val1 = hist2.get(k);
double val2 = hist.get(k);
ratio.put(k, val1 / (val2 + val1));
}
StatsWriter.writeHistogram(
ratio,
"k",
"p",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/k_ratio.txt");
logger.info("Mean num of cliques: " + numDistrNoZero.getMean());
logger.info("Mean size: " + sizeDistr.getMean());
logger.info("Median num of cliques: " + StatUtils.percentile(numDistrNoZero.getValues(), 50));
logger.info("Median size: " + StatUtils.percentile(sizeDistr.getValues(), 50));
TDoubleDoubleHashMap histNum =
Histogram.createHistogram(
numDistrNoZero,
FixedSampleSizeDiscretizer.create(numDistrNoZero.getValues(), 2, 20),
true);
Histogram.normalize(histNum);
StatsWriter.writeHistogram(
histNum,
"num",
"freq",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/numCliques.txt");
TDoubleDoubleHashMap histSize =
Histogram.createHistogram(
sizeDistr, FixedSampleSizeDiscretizer.create(sizeDistr.getValues(), 2, 20), true);
Histogram.normalize(histSize);
StatsWriter.writeHistogram(
histSize,
"size",
"freq",
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/numPersons.txt");
Discretizer discretizer =
FixedSampleSizeDiscretizer.create(kSizeValues.toNativeArray(), 20, 20);
TDoubleArrayList valuesX = new TDoubleArrayList();
for (int i = 0; i < kSizeValues.size(); i++) {
valuesX.add(discretizer.discretize(kSizeValues.get(i)));
}
Correlations.writeToFile(
Correlations.mean(valuesX.toNativeArray(), sizeValues.toNativeArray()),
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/size_k.txt",
"k",
"size");
discretizer = FixedSampleSizeDiscretizer.create(kNumValues.toNativeArray(), 20, 20);
valuesX = new TDoubleArrayList();
for (int i = 0; i < kNumValues.size(); i++) {
valuesX.add(discretizer.discretize(kNumValues.get(i)));
}
Correlations.writeToFile(
Correlations.mean(valuesX.toNativeArray(), numValues.toNativeArray()),
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/num_k.txt",
"k",
"n");
Correlations.writeToFile(
Correlations.mean(numValues2.toNativeArray(), sizeValues.toNativeArray()),
"/Users/jillenberger/Work/socialnets/data/ivt2009/11-2011/augmented/size_num.txt",
"num",
"size");
} catch (FileNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
logger.info(
String.format("Inserted %1$s edges, %2$s edges already present.", edgecnt, doublecnt));
logger.info(String.format("Inserted %1$s edges between at least one ego.", egoEdge));
}