public void loadFromLocalFS(String fn, java.util.Vector<Integer> ids) throws IOException {
Configuration config = new Configuration();
if (useHDFS) {
System.out.println(">>HDFS<< :: " + fn);
/** define HDFS */
config.addResource(new Path("/etc/hadoop/conf/core-site.xml"));
config.addResource(new Path("/etc/hadoop/conf/hdfs-site.xml"));
} else {
System.out.println("<<LocalFS>> :: " + fn);
}
FileSystem fs = FileSystem.get(config);
Path path = new Path(fn);
// write a SequenceFile form a Vector
SequenceFile.Reader reader = new SequenceFile.Reader(fs, path, config);
System.out.println("--> process bucket : " + fn);
System.out.println("--> compression-codes : " + reader.getCompressionCodec());
System.out.println("--> key-classename : " + reader.getKeyClassName());
System.out.println("--> value-classname : " + reader.getValueClassName());
// TSBucket.setRecordMode( "page-id" );
TSBucket.setRecordMode("counter");
boolean goOn = true;
int i = 1;
while (goOn && i <= LIMIT) {
Text key = new Text();
VectorWritable vec = new VectorWritable();
goOn = reader.next(key);
int pageID = TSBucket.getRecordID(key.toString());
boolean skipped = false;
// bearbeite nur die, die im FILTER file stehen ...
if (ids != null) {
if (ids.contains(pageID)) {
skipped = false;
} else {
skipped = true;
}
}
reader.getCurrentValue(vec);
Messreihe mr = new Messreihe();
mr.setDescription(i + " ) " + fn + "_[" + key.toString() + "]");
mr.setLabel(pageID + " ");
int c = 0;
NamedVector vector = (NamedVector) vec.get();
while (c < vector.size()) {
double value = vector.get(c);
// System.out.println( c + "\t" + value );
mr.addValue(value);
c++;
}
try {
Messreihe m = null;
if (tst != null) {
m = tst.processReihe(fw, mr, fwe);
}
} catch (Exception ex) {
Logger.getLogger(TSBucket.class.getName()).log(Level.SEVERE, null, ex);
}
i = i + 1;
int code = 1;
if (skipped) {
code = 0;
}
if (inMEM) {
bucketData.add(mr);
}
// System.out.println( code + "\t" + mr.getLabel() + "\t" + mr.xValues.size() + "\t" +
// mr.summeY() );
}
System.out.println("--> nr of records : " + (i - 1));
}
public static void calcCharts() throws Exception {
// alle Dateien einlesen im obigen Ordner
File[] files = new File(folderIN).listFiles();
int i = 0;
Messreihe[] RQs = new Messreihe[zINTER];
Messreihe[] RISs = new Messreihe[zINTER];
Vector<Messreihe>[] vRISs = new Vector[zINTER];
Vector<Messreihe>[] vFITDATA = new Vector[zINTER];
ReturnIntervallStatistik[] ris = new ReturnIntervallStatistik[zINTER];
// 58 files and 9 sections ...
double tTEST_DATA[][][] = new double[files.length][zINTER][5];
// 0=RQs, 1=m 2=RR 3=FEHLENDE-m 4=FEHLENDE-RR
for (int ii = 0; ii < zINTER; ii++) {
ris[ii] = new ReturnIntervallStatistik(pScale, pBinning);
// ris[ii].scaleY_LN = sLN;
// ris[ii].scaleY_LOG = sLOG;
RQs[ii] = new Messreihe("RQ_" + ii);
RISs[ii] = new Messreihe("RIS_" + ii);
vRISs[ii] = new Vector<Messreihe>();
vFITDATA[ii] = new Vector<Messreihe>();
}
Vector<Messreihe> all = new Vector<Messreihe>();
double linFitMIN;
double linFitMAX;
for (File f : files) {
if (f.getName().endsWith(".txt")) {
DecimalFormat nf = new DecimalFormat("0.0000000");
Messreihe data =
MessreihenLoader.getLoader().loadMessreihe(new File(folderIN + f.getName()));
Messreihe[] parts = data.split(zL, zINTER);
int j = 0;
// r ist die Messreihe mit den normalen Daten
for (Messreihe r : parts) {
double vRQ = r.getAvarage();
// System.out.print( "file -> " + i + " part -> " + j + "\t" + nf.format( vRQ ) + "\t" );
// System.out.print( r.toString() );
tTEST_DATA[i][j][0] = vRQ;
vRISs[j].add(r);
RQs[j].addValue(vRQ);
ReturnIntervallStatistik rr = new ReturnIntervallStatistik(pScale, pBinning);
rr.doScale_by_Rq = scaleRQ;
// rr.scaleY_LN = sLN;
// rr.scaleY_LOG = sLOG;
rr.setRQ(vRQ);
rr.addDistData(r.getYValues());
if (scaleRQ) {
linFitMIN = linFit_MIN * vRQ;
linFitMAX = linFit_MAX * vRQ;
// System.out.println( "( " + linFit_MIN + ", " + linFit_MAX + ") ");
} else {
linFitMIN = linFit_MIN;
linFitMAX = linFit_MAX;
// System.out.println( "( " + linFitMIN + ", " + linFitMAX + ") ");
}
;
// hier sollten nun die Daten für jede Reihe da sein ...
/** Problem: in den einzelnen Reihen kann man nicht sinnvoll den Anstieg bestimmen !!! */
Messreihe toFit = rr.mrHaeufigkeit.shrinkX(linFit_MIN, linFit_MAX);
SimpleRegression reg;
try {
reg = toFit.linFit(linFit_MIN, linFit_MAX);
double slope = reg.getSlope();
double RR = reg.getRSquare();
// System.out.println("m=" + slope + " \tRR=" + RR);
tTEST_DATA[i][j][1] = slope;
tTEST_DATA[i][j][2] = RR;
all.add(toFit);
} catch (Exception ex) {
// Logger.getLogger(RISAnalyse3.class.getName()).log(Level.SEVERE, null, ex);
}
vFITDATA[j].add(rr.mrHaeufigkeit);
try {
ris[j].add(rr);
} catch (Exception ex) {
Logger.getLogger(RISAnalyse3.class.getName()).log(Level.SEVERE, null, ex);
}
j++;
}
i++;
}
}
if (showRAWData) {
for (int i3 = 0; i3 < zINTER; i3++) {
MultiChart.openAndStore(
vRISs[i3],
label + " Intervall: " + i3 + " - scale=" + pScale + ", binning=" + pBinning,
"r",
"P(r)",
false,
folderOUT,
label + "_P(r)",
getParameterSet());
}
}
System.out.print(folderOUT);
Messreihe rq = new Messreihe("Rq(i)");
Messreihe rqSIGMA = new Messreihe("sigma(Rq(i))");
Vector<Messreihe> chart2Data = new Vector<Messreihe>();
chart2Data.add(rq);
chart2Data.add(rqSIGMA);
Vector<Messreihe> vMR = new Vector<Messreihe>();
// für alle Intervalle des Files ...
for (int d = 0; d < zINTER; d++) {
try {
ris[d].calcMessreihen();
} catch (Exception ex) {
Logger.getLogger(RISAnalyse3.class.getName()).log(Level.SEVERE, null, ex);
}
RISs[d] = ris[d].mrHaeufigkeit;
// System.out.println( "(b)\n" + ris[d].toString() );
ris[d].mrVerteilung.setLabel("Int_" + (d + 1));
vMR.add(ris[d].mrVerteilung);
RQs[d].calcAverage();
rq.addValuePair((double) d, RQs[d].getAvarage());
rqSIGMA.addValuePair((double) d, RQs[d].getStddev());
}
rq.normalize();
rqSIGMA.normalize();
// auf VMR nun den Fit ANWENDEN ...
Messreihe fitData1 = new Messreihe();
fitData1.setLabel("m");
Messreihe fitData2 = new Messreihe();
fitData2.setLabel("RR");
Vector<Messreihe> vFITS = new Vector<Messreihe>();
SimpleRegression reg2;
int c = 0;
for (Messreihe mr : vMR) {
try {
reg2 = mr.linFit(linFit_MIN, linFit_MAX);
double slope = reg2.getSlope();
double RR = reg2.getRSquare();
fitData1.addValuePair(c, slope);
fitData2.addValuePair(c, RR);
System.out.println("\tm=" + slope + " \tRR=" + RR);
c++;
} catch (Exception ex) {
}
}
;
fitData1.normalize();
fitData2.normalize();
vFITS.add(fitData1);
vFITS.add(fitData2);
if (showRAWRIS) {
for (int i3 = 0; i3 < zINTER; i3++) {
MultiChart.openAndStore(
vFITDATA[i3],
"vFIT " + label + " Intervall: " + i3 + " - scale=" + pScale + ", binning=" + pBinning,
"r",
"P(r)",
false,
folderOUT,
label + "_P(r)",
getParameterSet());
}
}
String labelX = "r/Rq";
String labelY = "P(r)*Rq";
if (!scaleRQ) {
labelX = "r";
labelY = "P(r)";
}
if (sLOG) {
labelY = "log(" + labelY + ")";
}
if (sLN) {
labelY = "ln(" + labelY + ")";
}
if (showCharts) {
MultiChart.openAndStore(
vMR,
label + " - scale=" + pScale + ", binning=" + pBinning,
labelX,
labelY,
true,
folderOUT,
label + "_P(r)",
getParameterSet());
MultiChart.openAndStore(
vFITS,
"FIT " + label + " - scale=" + pScale + ", binning=" + pBinning,
"i",
"m(i) RR(i)",
true,
folderOUT,
label + "_m(i)_RR(i)",
getParameterSet());
MultiChart.openAndStore(
chart2Data,
label + " - Rq(i)",
"i",
"<Rq>, <sigma(Rq)>",
true,
folderOUT,
label + "_Rq(i)",
getParameterSet());
MultiChart.open(
all,
"ALL " + label + " - scale=" + pScale + ", binning=" + pBinning,
"x",
"y",
false,
getParameterSet());
}
MesswertTabelle tab1 = new MesswertTabelle();
tab1.setHeader(getParameterSet());
tab1.setLabel(label + "_P(r)");
tab1.setMessReihen(vMR);
tab1.writeToFile(new File(folderOUT + File.separator + tab1.getLabel() + ".dat"));
MesswertTabelle tab6 = new MesswertTabelle();
tab6.setHeader(getParameterSet());
tab6.setLabel(label + "_m(i)_RR(i)");
tab6.setMessReihen(vFITS);
tab6.writeToFile(
new File(folderOUT + File.separator + "out" + File.separator + tab6.getLabel() + ".dat"));
MesswertTabelle tab2 = new MesswertTabelle();
tab2.setHeader(getParameterSet());
tab2.setLabel(label + "_Rq(i)");
tab2.setMessReihen(chart2Data);
tab2.writeToFile(new File(folderOUT + File.separator + tab2.getLabel() + ".dat"));
String fn = folderOUT + "/tTests_" + label + ".xls";
String fnTEMPL = folderOUT + "/tTests.xls";
HSSFWorkbook wb = null;
HSSFSheet[] sheets = new HSSFSheet[3];
File f2 = new File(fn);
File f1 = new File(fnTEMPL);
// Lesen der bestehenden Tabelle
FileInputStream fis = null;
try {
//
// Create a FileInputStream that will be use to read the excel file.
//
fis = new FileInputStream(fn);
//
// Create an excel workbook from the file system.
//
wb = new HSSFWorkbook(fis);
//
// Get the first sheet on the workbook.
//
} catch (Exception ex) {
ex.printStackTrace();
System.err.println(">>> " + ex.getMessage());
System.out.println("> Erzeuge neue XLS Datei ... ");
wb = new HSSFWorkbook();
sheets[0] = wb.createSheet("R_q");
sheets[1] = wb.createSheet("m");
sheets[2] = wb.createSheet("RR");
}
Messreihe[] mrRESULTS = new Messreihe[5];
try {
sheets[0] = wb.getSheet("R_q");
sheets[1] = wb.getSheet("m");
sheets[2] = wb.getSheet("RR");
// zur Sammlung der Mittelwerte für die Resultate ...
mrRESULTS[0] = new Messreihe("<R_q>");
mrRESULTS[1] = new Messreihe("<m>");
mrRESULTS[2] = new Messreihe("<RR>");
mrRESULTS[3] = new Messreihe("{m}");
mrRESULTS[4] = new Messreihe("{RR}");
// SCHRANKE für auszulassende Werte
double epsilon = 1e-6;
// ANZAHL der Reihen
for (int i2 = 1; i2 < files.length; i2++) {
// ANZAHL der INTERVALLE
for (int j2 = 0; j2 < zINTER; j2++) {
// ANZAHL der MESSSGRÖSSEN
for (int k2 = 0; k2 < 3; k2++) {
HSSFRow row = sheets[k2].getRow(i2);
if (row == null) row = sheets[k2].createRow(i2);
row.createCell(1).setCellValue(i2);
double value = tTEST_DATA[i2][j2][k2];
if (!Double.isNaN(value)) {
if (value > epsilon) {
row.createCell(j2 + 1).setCellValue(value);
}
} else {
row.createCell(j2 + 1).setCellValue("");
}
// System.out.println( i2 +"\t" + j2 + "\t" + k2 + "\t" + tTEST_DATA[i2][j2][k2]);
}
}
}
// MW für alle Spalten berechnen ... für drei Messwerte ...
for (int x = 0; x < 3; x++) {
double eps = 1e-3;
double anz = 0.0;
double summe = 0.0;
int counterNAN_values = 0;
// ANZAHL der INTERVALLE
for (int j2 = 0; j2 < zINTER; j2++) {
// ANZAHL der Reihen
for (int i2 = 1; i2 < files.length; i2++) {
double value = tTEST_DATA[i2][j2][x];
double RR = tTEST_DATA[i2][j2][2];
if (!Double.isNaN(value)) {
// if ( value > eps ) { // Variante 1
// if ( RR > 0.75) { // Variante 2
summe = summe + value;
anz++;
// }
} else {
counterNAN_values++;
}
}
double mw = summe / anz;
mrRESULTS[x].addValuePair(j2, mw);
if (x > 0) mrRESULTS[x + 2].addValuePair(j2, counterNAN_values);
anz = 0.0;
summe = 0.0;
counterNAN_values = 0;
}
}
mrRESULTS[3].normalize();
mrRESULTS[4].normalize();
Vector<Messreihe> v = new Vector<Messreihe>();
v.add(mrRESULTS[0]);
v.add(mrRESULTS[1]);
v.add(mrRESULTS[2]);
v.add(mrRESULTS[3]);
v.add(mrRESULTS[4]);
if (showCharts) {
MultiChart.openAndStore(
v,
label + " - scale=" + pScale + ", binning=" + pBinning,
"i",
"f(i)",
true,
folderOUT,
label + "_chart3",
getParameterSet());
}
FileOutputStream fout2 = new FileOutputStream(f2.getAbsolutePath());
wb.write(fout2);
fout2.close();
String[] titles = {"P(r)", "Rq(i)", "chart3", "m(i)_RR(i)"};
report.createResultlinePNG(label, titles, getParameterSetHTML());
System.out.println("[READY] " + label + "");
} catch (Exception ex) {
ex.printStackTrace();
}
}