public List<Point2D.Double> debugColumnVersion(
String timestamp, double latitude, double longitude, double radius) {
this.getStatLog(this.STAT_FILE_NAME);
long sTime = System.currentTimeMillis();
// build up a raster
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
Point2D.Double point = new Point2D.Double(latitude, longitude);
ResultScanner rScanner = null;
// return result
HashMap<String, String> results = new HashMap<String, String>();
ArrayList<Point2D.Double> returnPoints = new ArrayList<Point2D.Double>();
try {
// match rect to find the subspace it belongs to
XBox[] match_boxes = raster.match(latitude, longitude, radius);
String[] rowRange = new String[2];
rowRange[0] = match_boxes[0].getRow();
rowRange[1] = match_boxes[1].getRow() + "0";
String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
// the version here is harded coded, because i cannot get how many
// objects in one cell now
FilterList fList = new FilterList();
fList.addFilter(this.hbaseUtil.getInclusiveFilter(rowRange[1]));
rScanner = this.hbaseUtil.getResultSet(rowRange, fList, this.familyName, c, 1000000);
BixiReader reader = new BixiReader();
int count = 0;
int accepted = 0;
int max_column = 0;
int min_column = 10000;
int max_version = 0;
int min_version = 10000;
int row_count = 0;
int byte_lenght = 0;
for (Result r : rScanner) {
byte_lenght = r.getBytes().getLength();
row_count++;
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
r.getMap();
int count_column = 0;
for (byte[] family : resultMap.keySet()) {
NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
count_column = 0;
for (byte[] col : columns.keySet()) {
NavigableMap<Long, byte[]> values = columns.get(col);
count_column++;
if (values.values().size() > max_version) {
max_version = values.values().size();
}
if (values.values().size() < min_version) {
min_version = values.values().size();
}
for (Long version : values.keySet()) {
count++;
// get the distance between this point and the given
// point
XStation station = reader.getStationFromJson(Bytes.toString(values.get(version)));
Point2D.Double resPoint =
new Point2D.Double(station.getLatitude(), station.getlongitude());
double distance = resPoint.distance(point);
if (Bytes.toString(col).equals("0011")) {
/* System.out.println("!!!! key=>"+Bytes.toString(r.getRow())+
";column=>"+Bytes.toString(col)+
";version=>"+version+
";point=>"+resPoint.toString());*/
}
if (distance <= radius) {
returnPoints.add(resPoint);
// System.out.println("row=>"+Bytes.toString(r.getRow())
// +
// ";colum=>"+Bytes.toString(col)+";version=>"+version+
// ";station=>"+station.getId()+";distance=>"+distance);
accepted++;
results.put(station.getId(), String.valueOf(distance));
}
}
}
if (count_column > max_column) max_column = count_column;
if (count_column < min_column) min_column = count_column;
}
}
System.out.println("byte_length=>" + byte_lenght + ";row_count=>" + row_count);
System.out.println(
"max_column=>"
+ max_column
+ ";min_column=>"
+ min_column
+ ";max_version=>"
+ max_version
+ ";min_version=>"
+ min_version);
long eTime = System.currentTimeMillis();
System.out.println(
"count=>" + count + ";accepted=>" + accepted + ";time=>" + (eTime - sTime));
String outStr =
"radius=>"
+ radius
+ ";count=>"
+ count
+ ";accepted=>"
+ accepted
+ ";time=>"
+ (eTime - sTime)
+ ";row_stride=>"
+ this.min_size_of_height
+ ";columns=>"
+ this.max_num_of_column;
this.writeStat(outStr);
} catch (Exception e) {
e.printStackTrace();
} finally {
this.hbaseUtil.closeTableHandler();
this.closeStatLog();
}
return returnPoints;
}
@Override
public List<String> copQueryAvailableNear(
String timestamp, final double latitude, final double longitude, final double radius) {
this.getStatLog(STAT_FILE_NAME);
this.getCSVLog(this.FILE_NAME_PREFIX, 0);
this.getCSVLog(this.FILE_NAME_PREFIX, 1);
this.getCSVLog(this.FILE_NAME_PREFIX, 2);
this.timePhase.clear();
try {
/** Step1** Call back class definition * */
class BixiCallBack implements Batch.Callback<RCopResult> {
RCopResult res = new RCopResult();
int count = 0;
QueryAbstraction query = null;
public BixiCallBack(QueryAbstraction query) {
this.query = query;
}
@Override
public void update(byte[] region, byte[] row, RCopResult result) {
long current = System.currentTimeMillis();
count++;
res.getRes().addAll(result.getRes()); // to verify the error when large data
res.setStart(result.getStart());
res.setEnd(result.getEnd());
res.setRows((res.getRows() + result.getRows()));
res.setCells(res.getCells() + result.getCells());
// System.out.println((count++) + ": come back region: "+ Bytes.toString(region) + ";
// result: "+ result.size());
/* String outStr="count=>"+count+";start=>"+result.getStart()+";end=>"+result.getEnd()+";process=>"+(result.getEnd()-result.getStart())+
";tranmission=>"+(System.currentTimeMillis()-result.getEnd())+
";region=>"+Bytes.toString(region)+
";row=>"+result.getRows()+";result=>"+result.getRes().size();
this.query.writeStat(outStr);
outStr = query.regions.get(Bytes.toString(region)).toString();
this.query.writeStat(outStr); */
// write them into csv file
String outStr = "";
outStr +=
"within,"
+ "cop,"
+ result.getParameter()
+ ","
+ result.getStart()
+ ","
+ result.getEnd()
+ ","
+ current
+ ","
+ result.getRows()
+ ","
+ result.getCells()
+ ","
+ ","
+ result.getKvLength()
+ result.getRes().size()
+ ","
+ this.query.regionAndRS.get(Bytes.toString(region))
+ ","
+ Bytes.toString(region);
this.query.writeCSVLog(outStr, 1);
}
}
BixiCallBack callBack = new BixiCallBack(this);
/** Step2** generate the scan ********** */
long s_time = System.currentTimeMillis();
this.timePhase.add(s_time);
// build up the Raster
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
// match the query area in Raster to get the row range and column
// range
long match_s = System.currentTimeMillis();
XBox[] match_boxes = raster.match(latitude, longitude, radius);
long match_time = System.currentTimeMillis() - match_s;
String[] rowRange = new String[2];
rowRange[0] = match_boxes[0].getRow();
rowRange[1] = match_boxes[1].getRow() + "-*";
String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
// generate the scan
final Scan scan =
hbaseUtil.generateScan(
rowRange, null, new String[] {BixiConstant.LOCATION_FAMILY_NAME}, c, 1000000);
/** Step3** send out the query to trigger the corresponding function in Coprocessor*** */
this.timePhase.add(System.currentTimeMillis());
hbaseUtil
.getHTable()
.coprocessorExec(
BixiProtocol.class,
scan.getStartRow(),
scan.getStopRow(),
new Batch.Call<BixiProtocol, RCopResult>() {
public RCopResult call(BixiProtocol instance) throws IOException {
return instance.copQueryNeighbor4LS2(scan, latitude, longitude, radius);
};
},
callBack);
long e_time = System.currentTimeMillis();
this.timePhase.add(e_time);
long exe_time = e_time - s_time;
/* System.out.println("exe_time=>"+exe_time+";result=>"+callBack.res.getRes().size());
String outStr = "m=>cop;"+"radius=>"+radius+";exe_time=>"+exe_time+";result=>"+callBack.res.getRes().size();
this.writeStat(outStr);*/
// write to csv file
String outStr = "";
outStr +=
"within,"
+ "cop,"
+ callBack.res.getCells()
+ ","
+ callBack.res.getRes().size()
+ ","
+ exe_time
+ ","
+ callBack.res.getRows()
+ ","
+ match_time
+ ","
+ this.min_size_of_height
+ ","
+ radius;
this.writeCSVLog(outStr, 0);
String timeStr = "within,cop," + radius + ",";
for (int i = 0; i < this.timePhase.size(); i++) {
timeStr += this.timePhase.get(i) + ",";
}
this.writeCSVLog(timeStr, 2);
return callBack.res.getRes();
} catch (Exception e) {
e.printStackTrace();
} catch (Throwable ee) {
ee.printStackTrace();
} finally {
hbaseUtil.closeTableHandler();
this.closeStatLog();
this.closeCSVLog();
}
return null;
}
@Override
public TreeMap<Double, String> scanQueryAvailableKNN(
String timestamp, double latitude, double longitude, int k) {
this.getStatLog(STAT_FILE_NAME);
long sTime = System.currentTimeMillis();
// Step1: estimate the window circle for the first time
int total_points =
Integer.valueOf(this.conf.getProperty("total_num_of_points")); // 1000000000;//
double areaOfMBB = this.space.width * this.space.height;
double DensityOfMBB = total_points / areaOfMBB;
double init_radius = Math.sqrt(k / DensityOfMBB);
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
longitude = Math.abs(longitude);
int count = 0;
int accepted = 0;
ResultScanner rScanner = null;
List<String> resultsList = new ArrayList<String>();
BixiReader reader = new BixiReader();
TreeMap<Double, String> sorted = null;
try {
// Step2: trigger a scan to get the points based on the above window
int iteration = 1;
long match_s = System.currentTimeMillis();
double radius =
(init_radius > this.min_size_of_height) ? init_radius : this.min_size_of_height;
do {
String str = "iteration" + iteration + "; count=>" + count + ";radius=>" + radius;
this.writeStat(str);
// match rect to find the subspace it belongs to
XBox[] match_boxes = raster.match(latitude, longitude, radius);
String[] rowRange = new String[2];
rowRange[0] = match_boxes[0].getRow();
rowRange[1] = match_boxes[1].getRow() + "-*";
String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
rScanner = this.hbaseUtil.getResultSet(rowRange, null, this.familyName, c, 1000000);
count = 0;
resultsList.clear();
for (Result r : rScanner) {
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
r.getMap();
for (byte[] family : resultMap.keySet()) {
NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
for (byte[] col : columns.keySet()) {
NavigableMap<Long, byte[]> values = columns.get(col);
for (Long version : values.keySet()) {
resultsList.add(Bytes.toString(values.get(version)));
count++;
}
}
}
}
radius = init_radius * (iteration + 1);
// Step3: get the result,estimate the window circle next depending on the previous step
// result, util we got the K nodes
/* if(count == 0 && iteration ==1){ // when the first time count == 0
radius = radius * 2;
}else if(count > 0 && iteration >0){ // when the first time count >0 && count < k
areaOfMBB = radius * radius;
DensityOfMBB = count / areaOfMBB;
radius = Math.sqrt(k / DensityOfMBB);
}*/
} while (count < k && (++iteration > 0));
String str = "iteration" + iteration + "; count=>" + count + ";radius=>" + radius;
this.writeStat(str);
long match_time = System.currentTimeMillis() - match_s;
// Step4: get all possible points and sort them by the distance and get the top K
Point2D.Double point = new Point2D.Double(latitude, longitude);
// result container
HashMap<Double, String> distanceMap = new HashMap<Double, String>();
for (String value : resultsList) {
XStation station = reader.getStationFromJson(value);
Point2D.Double resPoint = new Point2D.Double(station.getLatitude(), station.getlongitude());
double distance = resPoint.distance(point);
distanceMap.put(distance, station.getId());
}
sorted = new TreeMap<Double, String>(distanceMap);
long eTime = System.currentTimeMillis();
String outStr =
"q=knn;m=>scan;"
+ ";count=>"
+ count
+ ";accepted=>"
+ accepted
+ ";time=>"
+ (eTime - sTime)
+ ";k=>"
+ k;
this.writeStat(outStr);
} catch (Exception e) {
e.printStackTrace();
} finally {
this.hbaseUtil.closeTableHandler();
this.closeStatLog();
}
return sorted;
}
@Override
public String scanQueryPoint(double latitude, double longitude) {
this.getStatLog(STAT_FILE_NAME);
long sTime = System.currentTimeMillis();
// build up a raster
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
ResultScanner rScanner = null;
try {
// match rect to find the subspace it belongs to
XBox match_box = raster.locate(latitude, Math.abs(longitude));
System.out.println("match_box is : " + match_box.toString());
String[] rowRange = new String[2];
rowRange[0] = match_box.getRow();
rowRange[1] = match_box.getRow() + "-*";
// the version here is harded coded, because i cannot get how many
// objects in one cell now
rScanner =
this.hbaseUtil.getResultSet(
rowRange, null, this.familyName, new String[] {match_box.getColumn()}, 1000000);
BixiReader reader = new BixiReader();
int count = 0;
int row = 0;
String stationName = null;
for (Result r : rScanner) {
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
r.getMap();
row++;
for (byte[] family : resultMap.keySet()) {
NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
for (byte[] col : columns.keySet()) {
NavigableMap<Long, byte[]> values = columns.get(col);
for (Long version : values.keySet()) {
count++;
// get the distance between this point and the given
// point
XStation station = reader.getStationFromJson(Bytes.toString(values.get(version)));
if ((station.getLatitude() == latitude && station.getlongitude() == longitude)) {
stationName = station.getId();
break;
}
}
if (stationName != null) break;
}
if (stationName != null) break;
}
if (stationName != null) break;
}
long eTime = System.currentTimeMillis();
System.out.println(
"count=>" + count + ";time=>" + (eTime - sTime) + ";result=>" + stationName);
String outStr =
"q=point;m=scan;count=>"
+ count
+ ";time=>"
+ (eTime - sTime)
+ ";row=>"
+ row
+ ";result=>"
+ stationName;
this.writeStat(outStr);
return stationName;
} catch (Exception e) {
e.printStackTrace();
} finally {
this.hbaseUtil.closeTableHandler();
this.closeStatLog();
}
return null;
}
@Override
public String copQueryPoint(final double latitude, final double longitude) {
this.getStatLog(STAT_FILE_NAME);
long s_time = System.currentTimeMillis();
try {
/** Step1** Call back class definition * */
class BixiCallBack implements Batch.Callback<String> {
String res = null;
int count = 0;
QueryAbstraction query = null;
public BixiCallBack(QueryAbstraction query) {
this.query = query;
}
@Override
public void update(byte[] region, byte[] row, String result) {
count++;
String outStr =
"endTime=>"
+ System.currentTimeMillis()
+ ";region=>"
+ Bytes.toString(region)
+ ";count=>"
+ count
+ ";result=>"
+ result;
this.query.writeStat(outStr);
outStr = query.regions.get(Bytes.toString(region)).toString();
this.query.writeStat(outStr);
res = result;
}
}
BixiCallBack callBack = new BixiCallBack(this);
/** Step2** generate the scan ********** */
// build up the Raster
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
XBox match_box = raster.locate(latitude, Math.abs(longitude));
System.out.println("match_box is : " + match_box.toString());
String[] rowRange = new String[2];
rowRange[0] = match_box.getRow();
rowRange[1] = match_box.getRow() + "0";
// generate the scan
final Scan scan =
hbaseUtil.generateScan(
rowRange,
null,
new String[] {BixiConstant.LOCATION_FAMILY_NAME},
new String[] {match_box.getColumn()},
100000);
/** Step3** send out the query to trigger the corresponding function in Coprocessor*** */
hbaseUtil
.getHTable()
.coprocessorExec(
BixiProtocol.class,
scan.getStartRow(),
scan.getStopRow(),
new Batch.Call<BixiProtocol, String>() {
public String call(BixiProtocol instance) throws IOException {
return instance.copQueryPoint4LS2(scan, latitude, longitude);
};
},
callBack);
long e_time = System.currentTimeMillis();
long exe_time = e_time - s_time;
// TODO store the time into database
System.out.println("exe_time=>" + exe_time + ";result=>" + callBack.res);
String outStr = "q=point;m=>cop;" + ";exe_time=>" + exe_time + ";result=>" + callBack.res;
this.writeStat(outStr);
return callBack.res;
} catch (Exception e) {
e.printStackTrace();
} catch (Throwable ee) {
ee.printStackTrace();
} finally {
hbaseUtil.closeTableHandler();
this.closeStatLog();
}
return null;
}
@Override
public HashMap<String, String> scanQueryAvailableNear(
String timestamp, double latitude, double longitude, double radius) {
this.getStatLog(STAT_FILE_NAME);
this.getCSVLog(FILE_NAME_PREFIX, 0);
this.getCSVLog(FILE_NAME_PREFIX, 2);
this.timePhase.clear();
long sTime = System.currentTimeMillis();
this.timePhase.add(sTime);
// build up a raster
XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
Point2D.Double point = new Point2D.Double(latitude, longitude);
ResultScanner rScanner = null;
// return result
HashMap<String, String> results = new HashMap<String, String>();
try {
// match rect to find the subspace it belongs to
long match_s = System.currentTimeMillis();
XBox[] match_boxes = raster.match(latitude, longitude, radius);
long match_time = System.currentTimeMillis() - match_s;
String[] rowRange = new String[2];
rowRange[0] = match_boxes[0].getRow();
rowRange[1] = match_boxes[1].getRow() + "-*";
String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
/*
* System.out.println(rowRange[0]+":"+rowRange[1]); for(int
* i=0;i<c.length;i++){ System.out.print(c[i]+";"); }
* System.out.println();
*/
// the version here is harded coded, because i cannot get how many
// objects in one cell now
this.timePhase.add(System.currentTimeMillis());
rScanner = this.hbaseUtil.getResultSet(rowRange, null, this.familyName, c, 1000000);
BixiReader reader = new BixiReader();
int count = 0;
int row = 0;
int accepted = 0;
for (Result r : rScanner) {
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
r.getMap();
for (byte[] family : resultMap.keySet()) {
NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
for (byte[] col : columns.keySet()) {
NavigableMap<Long, byte[]> values = columns.get(col);
for (Long version : values.keySet()) {
count++;
// get the distance between this point and the given
// point
XStation station = reader.getStationFromJson(Bytes.toString(values.get(version)));
Point2D.Double resPoint =
new Point2D.Double(station.getLatitude(), station.getlongitude());
double distance = resPoint.distance(point);
if (distance <= radius) {
// System.out.println("row=>"+Bytes.toString(r.getRow())
// +
// ";colum=>"+Bytes.toString(col)+";version=>"+version+
// ";station=>"+station.getId()+";distance=>"+distance);
accepted++;
results.put(station.getId(), String.valueOf(distance));
}
}
}
}
}
long eTime = System.currentTimeMillis();
this.timePhase.add(eTime);
String outStr =
"m=scan;"
+ "radius=>"
+ radius
+ ";count=>"
+ count
+ ";accepted=>"
+ accepted
+ ";time=>"
+ (eTime - sTime)
+ ";row=>"
+ row
+ "row_stride=>"
+ this.min_size_of_height
+ ";columns=>"
+ this.max_num_of_column;
;
this.writeStat(outStr);
// write to csv file
outStr = "";
outStr +=
"within,"
+ "scan,"
+ count
+ ","
+ accepted
+ ","
+ (eTime - sTime)
+ ","
+ row
+ ","
+ match_time
+ ","
+ this.min_size_of_height
+ ","
+ radius;
this.writeCSVLog(outStr, 0);
String timeStr = "within,scan," + radius + ",";
for (int i = 0; i < this.timePhase.size(); i++) {
timeStr += this.timePhase.get(i) + ",";
}
this.writeCSVLog(timeStr, 2);
} catch (Exception e) {
e.printStackTrace();
} finally {
this.hbaseUtil.closeTableHandler();
this.closeStatLog();
this.closeCSVLog();
}
return results;
}