public static void main(String argv[]) {
modshogun.init_shogun_with_defaults();
DoubleMatrix traindata_real = Load.load_numbers("../data/fm_train_real.dat");
DoubleMatrix testdata_real = Load.load_numbers("../data/fm_test_real.dat");
RealFeatures feats_train = new RealFeatures(traindata_real);
RealFeatures feats_test = new RealFeatures(testdata_real);
EuclideanDistance distance = new EuclideanDistance(feats_train, feats_train);
DoubleMatrix dm_train = distance.get_distance_matrix();
distance.init(feats_train, feats_test);
DoubleMatrix dm_test = distance.get_distance_matrix();
System.out.println(dm_train.toString());
System.out.println(dm_test.toString());
}
@Override
public ClusteredWindow forecastNextWindow(
ClusteredWindow originalWindow, EuclideanDistance targetDistance) throws Exception {
if (targetDistance == null) {
throw new Exception("You need to provide a non-null EuclideanDistance");
}
EuclideanDistanceWithSimpleUnionAndLatency distance;
if (targetDistance instanceof EuclideanDistanceWithSimpleUnionAndLatency) {
distance = (EuclideanDistanceWithSimpleUnionAndLatency) targetDistance;
} else {
throw new Exception(
"The EuclideanDistance given must be of type EuclideanDistanceWithSimpleUnionAndLatency");
}
if (originalWindow == null) {
throw new Exception("We cannot forecast the next window from a null window!");
}
penaltyForGoingFromZeroToNonZero = distance.getPenaltyForGoingFromZeroToNonZero();
int listSize = exampleSqlQueries.size();
double dist = targetDistance.getDistance();
int originalWindowSize = originalWindow.totalNumberOfQueries();
Map<String, Schema> schemaMap = getSchemaMap();
EuclideanDistanceWithSimpleUnionAndLatency.Generator distanceGenerator =
new EuclideanDistanceWithSimpleUnionAndLatency.Generator(
schemaMap,
distance.getPenaltyForGoingFromZeroToNonZero(),
whichClauses,
distance.getPenaltyForLatency(),
latencyMeter,
useExplainInsteadOfRunningQueries);
Clustering_QueryEquality clusteringQueryEquality = new Clustering_QueryEquality();
List<Query> listOfOriginalWindow = originalWindow.getAllQueries();
int numOfTrialsToFindBeta = 5000; // Change n here
double muchLargerDistanceThanRequested = 0;
List<Query> qList = new ArrayList<Query>();
Set<Query> setOfCurWindow = new HashSet<Query>();
List<Query> qStarList = new ArrayList<Query>();
int count = 0;
double bestSeenSofar = -1.0;
// Populate set of current window
for (Query q : listOfOriginalWindow) {
setOfCurWindow.add(q);
}
// store latency
latencys = new HashMap<Query, Long>();
List<PhysicalStructure> emptyDesign = new ArrayList<PhysicalStructure>();
// Set of q in T\W1
for (Query q : exampleSqlQueries) {
assert (!(q instanceof Query_SWGO));
if (!latencys.containsKey(q)) {
String query = q.getSql();
List<String> oneQuery = new ArrayList<String>(1);
oneQuery.add(query);
long oneLatency =
latencyMeter.measureSumLatency(
oneQuery, emptyDesign, useExplainInsteadOfRunningQueries);
if (oneLatency > 0) {
latencys.put(q, oneLatency);
qStarList.add(q);
}
}
}
// System.out.println("Done!");
// System.out.println("Size of qStarList = " + qStarList.size());
// System.out.println("Summary:" + qStarList.toString() + "\n\n");
// set up new alpha
Double alpha = dist, omega = distance.getPenaltyForLatency();
List<Query> bestQList = new ArrayList<Query>();
if (alpha >= omega) {
dist = (alpha - omega) / (1 - omega);
} else {
dist = (alpha) / (1 - omega);
}
for (; count < numOfTrialsToFindBeta; count++) {
qList = new ArrayList<Query>();
for (int i = 0; i < numOfNewQueries; i++) {
int randomC = Randomness.randGen.nextInt(listSize);
Query q = qStarList.get(randomC);
if (!setOfCurWindow.contains(q)) qList.add(q);
else i--;
}
EuclideanDistanceWithSimpleUnionAndLatency ed =
fast_distance(
qList,
listOfOriginalWindow,
latencys,
distance.getPenaltyForGoingFromZeroToNonZero(),
distance.getPenaltyForLatency());
muchLargerDistanceThanRequested = ed.getDistance();
if (alpha >= omega) {
if (muchLargerDistanceThanRequested < dist) {
bestSeenSofar =
(bestSeenSofar < muchLargerDistanceThanRequested
? muchLargerDistanceThanRequested
: bestSeenSofar);
} else {
if (bestSeenSofar < dist) {
bestSeenSofar = muchLargerDistanceThanRequested;
} else {
bestSeenSofar =
((bestSeenSofar - dist) < (muchLargerDistanceThanRequested - dist)
? bestSeenSofar
: muchLargerDistanceThanRequested);
}
}
} else {
bestSeenSofar =
(bestSeenSofar < muchLargerDistanceThanRequested
? muchLargerDistanceThanRequested
: bestSeenSofar);
}
if (bestSeenSofar == muchLargerDistanceThanRequested) {
bestQList.addAll(qList);
}
}
if ((bestSeenSofar < dist) || qList.isEmpty() || bestQList.isEmpty()) {
throw new Exception(
"The log file is too similar with the current window or WorkloadGeneratorJingkui not applicable: "
+ bestSeenSofar
+ " < "
+ dist
+ "\nAnd CurrentWindow is "
+ originalWindow);
}
// System.out.println("Done!");
// System.out.println("Size of bestQList = " + bestQList.size());
// System.out.println("Summary:" + bestQList.toString() + "\n\n");
// found a window with large distance
double lambda = Math.sqrt(dist / bestSeenSofar);
int copyN = (int) (originalWindowSize * lambda / ((1.0 - lambda) * numOfNewQueries));
for (int k = 0; k < numOfNewQueries; k++) {
Query q = bestQList.get(k);
for (int i = 0; i < copyN; ++i) {
listOfOriginalWindow.add(q);
}
}
if (listOfOriginalWindow.size() == originalWindowSize)
throw new Exception("No new queries added!!");
return clusteringQueryEquality.cluster(listOfOriginalWindow);
}
