@Override
protected void map(WritableComparable<?> key, Text point, Context context)
throws IOException, InterruptedException {
Cluster nearestCluster = null;
double nearestDistance = Double.MAX_VALUE;
Vector pointv = parse.parseVector(point.toString());
if (pointv == null) {
return;
}
pointv.setNumPoints(1);
for (Cluster cluster : clusters) {
Vector clusterCenter = cluster.getCenter();
boolean isDeny = pointv.Deny(clusterCenter);
if (isDeny) {
continue;
}
double distance = clusterCenter.distiance(pointv);
context.getCounter("Clustering", "similar").increment(1);
if (distance <= nearestDistance || nearestCluster == null) {
nearestCluster = cluster;
nearestDistance = distance;
}
}
if (nearestCluster != null) {
context.write(new Text(String.valueOf(nearestCluster.getId())), pointv);
}
}
private void setupAndStartImpl() throws Exception {
if (cluster.size() == 0) {
return;
}
cluster.assignRoles();
setupAndStartCluster();
}
protected void cleanup(Context context) throws IOException, InterruptedException {
for (Cluster cluster : clusters) {
Vector clusterCenter = cluster.getCenter();
clusterCenter.setNumPoints(0);
context.write(new Text(String.valueOf(cluster.getId())), clusterCenter);
}
}
private void installFreeEed() throws Exception {
String url = Settings.getSettings().getDownloadLink();
logger.info("Installing FreeEed software from " + url);
String cmd =
"rm FreeEed.zip; "
+ "wget "
+ url
+ " -O FreeEed.zip --no-check-certificate; "
+ "rm -fr FreeEed; "
+ "unzip -P 4ushH7XZT1 FreeEed.zip";
SSHAgent sshAgent = new SSHAgent();
sshAgent.setUser(ParameterProcessing.CLUSTER_USER_NAME);
sshAgent.setKey(ParameterProcessing.PEM_CERTIFICATE_NAME);
sshAgent.setHost(cluster.getJobTracker().getDnsName());
sshAgent.executeCommand(cmd);
logger.info("Successfully installed FreeEed");
// copy the settings to jobtracker
Server server = cluster.getJobTracker();
sshAgent.setHost(server.getDnsName());
Settings cloneForS3 = Settings.getSettings().cloneForS3();
String settingsFileToUse = "settings.properties.s3";
Util.writeTextFile(settingsFileToUse, cloneForS3.toString());
logger.info("Copying settings file: {}", settingsFileToUse);
// TODO change passing the settings to the cloud
// sshAgent.scpTo(settingsFileToUse, "FreeEed/" + ParameterProcessing.DEFAULT_SETTINGS);
}
private void checkHealthImpl() throws Exception {
if (cluster.size() == 0) {
return;
}
cluster.assignRoles();
verifyOperation();
}
public static void main(String[] args) throws Exception {
HazelcastInstance hz = Hazelcast.newHazelcastInstance();
Cluster cluster = hz.getCluster();
cluster.addMembershipListener(new MembershipListener() {
@Override
public void memberAdded(MembershipEvent membershipEvent) {
System.out.println( "********** MemberAdded " + membershipEvent );
}
@Override
public void memberRemoved(MembershipEvent membershipEvent) {
System.out.println( "********** MemberRemoved " + membershipEvent );
}
@Override
public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) {
System.out.println( "********** MemberAttributeChanged " + memberAttributeEvent );
}
});
Member localMember = cluster.getLocalMember();
System.out.println ( "********** my inetAddress= " + localMember.getInetSocketAddress() );
}
@Test
public void testSeqFileClusterIteratorKMeans() throws IOException {
Path pointsPath = getTestTempDirPath("points");
Path priorPath = getTestTempDirPath("prior");
Path outPath = getTestTempDirPath("output");
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
List<VectorWritable> points =
TestKmeansClustering.getPointsWritable(TestKmeansClustering.REFERENCE);
ClusteringTestUtils.writePointsToFile(points, new Path(pointsPath, "file1"), fs, conf);
Path path = new Path(priorPath, "priorClassifier");
ClusterClassifier prior = newClusterClassifier();
writeClassifier(prior, conf, path, fs);
assertEquals(3, prior.getModels().size());
System.out.println("Prior");
for (Cluster cluster : prior.getModels()) {
System.out.println(cluster.asFormatString(null));
}
ClusteringPolicy policy = new KMeansClusteringPolicy();
ClusterIterator iterator = new ClusterIterator(policy);
iterator.iterate(pointsPath, path, outPath, 5);
for (int i = 1; i <= 5; i++) {
System.out.println("Classifier-" + i);
ClusterClassifier posterior = readClassifier(conf, new Path(outPath, "classifier-" + i), fs);
assertEquals(3, posterior.getModels().size());
for (Cluster cluster : posterior.getModels()) {
System.out.println(cluster.asFormatString(null));
}
}
}
protected Site getSite(int site_id) {
assertNotNull(catalog);
Cluster catalog_clus = this.getCluster();
Site catalog_site = catalog_clus.getSites().get("id", site_id);
assert (catalog_site != null) : "Failed to retrieve Site #" + site_id + " from catalog";
return (catalog_site);
}
public static int[] calculateVectorSpace(BufferedImage image) {
clusters = createClusters(image);
int[] vectorSpace = new int[IMAGE_WIDTH * IMAGE_HEIGHT];
Arrays.fill(vectorSpace, -1);
boolean refineNeeded = true;
int loops = 0;
while (refineNeeded) {
refineNeeded = false;
loops++;
for (int y = 0; y < IMAGE_HEIGHT; y++) {
for (int x = 0; x < IMAGE_WIDTH; x++) {
int pixel = image.getRGB(x, y);
Cluster cluster = getMinCluster(pixel);
if (vectorSpace[IMAGE_WIDTH * y + x] != cluster.getId()) {
if (vectorSpace[IMAGE_WIDTH * y + x] != -1) {
clusters[vectorSpace[IMAGE_WIDTH * y + x]].removePixel(pixel);
}
cluster.addPixel(pixel);
refineNeeded = true;
vectorSpace[IMAGE_WIDTH * y + x] = cluster.getId();
}
}
}
}
System.out.println("Took " + loops + " loops.");
return vectorSpace;
}
public void writeMap(Cluster cluster, float yBar) {
Set<Submission> subSet = new HashSet<Submission>(cluster.size());
String documents = "";
for (int i = 0; i < cluster.size(); i++) {
Submission sub = submissions.elementAt(cluster.getSubmissionAt(i));
documents += sub.name + " ";
subSet.add(sub);
}
documents = documents.trim();
String theme =
ThemeGenerator.generateThemes(subSet, this.program.get_themewords(), false, this.program);
mapString +=
"<area shape=\"rect\" coords=\""
+ (cluster.x - 2)
+ ","
+ (yBar)
+ ","
+ (cluster.x + 2)
+ ","
+ (cluster.y + 2)
+ "\" onMouseover=\"set('"
+ cluster.size()
+ "','"
+ trimStringToLength(String.valueOf(cluster.getSimilarity()), 6)
+ "','"
+ trimStringToLength(documents, 50)
+ "','"
+ theme
+ "')\" ";
// if (cluster.size() == 1)
// mapString += "href=\"submission"+cluster.getSubmissionAt(0)+".html\">\n";
// else
mapString += "nohref>\n";
}
public static String getClusterName(Cluster cluster) {
StringBuilder sb = new StringBuilder("{");
for (int i = 0; i < cluster.getPoints().size(); i++) {
sb.append(cluster.getPoints().get(i).getPointName()).append(",");
}
return sb.substring(0, sb.length() - 1) + "}";
}
/** Loads info from xml that is supplied as an argument to the internal data objects. */
public void loadXML(final String xml) {
final Document document = getXMLDocument(xml);
if (document == null) {
return;
}
/* get root <drbdgui> */
final Node rootNode = getChildNode(document, "drbdgui");
final Map<String, List<Host>> hostMap = new LinkedHashMap<String, List<Host>>();
if (rootNode != null) {
/* download area */
final String downloadUser = getAttribute(rootNode, DOWNLOAD_USER_ATTR);
final String downloadPasswd = getAttribute(rootNode, DOWNLOAD_PASSWD_ATTR);
if (downloadUser != null && downloadPasswd != null) {
Tools.getConfigData().setDownloadLogin(downloadUser, downloadPasswd, true);
}
/* hosts */
final Node hostsNode = getChildNode(rootNode, "hosts");
if (hostsNode != null) {
final NodeList hosts = hostsNode.getChildNodes();
if (hosts != null) {
for (int i = 0; i < hosts.getLength(); i++) {
final Node hostNode = hosts.item(i);
if (hostNode.getNodeName().equals(HOST_NODE_STRING)) {
final String nodeName = getAttribute(hostNode, HOST_NAME_ATTR);
final String sshPort = getAttribute(hostNode, HOST_SSHPORT_ATTR);
final String color = getAttribute(hostNode, HOST_COLOR_ATTR);
final String useSudo = getAttribute(hostNode, HOST_USESUDO_ATTR);
final Node ipNode = getChildNode(hostNode, "ip");
String ip = null;
if (ipNode != null) {
ip = getText(ipNode);
}
final Node usernameNode = getChildNode(hostNode, "user");
final String username = getText(usernameNode);
setHost(
hostMap, username, nodeName, ip, sshPort, color, "true".equals(useSudo), true);
}
}
}
}
/* clusters */
final Node clustersNode = getChildNode(rootNode, "clusters");
if (clustersNode != null) {
final NodeList clusters = clustersNode.getChildNodes();
if (clusters != null) {
for (int i = 0; i < clusters.getLength(); i++) {
final Node clusterNode = clusters.item(i);
if (clusterNode.getNodeName().equals("cluster")) {
final String clusterName = getAttribute(clusterNode, CLUSTER_NAME_ATTR);
final Cluster cluster = new Cluster();
cluster.setName(clusterName);
Tools.getConfigData().addClusterToClusters(cluster);
loadClusterHosts(clusterNode, cluster, hostMap);
}
}
}
}
}
}
/**
* ���ýڵ����
*
* @param clusteringFeature ��ǰ�ڵ�
* @param level ��ǰ���ֵ
*/
private void setTreeLevel(ClusteringFeature clusteringFeature, int level) {
LeafNode leafNode = null;
NonLeafNode nonLeafNode = null;
if (clusteringFeature instanceof LeafNode) {
leafNode = (LeafNode) clusteringFeature;
} else if (clusteringFeature instanceof NonLeafNode) {
nonLeafNode = (NonLeafNode) clusteringFeature;
}
if (nonLeafNode != null) {
nonLeafNode.setLevel(level);
level++;
// �����ӽڵ�
if (nonLeafNode.getNonLeafChilds() != null) {
for (NonLeafNode n1 : nonLeafNode.getNonLeafChilds()) {
setTreeLevel(n1, level);
}
} else {
for (LeafNode n2 : nonLeafNode.getLeafChilds()) {
setTreeLevel(n2, level);
}
}
} else {
leafNode.setLevel(level);
level++;
// �����Ӿ۴�
for (Cluster c : leafNode.getClusterChilds()) {
c.setLevel(level);
}
}
}
/**
* @param attr
* @param volume
*/
private static void setClusterIfAvailable(HashMap<String, String> attr, Volume volume) {
if (attr.get("cluster-id") != null) {
Cluster rcluster = new Cluster();
rcluster.setId(attr.get("cluster-id"));
volume.setCluster(rcluster);
}
}
// show the old centroid and the new centroid
private void displayCentroidMigration() {
System.out.println("----------------------Centroid Movement---------------------");
for (Cluster c : clusters) {
System.out.print(c.getOldCentroid() + "->");
System.out.print(c.getCentroid() + "\n");
}
}
/** Generate the hierarchical cluster and display it as a denogram in the graph */
public TreeNode makeTree(ClutoSolution cs) {
int[] tsize = cs.getTreeCounts();
int[][] ftree = cs.getForwardTree();
int nnrows = tsize.length;
int nrows = cs.getMatrix().getRowCount();
// for (int i = 0; i < nnrows-1; i++) {
// String s = "ftree" + "\t" + i + "\t" + ftree[i][0] + "\t" + ftree[i][1] + "\t" + tsize[i];
// System.out.println(s);
// }
Cluster[] ca = new Cluster[nnrows];
for (int i = 0; i < nnrows - 1; i++) {
if (!true) {
String s = "ftree" + "\t" + i + "\t" + ftree[i][0] + "\t" + ftree[i][1] + "\t" + tsize[i];
System.out.println(s);
}
Cluster cn = i < nrows ? (Cluster) new RowCluster(tm, i, null) : new CompositeCluster();
cn.setSimilarity(Math.abs(tsize[i]));
ca[i] = cn;
if (ftree[i][0] > -1) {
cn.add(ca[ftree[i][0]]);
}
if (ftree[i][0] > -1) {
cn.add(ca[ftree[i][1]]);
}
rootNode = cn;
}
return rootNode;
}
/** The plugin provides a set of logs associated with each identified cluster */
public ProvidedObject[] getProvidedObjects() {
ProvidedObject[] p = null;
ArrayList a = new ArrayList();
if (selectedCluster != null) {
if (selectedCluster.getHeuristicsNet() != null) {
a.add(
new ProvidedObject(
"Heuristics net with " + selectedCluster.getName(),
new Object[] {selectedCluster.getHeuristicsNet(), selectedCluster.getLog()}));
a.add(
new ProvidedObject(
"Heuristics net with whole log",
new Object[] {selectedCluster.getHeuristicsNet(), root.getLog()}));
}
}
/*
* if (root.getChildren().size()>0){ for (int
* i=0;i<root.getChildren().size();i++){ Cluster
* c=(Cluster)root.getChildren().get(i); LogReader logc=c.getLog();
* a.add(new ProvidedObject(c.getName(), new Object[] { logc })); } }
*/
if (selectedCluster != null && selectedCluster.getHeuristicsNet() == null) {
Cluster c = selectedCluster;
LogReader logc = c.getLog();
a.add(new ProvidedObject(c.getName(), new Object[] {logc}));
}
p = new ProvidedObject[a.size()];
for (int i = 0; i < a.size(); i++) {
p[i] = (ProvidedObject) a.get(i);
}
return p;
}
public static void handleClusterRemoved(ClusterContext ctxt) {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(ctxt.getCartridgeType());
String deploymentPolicy;
if (service == null) {
log.warn(String.format("Service %s does not exist", ctxt.getCartridgeType()));
return;
}
if (!service.clusterExists(ctxt.getClusterId())) {
log.warn(
String.format(
"Cluster %s does not exist for service %s",
ctxt.getClusterId(), ctxt.getCartridgeType()));
return;
}
try {
TopologyManager.acquireWriteLock();
Cluster cluster = service.removeCluster(ctxt.getClusterId());
deploymentPolicy = cluster.getDeploymentPolicyName();
TopologyManager.updateTopology(topology);
} finally {
TopologyManager.releaseWriteLock();
}
TopologyEventPublisher.sendClusterRemovedEvent(ctxt, deploymentPolicy);
}
public void drawCluster(Cluster cluster) {
int index = clusters.indexOf(cluster);
if (index != -1) {
cluster.y = maxY;
cluster.x = minX + index * factor;
if (cluster.size() > 1) g.setColor(Color.RED);
else g.setColor(Color.BLACK);
g.drawRect(cluster.x - 1, cluster.y - cluster.size(), 2, 1 + cluster.size());
} else {
Cluster left = cluster.getLeft();
Cluster right = cluster.getRight();
drawCluster(left);
drawCluster(right);
int yBar = minY + (int) ((maxY - minY) * (cluster.getSimilarity() / threshold));
g.setColor(Color.DARK_GRAY);
if (left.y > yBar) {
g.drawLine(left.x, left.y - 1, left.x, yBar);
writeMap(left, yBar);
}
if (right.y > yBar) {
g.drawLine(right.x, right.y - 1, right.x, yBar);
writeMap(right, yBar);
}
g.setColor(Color.BLACK);
g.drawLine(left.x, yBar, right.x, yBar);
cluster.x = (right.x + left.x) / 2;
cluster.y = yBar;
}
}
@Override
public void run() {
while (running.get()) {
switch (random.nextInt() % 2) {
case 0: // start a server
try {
cluster.startServer();
} catch (Exception e) {
LOG.warn(e);
exception.compareAndSet(null, e);
}
break;
case 1: // stop a server
try {
cluster.stopRandomServer();
} catch (Exception e) {
LOG.warn(e);
exception.compareAndSet(null, e);
}
default:
}
Threads.sleep(100);
}
}
private Cluster findCluster(SSA ssa) {
for (Cluster cluster : clusters) {
if (cluster.getSSAList().contains(ssa)) {
return cluster;
}
}
return null;
}
public Cluster locateUser(Integer id) {
for (Cluster myCluster : clusters) {
if (myCluster.getUsers().containsKey(id)) {
return myCluster;
}
}
return null;
}
public Cluster getArbitraryCluster(int regLimit) {
for (Cluster cluster : clusters) {
if (cluster.getNeighborNum() < regLimit) {
return cluster;
}
}
return null;
}
public static void handleClusterTerminatedEvent(ClusterStatusClusterTerminatedEvent event) {
TopologyManager.acquireWriteLock();
try {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(event.getServiceName());
// update the status of the cluster
if (service == null) {
log.warn(String.format("Service %s does not exist", event.getServiceName()));
return;
}
Cluster cluster = service.getCluster(event.getClusterId());
if (cluster == null) {
log.warn(String.format("Cluster %s does not exist", event.getClusterId()));
return;
}
ClusterInstance context = cluster.getInstanceContexts(event.getInstanceId());
if (context == null) {
log.warn(
"Cluster Instance Context is not found for [cluster] "
+ event.getClusterId()
+ " [instance-id] "
+ event.getInstanceId());
return;
}
ClusterStatus status = ClusterStatus.Terminated;
if (context.isStateTransitionValid(status)) {
context.setStatus(status);
log.info(
"Cluster Terminated adding status started for and removing the cluster instance"
+ cluster.getClusterId());
cluster.removeInstanceContext(event.getInstanceId());
TopologyManager.updateTopology(topology);
// publishing data
ClusterInstanceTerminatedEvent clusterTerminatedEvent =
new ClusterInstanceTerminatedEvent(
event.getAppId(),
event.getServiceName(),
event.getClusterId(),
event.getInstanceId());
TopologyEventPublisher.sendClusterTerminatedEvent(clusterTerminatedEvent);
} else {
log.error(
String.format(
"Cluster state transition is not valid: [cluster-id] %s "
+ " [instance-id] %s [current-status] %s [status-requested] %s",
event.getClusterId(), event.getInstanceId(), context.getStatus(), status));
return;
}
} finally {
TopologyManager.releaseWriteLock();
}
}
public String getCurrentStatus(String clusterName) throws IOException {
Cluster cluster = clusters.get(clusterName);
if (cluster == null) {
throw new IOException("Cluster " + clusterName + " does not exit " + "in this configuration");
}
return cluster.getStatus();
}
public static void handleClusterTerminatingEvent(ClusterStatusClusterTerminatingEvent event) {
TopologyManager.acquireWriteLock();
try {
Topology topology = TopologyManager.getTopology();
Cluster cluster =
topology.getService(event.getServiceName()).getCluster(event.getClusterId());
if (!cluster.isStateTransitionValid(ClusterStatus.Terminating, event.getInstanceId())) {
log.error(
"Invalid state transfer from "
+ cluster.getStatus(event.getInstanceId())
+ " to "
+ ClusterStatus.Terminating);
}
ClusterInstance context = cluster.getInstanceContexts(event.getInstanceId());
if (context == null) {
log.warn(
"Cluster Instance Context is not found for [cluster] "
+ event.getClusterId()
+ " [instance-id] "
+ event.getInstanceId());
return;
}
ClusterStatus status = ClusterStatus.Terminating;
if (context.isStateTransitionValid(status)) {
context.setStatus(status);
log.info("Cluster Terminating started for " + cluster.getClusterId());
TopologyManager.updateTopology(topology);
// publishing data
ClusterInstanceTerminatingEvent clusterTerminaingEvent =
new ClusterInstanceTerminatingEvent(
event.getAppId(),
event.getServiceName(),
event.getClusterId(),
event.getInstanceId());
TopologyEventPublisher.sendClusterTerminatingEvent(clusterTerminaingEvent);
// Remove kubernetes services if available
ClusterContext clusterContext =
CloudControllerContext.getInstance().getClusterContext(event.getClusterId());
if (StringUtils.isNotBlank(clusterContext.getKubernetesClusterId())) {
KubernetesIaas.removeKubernetesServices(event.getAppId(), event.getClusterId());
}
} else {
log.error(
String.format(
"Cluster state transition is not valid: [cluster-id] %s "
+ " [instance-id] %s [current-status] %s [status-requested] %s",
event.getClusterId(), event.getInstanceId(), context.getStatus(), status));
}
} finally {
TopologyManager.releaseWriteLock();
}
}
private void prepareSciptForCluster(Cluster cluster) {
if (myDefaultScript != null) {
cluster.setScript(myDefaultScript);
} else {
ScriptBuilder builder = cluster.createScriptBuilder();
cluster.setScript(
(myMakeSession == null ? builder.toScript() : myMakeSession.toScript(builder)));
}
}
// ----------------------------------------------------------------------------------------
public void broadcastCluster(String clusterName, String[] rowNames, String[] columnNames) {
try {
Cluster cluster = new Cluster(clusterName, getSpecies(), rowNames, columnNames);
cluster.setSpecies(defaultSpecies);
gaggleBoss.broadcastCluster(myGaggleName, targetGoose, cluster);
} catch (RemoteException rex) {
System.err.println("broadcast cluster error, in RShellGoose " + myGaggleName);
}
} // broadcastCluster
@Start
public void startHazel() {
Config configApp = new Config();
configApp.getGroupConfig().setName("KGroup_HazelGroup_" + getName());
configApp.setInstanceName("KGroup_HazelGroup_" + getNodeName());
hazelInstance = Hazelcast.newHazelcastInstance(configApp);
Cluster cluster = hazelInstance.getCluster();
cluster.addMembershipListener(this);
}
public static void handleMemberMaintenance(
InstanceMaintenanceModeEvent instanceMaintenanceModeEvent)
throws InvalidMemberException, InvalidCartridgeTypeException {
Topology topology = TopologyManager.getTopology();
Service service = topology.getService(instanceMaintenanceModeEvent.getServiceName());
// update the status of the member
if (service == null) {
log.warn(
String.format(
"Service %s does not exist", instanceMaintenanceModeEvent.getServiceName()));
return;
}
Cluster cluster = service.getCluster(instanceMaintenanceModeEvent.getClusterId());
if (cluster == null) {
log.warn(
String.format("Cluster %s does not exist", instanceMaintenanceModeEvent.getClusterId()));
return;
}
Member member = cluster.getMember(instanceMaintenanceModeEvent.getMemberId());
if (member == null) {
log.warn(
String.format("Member %s does not exist", instanceMaintenanceModeEvent.getMemberId()));
return;
}
MemberMaintenanceModeEvent memberMaintenanceModeEvent =
new MemberMaintenanceModeEvent(
instanceMaintenanceModeEvent.getServiceName(),
instanceMaintenanceModeEvent.getClusterId(),
instanceMaintenanceModeEvent.getClusterInstanceId(),
instanceMaintenanceModeEvent.getMemberId(),
instanceMaintenanceModeEvent.getNetworkPartitionId(),
instanceMaintenanceModeEvent.getPartitionId());
try {
TopologyManager.acquireWriteLock();
// try update lifecycle state
if (!member.isStateTransitionValid(MemberStatus.In_Maintenance)) {
log.error(
"Invalid State Transition from "
+ member.getStatus()
+ " to "
+ MemberStatus.In_Maintenance);
return;
}
member.setStatus(MemberStatus.In_Maintenance);
log.info("member maintenance mode event adding status started");
TopologyManager.updateTopology(topology);
} finally {
TopologyManager.releaseWriteLock();
}
// publishing data
TopologyEventPublisher.sendMemberMaintenanceModeEvent(memberMaintenanceModeEvent);
}
