@Override protected void map(WritableComparable<?> key, Text point, Context context) throws IOException, InterruptedException { Cluster nearestCluster = null; double nearestDistance = Double.MAX_VALUE; Vector pointv = parse.parseVector(point.toString()); if (pointv == null) { return; } pointv.setNumPoints(1); for (Cluster cluster : clusters) { Vector clusterCenter = cluster.getCenter(); boolean isDeny = pointv.Deny(clusterCenter); if (isDeny) { continue; } double distance = clusterCenter.distiance(pointv); context.getCounter("Clustering", "similar").increment(1); if (distance <= nearestDistance || nearestCluster == null) { nearestCluster = cluster; nearestDistance = distance; } } if (nearestCluster != null) { context.write(new Text(String.valueOf(nearestCluster.getId())), pointv); } }
private void setupAndStartImpl() throws Exception { if (cluster.size() == 0) { return; } cluster.assignRoles(); setupAndStartCluster(); }
protected void cleanup(Context context) throws IOException, InterruptedException { for (Cluster cluster : clusters) { Vector clusterCenter = cluster.getCenter(); clusterCenter.setNumPoints(0); context.write(new Text(String.valueOf(cluster.getId())), clusterCenter); } }
private void installFreeEed() throws Exception { String url = Settings.getSettings().getDownloadLink(); logger.info("Installing FreeEed software from " + url); String cmd = "rm FreeEed.zip; " + "wget " + url + " -O FreeEed.zip --no-check-certificate; " + "rm -fr FreeEed; " + "unzip -P 4ushH7XZT1 FreeEed.zip"; SSHAgent sshAgent = new SSHAgent(); sshAgent.setUser(ParameterProcessing.CLUSTER_USER_NAME); sshAgent.setKey(ParameterProcessing.PEM_CERTIFICATE_NAME); sshAgent.setHost(cluster.getJobTracker().getDnsName()); sshAgent.executeCommand(cmd); logger.info("Successfully installed FreeEed"); // copy the settings to jobtracker Server server = cluster.getJobTracker(); sshAgent.setHost(server.getDnsName()); Settings cloneForS3 = Settings.getSettings().cloneForS3(); String settingsFileToUse = "settings.properties.s3"; Util.writeTextFile(settingsFileToUse, cloneForS3.toString()); logger.info("Copying settings file: {}", settingsFileToUse); // TODO change passing the settings to the cloud // sshAgent.scpTo(settingsFileToUse, "FreeEed/" + ParameterProcessing.DEFAULT_SETTINGS); }
private void checkHealthImpl() throws Exception { if (cluster.size() == 0) { return; } cluster.assignRoles(); verifyOperation(); }
public static void main(String[] args) throws Exception { HazelcastInstance hz = Hazelcast.newHazelcastInstance(); Cluster cluster = hz.getCluster(); cluster.addMembershipListener(new MembershipListener() { @Override public void memberAdded(MembershipEvent membershipEvent) { System.out.println( "********** MemberAdded " + membershipEvent ); } @Override public void memberRemoved(MembershipEvent membershipEvent) { System.out.println( "********** MemberRemoved " + membershipEvent ); } @Override public void memberAttributeChanged(MemberAttributeEvent memberAttributeEvent) { System.out.println( "********** MemberAttributeChanged " + memberAttributeEvent ); } }); Member localMember = cluster.getLocalMember(); System.out.println ( "********** my inetAddress= " + localMember.getInetSocketAddress() ); }
@Test public void testSeqFileClusterIteratorKMeans() throws IOException { Path pointsPath = getTestTempDirPath("points"); Path priorPath = getTestTempDirPath("prior"); Path outPath = getTestTempDirPath("output"); Configuration conf = new Configuration(); FileSystem fs = FileSystem.get(conf); List<VectorWritable> points = TestKmeansClustering.getPointsWritable(TestKmeansClustering.REFERENCE); ClusteringTestUtils.writePointsToFile(points, new Path(pointsPath, "file1"), fs, conf); Path path = new Path(priorPath, "priorClassifier"); ClusterClassifier prior = newClusterClassifier(); writeClassifier(prior, conf, path, fs); assertEquals(3, prior.getModels().size()); System.out.println("Prior"); for (Cluster cluster : prior.getModels()) { System.out.println(cluster.asFormatString(null)); } ClusteringPolicy policy = new KMeansClusteringPolicy(); ClusterIterator iterator = new ClusterIterator(policy); iterator.iterate(pointsPath, path, outPath, 5); for (int i = 1; i <= 5; i++) { System.out.println("Classifier-" + i); ClusterClassifier posterior = readClassifier(conf, new Path(outPath, "classifier-" + i), fs); assertEquals(3, posterior.getModels().size()); for (Cluster cluster : posterior.getModels()) { System.out.println(cluster.asFormatString(null)); } } }
protected Site getSite(int site_id) { assertNotNull(catalog); Cluster catalog_clus = this.getCluster(); Site catalog_site = catalog_clus.getSites().get("id", site_id); assert (catalog_site != null) : "Failed to retrieve Site #" + site_id + " from catalog"; return (catalog_site); }
public static int[] calculateVectorSpace(BufferedImage image) { clusters = createClusters(image); int[] vectorSpace = new int[IMAGE_WIDTH * IMAGE_HEIGHT]; Arrays.fill(vectorSpace, -1); boolean refineNeeded = true; int loops = 0; while (refineNeeded) { refineNeeded = false; loops++; for (int y = 0; y < IMAGE_HEIGHT; y++) { for (int x = 0; x < IMAGE_WIDTH; x++) { int pixel = image.getRGB(x, y); Cluster cluster = getMinCluster(pixel); if (vectorSpace[IMAGE_WIDTH * y + x] != cluster.getId()) { if (vectorSpace[IMAGE_WIDTH * y + x] != -1) { clusters[vectorSpace[IMAGE_WIDTH * y + x]].removePixel(pixel); } cluster.addPixel(pixel); refineNeeded = true; vectorSpace[IMAGE_WIDTH * y + x] = cluster.getId(); } } } } System.out.println("Took " + loops + " loops."); return vectorSpace; }
public void writeMap(Cluster cluster, float yBar) { Set<Submission> subSet = new HashSet<Submission>(cluster.size()); String documents = ""; for (int i = 0; i < cluster.size(); i++) { Submission sub = submissions.elementAt(cluster.getSubmissionAt(i)); documents += sub.name + " "; subSet.add(sub); } documents = documents.trim(); String theme = ThemeGenerator.generateThemes(subSet, this.program.get_themewords(), false, this.program); mapString += "<area shape=\"rect\" coords=\"" + (cluster.x - 2) + "," + (yBar) + "," + (cluster.x + 2) + "," + (cluster.y + 2) + "\" onMouseover=\"set('" + cluster.size() + "','" + trimStringToLength(String.valueOf(cluster.getSimilarity()), 6) + "','" + trimStringToLength(documents, 50) + "','" + theme + "')\" "; // if (cluster.size() == 1) // mapString += "href=\"submission"+cluster.getSubmissionAt(0)+".html\">\n"; // else mapString += "nohref>\n"; }
public static String getClusterName(Cluster cluster) { StringBuilder sb = new StringBuilder("{"); for (int i = 0; i < cluster.getPoints().size(); i++) { sb.append(cluster.getPoints().get(i).getPointName()).append(","); } return sb.substring(0, sb.length() - 1) + "}"; }
/** Loads info from xml that is supplied as an argument to the internal data objects. */ public void loadXML(final String xml) { final Document document = getXMLDocument(xml); if (document == null) { return; } /* get root <drbdgui> */ final Node rootNode = getChildNode(document, "drbdgui"); final Map<String, List<Host>> hostMap = new LinkedHashMap<String, List<Host>>(); if (rootNode != null) { /* download area */ final String downloadUser = getAttribute(rootNode, DOWNLOAD_USER_ATTR); final String downloadPasswd = getAttribute(rootNode, DOWNLOAD_PASSWD_ATTR); if (downloadUser != null && downloadPasswd != null) { Tools.getConfigData().setDownloadLogin(downloadUser, downloadPasswd, true); } /* hosts */ final Node hostsNode = getChildNode(rootNode, "hosts"); if (hostsNode != null) { final NodeList hosts = hostsNode.getChildNodes(); if (hosts != null) { for (int i = 0; i < hosts.getLength(); i++) { final Node hostNode = hosts.item(i); if (hostNode.getNodeName().equals(HOST_NODE_STRING)) { final String nodeName = getAttribute(hostNode, HOST_NAME_ATTR); final String sshPort = getAttribute(hostNode, HOST_SSHPORT_ATTR); final String color = getAttribute(hostNode, HOST_COLOR_ATTR); final String useSudo = getAttribute(hostNode, HOST_USESUDO_ATTR); final Node ipNode = getChildNode(hostNode, "ip"); String ip = null; if (ipNode != null) { ip = getText(ipNode); } final Node usernameNode = getChildNode(hostNode, "user"); final String username = getText(usernameNode); setHost( hostMap, username, nodeName, ip, sshPort, color, "true".equals(useSudo), true); } } } } /* clusters */ final Node clustersNode = getChildNode(rootNode, "clusters"); if (clustersNode != null) { final NodeList clusters = clustersNode.getChildNodes(); if (clusters != null) { for (int i = 0; i < clusters.getLength(); i++) { final Node clusterNode = clusters.item(i); if (clusterNode.getNodeName().equals("cluster")) { final String clusterName = getAttribute(clusterNode, CLUSTER_NAME_ATTR); final Cluster cluster = new Cluster(); cluster.setName(clusterName); Tools.getConfigData().addClusterToClusters(cluster); loadClusterHosts(clusterNode, cluster, hostMap); } } } } } }
/** * ���ýڵ���� * * @param clusteringFeature ��ǰ�ڵ� * @param level ��ǰ���ֵ */ private void setTreeLevel(ClusteringFeature clusteringFeature, int level) { LeafNode leafNode = null; NonLeafNode nonLeafNode = null; if (clusteringFeature instanceof LeafNode) { leafNode = (LeafNode) clusteringFeature; } else if (clusteringFeature instanceof NonLeafNode) { nonLeafNode = (NonLeafNode) clusteringFeature; } if (nonLeafNode != null) { nonLeafNode.setLevel(level); level++; // �����ӽڵ� if (nonLeafNode.getNonLeafChilds() != null) { for (NonLeafNode n1 : nonLeafNode.getNonLeafChilds()) { setTreeLevel(n1, level); } } else { for (LeafNode n2 : nonLeafNode.getLeafChilds()) { setTreeLevel(n2, level); } } } else { leafNode.setLevel(level); level++; // �����Ӿ۴� for (Cluster c : leafNode.getClusterChilds()) { c.setLevel(level); } } }
/** * @param attr * @param volume */ private static void setClusterIfAvailable(HashMap<String, String> attr, Volume volume) { if (attr.get("cluster-id") != null) { Cluster rcluster = new Cluster(); rcluster.setId(attr.get("cluster-id")); volume.setCluster(rcluster); } }
// show the old centroid and the new centroid private void displayCentroidMigration() { System.out.println("----------------------Centroid Movement---------------------"); for (Cluster c : clusters) { System.out.print(c.getOldCentroid() + "->"); System.out.print(c.getCentroid() + "\n"); } }
/** Generate the hierarchical cluster and display it as a denogram in the graph */ public TreeNode makeTree(ClutoSolution cs) { int[] tsize = cs.getTreeCounts(); int[][] ftree = cs.getForwardTree(); int nnrows = tsize.length; int nrows = cs.getMatrix().getRowCount(); // for (int i = 0; i < nnrows-1; i++) { // String s = "ftree" + "\t" + i + "\t" + ftree[i][0] + "\t" + ftree[i][1] + "\t" + tsize[i]; // System.out.println(s); // } Cluster[] ca = new Cluster[nnrows]; for (int i = 0; i < nnrows - 1; i++) { if (!true) { String s = "ftree" + "\t" + i + "\t" + ftree[i][0] + "\t" + ftree[i][1] + "\t" + tsize[i]; System.out.println(s); } Cluster cn = i < nrows ? (Cluster) new RowCluster(tm, i, null) : new CompositeCluster(); cn.setSimilarity(Math.abs(tsize[i])); ca[i] = cn; if (ftree[i][0] > -1) { cn.add(ca[ftree[i][0]]); } if (ftree[i][0] > -1) { cn.add(ca[ftree[i][1]]); } rootNode = cn; } return rootNode; }
/** The plugin provides a set of logs associated with each identified cluster */ public ProvidedObject[] getProvidedObjects() { ProvidedObject[] p = null; ArrayList a = new ArrayList(); if (selectedCluster != null) { if (selectedCluster.getHeuristicsNet() != null) { a.add( new ProvidedObject( "Heuristics net with " + selectedCluster.getName(), new Object[] {selectedCluster.getHeuristicsNet(), selectedCluster.getLog()})); a.add( new ProvidedObject( "Heuristics net with whole log", new Object[] {selectedCluster.getHeuristicsNet(), root.getLog()})); } } /* * if (root.getChildren().size()>0){ for (int * i=0;i<root.getChildren().size();i++){ Cluster * c=(Cluster)root.getChildren().get(i); LogReader logc=c.getLog(); * a.add(new ProvidedObject(c.getName(), new Object[] { logc })); } } */ if (selectedCluster != null && selectedCluster.getHeuristicsNet() == null) { Cluster c = selectedCluster; LogReader logc = c.getLog(); a.add(new ProvidedObject(c.getName(), new Object[] {logc})); } p = new ProvidedObject[a.size()]; for (int i = 0; i < a.size(); i++) { p[i] = (ProvidedObject) a.get(i); } return p; }
public static void handleClusterRemoved(ClusterContext ctxt) { Topology topology = TopologyManager.getTopology(); Service service = topology.getService(ctxt.getCartridgeType()); String deploymentPolicy; if (service == null) { log.warn(String.format("Service %s does not exist", ctxt.getCartridgeType())); return; } if (!service.clusterExists(ctxt.getClusterId())) { log.warn( String.format( "Cluster %s does not exist for service %s", ctxt.getClusterId(), ctxt.getCartridgeType())); return; } try { TopologyManager.acquireWriteLock(); Cluster cluster = service.removeCluster(ctxt.getClusterId()); deploymentPolicy = cluster.getDeploymentPolicyName(); TopologyManager.updateTopology(topology); } finally { TopologyManager.releaseWriteLock(); } TopologyEventPublisher.sendClusterRemovedEvent(ctxt, deploymentPolicy); }
public void drawCluster(Cluster cluster) { int index = clusters.indexOf(cluster); if (index != -1) { cluster.y = maxY; cluster.x = minX + index * factor; if (cluster.size() > 1) g.setColor(Color.RED); else g.setColor(Color.BLACK); g.drawRect(cluster.x - 1, cluster.y - cluster.size(), 2, 1 + cluster.size()); } else { Cluster left = cluster.getLeft(); Cluster right = cluster.getRight(); drawCluster(left); drawCluster(right); int yBar = minY + (int) ((maxY - minY) * (cluster.getSimilarity() / threshold)); g.setColor(Color.DARK_GRAY); if (left.y > yBar) { g.drawLine(left.x, left.y - 1, left.x, yBar); writeMap(left, yBar); } if (right.y > yBar) { g.drawLine(right.x, right.y - 1, right.x, yBar); writeMap(right, yBar); } g.setColor(Color.BLACK); g.drawLine(left.x, yBar, right.x, yBar); cluster.x = (right.x + left.x) / 2; cluster.y = yBar; } }
@Override public void run() { while (running.get()) { switch (random.nextInt() % 2) { case 0: // start a server try { cluster.startServer(); } catch (Exception e) { LOG.warn(e); exception.compareAndSet(null, e); } break; case 1: // stop a server try { cluster.stopRandomServer(); } catch (Exception e) { LOG.warn(e); exception.compareAndSet(null, e); } default: } Threads.sleep(100); } }
private Cluster findCluster(SSA ssa) { for (Cluster cluster : clusters) { if (cluster.getSSAList().contains(ssa)) { return cluster; } } return null; }
public Cluster locateUser(Integer id) { for (Cluster myCluster : clusters) { if (myCluster.getUsers().containsKey(id)) { return myCluster; } } return null; }
public Cluster getArbitraryCluster(int regLimit) { for (Cluster cluster : clusters) { if (cluster.getNeighborNum() < regLimit) { return cluster; } } return null; }
public static void handleClusterTerminatedEvent(ClusterStatusClusterTerminatedEvent event) { TopologyManager.acquireWriteLock(); try { Topology topology = TopologyManager.getTopology(); Service service = topology.getService(event.getServiceName()); // update the status of the cluster if (service == null) { log.warn(String.format("Service %s does not exist", event.getServiceName())); return; } Cluster cluster = service.getCluster(event.getClusterId()); if (cluster == null) { log.warn(String.format("Cluster %s does not exist", event.getClusterId())); return; } ClusterInstance context = cluster.getInstanceContexts(event.getInstanceId()); if (context == null) { log.warn( "Cluster Instance Context is not found for [cluster] " + event.getClusterId() + " [instance-id] " + event.getInstanceId()); return; } ClusterStatus status = ClusterStatus.Terminated; if (context.isStateTransitionValid(status)) { context.setStatus(status); log.info( "Cluster Terminated adding status started for and removing the cluster instance" + cluster.getClusterId()); cluster.removeInstanceContext(event.getInstanceId()); TopologyManager.updateTopology(topology); // publishing data ClusterInstanceTerminatedEvent clusterTerminatedEvent = new ClusterInstanceTerminatedEvent( event.getAppId(), event.getServiceName(), event.getClusterId(), event.getInstanceId()); TopologyEventPublisher.sendClusterTerminatedEvent(clusterTerminatedEvent); } else { log.error( String.format( "Cluster state transition is not valid: [cluster-id] %s " + " [instance-id] %s [current-status] %s [status-requested] %s", event.getClusterId(), event.getInstanceId(), context.getStatus(), status)); return; } } finally { TopologyManager.releaseWriteLock(); } }
public String getCurrentStatus(String clusterName) throws IOException { Cluster cluster = clusters.get(clusterName); if (cluster == null) { throw new IOException("Cluster " + clusterName + " does not exit " + "in this configuration"); } return cluster.getStatus(); }
public static void handleClusterTerminatingEvent(ClusterStatusClusterTerminatingEvent event) { TopologyManager.acquireWriteLock(); try { Topology topology = TopologyManager.getTopology(); Cluster cluster = topology.getService(event.getServiceName()).getCluster(event.getClusterId()); if (!cluster.isStateTransitionValid(ClusterStatus.Terminating, event.getInstanceId())) { log.error( "Invalid state transfer from " + cluster.getStatus(event.getInstanceId()) + " to " + ClusterStatus.Terminating); } ClusterInstance context = cluster.getInstanceContexts(event.getInstanceId()); if (context == null) { log.warn( "Cluster Instance Context is not found for [cluster] " + event.getClusterId() + " [instance-id] " + event.getInstanceId()); return; } ClusterStatus status = ClusterStatus.Terminating; if (context.isStateTransitionValid(status)) { context.setStatus(status); log.info("Cluster Terminating started for " + cluster.getClusterId()); TopologyManager.updateTopology(topology); // publishing data ClusterInstanceTerminatingEvent clusterTerminaingEvent = new ClusterInstanceTerminatingEvent( event.getAppId(), event.getServiceName(), event.getClusterId(), event.getInstanceId()); TopologyEventPublisher.sendClusterTerminatingEvent(clusterTerminaingEvent); // Remove kubernetes services if available ClusterContext clusterContext = CloudControllerContext.getInstance().getClusterContext(event.getClusterId()); if (StringUtils.isNotBlank(clusterContext.getKubernetesClusterId())) { KubernetesIaas.removeKubernetesServices(event.getAppId(), event.getClusterId()); } } else { log.error( String.format( "Cluster state transition is not valid: [cluster-id] %s " + " [instance-id] %s [current-status] %s [status-requested] %s", event.getClusterId(), event.getInstanceId(), context.getStatus(), status)); } } finally { TopologyManager.releaseWriteLock(); } }
private void prepareSciptForCluster(Cluster cluster) { if (myDefaultScript != null) { cluster.setScript(myDefaultScript); } else { ScriptBuilder builder = cluster.createScriptBuilder(); cluster.setScript( (myMakeSession == null ? builder.toScript() : myMakeSession.toScript(builder))); } }
// ---------------------------------------------------------------------------------------- public void broadcastCluster(String clusterName, String[] rowNames, String[] columnNames) { try { Cluster cluster = new Cluster(clusterName, getSpecies(), rowNames, columnNames); cluster.setSpecies(defaultSpecies); gaggleBoss.broadcastCluster(myGaggleName, targetGoose, cluster); } catch (RemoteException rex) { System.err.println("broadcast cluster error, in RShellGoose " + myGaggleName); } } // broadcastCluster
@Start public void startHazel() { Config configApp = new Config(); configApp.getGroupConfig().setName("KGroup_HazelGroup_" + getName()); configApp.setInstanceName("KGroup_HazelGroup_" + getNodeName()); hazelInstance = Hazelcast.newHazelcastInstance(configApp); Cluster cluster = hazelInstance.getCluster(); cluster.addMembershipListener(this); }
public static void handleMemberMaintenance( InstanceMaintenanceModeEvent instanceMaintenanceModeEvent) throws InvalidMemberException, InvalidCartridgeTypeException { Topology topology = TopologyManager.getTopology(); Service service = topology.getService(instanceMaintenanceModeEvent.getServiceName()); // update the status of the member if (service == null) { log.warn( String.format( "Service %s does not exist", instanceMaintenanceModeEvent.getServiceName())); return; } Cluster cluster = service.getCluster(instanceMaintenanceModeEvent.getClusterId()); if (cluster == null) { log.warn( String.format("Cluster %s does not exist", instanceMaintenanceModeEvent.getClusterId())); return; } Member member = cluster.getMember(instanceMaintenanceModeEvent.getMemberId()); if (member == null) { log.warn( String.format("Member %s does not exist", instanceMaintenanceModeEvent.getMemberId())); return; } MemberMaintenanceModeEvent memberMaintenanceModeEvent = new MemberMaintenanceModeEvent( instanceMaintenanceModeEvent.getServiceName(), instanceMaintenanceModeEvent.getClusterId(), instanceMaintenanceModeEvent.getClusterInstanceId(), instanceMaintenanceModeEvent.getMemberId(), instanceMaintenanceModeEvent.getNetworkPartitionId(), instanceMaintenanceModeEvent.getPartitionId()); try { TopologyManager.acquireWriteLock(); // try update lifecycle state if (!member.isStateTransitionValid(MemberStatus.In_Maintenance)) { log.error( "Invalid State Transition from " + member.getStatus() + " to " + MemberStatus.In_Maintenance); return; } member.setStatus(MemberStatus.In_Maintenance); log.info("member maintenance mode event adding status started"); TopologyManager.updateTopology(topology); } finally { TopologyManager.releaseWriteLock(); } // publishing data TopologyEventPublisher.sendMemberMaintenanceModeEvent(memberMaintenanceModeEvent); }