@Override public void run() { if (LOG.isDebugEnabled()) { LOG.debug("==> ConsumerRunnable.run()"); } while (true) { try { if (hasNext()) { EntityNotification notification = consumer.peek(); if (notification != null) { if (LOG.isDebugEnabled()) { LOG.debug("Notification=" + getPrintableEntityNotification(notification)); } ServiceTags serviceTags = AtlasNotificationMapper.processEntityNotification(notification); if (serviceTags == null) { LOG.error( "No ServiceTags built for notification :" + getPrintableEntityNotification(notification)); } else { updateSink(serviceTags); } } else { LOG.error("Null entityNotification received from Kafka!! Ignoring.."); } // Move iterator forward consumer.next(); } } catch (Exception exception) { LOG.error("Caught exception..: ", exception); return; } } }
@Override public void visitReferenceExpression(GrReferenceExpression expression) { super.visitReferenceExpression(expression); if (org.jetbrains.plugins.groovy.lang.psi.util.PsiUtil.isSuperReference(expression)) { final GrExpression qualifier = expression.getQualifier(); if (qualifier instanceof GrReferenceExpression && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) { try { expression.putCopyableUserData(SUPER_REF, Boolean.TRUE); } catch (IncorrectOperationException e) { LOG.error(e); } } } else if (org.jetbrains.plugins.groovy.lang.psi.util.PsiUtil.isThisReference(expression)) { final GrExpression qualifier = expression.getQualifier(); if (qualifier instanceof GrReferenceExpression && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) { try { expression.putCopyableUserData(THIS_REF, Boolean.TRUE); } catch (IncorrectOperationException e) { LOG.error(e); } } } }
@Override public void visitReferenceExpression(GrReferenceExpression expression) { super.visitReferenceExpression(expression); if (expression.getCopyableUserData(SUPER_REF) != null) { expression.putCopyableUserData(SUPER_REF, null); final GrExpression qualifier = expression.getQualifier(); if (qualifier instanceof GrReferenceExpression && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) { try { GroovyPsiElementFactory factory = GroovyPsiElementFactory.getInstance(myProject); GrExpression newExpr = factory.createExpressionFromText(myTargetSuperClass.getName() + ".this", null); expression.replace(newExpr); } catch (IncorrectOperationException e) { LOG.error(e); } } } else if (expression.getCopyableUserData(THIS_REF) != null) { expression.putCopyableUserData(THIS_REF, null); final GrExpression qualifier = expression.getQualifier(); if (qualifier instanceof GrReferenceExpression && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) { try { ((GrReferenceExpression) qualifier).bindToElement(myTargetSuperClass); GroovyChangeContextUtil.clearContextInfo(qualifier); } catch (IncorrectOperationException e) { LOG.error(e); } } } }
private Map<String, SpringResource> generateResourceMap(Set<Class<?>> validClasses) throws GenerateException { Map<String, SpringResource> resourceMap = new HashMap<String, SpringResource>(); for (Class<?> c : validClasses) { RequestMapping requestMapping = c.getAnnotation(RequestMapping.class); String description = ""; // This try/catch block is to stop a bamboo build from failing due to NoClassDefFoundError // This occurs when a class or method loaded by reflections contains a type that has no // dependency try { resourceMap = analyzeController(c, resourceMap, description); List<Method> mList = new ArrayList<Method>(Arrays.asList(c.getMethods())); if (c.getSuperclass() != null) { mList.addAll(Arrays.asList(c.getSuperclass().getMethods())); } } catch (NoClassDefFoundError e) { LOG.error(e.getMessage()); LOG.info(c.getName()); // exception occurs when a method type or annotation is not recognized by the plugin } catch (ClassNotFoundException e) { LOG.error(e.getMessage()); LOG.info(c.getName()); } } return resourceMap; }
public ComponentDomainClass(Class type) { super(type, ""); PropertyDescriptor[] descriptors; try { descriptors = java.beans.Introspector.getBeanInfo(type).getPropertyDescriptors(); } catch (IntrospectionException e) { throw new GrailsDomainException( "Failed to use class [" + type + "] as a component. Cannot introspect! " + e.getMessage()); } List tmp = (List) getPropertyOrStaticPropertyOrFieldValue( GrailsDomainClassProperty.TRANSIENT, List.class); if (tmp != null) this.transients = tmp; this.properties = createDomainClassProperties(this, descriptors); try { this.constraints = GrailsDomainConfigurationUtil.evaluateConstraints( getReference().getWrappedInstance(), properties); } catch (IntrospectionException e) { LOG.error( "Error reading embedded component [" + getClazz() + "] constraints: " + e.getMessage(), e); } }
@Override public void consume(Object o) { if (!(o instanceof GroovyResolveResult)) { LOG.error(o); return; } GroovyResolveResult result = (GroovyResolveResult) o; if (!result.isStaticsOK()) { if (myInapplicable == null) myInapplicable = ContainerUtil.newArrayList(); myInapplicable.add(result); return; } if (!result.isAccessible() && myParameters.getInvocationCount() < 2) return; if (mySkipPackages && result.getElement() instanceof PsiPackage) return; PsiElement element = result.getElement(); if (element instanceof PsiVariable && !myMatcher.prefixMatches(((PsiVariable) element).getName())) { return; } if (element instanceof GrReflectedMethod) { element = ((GrReflectedMethod) element).getBaseMethod(); if (!myProcessedMethodWithOptionalParams.add((GrMethod) element)) return; result = new GroovyResolveResultImpl( element, result.getCurrentFileResolveContext(), result.getSpreadState(), result.getSubstitutor(), result.isAccessible(), result.isStaticsOK(), result.isInvokedOnProperty(), result.isValidResult()); } if (myFieldPointerOperator && !(element instanceof PsiVariable)) { return; } if (myMethodPointerOperator && !(element instanceof PsiMethod)) { return; } addCandidate(result); if (!myFieldPointerOperator && !myMethodPointerOperator) { if (element instanceof PsiMethod) { processProperty((PsiMethod) element, result); } else if (element instanceof GrField) { if (((GrField) element).isProperty()) { processPropertyFromField((GrField) element, result); } } } if (element instanceof GrVariable && !(element instanceof GrField)) { myLocalVars.add(((GrVariable) element).getName()); } }
@Override public StepExecutionResult executeImplementation(StepExecutionContext context) throws MuseExecutionError { StepConfiguration config = getConfiguration(); HashMap<String, Object> values = new HashMap<>(); for (String name : config.getSourceNames()) { MuseValueSource source = getValueSource(config, name, false, context.getProject()); values.put(name, getValue(source, context, true, Object.class)); } try { JavascriptRunner runner = new JavascriptStepRunner(); runner.evalScript(_origin); Object result = runner.invokeFunction(EXECUTE_FUNCTION, context, values); if (result instanceof StepExecutionResult) return (StepExecutionResult) result; else context.raiseEvent( new ScriptFailureEvent( "Script did not return a StepExecutionResult. Instead, it returned: " + result, null)); // TODO do something better than n/a } catch (Throwable t) { LOG.error("unable to execute script: ", t); context.raiseEvent( new ScriptFailureEvent( "Script threw an exception: " + t.getMessage(), t)); // TODO do something better than n/a } return new BasicStepExecutionResult(StepExecutionStatus.FAILURE); }
@Override protected void close() { try { super.close(); } catch (IOException e) { LOG.error(e); } }
@Override public void processPacket(Packet packet) { try { doProcessPacket(packet); } catch (Throwable e) { LOG.error(e.getMessage(), e); } }
private boolean hasNext() { boolean ret = false; try { ret = consumer.hasNext(); } catch (Exception exception) { LOG.error("EntityNotification consumer threw exception, IGNORING...:", exception); } return ret; }
public void refreshConstraints() { try { GrailsDomainClassProperty[] props = getPersistentProperties(); this.constraints = GrailsDomainConfigurationUtil.evaluateConstraints( getReference().getWrappedInstance(), props); } catch (IntrospectionException e) { LOG.error("Error reading class [" + getClazz() + "] constraints: " + e.getMessage(), e); } }
public void process(WatchedEvent event) { if (event.getType() == Event.EventType.NodeChildrenChanged) { LOG.debug("Running children changed [" + event.getPath() + "]"); try { getZkRunning(); } catch (Exception e) { e.printStackTrace(); LOG.error(e); } } else if (event.getType() == Event.EventType.NodeDeleted) { String znodePath = event.getPath(); LOG.debug("Running znode deleted [" + znodePath + "]"); try { restartServer(znodePath); } catch (Exception e) { e.printStackTrace(); LOG.error(e); } } }
public void reportTo(CompileContext compileContext) { compileContext.addMessage( messageCategory, message, url == null ? "" : url, line == null ? -1 : line, column == null ? -1 : column); if (isException) { LOG.error(message); } }
@Override public void run() { try { LOG.debug("Posting queueName: " + queueName + " and data: " + data); pushedQueueMap.put(queueName, data); this.lucasMessageBroadcastService.sendMessage(queueName, bindKey, data); } catch (Exception e) { LOG.error("Exception Generated ", e); throw new LucasRuntimeException(e); } }
@Override public void onTextAvailable(ProcessEvent event, Key outputType) { if (outputType == ProcessOutputTypes.STDERR) { LOG.warn(event.getText().trim()); } if (outputType != ProcessOutputTypes.STDOUT) { return; } final String line = event.getText().trim(); if (LOG.isDebugEnabled()) { LOG.debug(">> " + line); } if (myLastOp == null) { final WatcherOp watcherOp; try { watcherOp = WatcherOp.valueOf(line); } catch (IllegalArgumentException e) { LOG.error("Illegal watcher command: " + line); return; } if (watcherOp == WatcherOp.GIVEUP) { notifyOnFailure(ApplicationBundle.message("watcher.gave.up"), null); myIsShuttingDown = true; } else if (watcherOp == WatcherOp.RESET) { reset(); } else { myLastOp = watcherOp; } } else if (myLastOp == WatcherOp.MESSAGE) { notifyOnFailure(line, NotificationListener.URL_OPENING_LISTENER); myLastOp = null; } else if (myLastOp == WatcherOp.REMAP || myLastOp == WatcherOp.UNWATCHEABLE) { if ("#".equals(line)) { if (myLastOp == WatcherOp.REMAP) { processRemap(); } else { mySettingRoots.decrementAndGet(); processUnwatchable(); } myLines.clear(); myLastOp = null; } else { myLines.add(line); } } else { String path = line.replace('\0', '\n'); // unescape processChange(path, myLastOp); myLastOp = null; } }
@Override public void visitThisExpression(PsiThisExpression expression) { super.visitThisExpression(expression); final PsiJavaCodeReferenceElement qualifier = expression.getQualifier(); if (qualifier != null && qualifier.isReferenceTo(mySourceClass)) { try { qualifier.bindToElement(myTargetSuperClass); } catch (IncorrectOperationException e) { LOG.error(e); } } }
@Override public void visitSuperExpression(PsiSuperExpression expression) { super.visitSuperExpression(expression); final PsiJavaCodeReferenceElement qualifier = expression.getQualifier(); if (qualifier != null && qualifier.isReferenceTo(mySourceClass)) { try { expression.replace( JavaPsiFacade.getElementFactory(myProject) .createExpressionFromText(myTargetSuperClass.getName() + ".this", null)); } catch (IncorrectOperationException e) { LOG.error(e); } } }
@Override public void setSelected(AnActionEvent e, boolean state) { mySelected = state; if (mySelected) { try { mySession = connectToDebugger(); } catch (Exception e1) { LOG.error(e1); Messages.showErrorDialog("Can't connect to debugger", "Error Connecting Debugger"); } } else { // TODO: disable debugging } }
@Override public void run() { try { String[] stringArray = this.lucasMessageBroadcastClientService.receiveAllMessagesWithinTimePeriod( this.lucasBroadCastQueue, this.lucasBroadCastBinding, 1L, 5000L); if (stringArray != null) { for (String string : stringArray) { pulledQueueMap.put(this.lucasBroadCastQueue, string); } } } catch (Exception e) { LOG.error("Exception Generated ", e); throw new LucasRuntimeException(e); } }
public void run() { processing = true; try { while (true) { FileObject path = myPaths.take(); if (path == myStopThreadToken) { break; } processPath(path, myProject); } } catch (InterruptedException e) { LOG.error(e); } catch (CacheCorruptedException e) { myError = e; } finally { processing = false; } }
@Override protected VirtualFile[] doAddItems() { Project project = CommonDataKeys.PROJECT.getData(DataManager.getInstance().getDataContext(myPanel)); try { String[] files = PythonRemoteInterpreterManager.getInstance() .chooseRemoteFiles( project, (PyRemoteSdkAdditionalDataBase) mySdk.getSdkAdditionalData(), false); final String sourcesLocalPath = PySdkUtil.getRemoteSourcesLocalPath(mySdk.getHomePath()); VirtualFile[] vFiles = new VirtualFile[files.length]; int i = 0; for (String file : files) { String localRoot = PyRemoteSourceItem.localPathForRemoteRoot(sourcesLocalPath, file); myNewMappings.add(new PathMappingSettings.PathMapping(localRoot, file)); myRemoteSdkData.getPathMappings().addMappingCheckUnique(localRoot, file); if (!new File(localRoot).exists()) { new File(localRoot).mkdirs(); } vFiles[i++] = LocalFileSystem.getInstance().refreshAndFindFileByIoFile(new File(localRoot)); } vFiles = adjustAddedFileSet(myPanel, vFiles); List<VirtualFile> added = new ArrayList<VirtualFile>(vFiles.length); for (VirtualFile vFile : vFiles) { if (addElement(vFile)) { added.add(vFile); } } return VfsUtilCore.toVirtualFileArray(added); } catch (Exception e) { LOG.error(e); } return new VirtualFile[0]; }
private void processResponse(final String line) { try { final ProtocolFrame frame = new ProtocolFrame(line); logFrame(frame, false); if (AbstractThreadCommand.isThreadCommand(frame.getCommand())) { processThreadEvent(frame); } else if (AbstractCommand.isWriteToConsole(frame.getCommand())) { writeToConsole(ProtocolParser.parseIo(frame.getPayload())); } else if (AbstractCommand.isExitEvent(frame.getCommand())) { fireCommunicationError(); } else if (AbstractCommand.isCallSignatureTrace(frame.getCommand())) { recordCallSignature(ProtocolParser.parseCallSignature(frame.getPayload())); } else { placeResponse(frame.getSequence(), frame); } } catch (Throwable t) { // shouldn't interrupt reader thread LOG.error(t); } }
@Override protected JPanel processFiles(@NotNull List<Element> fileElements, final JPanel context) { final Ref<EditorWindow> windowRef = new Ref<EditorWindow>(); UIUtil.invokeAndWaitIfNeeded( new Runnable() { @Override public void run() { windowRef.set(context == null ? createEditorWindow() : findWindowWith(context)); } }); final EditorWindow window = windowRef.get(); LOG.assertTrue(window != null); VirtualFile focusedFile = null; for (int i = 0; i < fileElements.size(); i++) { final Element file = fileElements.get(i); if (i == 0) { EditorTabbedContainer tabbedPane = window.getTabbedPane(); if (tabbedPane != null) { try { int limit = Integer.parseInt( file.getParentElement() .getAttributeValue( JBTabsImpl.SIDE_TABS_SIZE_LIMIT_KEY.toString(), String.valueOf(JBTabsImpl.DEFAULT_MAX_TAB_WIDTH))); UIUtil.putClientProperty( tabbedPane.getComponent(), JBTabsImpl.SIDE_TABS_SIZE_LIMIT_KEY, limit); } catch (NumberFormatException e) { // ignore } } } try { final FileEditorManagerImpl fileEditorManager = getManager(); Element historyElement = file.getChild(HistoryEntry.TAG); final HistoryEntry entry = HistoryEntry.createLight(fileEditorManager.getProject(), historyElement); final VirtualFile virtualFile = entry.getFile(); if (virtualFile == null) throw new InvalidDataException("No file exists: " + entry.getFilePointer().getUrl()); Document document = ApplicationManager.getApplication() .runReadAction( new Computable<Document>() { @Override public Document compute() { return virtualFile.isValid() ? FileDocumentManager.getInstance().getDocument(virtualFile) : null; } }); final boolean isCurrentInTab = Boolean.valueOf(file.getAttributeValue(CURRENT_IN_TAB)).booleanValue(); Boolean pin = Boolean.valueOf(file.getAttributeValue(PINNED)); fileEditorManager.openFileImpl4( window, virtualFile, entry, isCurrentInTab, isCurrentInTab, pin, i); if (isCurrentInTab) { focusedFile = virtualFile; } if (document != null) { // This is just to make sure document reference is kept on stack till this point // so that document is available for folding state deserialization in HistoryEntry // constructor // and that document will be created only once during file opening document.putUserData(DUMMY_KEY, null); } updateProgress(); } catch (InvalidDataException e) { if (ApplicationManager.getApplication().isUnitTestMode()) { LOG.error(e); } } } if (focusedFile != null) { getManager().addSelectionRecord(focusedFile, window); } return window.myPanel; }
// todo: extract response processing private void processThreadEvent(ProtocolFrame frame) throws PyDebuggerException { switch (frame.getCommand()) { case AbstractCommand.CREATE_THREAD: { final PyThreadInfo thread = parseThreadEvent(frame); if (!thread.isPydevThread()) { // ignore pydevd threads myThreads.put(thread.getId(), thread); } break; } case AbstractCommand.SUSPEND_THREAD: { final PyThreadInfo event = parseThreadEvent(frame); PyThreadInfo thread = myThreads.get(event.getId()); if (thread == null) { LOG.error( "Trying to stop on non-existent thread: " + event.getId() + ", " + event.getStopReason() + ", " + event.getMessage()); myThreads.put(event.getId(), event); thread = event; } thread.updateState(PyThreadInfo.State.SUSPENDED, event.getFrames()); thread.setStopReason(event.getStopReason()); thread.setMessage(event.getMessage()); myDebugProcess.threadSuspended(thread); break; } case AbstractCommand.RESUME_THREAD: { final String id = ProtocolParser.getThreadId(frame.getPayload()); final PyThreadInfo thread = myThreads.get(id); if (thread != null) { thread.updateState(PyThreadInfo.State.RUNNING, null); myDebugProcess.threadResumed(thread); } break; } case AbstractCommand.KILL_THREAD: { final String id = frame.getPayload(); final PyThreadInfo thread = myThreads.get(id); if (thread != null) { thread.updateState(PyThreadInfo.State.KILLED, null); myThreads.remove(id); } break; } case AbstractCommand.SHOW_CONSOLE: { final PyThreadInfo event = parseThreadEvent(frame); PyThreadInfo thread = myThreads.get(event.getId()); if (thread == null) { myThreads.put(event.getId(), event); thread = event; } thread.updateState(PyThreadInfo.State.SUSPENDED, event.getFrames()); thread.setStopReason(event.getStopReason()); thread.setMessage(event.getMessage()); myDebugProcess.showConsole(thread); break; } } }
private void processChange(String path, WatcherOp op) { if (SystemInfo.isWindows && op == WatcherOp.RECDIRTY && path.length() == 3 && Character.isLetter(path.charAt(0))) { VirtualFile root = LocalFileSystem.getInstance().findFileByPath(path); if (root != null) { myNotificationSink.notifyPathsRecursive(list(root.getPresentableUrl())); } notifyOnAnyEvent(); return; } if (op == WatcherOp.CHANGE) { // collapse subsequent change file change notifications that happen once we copy large file, // this allows reduction of path checks at least 20% for Windows synchronized (myLock) { for (int i = 0; i < myLastChangedPaths.length; ++i) { int last = myLastChangedPathIndex - i - 1; if (last < 0) last += myLastChangedPaths.length; String lastChangedPath = myLastChangedPaths[last]; if (lastChangedPath != null && lastChangedPath.equals(path)) { return; } } myLastChangedPaths[myLastChangedPathIndex++] = path; if (myLastChangedPathIndex == myLastChangedPaths.length) myLastChangedPathIndex = 0; } } int length = path.length(); if (length > 1 && path.charAt(length - 1) == '/') path = path.substring(0, length - 1); boolean exactPath = op != WatcherOp.DIRTY && op != WatcherOp.RECDIRTY; Collection<String> paths = checkWatchable(path, exactPath, false); if (paths.isEmpty()) { if (LOG.isDebugEnabled()) { LOG.debug("Not watchable, filtered: " + path); } return; } switch (op) { case STATS: case CHANGE: myNotificationSink.notifyDirtyPaths(paths); break; case CREATE: case DELETE: myNotificationSink.notifyPathsCreatedOrDeleted(paths); break; case DIRTY: myNotificationSink.notifyDirtyDirectories(paths); break; case RECDIRTY: myNotificationSink.notifyPathsRecursive(paths); break; default: LOG.error("Unexpected op: " + op); } notifyOnAnyEvent(); }
public void assignToNonLeafTasks(LinkedList<TaskRequestEvent> taskRequests) { Collections.shuffle(taskRequests); String queryMasterHostAndPort = context .getMasterContext() .getQueryMasterContext() .getWorkerContext() .getConnectionInfo() .getHostAndQMPort(); TaskRequestEvent taskRequest; while (!taskRequests.isEmpty()) { taskRequest = taskRequests.pollFirst(); LOG.debug("assignToNonLeafTasks: " + taskRequest.getExecutionBlockId()); TaskAttemptId attemptId; // random allocation if (nonLeafTasks.size() > 0) { synchronized (nonLeafTasks) { attemptId = nonLeafTasks.iterator().next(); nonLeafTasks.remove(attemptId); } LOG.debug("Assigned based on * match"); Task task; task = stage.getTask(attemptId.getTaskId()); TaskRequest taskAssign = new TaskRequestImpl( attemptId, Lists.newArrayList(task.getAllFragments()), "", false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer(), queryMasterHostAndPort); if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) { taskAssign.setInterQuery(); } for (Map.Entry<String, Set<FetchImpl>> entry : task.getFetchMap().entrySet()) { Collection<FetchImpl> fetches = entry.getValue(); if (fetches != null) { for (FetchImpl fetch : fetches) { taskAssign.addFetch(entry.getKey(), fetch); } } } WorkerConnectionInfo connectionInfo = context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId()); // TODO send batch request BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.addTaskRequest( TaskAllocationProto.newBuilder() .setResource(taskRequest.getResponseProto().getResource()) .setTaskRequest(taskAssign.getProto()) .build()); requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto()); context .getMasterContext() .getEventHandler() .handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo)); CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId()); if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort()); AsyncRpcClient tajoWorkerRpc; try { tajoWorkerRpc = RpcClientManager.getInstance() .getClient(addr, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks( callFuture.getController(), requestProto.build(), callFuture); BatchAllocationResponse responseProto = callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); if (responseProto.getCancellationTaskCount() > 0) { for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) { cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId()))); cancellation++; } if (LOG.isDebugEnabled()) { LOG.debug( "Canceled requests: " + responseProto.getCancellationTaskCount() + " from " + addr); } continue; } totalAssigned++; scheduledObjectNum--; } catch (Exception e) { LOG.error(e); } } } }
private void doRun() { if (myUpdateProgress.isCanceled()) return; log(myUpdateProgress, myPass, "Started. "); for (ScheduledPass successor : mySuccessorsOnSubmit) { int predecessorsToRun = successor.myRunningPredecessorsCount.decrementAndGet(); if (predecessorsToRun == 0) { submit(successor); } } ProgressManager.getInstance() .executeProcessUnderProgress( () -> { boolean success = ApplicationManagerEx.getApplicationEx() .tryRunReadAction( () -> { try { if (DumbService.getInstance(myProject).isDumb() && !DumbService.isDumbAware(myPass)) { return; } if (!myUpdateProgress.isCanceled() && !myProject.isDisposed()) { myPass.collectInformation(myUpdateProgress); } } catch (ProcessCanceledException e) { log(myUpdateProgress, myPass, "Canceled "); if (!myUpdateProgress.isCanceled()) { myUpdateProgress.cancel( e); // in case when some smart asses throw PCE just for fun } } catch (RuntimeException | Error e) { myUpdateProgress.cancel(e); LOG.error(e); throw e; } }); if (!success) { myUpdateProgress.cancel(); } }, myUpdateProgress); log(myUpdateProgress, myPass, "Finished. "); if (!myUpdateProgress.isCanceled()) { applyInformationToEditorsLater( myFileEditor, myPass, myUpdateProgress, myThreadsToStartCountdown); for (ScheduledPass successor : mySuccessorsOnCompletion) { int predecessorsToRun = successor.myRunningPredecessorsCount.decrementAndGet(); if (predecessorsToRun == 0) { submit(successor); } } } }
public void assignToLeafTasks(LinkedList<TaskRequestEvent> taskRequests) { Collections.shuffle(taskRequests); LinkedList<TaskRequestEvent> remoteTaskRequests = new LinkedList<>(); String queryMasterHostAndPort = context .getMasterContext() .getQueryMasterContext() .getWorkerContext() .getConnectionInfo() .getHostAndQMPort(); TaskRequestEvent taskRequest; while (leafTasks.size() > 0 && (!taskRequests.isEmpty() || !remoteTaskRequests.isEmpty())) { int localAssign = 0; int rackAssign = 0; taskRequest = taskRequests.pollFirst(); if (taskRequest == null) { // if there are only remote task requests taskRequest = remoteTaskRequests.pollFirst(); } // checking if this container is still alive. // If not, ignore the task request and stop the task runner WorkerConnectionInfo connectionInfo = context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId()); if (connectionInfo == null) continue; // getting the hostname of requested node String host = connectionInfo.getHost(); // if there are no worker matched to the hostname a task request if (!leafTaskHostMapping.containsKey(host) && !taskRequests.isEmpty()) { String normalizedHost = NetUtils.normalizeHost(host); if (!leafTaskHostMapping.containsKey(normalizedHost)) { // this case means one of either cases: // * there are no blocks which reside in this node. // * all blocks which reside in this node are consumed, and this task runner requests a // remote task. // In this case, we transfer the task request to the remote task request list, and skip // the followings. remoteTaskRequests.add(taskRequest); continue; } else { host = normalizedHost; } } if (LOG.isDebugEnabled()) { LOG.debug( "assignToLeafTasks: " + taskRequest.getExecutionBlockId() + "," + "worker=" + connectionInfo.getHostAndPeerRpcPort()); } ////////////////////////////////////////////////////////////////////// // disk or host-local allocation ////////////////////////////////////////////////////////////////////// TaskAttemptId attemptId = allocateLocalTask(host); if (attemptId == null) { // if a local task cannot be found HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host); if (!taskRequests .isEmpty()) { // if other requests remains, move to remote list for better locality remoteTaskRequests.add(taskRequest); candidateWorkers.remove(connectionInfo.getId()); continue; } else { if (hostVolumeMapping != null) { int nodes = context.getMasterContext().getWorkerMap().size(); // this part is to control the assignment of tail and remote task balancing per node int tailLimit = 1; if (remainingScheduledObjectNum() > 0 && nodes > 0) { tailLimit = Math.max(remainingScheduledObjectNum() / nodes, 1); } if (hostVolumeMapping.getRemoteConcurrency() >= tailLimit) { // remote task throttling per node continue; } else { // assign to remote volume hostVolumeMapping.increaseConcurrency(HostVolumeMapping.REMOTE); } } } ////////////////////////////////////////////////////////////////////// // rack-local allocation ////////////////////////////////////////////////////////////////////// attemptId = allocateRackTask(host); ////////////////////////////////////////////////////////////////////// // random node allocation ////////////////////////////////////////////////////////////////////// if (attemptId == null && leafTaskNum() > 0) { synchronized (leafTasks) { attemptId = leafTasks.iterator().next(); leafTasks.remove(attemptId); } } if (attemptId != null && hostVolumeMapping != null) { hostVolumeMapping.lastAssignedVolumeId.put(attemptId, HostVolumeMapping.REMOTE); } rackAssign++; } else { localAssign++; } if (attemptId != null) { Task task = stage.getTask(attemptId.getTaskId()); TaskRequest taskAssign = new TaskRequestImpl( attemptId, new ArrayList<>(task.getAllFragments()), "", false, LogicalNodeSerializer.serialize(task.getLogicalPlan()), context.getMasterContext().getQueryContext(), stage.getDataChannel(), stage.getBlock().getEnforcer(), queryMasterHostAndPort); if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) { taskAssign.setInterQuery(); } // TODO send batch request BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder(); requestProto.addTaskRequest( TaskAllocationProto.newBuilder() .setResource(taskRequest.getResponseProto().getResource()) .setTaskRequest(taskAssign.getProto()) .build()); requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto()); context .getMasterContext() .getEventHandler() .handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo)); InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId()); if (addr == null) addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort()); AsyncRpcClient tajoWorkerRpc = null; CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>(); totalAttempts++; try { tajoWorkerRpc = RpcClientManager.getInstance() .getClient(addr, TajoWorkerProtocol.class, true, rpcParams); TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient = tajoWorkerRpc.getStub(); tajoWorkerRpcClient.allocateTasks( callFuture.getController(), requestProto.build(), callFuture); BatchAllocationResponse responseProto = callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS); if (responseProto.getCancellationTaskCount() > 0) { for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) { cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId()))); cancellation++; } if (LOG.isDebugEnabled()) { LOG.debug( "Canceled requests: " + responseProto.getCancellationTaskCount() + " from " + addr); } continue; } } catch (Exception e) { LOG.error(e); } scheduledObjectNum--; totalAssigned++; hostLocalAssigned += localAssign; rackLocalAssigned += rackAssign; if (rackAssign > 0) { LOG.info( String.format( "Assigned Local/Rack/Total: (%d/%d/%d), " + "Attempted Cancel/Assign/Total: (%d/%d/%d), " + "Locality: %.2f%%, Rack host: %s", hostLocalAssigned, rackLocalAssigned, totalAssigned, cancellation, totalAssigned, totalAttempts, ((double) hostLocalAssigned / (double) totalAssigned) * 100, host)); } } else { throw new RuntimeException("Illegal State!!!!!!!!!!!!!!!!!!!!!"); } } }
private ReturnResult checkUserName() { Project project = myPanel.getProject(); GitVcs vcs = GitVcs.getInstance(project); assert vcs != null; Collection<VirtualFile> notDefined = new ArrayList<VirtualFile>(); Map<VirtualFile, Pair<String, String>> defined = new HashMap<VirtualFile, Pair<String, String>>(); Collection<VirtualFile> allRoots = new ArrayList<VirtualFile>( Arrays.asList(ProjectLevelVcsManager.getInstance(project).getRootsUnderVcs(vcs))); Collection<VirtualFile> affectedRoots = getSelectedRoots(); for (VirtualFile root : affectedRoots) { try { Pair<String, String> nameAndEmail = getUserNameAndEmailFromGitConfig(project, root); String name = nameAndEmail.getFirst(); String email = nameAndEmail.getSecond(); if (name == null || email == null) { notDefined.add(root); } else { defined.put(root, nameAndEmail); } } catch (VcsException e) { LOG.error("Couldn't get user.name and user.email for root " + root, e); // doing nothing - let commit with possibly empty user.name/email } } if (notDefined.isEmpty()) { return ReturnResult.COMMIT; } GitVersion version = vcs.getVersion(); if (System.getenv("HOME") == null && GitVersionSpecialty.DOESNT_DEFINE_HOME_ENV_VAR.existsIn(version)) { Messages.showErrorDialog( project, "You are using Git " + version + " which doesn't define %HOME% environment variable properly.\n" + "Consider updating Git to a newer version " + "or define %HOME% to point to the place where the global .gitconfig is stored \n" + "(it is usually %USERPROFILE% or %HOMEDRIVE%%HOMEPATH%).", "HOME Variable Is Not Defined"); return ReturnResult.CANCEL; } if (defined.isEmpty() && allRoots.size() > affectedRoots.size()) { allRoots.removeAll(affectedRoots); for (VirtualFile root : allRoots) { try { Pair<String, String> nameAndEmail = getUserNameAndEmailFromGitConfig(project, root); String name = nameAndEmail.getFirst(); String email = nameAndEmail.getSecond(); if (name != null && email != null) { defined.put(root, nameAndEmail); break; } } catch (VcsException e) { LOG.error("Couldn't get user.name and user.email for root " + root, e); // doing nothing - not critical not to find the values for other roots not affected by // commit } } } GitUserNameNotDefinedDialog dialog = new GitUserNameNotDefinedDialog(project, notDefined, affectedRoots, defined); dialog.show(); if (dialog.isOK()) { try { if (dialog.isGlobal()) { GitConfigUtil.setValue( project, notDefined.iterator().next(), GitConfigUtil.USER_NAME, dialog.getUserName(), "--global"); GitConfigUtil.setValue( project, notDefined.iterator().next(), GitConfigUtil.USER_EMAIL, dialog.getUserEmail(), "--global"); } else { for (VirtualFile root : notDefined) { GitConfigUtil.setValue(project, root, GitConfigUtil.USER_NAME, dialog.getUserName()); GitConfigUtil.setValue( project, root, GitConfigUtil.USER_EMAIL, dialog.getUserEmail()); } } } catch (VcsException e) { String message = "Couldn't set user.name and user.email"; LOG.error(message, e); Messages.showErrorDialog(myPanel.getComponent(), message); return ReturnResult.CANCEL; } return ReturnResult.COMMIT; } return ReturnResult.CLOSE_WINDOW; }
@Override public ScriptContext call() throws Exception { try { Scanner scn = new Scanner(znodePath); scn.useDelimiter(":"); String hostName = scn.next(); // host name String instance = scn.next(); // instance int infoPort = Integer.parseInt(scn.next()); // UI info port long serverStartTimestamp = Long.parseLong(scn.next()); scn.close(); // Get the --config property from classpath...it's always first // in the classpath String cp = System.getProperty("java.class.path"); scn = new Scanner(cp); scn.useDelimiter(":"); String confDir = scn.next(); scn.close(); LOG.debug("conf dir [" + confDir + "]"); // Get -Dwms.home.dir String wmsHome = System.getProperty("wms.home.dir"); // If stop-wms.sh is executed and WMS_MANAGES_ZK then zookeeper // is stopped abruptly. // Second scenario is when ZooKeeper fails for some reason // regardless of whether WMS // manages it. When either happens the WmsServer running znodes // still exist in ZooKeeper // and we see them at next startup. When they eventually timeout // we get node deleted events for a server that no longer // exists. So, only recognize // WmsServer running znodes that have timestamps after last // WmsMaster startup. if (serverStartTimestamp > startupTimestamp) { scriptContext.setHostName(hostName); scriptContext.setScriptName("sys_shell.py"); if (hostName.equalsIgnoreCase(ia.getCanonicalHostName())) scriptContext.setCommand( "bin/wms-daemon.sh --config " + confDir + " start server " + instance); else scriptContext.setCommand( "pdsh -w " + hostName + " \"cd " + wmsHome + ";bin/wms-daemon.sh --config " + confDir + " start server " + instance + "\""); RetryCounter retryCounter = retryCounterFactory.create(); while (true) { if (scriptContext.getStdOut().length() > 0) scriptContext.getStdOut().delete(0, scriptContext.getStdOut().length()); if (scriptContext.getStdErr().length() > 0) scriptContext.getStdErr().delete(0, scriptContext.getStdErr().length()); LOG.info( "Restarting WmsServer [" + hostName + ":" + instance + "], script [ " + scriptContext.toString() + " ]"); ScriptManager.getInstance().runScript(scriptContext); if (scriptContext.getExitCode() == 0) { LOG.info("WmsServer [" + hostName + ":" + instance + "] restarted"); break; } else { StringBuilder sb = new StringBuilder(); sb.append("exit code [" + scriptContext.getExitCode() + "]"); if (!scriptContext.getStdOut().toString().isEmpty()) sb.append(", stdout [" + scriptContext.getStdOut().toString() + "]"); if (!scriptContext.getStdErr().toString().isEmpty()) sb.append(", stderr [" + scriptContext.getStdErr().toString() + "]"); LOG.error(sb.toString()); if (!retryCounter.shouldRetry()) { LOG.error( "WmsServer [" + hostName + ":" + instance + "] restart failed after " + retryCounter.getMaxRetries() + " retries"); break; } else { retryCounter.sleepUntilNextRetry(); retryCounter.useRetry(); } } } } else { LOG.debug( "No restart for " + znodePath + "\nbecause WmsServer start time [" + DateFormat.getDateTimeInstance().format(new Date(serverStartTimestamp)) + "] was before WmsMaster start time [" + DateFormat.getDateTimeInstance().format(new Date(startupTimestamp)) + "]"); } } catch (Exception e) { e.printStackTrace(); LOG.error(e); } return scriptContext; }