@Override
public void run() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ConsumerRunnable.run()");
}
while (true) {
try {
if (hasNext()) {
EntityNotification notification = consumer.peek();
if (notification != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Notification=" + getPrintableEntityNotification(notification));
}
ServiceTags serviceTags =
AtlasNotificationMapper.processEntityNotification(notification);
if (serviceTags == null) {
LOG.error(
"No ServiceTags built for notification :"
+ getPrintableEntityNotification(notification));
} else {
updateSink(serviceTags);
}
} else {
LOG.error("Null entityNotification received from Kafka!! Ignoring..");
}
// Move iterator forward
consumer.next();
}
} catch (Exception exception) {
LOG.error("Caught exception..: ", exception);
return;
}
}
}
@Override
public void visitReferenceExpression(GrReferenceExpression expression) {
super.visitReferenceExpression(expression);
if (org.jetbrains.plugins.groovy.lang.psi.util.PsiUtil.isSuperReference(expression)) {
final GrExpression qualifier = expression.getQualifier();
if (qualifier instanceof GrReferenceExpression
&& ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
try {
expression.putCopyableUserData(SUPER_REF, Boolean.TRUE);
} catch (IncorrectOperationException e) {
LOG.error(e);
}
}
} else if (org.jetbrains.plugins.groovy.lang.psi.util.PsiUtil.isThisReference(expression)) {
final GrExpression qualifier = expression.getQualifier();
if (qualifier instanceof GrReferenceExpression
&& ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
try {
expression.putCopyableUserData(THIS_REF, Boolean.TRUE);
} catch (IncorrectOperationException e) {
LOG.error(e);
}
}
}
}
@Override
public void visitReferenceExpression(GrReferenceExpression expression) {
super.visitReferenceExpression(expression);
if (expression.getCopyableUserData(SUPER_REF) != null) {
expression.putCopyableUserData(SUPER_REF, null);
final GrExpression qualifier = expression.getQualifier();
if (qualifier instanceof GrReferenceExpression
&& ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
try {
GroovyPsiElementFactory factory = GroovyPsiElementFactory.getInstance(myProject);
GrExpression newExpr =
factory.createExpressionFromText(myTargetSuperClass.getName() + ".this", null);
expression.replace(newExpr);
} catch (IncorrectOperationException e) {
LOG.error(e);
}
}
} else if (expression.getCopyableUserData(THIS_REF) != null) {
expression.putCopyableUserData(THIS_REF, null);
final GrExpression qualifier = expression.getQualifier();
if (qualifier instanceof GrReferenceExpression
&& ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
try {
((GrReferenceExpression) qualifier).bindToElement(myTargetSuperClass);
GroovyChangeContextUtil.clearContextInfo(qualifier);
} catch (IncorrectOperationException e) {
LOG.error(e);
}
}
}
}
private Map<String, SpringResource> generateResourceMap(Set<Class<?>> validClasses)
throws GenerateException {
Map<String, SpringResource> resourceMap = new HashMap<String, SpringResource>();
for (Class<?> c : validClasses) {
RequestMapping requestMapping = c.getAnnotation(RequestMapping.class);
String description = "";
// This try/catch block is to stop a bamboo build from failing due to NoClassDefFoundError
// This occurs when a class or method loaded by reflections contains a type that has no
// dependency
try {
resourceMap = analyzeController(c, resourceMap, description);
List<Method> mList = new ArrayList<Method>(Arrays.asList(c.getMethods()));
if (c.getSuperclass() != null) {
mList.addAll(Arrays.asList(c.getSuperclass().getMethods()));
}
} catch (NoClassDefFoundError e) {
LOG.error(e.getMessage());
LOG.info(c.getName());
// exception occurs when a method type or annotation is not recognized by the plugin
} catch (ClassNotFoundException e) {
LOG.error(e.getMessage());
LOG.info(c.getName());
}
}
return resourceMap;
}
public ComponentDomainClass(Class type) {
super(type, "");
PropertyDescriptor[] descriptors;
try {
descriptors = java.beans.Introspector.getBeanInfo(type).getPropertyDescriptors();
} catch (IntrospectionException e) {
throw new GrailsDomainException(
"Failed to use class ["
+ type
+ "] as a component. Cannot introspect! "
+ e.getMessage());
}
List tmp =
(List)
getPropertyOrStaticPropertyOrFieldValue(
GrailsDomainClassProperty.TRANSIENT, List.class);
if (tmp != null) this.transients = tmp;
this.properties = createDomainClassProperties(this, descriptors);
try {
this.constraints =
GrailsDomainConfigurationUtil.evaluateConstraints(
getReference().getWrappedInstance(), properties);
} catch (IntrospectionException e) {
LOG.error(
"Error reading embedded component [" + getClazz() + "] constraints: " + e.getMessage(),
e);
}
}
@Override
public void consume(Object o) {
if (!(o instanceof GroovyResolveResult)) {
LOG.error(o);
return;
}
GroovyResolveResult result = (GroovyResolveResult) o;
if (!result.isStaticsOK()) {
if (myInapplicable == null) myInapplicable = ContainerUtil.newArrayList();
myInapplicable.add(result);
return;
}
if (!result.isAccessible() && myParameters.getInvocationCount() < 2) return;
if (mySkipPackages && result.getElement() instanceof PsiPackage) return;
PsiElement element = result.getElement();
if (element instanceof PsiVariable
&& !myMatcher.prefixMatches(((PsiVariable) element).getName())) {
return;
}
if (element instanceof GrReflectedMethod) {
element = ((GrReflectedMethod) element).getBaseMethod();
if (!myProcessedMethodWithOptionalParams.add((GrMethod) element)) return;
result =
new GroovyResolveResultImpl(
element,
result.getCurrentFileResolveContext(),
result.getSpreadState(),
result.getSubstitutor(),
result.isAccessible(),
result.isStaticsOK(),
result.isInvokedOnProperty(),
result.isValidResult());
}
if (myFieldPointerOperator && !(element instanceof PsiVariable)) {
return;
}
if (myMethodPointerOperator && !(element instanceof PsiMethod)) {
return;
}
addCandidate(result);
if (!myFieldPointerOperator && !myMethodPointerOperator) {
if (element instanceof PsiMethod) {
processProperty((PsiMethod) element, result);
} else if (element instanceof GrField) {
if (((GrField) element).isProperty()) {
processPropertyFromField((GrField) element, result);
}
}
}
if (element instanceof GrVariable && !(element instanceof GrField)) {
myLocalVars.add(((GrVariable) element).getName());
}
}
@Override
public StepExecutionResult executeImplementation(StepExecutionContext context)
throws MuseExecutionError {
StepConfiguration config = getConfiguration();
HashMap<String, Object> values = new HashMap<>();
for (String name : config.getSourceNames()) {
MuseValueSource source = getValueSource(config, name, false, context.getProject());
values.put(name, getValue(source, context, true, Object.class));
}
try {
JavascriptRunner runner = new JavascriptStepRunner();
runner.evalScript(_origin);
Object result = runner.invokeFunction(EXECUTE_FUNCTION, context, values);
if (result instanceof StepExecutionResult) return (StepExecutionResult) result;
else
context.raiseEvent(
new ScriptFailureEvent(
"Script did not return a StepExecutionResult. Instead, it returned: " + result,
null)); // TODO do something better than n/a
} catch (Throwable t) {
LOG.error("unable to execute script: ", t);
context.raiseEvent(
new ScriptFailureEvent(
"Script threw an exception: " + t.getMessage(),
t)); // TODO do something better than n/a
}
return new BasicStepExecutionResult(StepExecutionStatus.FAILURE);
}
@Override
protected void close() {
try {
super.close();
} catch (IOException e) {
LOG.error(e);
}
}
@Override
public void processPacket(Packet packet) {
try {
doProcessPacket(packet);
} catch (Throwable e) {
LOG.error(e.getMessage(), e);
}
}
private boolean hasNext() {
boolean ret = false;
try {
ret = consumer.hasNext();
} catch (Exception exception) {
LOG.error("EntityNotification consumer threw exception, IGNORING...:", exception);
}
return ret;
}
public void refreshConstraints() {
try {
GrailsDomainClassProperty[] props = getPersistentProperties();
this.constraints =
GrailsDomainConfigurationUtil.evaluateConstraints(
getReference().getWrappedInstance(), props);
} catch (IntrospectionException e) {
LOG.error("Error reading class [" + getClazz() + "] constraints: " + e.getMessage(), e);
}
}
public void process(WatchedEvent event) {
if (event.getType() == Event.EventType.NodeChildrenChanged) {
LOG.debug("Running children changed [" + event.getPath() + "]");
try {
getZkRunning();
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
} else if (event.getType() == Event.EventType.NodeDeleted) {
String znodePath = event.getPath();
LOG.debug("Running znode deleted [" + znodePath + "]");
try {
restartServer(znodePath);
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
}
}
public void reportTo(CompileContext compileContext) {
compileContext.addMessage(
messageCategory,
message,
url == null ? "" : url,
line == null ? -1 : line,
column == null ? -1 : column);
if (isException) {
LOG.error(message);
}
}
@Override
public void run() {
try {
LOG.debug("Posting queueName: " + queueName + " and data: " + data);
pushedQueueMap.put(queueName, data);
this.lucasMessageBroadcastService.sendMessage(queueName, bindKey, data);
} catch (Exception e) {
LOG.error("Exception Generated ", e);
throw new LucasRuntimeException(e);
}
}
@Override
public void onTextAvailable(ProcessEvent event, Key outputType) {
if (outputType == ProcessOutputTypes.STDERR) {
LOG.warn(event.getText().trim());
}
if (outputType != ProcessOutputTypes.STDOUT) {
return;
}
final String line = event.getText().trim();
if (LOG.isDebugEnabled()) {
LOG.debug(">> " + line);
}
if (myLastOp == null) {
final WatcherOp watcherOp;
try {
watcherOp = WatcherOp.valueOf(line);
} catch (IllegalArgumentException e) {
LOG.error("Illegal watcher command: " + line);
return;
}
if (watcherOp == WatcherOp.GIVEUP) {
notifyOnFailure(ApplicationBundle.message("watcher.gave.up"), null);
myIsShuttingDown = true;
} else if (watcherOp == WatcherOp.RESET) {
reset();
} else {
myLastOp = watcherOp;
}
} else if (myLastOp == WatcherOp.MESSAGE) {
notifyOnFailure(line, NotificationListener.URL_OPENING_LISTENER);
myLastOp = null;
} else if (myLastOp == WatcherOp.REMAP || myLastOp == WatcherOp.UNWATCHEABLE) {
if ("#".equals(line)) {
if (myLastOp == WatcherOp.REMAP) {
processRemap();
} else {
mySettingRoots.decrementAndGet();
processUnwatchable();
}
myLines.clear();
myLastOp = null;
} else {
myLines.add(line);
}
} else {
String path = line.replace('\0', '\n'); // unescape
processChange(path, myLastOp);
myLastOp = null;
}
}
@Override
public void visitThisExpression(PsiThisExpression expression) {
super.visitThisExpression(expression);
final PsiJavaCodeReferenceElement qualifier = expression.getQualifier();
if (qualifier != null && qualifier.isReferenceTo(mySourceClass)) {
try {
qualifier.bindToElement(myTargetSuperClass);
} catch (IncorrectOperationException e) {
LOG.error(e);
}
}
}
@Override
public void visitSuperExpression(PsiSuperExpression expression) {
super.visitSuperExpression(expression);
final PsiJavaCodeReferenceElement qualifier = expression.getQualifier();
if (qualifier != null && qualifier.isReferenceTo(mySourceClass)) {
try {
expression.replace(
JavaPsiFacade.getElementFactory(myProject)
.createExpressionFromText(myTargetSuperClass.getName() + ".this", null));
} catch (IncorrectOperationException e) {
LOG.error(e);
}
}
}
@Override
public void setSelected(AnActionEvent e, boolean state) {
mySelected = state;
if (mySelected) {
try {
mySession = connectToDebugger();
} catch (Exception e1) {
LOG.error(e1);
Messages.showErrorDialog("Can't connect to debugger", "Error Connecting Debugger");
}
} else {
// TODO: disable debugging
}
}
@Override
public void run() {
try {
String[] stringArray =
this.lucasMessageBroadcastClientService.receiveAllMessagesWithinTimePeriod(
this.lucasBroadCastQueue, this.lucasBroadCastBinding, 1L, 5000L);
if (stringArray != null) {
for (String string : stringArray) {
pulledQueueMap.put(this.lucasBroadCastQueue, string);
}
}
} catch (Exception e) {
LOG.error("Exception Generated ", e);
throw new LucasRuntimeException(e);
}
}
public void run() {
processing = true;
try {
while (true) {
FileObject path = myPaths.take();
if (path == myStopThreadToken) {
break;
}
processPath(path, myProject);
}
} catch (InterruptedException e) {
LOG.error(e);
} catch (CacheCorruptedException e) {
myError = e;
} finally {
processing = false;
}
}
@Override
protected VirtualFile[] doAddItems() {
Project project =
CommonDataKeys.PROJECT.getData(DataManager.getInstance().getDataContext(myPanel));
try {
String[] files =
PythonRemoteInterpreterManager.getInstance()
.chooseRemoteFiles(
project, (PyRemoteSdkAdditionalDataBase) mySdk.getSdkAdditionalData(), false);
final String sourcesLocalPath = PySdkUtil.getRemoteSourcesLocalPath(mySdk.getHomePath());
VirtualFile[] vFiles = new VirtualFile[files.length];
int i = 0;
for (String file : files) {
String localRoot = PyRemoteSourceItem.localPathForRemoteRoot(sourcesLocalPath, file);
myNewMappings.add(new PathMappingSettings.PathMapping(localRoot, file));
myRemoteSdkData.getPathMappings().addMappingCheckUnique(localRoot, file);
if (!new File(localRoot).exists()) {
new File(localRoot).mkdirs();
}
vFiles[i++] =
LocalFileSystem.getInstance().refreshAndFindFileByIoFile(new File(localRoot));
}
vFiles = adjustAddedFileSet(myPanel, vFiles);
List<VirtualFile> added = new ArrayList<VirtualFile>(vFiles.length);
for (VirtualFile vFile : vFiles) {
if (addElement(vFile)) {
added.add(vFile);
}
}
return VfsUtilCore.toVirtualFileArray(added);
} catch (Exception e) {
LOG.error(e);
}
return new VirtualFile[0];
}
private void processResponse(final String line) {
try {
final ProtocolFrame frame = new ProtocolFrame(line);
logFrame(frame, false);
if (AbstractThreadCommand.isThreadCommand(frame.getCommand())) {
processThreadEvent(frame);
} else if (AbstractCommand.isWriteToConsole(frame.getCommand())) {
writeToConsole(ProtocolParser.parseIo(frame.getPayload()));
} else if (AbstractCommand.isExitEvent(frame.getCommand())) {
fireCommunicationError();
} else if (AbstractCommand.isCallSignatureTrace(frame.getCommand())) {
recordCallSignature(ProtocolParser.parseCallSignature(frame.getPayload()));
} else {
placeResponse(frame.getSequence(), frame);
}
} catch (Throwable t) {
// shouldn't interrupt reader thread
LOG.error(t);
}
}
@Override
protected JPanel processFiles(@NotNull List<Element> fileElements, final JPanel context) {
final Ref<EditorWindow> windowRef = new Ref<EditorWindow>();
UIUtil.invokeAndWaitIfNeeded(
new Runnable() {
@Override
public void run() {
windowRef.set(context == null ? createEditorWindow() : findWindowWith(context));
}
});
final EditorWindow window = windowRef.get();
LOG.assertTrue(window != null);
VirtualFile focusedFile = null;
for (int i = 0; i < fileElements.size(); i++) {
final Element file = fileElements.get(i);
if (i == 0) {
EditorTabbedContainer tabbedPane = window.getTabbedPane();
if (tabbedPane != null) {
try {
int limit =
Integer.parseInt(
file.getParentElement()
.getAttributeValue(
JBTabsImpl.SIDE_TABS_SIZE_LIMIT_KEY.toString(),
String.valueOf(JBTabsImpl.DEFAULT_MAX_TAB_WIDTH)));
UIUtil.putClientProperty(
tabbedPane.getComponent(), JBTabsImpl.SIDE_TABS_SIZE_LIMIT_KEY, limit);
} catch (NumberFormatException e) {
// ignore
}
}
}
try {
final FileEditorManagerImpl fileEditorManager = getManager();
Element historyElement = file.getChild(HistoryEntry.TAG);
final HistoryEntry entry =
HistoryEntry.createLight(fileEditorManager.getProject(), historyElement);
final VirtualFile virtualFile = entry.getFile();
if (virtualFile == null)
throw new InvalidDataException("No file exists: " + entry.getFilePointer().getUrl());
Document document =
ApplicationManager.getApplication()
.runReadAction(
new Computable<Document>() {
@Override
public Document compute() {
return virtualFile.isValid()
? FileDocumentManager.getInstance().getDocument(virtualFile)
: null;
}
});
final boolean isCurrentInTab =
Boolean.valueOf(file.getAttributeValue(CURRENT_IN_TAB)).booleanValue();
Boolean pin = Boolean.valueOf(file.getAttributeValue(PINNED));
fileEditorManager.openFileImpl4(
window, virtualFile, entry, isCurrentInTab, isCurrentInTab, pin, i);
if (isCurrentInTab) {
focusedFile = virtualFile;
}
if (document != null) {
// This is just to make sure document reference is kept on stack till this point
// so that document is available for folding state deserialization in HistoryEntry
// constructor
// and that document will be created only once during file opening
document.putUserData(DUMMY_KEY, null);
}
updateProgress();
} catch (InvalidDataException e) {
if (ApplicationManager.getApplication().isUnitTestMode()) {
LOG.error(e);
}
}
}
if (focusedFile != null) {
getManager().addSelectionRecord(focusedFile, window);
}
return window.myPanel;
}
// todo: extract response processing
private void processThreadEvent(ProtocolFrame frame) throws PyDebuggerException {
switch (frame.getCommand()) {
case AbstractCommand.CREATE_THREAD:
{
final PyThreadInfo thread = parseThreadEvent(frame);
if (!thread.isPydevThread()) { // ignore pydevd threads
myThreads.put(thread.getId(), thread);
}
break;
}
case AbstractCommand.SUSPEND_THREAD:
{
final PyThreadInfo event = parseThreadEvent(frame);
PyThreadInfo thread = myThreads.get(event.getId());
if (thread == null) {
LOG.error(
"Trying to stop on non-existent thread: "
+ event.getId()
+ ", "
+ event.getStopReason()
+ ", "
+ event.getMessage());
myThreads.put(event.getId(), event);
thread = event;
}
thread.updateState(PyThreadInfo.State.SUSPENDED, event.getFrames());
thread.setStopReason(event.getStopReason());
thread.setMessage(event.getMessage());
myDebugProcess.threadSuspended(thread);
break;
}
case AbstractCommand.RESUME_THREAD:
{
final String id = ProtocolParser.getThreadId(frame.getPayload());
final PyThreadInfo thread = myThreads.get(id);
if (thread != null) {
thread.updateState(PyThreadInfo.State.RUNNING, null);
myDebugProcess.threadResumed(thread);
}
break;
}
case AbstractCommand.KILL_THREAD:
{
final String id = frame.getPayload();
final PyThreadInfo thread = myThreads.get(id);
if (thread != null) {
thread.updateState(PyThreadInfo.State.KILLED, null);
myThreads.remove(id);
}
break;
}
case AbstractCommand.SHOW_CONSOLE:
{
final PyThreadInfo event = parseThreadEvent(frame);
PyThreadInfo thread = myThreads.get(event.getId());
if (thread == null) {
myThreads.put(event.getId(), event);
thread = event;
}
thread.updateState(PyThreadInfo.State.SUSPENDED, event.getFrames());
thread.setStopReason(event.getStopReason());
thread.setMessage(event.getMessage());
myDebugProcess.showConsole(thread);
break;
}
}
}
private void processChange(String path, WatcherOp op) {
if (SystemInfo.isWindows
&& op == WatcherOp.RECDIRTY
&& path.length() == 3
&& Character.isLetter(path.charAt(0))) {
VirtualFile root = LocalFileSystem.getInstance().findFileByPath(path);
if (root != null) {
myNotificationSink.notifyPathsRecursive(list(root.getPresentableUrl()));
}
notifyOnAnyEvent();
return;
}
if (op == WatcherOp.CHANGE) {
// collapse subsequent change file change notifications that happen once we copy large file,
// this allows reduction of path checks at least 20% for Windows
synchronized (myLock) {
for (int i = 0; i < myLastChangedPaths.length; ++i) {
int last = myLastChangedPathIndex - i - 1;
if (last < 0) last += myLastChangedPaths.length;
String lastChangedPath = myLastChangedPaths[last];
if (lastChangedPath != null && lastChangedPath.equals(path)) {
return;
}
}
myLastChangedPaths[myLastChangedPathIndex++] = path;
if (myLastChangedPathIndex == myLastChangedPaths.length) myLastChangedPathIndex = 0;
}
}
int length = path.length();
if (length > 1 && path.charAt(length - 1) == '/') path = path.substring(0, length - 1);
boolean exactPath = op != WatcherOp.DIRTY && op != WatcherOp.RECDIRTY;
Collection<String> paths = checkWatchable(path, exactPath, false);
if (paths.isEmpty()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Not watchable, filtered: " + path);
}
return;
}
switch (op) {
case STATS:
case CHANGE:
myNotificationSink.notifyDirtyPaths(paths);
break;
case CREATE:
case DELETE:
myNotificationSink.notifyPathsCreatedOrDeleted(paths);
break;
case DIRTY:
myNotificationSink.notifyDirtyDirectories(paths);
break;
case RECDIRTY:
myNotificationSink.notifyPathsRecursive(paths);
break;
default:
LOG.error("Unexpected op: " + op);
}
notifyOnAnyEvent();
}
public void assignToNonLeafTasks(LinkedList<TaskRequestEvent> taskRequests) {
Collections.shuffle(taskRequests);
String queryMasterHostAndPort =
context
.getMasterContext()
.getQueryMasterContext()
.getWorkerContext()
.getConnectionInfo()
.getHostAndQMPort();
TaskRequestEvent taskRequest;
while (!taskRequests.isEmpty()) {
taskRequest = taskRequests.pollFirst();
LOG.debug("assignToNonLeafTasks: " + taskRequest.getExecutionBlockId());
TaskAttemptId attemptId;
// random allocation
if (nonLeafTasks.size() > 0) {
synchronized (nonLeafTasks) {
attemptId = nonLeafTasks.iterator().next();
nonLeafTasks.remove(attemptId);
}
LOG.debug("Assigned based on * match");
Task task;
task = stage.getTask(attemptId.getTaskId());
TaskRequest taskAssign =
new TaskRequestImpl(
attemptId,
Lists.newArrayList(task.getAllFragments()),
"",
false,
LogicalNodeSerializer.serialize(task.getLogicalPlan()),
context.getMasterContext().getQueryContext(),
stage.getDataChannel(),
stage.getBlock().getEnforcer(),
queryMasterHostAndPort);
if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) {
taskAssign.setInterQuery();
}
for (Map.Entry<String, Set<FetchImpl>> entry : task.getFetchMap().entrySet()) {
Collection<FetchImpl> fetches = entry.getValue();
if (fetches != null) {
for (FetchImpl fetch : fetches) {
taskAssign.addFetch(entry.getKey(), fetch);
}
}
}
WorkerConnectionInfo connectionInfo =
context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId());
// TODO send batch request
BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
requestProto.addTaskRequest(
TaskAllocationProto.newBuilder()
.setResource(taskRequest.getResponseProto().getResource())
.setTaskRequest(taskAssign.getProto())
.build());
requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto());
context
.getMasterContext()
.getEventHandler()
.handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId());
if (addr == null)
addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());
AsyncRpcClient tajoWorkerRpc;
try {
tajoWorkerRpc =
RpcClientManager.getInstance()
.getClient(addr, TajoWorkerProtocol.class, true, rpcParams);
TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient =
tajoWorkerRpc.getStub();
tajoWorkerRpcClient.allocateTasks(
callFuture.getController(), requestProto.build(), callFuture);
BatchAllocationResponse responseProto =
callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS);
if (responseProto.getCancellationTaskCount() > 0) {
for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) {
cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId())));
cancellation++;
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Canceled requests: "
+ responseProto.getCancellationTaskCount()
+ " from "
+ addr);
}
continue;
}
totalAssigned++;
scheduledObjectNum--;
} catch (Exception e) {
LOG.error(e);
}
}
}
}
private void doRun() {
if (myUpdateProgress.isCanceled()) return;
log(myUpdateProgress, myPass, "Started. ");
for (ScheduledPass successor : mySuccessorsOnSubmit) {
int predecessorsToRun = successor.myRunningPredecessorsCount.decrementAndGet();
if (predecessorsToRun == 0) {
submit(successor);
}
}
ProgressManager.getInstance()
.executeProcessUnderProgress(
() -> {
boolean success =
ApplicationManagerEx.getApplicationEx()
.tryRunReadAction(
() -> {
try {
if (DumbService.getInstance(myProject).isDumb()
&& !DumbService.isDumbAware(myPass)) {
return;
}
if (!myUpdateProgress.isCanceled() && !myProject.isDisposed()) {
myPass.collectInformation(myUpdateProgress);
}
} catch (ProcessCanceledException e) {
log(myUpdateProgress, myPass, "Canceled ");
if (!myUpdateProgress.isCanceled()) {
myUpdateProgress.cancel(
e); // in case when some smart asses throw PCE just for fun
}
} catch (RuntimeException | Error e) {
myUpdateProgress.cancel(e);
LOG.error(e);
throw e;
}
});
if (!success) {
myUpdateProgress.cancel();
}
},
myUpdateProgress);
log(myUpdateProgress, myPass, "Finished. ");
if (!myUpdateProgress.isCanceled()) {
applyInformationToEditorsLater(
myFileEditor, myPass, myUpdateProgress, myThreadsToStartCountdown);
for (ScheduledPass successor : mySuccessorsOnCompletion) {
int predecessorsToRun = successor.myRunningPredecessorsCount.decrementAndGet();
if (predecessorsToRun == 0) {
submit(successor);
}
}
}
}
public void assignToLeafTasks(LinkedList<TaskRequestEvent> taskRequests) {
Collections.shuffle(taskRequests);
LinkedList<TaskRequestEvent> remoteTaskRequests = new LinkedList<>();
String queryMasterHostAndPort =
context
.getMasterContext()
.getQueryMasterContext()
.getWorkerContext()
.getConnectionInfo()
.getHostAndQMPort();
TaskRequestEvent taskRequest;
while (leafTasks.size() > 0 && (!taskRequests.isEmpty() || !remoteTaskRequests.isEmpty())) {
int localAssign = 0;
int rackAssign = 0;
taskRequest = taskRequests.pollFirst();
if (taskRequest == null) { // if there are only remote task requests
taskRequest = remoteTaskRequests.pollFirst();
}
// checking if this container is still alive.
// If not, ignore the task request and stop the task runner
WorkerConnectionInfo connectionInfo =
context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId());
if (connectionInfo == null) continue;
// getting the hostname of requested node
String host = connectionInfo.getHost();
// if there are no worker matched to the hostname a task request
if (!leafTaskHostMapping.containsKey(host) && !taskRequests.isEmpty()) {
String normalizedHost = NetUtils.normalizeHost(host);
if (!leafTaskHostMapping.containsKey(normalizedHost)) {
// this case means one of either cases:
// * there are no blocks which reside in this node.
// * all blocks which reside in this node are consumed, and this task runner requests a
// remote task.
// In this case, we transfer the task request to the remote task request list, and skip
// the followings.
remoteTaskRequests.add(taskRequest);
continue;
} else {
host = normalizedHost;
}
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"assignToLeafTasks: "
+ taskRequest.getExecutionBlockId()
+ ","
+ "worker="
+ connectionInfo.getHostAndPeerRpcPort());
}
//////////////////////////////////////////////////////////////////////
// disk or host-local allocation
//////////////////////////////////////////////////////////////////////
TaskAttemptId attemptId = allocateLocalTask(host);
if (attemptId == null) { // if a local task cannot be found
HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);
if (!taskRequests
.isEmpty()) { // if other requests remains, move to remote list for better locality
remoteTaskRequests.add(taskRequest);
candidateWorkers.remove(connectionInfo.getId());
continue;
} else {
if (hostVolumeMapping != null) {
int nodes = context.getMasterContext().getWorkerMap().size();
// this part is to control the assignment of tail and remote task balancing per node
int tailLimit = 1;
if (remainingScheduledObjectNum() > 0 && nodes > 0) {
tailLimit = Math.max(remainingScheduledObjectNum() / nodes, 1);
}
if (hostVolumeMapping.getRemoteConcurrency()
>= tailLimit) { // remote task throttling per node
continue;
} else {
// assign to remote volume
hostVolumeMapping.increaseConcurrency(HostVolumeMapping.REMOTE);
}
}
}
//////////////////////////////////////////////////////////////////////
// rack-local allocation
//////////////////////////////////////////////////////////////////////
attemptId = allocateRackTask(host);
//////////////////////////////////////////////////////////////////////
// random node allocation
//////////////////////////////////////////////////////////////////////
if (attemptId == null && leafTaskNum() > 0) {
synchronized (leafTasks) {
attemptId = leafTasks.iterator().next();
leafTasks.remove(attemptId);
}
}
if (attemptId != null && hostVolumeMapping != null) {
hostVolumeMapping.lastAssignedVolumeId.put(attemptId, HostVolumeMapping.REMOTE);
}
rackAssign++;
} else {
localAssign++;
}
if (attemptId != null) {
Task task = stage.getTask(attemptId.getTaskId());
TaskRequest taskAssign =
new TaskRequestImpl(
attemptId,
new ArrayList<>(task.getAllFragments()),
"",
false,
LogicalNodeSerializer.serialize(task.getLogicalPlan()),
context.getMasterContext().getQueryContext(),
stage.getDataChannel(),
stage.getBlock().getEnforcer(),
queryMasterHostAndPort);
if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) {
taskAssign.setInterQuery();
}
// TODO send batch request
BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
requestProto.addTaskRequest(
TaskAllocationProto.newBuilder()
.setResource(taskRequest.getResponseProto().getResource())
.setTaskRequest(taskAssign.getProto())
.build());
requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto());
context
.getMasterContext()
.getEventHandler()
.handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));
InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId());
if (addr == null)
addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());
AsyncRpcClient tajoWorkerRpc = null;
CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
totalAttempts++;
try {
tajoWorkerRpc =
RpcClientManager.getInstance()
.getClient(addr, TajoWorkerProtocol.class, true, rpcParams);
TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient =
tajoWorkerRpc.getStub();
tajoWorkerRpcClient.allocateTasks(
callFuture.getController(), requestProto.build(), callFuture);
BatchAllocationResponse responseProto =
callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS);
if (responseProto.getCancellationTaskCount() > 0) {
for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) {
cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId())));
cancellation++;
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"Canceled requests: "
+ responseProto.getCancellationTaskCount()
+ " from "
+ addr);
}
continue;
}
} catch (Exception e) {
LOG.error(e);
}
scheduledObjectNum--;
totalAssigned++;
hostLocalAssigned += localAssign;
rackLocalAssigned += rackAssign;
if (rackAssign > 0) {
LOG.info(
String.format(
"Assigned Local/Rack/Total: (%d/%d/%d), "
+ "Attempted Cancel/Assign/Total: (%d/%d/%d), "
+ "Locality: %.2f%%, Rack host: %s",
hostLocalAssigned,
rackLocalAssigned,
totalAssigned,
cancellation,
totalAssigned,
totalAttempts,
((double) hostLocalAssigned / (double) totalAssigned) * 100,
host));
}
} else {
throw new RuntimeException("Illegal State!!!!!!!!!!!!!!!!!!!!!");
}
}
}
private ReturnResult checkUserName() {
Project project = myPanel.getProject();
GitVcs vcs = GitVcs.getInstance(project);
assert vcs != null;
Collection<VirtualFile> notDefined = new ArrayList<VirtualFile>();
Map<VirtualFile, Pair<String, String>> defined =
new HashMap<VirtualFile, Pair<String, String>>();
Collection<VirtualFile> allRoots =
new ArrayList<VirtualFile>(
Arrays.asList(ProjectLevelVcsManager.getInstance(project).getRootsUnderVcs(vcs)));
Collection<VirtualFile> affectedRoots = getSelectedRoots();
for (VirtualFile root : affectedRoots) {
try {
Pair<String, String> nameAndEmail = getUserNameAndEmailFromGitConfig(project, root);
String name = nameAndEmail.getFirst();
String email = nameAndEmail.getSecond();
if (name == null || email == null) {
notDefined.add(root);
} else {
defined.put(root, nameAndEmail);
}
} catch (VcsException e) {
LOG.error("Couldn't get user.name and user.email for root " + root, e);
// doing nothing - let commit with possibly empty user.name/email
}
}
if (notDefined.isEmpty()) {
return ReturnResult.COMMIT;
}
GitVersion version = vcs.getVersion();
if (System.getenv("HOME") == null
&& GitVersionSpecialty.DOESNT_DEFINE_HOME_ENV_VAR.existsIn(version)) {
Messages.showErrorDialog(
project,
"You are using Git "
+ version
+ " which doesn't define %HOME% environment variable properly.\n"
+ "Consider updating Git to a newer version "
+ "or define %HOME% to point to the place where the global .gitconfig is stored \n"
+ "(it is usually %USERPROFILE% or %HOMEDRIVE%%HOMEPATH%).",
"HOME Variable Is Not Defined");
return ReturnResult.CANCEL;
}
if (defined.isEmpty() && allRoots.size() > affectedRoots.size()) {
allRoots.removeAll(affectedRoots);
for (VirtualFile root : allRoots) {
try {
Pair<String, String> nameAndEmail = getUserNameAndEmailFromGitConfig(project, root);
String name = nameAndEmail.getFirst();
String email = nameAndEmail.getSecond();
if (name != null && email != null) {
defined.put(root, nameAndEmail);
break;
}
} catch (VcsException e) {
LOG.error("Couldn't get user.name and user.email for root " + root, e);
// doing nothing - not critical not to find the values for other roots not affected by
// commit
}
}
}
GitUserNameNotDefinedDialog dialog =
new GitUserNameNotDefinedDialog(project, notDefined, affectedRoots, defined);
dialog.show();
if (dialog.isOK()) {
try {
if (dialog.isGlobal()) {
GitConfigUtil.setValue(
project,
notDefined.iterator().next(),
GitConfigUtil.USER_NAME,
dialog.getUserName(),
"--global");
GitConfigUtil.setValue(
project,
notDefined.iterator().next(),
GitConfigUtil.USER_EMAIL,
dialog.getUserEmail(),
"--global");
} else {
for (VirtualFile root : notDefined) {
GitConfigUtil.setValue(project, root, GitConfigUtil.USER_NAME, dialog.getUserName());
GitConfigUtil.setValue(
project, root, GitConfigUtil.USER_EMAIL, dialog.getUserEmail());
}
}
} catch (VcsException e) {
String message = "Couldn't set user.name and user.email";
LOG.error(message, e);
Messages.showErrorDialog(myPanel.getComponent(), message);
return ReturnResult.CANCEL;
}
return ReturnResult.COMMIT;
}
return ReturnResult.CLOSE_WINDOW;
}
@Override
public ScriptContext call() throws Exception {
try {
Scanner scn = new Scanner(znodePath);
scn.useDelimiter(":");
String hostName = scn.next(); // host name
String instance = scn.next(); // instance
int infoPort = Integer.parseInt(scn.next()); // UI info port
long serverStartTimestamp = Long.parseLong(scn.next());
scn.close();
// Get the --config property from classpath...it's always first
// in the classpath
String cp = System.getProperty("java.class.path");
scn = new Scanner(cp);
scn.useDelimiter(":");
String confDir = scn.next();
scn.close();
LOG.debug("conf dir [" + confDir + "]");
// Get -Dwms.home.dir
String wmsHome = System.getProperty("wms.home.dir");
// If stop-wms.sh is executed and WMS_MANAGES_ZK then zookeeper
// is stopped abruptly.
// Second scenario is when ZooKeeper fails for some reason
// regardless of whether WMS
// manages it. When either happens the WmsServer running znodes
// still exist in ZooKeeper
// and we see them at next startup. When they eventually timeout
// we get node deleted events for a server that no longer
// exists. So, only recognize
// WmsServer running znodes that have timestamps after last
// WmsMaster startup.
if (serverStartTimestamp > startupTimestamp) {
scriptContext.setHostName(hostName);
scriptContext.setScriptName("sys_shell.py");
if (hostName.equalsIgnoreCase(ia.getCanonicalHostName()))
scriptContext.setCommand(
"bin/wms-daemon.sh --config " + confDir + " start server " + instance);
else
scriptContext.setCommand(
"pdsh -w "
+ hostName
+ " \"cd "
+ wmsHome
+ ";bin/wms-daemon.sh --config "
+ confDir
+ " start server "
+ instance
+ "\"");
RetryCounter retryCounter = retryCounterFactory.create();
while (true) {
if (scriptContext.getStdOut().length() > 0)
scriptContext.getStdOut().delete(0, scriptContext.getStdOut().length());
if (scriptContext.getStdErr().length() > 0)
scriptContext.getStdErr().delete(0, scriptContext.getStdErr().length());
LOG.info(
"Restarting WmsServer ["
+ hostName
+ ":"
+ instance
+ "], script [ "
+ scriptContext.toString()
+ " ]");
ScriptManager.getInstance().runScript(scriptContext);
if (scriptContext.getExitCode() == 0) {
LOG.info("WmsServer [" + hostName + ":" + instance + "] restarted");
break;
} else {
StringBuilder sb = new StringBuilder();
sb.append("exit code [" + scriptContext.getExitCode() + "]");
if (!scriptContext.getStdOut().toString().isEmpty())
sb.append(", stdout [" + scriptContext.getStdOut().toString() + "]");
if (!scriptContext.getStdErr().toString().isEmpty())
sb.append(", stderr [" + scriptContext.getStdErr().toString() + "]");
LOG.error(sb.toString());
if (!retryCounter.shouldRetry()) {
LOG.error(
"WmsServer ["
+ hostName
+ ":"
+ instance
+ "] restart failed after "
+ retryCounter.getMaxRetries()
+ " retries");
break;
} else {
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
}
}
}
} else {
LOG.debug(
"No restart for "
+ znodePath
+ "\nbecause WmsServer start time ["
+ DateFormat.getDateTimeInstance().format(new Date(serverStartTimestamp))
+ "] was before WmsMaster start time ["
+ DateFormat.getDateTimeInstance().format(new Date(startupTimestamp))
+ "]");
}
} catch (Exception e) {
e.printStackTrace();
LOG.error(e);
}
return scriptContext;
}