Example #1
0
    @Override
    public void run() {
      if (LOG.isDebugEnabled()) {
        LOG.debug("==> ConsumerRunnable.run()");
      }
      while (true) {
        try {
          if (hasNext()) {
            EntityNotification notification = consumer.peek();
            if (notification != null) {
              if (LOG.isDebugEnabled()) {
                LOG.debug("Notification=" + getPrintableEntityNotification(notification));
              }

              ServiceTags serviceTags =
                  AtlasNotificationMapper.processEntityNotification(notification);
              if (serviceTags == null) {
                LOG.error(
                    "No ServiceTags built for notification :"
                        + getPrintableEntityNotification(notification));
              } else {
                updateSink(serviceTags);
              }
            } else {
              LOG.error("Null entityNotification received from Kafka!! Ignoring..");
            }
            // Move iterator forward
            consumer.next();
          }
        } catch (Exception exception) {
          LOG.error("Caught exception..: ", exception);
          return;
        }
      }
    }
 @Override
 public void visitReferenceExpression(GrReferenceExpression expression) {
   super.visitReferenceExpression(expression);
   if (org.jetbrains.plugins.groovy.lang.psi.util.PsiUtil.isSuperReference(expression)) {
     final GrExpression qualifier = expression.getQualifier();
     if (qualifier instanceof GrReferenceExpression
         && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
       try {
         expression.putCopyableUserData(SUPER_REF, Boolean.TRUE);
       } catch (IncorrectOperationException e) {
         LOG.error(e);
       }
     }
   } else if (org.jetbrains.plugins.groovy.lang.psi.util.PsiUtil.isThisReference(expression)) {
     final GrExpression qualifier = expression.getQualifier();
     if (qualifier instanceof GrReferenceExpression
         && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
       try {
         expression.putCopyableUserData(THIS_REF, Boolean.TRUE);
       } catch (IncorrectOperationException e) {
         LOG.error(e);
       }
     }
   }
 }
 @Override
 public void visitReferenceExpression(GrReferenceExpression expression) {
   super.visitReferenceExpression(expression);
   if (expression.getCopyableUserData(SUPER_REF) != null) {
     expression.putCopyableUserData(SUPER_REF, null);
     final GrExpression qualifier = expression.getQualifier();
     if (qualifier instanceof GrReferenceExpression
         && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
       try {
         GroovyPsiElementFactory factory = GroovyPsiElementFactory.getInstance(myProject);
         GrExpression newExpr =
             factory.createExpressionFromText(myTargetSuperClass.getName() + ".this", null);
         expression.replace(newExpr);
       } catch (IncorrectOperationException e) {
         LOG.error(e);
       }
     }
   } else if (expression.getCopyableUserData(THIS_REF) != null) {
     expression.putCopyableUserData(THIS_REF, null);
     final GrExpression qualifier = expression.getQualifier();
     if (qualifier instanceof GrReferenceExpression
         && ((GrReferenceExpression) qualifier).isReferenceTo(mySourceClass)) {
       try {
         ((GrReferenceExpression) qualifier).bindToElement(myTargetSuperClass);
         GroovyChangeContextUtil.clearContextInfo(qualifier);
       } catch (IncorrectOperationException e) {
         LOG.error(e);
       }
     }
   }
 }
  private Map<String, SpringResource> generateResourceMap(Set<Class<?>> validClasses)
      throws GenerateException {
    Map<String, SpringResource> resourceMap = new HashMap<String, SpringResource>();
    for (Class<?> c : validClasses) {
      RequestMapping requestMapping = c.getAnnotation(RequestMapping.class);
      String description = "";
      // This try/catch block is to stop a bamboo build from failing due to NoClassDefFoundError
      // This occurs when a class or method loaded by reflections contains a type that has no
      // dependency
      try {
        resourceMap = analyzeController(c, resourceMap, description);
        List<Method> mList = new ArrayList<Method>(Arrays.asList(c.getMethods()));
        if (c.getSuperclass() != null) {
          mList.addAll(Arrays.asList(c.getSuperclass().getMethods()));
        }

      } catch (NoClassDefFoundError e) {
        LOG.error(e.getMessage());
        LOG.info(c.getName());
        // exception occurs when a method type or annotation is not recognized by the plugin
      } catch (ClassNotFoundException e) {
        LOG.error(e.getMessage());
        LOG.info(c.getName());
      }
    }

    return resourceMap;
  }
    public ComponentDomainClass(Class type) {
      super(type, "");

      PropertyDescriptor[] descriptors;

      try {
        descriptors = java.beans.Introspector.getBeanInfo(type).getPropertyDescriptors();
      } catch (IntrospectionException e) {
        throw new GrailsDomainException(
            "Failed to use class ["
                + type
                + "] as a component. Cannot introspect! "
                + e.getMessage());
      }

      List tmp =
          (List)
              getPropertyOrStaticPropertyOrFieldValue(
                  GrailsDomainClassProperty.TRANSIENT, List.class);
      if (tmp != null) this.transients = tmp;
      this.properties = createDomainClassProperties(this, descriptors);
      try {
        this.constraints =
            GrailsDomainConfigurationUtil.evaluateConstraints(
                getReference().getWrappedInstance(), properties);
      } catch (IntrospectionException e) {
        LOG.error(
            "Error reading embedded component [" + getClazz() + "] constraints: " + e.getMessage(),
            e);
      }
    }
Example #6
0
    @Override
    public void consume(Object o) {
      if (!(o instanceof GroovyResolveResult)) {
        LOG.error(o);
        return;
      }

      GroovyResolveResult result = (GroovyResolveResult) o;
      if (!result.isStaticsOK()) {
        if (myInapplicable == null) myInapplicable = ContainerUtil.newArrayList();
        myInapplicable.add(result);
        return;
      }
      if (!result.isAccessible() && myParameters.getInvocationCount() < 2) return;

      if (mySkipPackages && result.getElement() instanceof PsiPackage) return;

      PsiElement element = result.getElement();
      if (element instanceof PsiVariable
          && !myMatcher.prefixMatches(((PsiVariable) element).getName())) {
        return;
      }

      if (element instanceof GrReflectedMethod) {
        element = ((GrReflectedMethod) element).getBaseMethod();
        if (!myProcessedMethodWithOptionalParams.add((GrMethod) element)) return;

        result =
            new GroovyResolveResultImpl(
                element,
                result.getCurrentFileResolveContext(),
                result.getSpreadState(),
                result.getSubstitutor(),
                result.isAccessible(),
                result.isStaticsOK(),
                result.isInvokedOnProperty(),
                result.isValidResult());
      }

      if (myFieldPointerOperator && !(element instanceof PsiVariable)) {
        return;
      }
      if (myMethodPointerOperator && !(element instanceof PsiMethod)) {
        return;
      }
      addCandidate(result);

      if (!myFieldPointerOperator && !myMethodPointerOperator) {
        if (element instanceof PsiMethod) {
          processProperty((PsiMethod) element, result);
        } else if (element instanceof GrField) {
          if (((GrField) element).isProperty()) {
            processPropertyFromField((GrField) element, result);
          }
        }
      }
      if (element instanceof GrVariable && !(element instanceof GrField)) {
        myLocalVars.add(((GrVariable) element).getName());
      }
    }
Example #7
0
  @Override
  public StepExecutionResult executeImplementation(StepExecutionContext context)
      throws MuseExecutionError {
    StepConfiguration config = getConfiguration();
    HashMap<String, Object> values = new HashMap<>();
    for (String name : config.getSourceNames()) {
      MuseValueSource source = getValueSource(config, name, false, context.getProject());
      values.put(name, getValue(source, context, true, Object.class));
    }

    try {
      JavascriptRunner runner = new JavascriptStepRunner();
      runner.evalScript(_origin);

      Object result = runner.invokeFunction(EXECUTE_FUNCTION, context, values);
      if (result instanceof StepExecutionResult) return (StepExecutionResult) result;
      else
        context.raiseEvent(
            new ScriptFailureEvent(
                "Script did not return a StepExecutionResult. Instead, it returned: " + result,
                null)); // TODO do something better than n/a
    } catch (Throwable t) {
      LOG.error("unable to execute script: ", t);
      context.raiseEvent(
          new ScriptFailureEvent(
              "Script threw an exception: " + t.getMessage(),
              t)); // TODO do something better than n/a
    }
    return new BasicStepExecutionResult(StepExecutionStatus.FAILURE);
  }
 @Override
 protected void close() {
   try {
     super.close();
   } catch (IOException e) {
     LOG.error(e);
   }
 }
 @Override
 public void processPacket(Packet packet) {
   try {
     doProcessPacket(packet);
   } catch (Throwable e) {
     LOG.error(e.getMessage(), e);
   }
 }
Example #10
0
 private boolean hasNext() {
   boolean ret = false;
   try {
     ret = consumer.hasNext();
   } catch (Exception exception) {
     LOG.error("EntityNotification consumer threw exception, IGNORING...:", exception);
   }
   return ret;
 }
 public void refreshConstraints() {
   try {
     GrailsDomainClassProperty[] props = getPersistentProperties();
     this.constraints =
         GrailsDomainConfigurationUtil.evaluateConstraints(
             getReference().getWrappedInstance(), props);
   } catch (IntrospectionException e) {
     LOG.error("Error reading class [" + getClazz() + "] constraints: " + e.getMessage(), e);
   }
 }
 public void process(WatchedEvent event) {
   if (event.getType() == Event.EventType.NodeChildrenChanged) {
     LOG.debug("Running children changed [" + event.getPath() + "]");
     try {
       getZkRunning();
     } catch (Exception e) {
       e.printStackTrace();
       LOG.error(e);
     }
   } else if (event.getType() == Event.EventType.NodeDeleted) {
     String znodePath = event.getPath();
     LOG.debug("Running znode deleted [" + znodePath + "]");
     try {
       restartServer(znodePath);
     } catch (Exception e) {
       e.printStackTrace();
       LOG.error(e);
     }
   }
 }
Example #13
0
 public void reportTo(CompileContext compileContext) {
   compileContext.addMessage(
       messageCategory,
       message,
       url == null ? "" : url,
       line == null ? -1 : line,
       column == null ? -1 : column);
   if (isException) {
     LOG.error(message);
   }
 }
 @Override
 public void run() {
   try {
     LOG.debug("Posting queueName: " + queueName + " and data: " + data);
     pushedQueueMap.put(queueName, data);
     this.lucasMessageBroadcastService.sendMessage(queueName, bindKey, data);
   } catch (Exception e) {
     LOG.error("Exception Generated ", e);
     throw new LucasRuntimeException(e);
   }
 }
    @Override
    public void onTextAvailable(ProcessEvent event, Key outputType) {
      if (outputType == ProcessOutputTypes.STDERR) {
        LOG.warn(event.getText().trim());
      }
      if (outputType != ProcessOutputTypes.STDOUT) {
        return;
      }

      final String line = event.getText().trim();
      if (LOG.isDebugEnabled()) {
        LOG.debug(">> " + line);
      }

      if (myLastOp == null) {
        final WatcherOp watcherOp;
        try {
          watcherOp = WatcherOp.valueOf(line);
        } catch (IllegalArgumentException e) {
          LOG.error("Illegal watcher command: " + line);
          return;
        }

        if (watcherOp == WatcherOp.GIVEUP) {
          notifyOnFailure(ApplicationBundle.message("watcher.gave.up"), null);
          myIsShuttingDown = true;
        } else if (watcherOp == WatcherOp.RESET) {
          reset();
        } else {
          myLastOp = watcherOp;
        }
      } else if (myLastOp == WatcherOp.MESSAGE) {
        notifyOnFailure(line, NotificationListener.URL_OPENING_LISTENER);
        myLastOp = null;
      } else if (myLastOp == WatcherOp.REMAP || myLastOp == WatcherOp.UNWATCHEABLE) {
        if ("#".equals(line)) {
          if (myLastOp == WatcherOp.REMAP) {
            processRemap();
          } else {
            mySettingRoots.decrementAndGet();
            processUnwatchable();
          }
          myLines.clear();
          myLastOp = null;
        } else {
          myLines.add(line);
        }
      } else {
        String path = line.replace('\0', '\n'); // unescape
        processChange(path, myLastOp);
        myLastOp = null;
      }
    }
 @Override
 public void visitThisExpression(PsiThisExpression expression) {
   super.visitThisExpression(expression);
   final PsiJavaCodeReferenceElement qualifier = expression.getQualifier();
   if (qualifier != null && qualifier.isReferenceTo(mySourceClass)) {
     try {
       qualifier.bindToElement(myTargetSuperClass);
     } catch (IncorrectOperationException e) {
       LOG.error(e);
     }
   }
 }
 @Override
 public void visitSuperExpression(PsiSuperExpression expression) {
   super.visitSuperExpression(expression);
   final PsiJavaCodeReferenceElement qualifier = expression.getQualifier();
   if (qualifier != null && qualifier.isReferenceTo(mySourceClass)) {
     try {
       expression.replace(
           JavaPsiFacade.getElementFactory(myProject)
               .createExpressionFromText(myTargetSuperClass.getName() + ".this", null));
     } catch (IncorrectOperationException e) {
       LOG.error(e);
     }
   }
 }
    @Override
    public void setSelected(AnActionEvent e, boolean state) {
      mySelected = state;

      if (mySelected) {
        try {
          mySession = connectToDebugger();
        } catch (Exception e1) {
          LOG.error(e1);
          Messages.showErrorDialog("Can't connect to debugger", "Error Connecting Debugger");
        }
      } else {
        // TODO: disable debugging
      }
    }
      @Override
      public void run() {
        try {
          String[] stringArray =
              this.lucasMessageBroadcastClientService.receiveAllMessagesWithinTimePeriod(
                  this.lucasBroadCastQueue, this.lucasBroadCastBinding, 1L, 5000L);
          if (stringArray != null) {
            for (String string : stringArray) {
              pulledQueueMap.put(this.lucasBroadCastQueue, string);
            }
          }

        } catch (Exception e) {
          LOG.error("Exception Generated ", e);
          throw new LucasRuntimeException(e);
        }
      }
    public void run() {
      processing = true;
      try {
        while (true) {
          FileObject path = myPaths.take();

          if (path == myStopThreadToken) {
            break;
          }
          processPath(path, myProject);
        }
      } catch (InterruptedException e) {
        LOG.error(e);
      } catch (CacheCorruptedException e) {
        myError = e;
      } finally {
        processing = false;
      }
    }
    @Override
    protected VirtualFile[] doAddItems() {
      Project project =
          CommonDataKeys.PROJECT.getData(DataManager.getInstance().getDataContext(myPanel));
      try {
        String[] files =
            PythonRemoteInterpreterManager.getInstance()
                .chooseRemoteFiles(
                    project, (PyRemoteSdkAdditionalDataBase) mySdk.getSdkAdditionalData(), false);

        final String sourcesLocalPath = PySdkUtil.getRemoteSourcesLocalPath(mySdk.getHomePath());

        VirtualFile[] vFiles = new VirtualFile[files.length];

        int i = 0;
        for (String file : files) {
          String localRoot = PyRemoteSourceItem.localPathForRemoteRoot(sourcesLocalPath, file);

          myNewMappings.add(new PathMappingSettings.PathMapping(localRoot, file));
          myRemoteSdkData.getPathMappings().addMappingCheckUnique(localRoot, file);

          if (!new File(localRoot).exists()) {
            new File(localRoot).mkdirs();
          }
          vFiles[i++] =
              LocalFileSystem.getInstance().refreshAndFindFileByIoFile(new File(localRoot));
        }

        vFiles = adjustAddedFileSet(myPanel, vFiles);
        List<VirtualFile> added = new ArrayList<VirtualFile>(vFiles.length);
        for (VirtualFile vFile : vFiles) {
          if (addElement(vFile)) {
            added.add(vFile);
          }
        }
        return VfsUtilCore.toVirtualFileArray(added);
      } catch (Exception e) {
        LOG.error(e);
      }
      return new VirtualFile[0];
    }
    private void processResponse(final String line) {
      try {
        final ProtocolFrame frame = new ProtocolFrame(line);
        logFrame(frame, false);

        if (AbstractThreadCommand.isThreadCommand(frame.getCommand())) {
          processThreadEvent(frame);
        } else if (AbstractCommand.isWriteToConsole(frame.getCommand())) {
          writeToConsole(ProtocolParser.parseIo(frame.getPayload()));
        } else if (AbstractCommand.isExitEvent(frame.getCommand())) {
          fireCommunicationError();
        } else if (AbstractCommand.isCallSignatureTrace(frame.getCommand())) {
          recordCallSignature(ProtocolParser.parseCallSignature(frame.getPayload()));
        } else {
          placeResponse(frame.getSequence(), frame);
        }
      } catch (Throwable t) {
        // shouldn't interrupt reader thread
        LOG.error(t);
      }
    }
    @Override
    protected JPanel processFiles(@NotNull List<Element> fileElements, final JPanel context) {
      final Ref<EditorWindow> windowRef = new Ref<EditorWindow>();
      UIUtil.invokeAndWaitIfNeeded(
          new Runnable() {
            @Override
            public void run() {
              windowRef.set(context == null ? createEditorWindow() : findWindowWith(context));
            }
          });
      final EditorWindow window = windowRef.get();
      LOG.assertTrue(window != null);
      VirtualFile focusedFile = null;

      for (int i = 0; i < fileElements.size(); i++) {
        final Element file = fileElements.get(i);
        if (i == 0) {
          EditorTabbedContainer tabbedPane = window.getTabbedPane();
          if (tabbedPane != null) {
            try {
              int limit =
                  Integer.parseInt(
                      file.getParentElement()
                          .getAttributeValue(
                              JBTabsImpl.SIDE_TABS_SIZE_LIMIT_KEY.toString(),
                              String.valueOf(JBTabsImpl.DEFAULT_MAX_TAB_WIDTH)));
              UIUtil.putClientProperty(
                  tabbedPane.getComponent(), JBTabsImpl.SIDE_TABS_SIZE_LIMIT_KEY, limit);
            } catch (NumberFormatException e) {
              // ignore
            }
          }
        }
        try {
          final FileEditorManagerImpl fileEditorManager = getManager();
          Element historyElement = file.getChild(HistoryEntry.TAG);
          final HistoryEntry entry =
              HistoryEntry.createLight(fileEditorManager.getProject(), historyElement);
          final VirtualFile virtualFile = entry.getFile();
          if (virtualFile == null)
            throw new InvalidDataException("No file exists: " + entry.getFilePointer().getUrl());
          Document document =
              ApplicationManager.getApplication()
                  .runReadAction(
                      new Computable<Document>() {
                        @Override
                        public Document compute() {
                          return virtualFile.isValid()
                              ? FileDocumentManager.getInstance().getDocument(virtualFile)
                              : null;
                        }
                      });
          final boolean isCurrentInTab =
              Boolean.valueOf(file.getAttributeValue(CURRENT_IN_TAB)).booleanValue();
          Boolean pin = Boolean.valueOf(file.getAttributeValue(PINNED));
          fileEditorManager.openFileImpl4(
              window, virtualFile, entry, isCurrentInTab, isCurrentInTab, pin, i);
          if (isCurrentInTab) {
            focusedFile = virtualFile;
          }
          if (document != null) {
            // This is just to make sure document reference is kept on stack till this point
            // so that document is available for folding state deserialization in HistoryEntry
            // constructor
            // and that document will be created only once during file opening
            document.putUserData(DUMMY_KEY, null);
          }
          updateProgress();
        } catch (InvalidDataException e) {
          if (ApplicationManager.getApplication().isUnitTestMode()) {
            LOG.error(e);
          }
        }
      }
      if (focusedFile != null) {
        getManager().addSelectionRecord(focusedFile, window);
      }
      return window.myPanel;
    }
 // todo: extract response processing
 private void processThreadEvent(ProtocolFrame frame) throws PyDebuggerException {
   switch (frame.getCommand()) {
     case AbstractCommand.CREATE_THREAD:
       {
         final PyThreadInfo thread = parseThreadEvent(frame);
         if (!thread.isPydevThread()) { // ignore pydevd threads
           myThreads.put(thread.getId(), thread);
         }
         break;
       }
     case AbstractCommand.SUSPEND_THREAD:
       {
         final PyThreadInfo event = parseThreadEvent(frame);
         PyThreadInfo thread = myThreads.get(event.getId());
         if (thread == null) {
           LOG.error(
               "Trying to stop on non-existent thread: "
                   + event.getId()
                   + ", "
                   + event.getStopReason()
                   + ", "
                   + event.getMessage());
           myThreads.put(event.getId(), event);
           thread = event;
         }
         thread.updateState(PyThreadInfo.State.SUSPENDED, event.getFrames());
         thread.setStopReason(event.getStopReason());
         thread.setMessage(event.getMessage());
         myDebugProcess.threadSuspended(thread);
         break;
       }
     case AbstractCommand.RESUME_THREAD:
       {
         final String id = ProtocolParser.getThreadId(frame.getPayload());
         final PyThreadInfo thread = myThreads.get(id);
         if (thread != null) {
           thread.updateState(PyThreadInfo.State.RUNNING, null);
           myDebugProcess.threadResumed(thread);
         }
         break;
       }
     case AbstractCommand.KILL_THREAD:
       {
         final String id = frame.getPayload();
         final PyThreadInfo thread = myThreads.get(id);
         if (thread != null) {
           thread.updateState(PyThreadInfo.State.KILLED, null);
           myThreads.remove(id);
         }
         break;
       }
     case AbstractCommand.SHOW_CONSOLE:
       {
         final PyThreadInfo event = parseThreadEvent(frame);
         PyThreadInfo thread = myThreads.get(event.getId());
         if (thread == null) {
           myThreads.put(event.getId(), event);
           thread = event;
         }
         thread.updateState(PyThreadInfo.State.SUSPENDED, event.getFrames());
         thread.setStopReason(event.getStopReason());
         thread.setMessage(event.getMessage());
         myDebugProcess.showConsole(thread);
         break;
       }
   }
 }
    private void processChange(String path, WatcherOp op) {
      if (SystemInfo.isWindows
          && op == WatcherOp.RECDIRTY
          && path.length() == 3
          && Character.isLetter(path.charAt(0))) {
        VirtualFile root = LocalFileSystem.getInstance().findFileByPath(path);
        if (root != null) {
          myNotificationSink.notifyPathsRecursive(list(root.getPresentableUrl()));
        }
        notifyOnAnyEvent();
        return;
      }

      if (op == WatcherOp.CHANGE) {
        // collapse subsequent change file change notifications that happen once we copy large file,
        // this allows reduction of path checks at least 20% for Windows
        synchronized (myLock) {
          for (int i = 0; i < myLastChangedPaths.length; ++i) {
            int last = myLastChangedPathIndex - i - 1;
            if (last < 0) last += myLastChangedPaths.length;
            String lastChangedPath = myLastChangedPaths[last];
            if (lastChangedPath != null && lastChangedPath.equals(path)) {
              return;
            }
          }
          myLastChangedPaths[myLastChangedPathIndex++] = path;
          if (myLastChangedPathIndex == myLastChangedPaths.length) myLastChangedPathIndex = 0;
        }
      }

      int length = path.length();
      if (length > 1 && path.charAt(length - 1) == '/') path = path.substring(0, length - 1);
      boolean exactPath = op != WatcherOp.DIRTY && op != WatcherOp.RECDIRTY;
      Collection<String> paths = checkWatchable(path, exactPath, false);

      if (paths.isEmpty()) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("Not watchable, filtered: " + path);
        }
        return;
      }

      switch (op) {
        case STATS:
        case CHANGE:
          myNotificationSink.notifyDirtyPaths(paths);
          break;

        case CREATE:
        case DELETE:
          myNotificationSink.notifyPathsCreatedOrDeleted(paths);
          break;

        case DIRTY:
          myNotificationSink.notifyDirtyDirectories(paths);
          break;

        case RECDIRTY:
          myNotificationSink.notifyPathsRecursive(paths);
          break;

        default:
          LOG.error("Unexpected op: " + op);
      }

      notifyOnAnyEvent();
    }
Example #26
0
    public void assignToNonLeafTasks(LinkedList<TaskRequestEvent> taskRequests) {
      Collections.shuffle(taskRequests);
      String queryMasterHostAndPort =
          context
              .getMasterContext()
              .getQueryMasterContext()
              .getWorkerContext()
              .getConnectionInfo()
              .getHostAndQMPort();

      TaskRequestEvent taskRequest;
      while (!taskRequests.isEmpty()) {
        taskRequest = taskRequests.pollFirst();
        LOG.debug("assignToNonLeafTasks: " + taskRequest.getExecutionBlockId());

        TaskAttemptId attemptId;
        // random allocation
        if (nonLeafTasks.size() > 0) {
          synchronized (nonLeafTasks) {
            attemptId = nonLeafTasks.iterator().next();
            nonLeafTasks.remove(attemptId);
          }
          LOG.debug("Assigned based on * match");

          Task task;
          task = stage.getTask(attemptId.getTaskId());

          TaskRequest taskAssign =
              new TaskRequestImpl(
                  attemptId,
                  Lists.newArrayList(task.getAllFragments()),
                  "",
                  false,
                  LogicalNodeSerializer.serialize(task.getLogicalPlan()),
                  context.getMasterContext().getQueryContext(),
                  stage.getDataChannel(),
                  stage.getBlock().getEnforcer(),
                  queryMasterHostAndPort);

          if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) {
            taskAssign.setInterQuery();
          }
          for (Map.Entry<String, Set<FetchImpl>> entry : task.getFetchMap().entrySet()) {
            Collection<FetchImpl> fetches = entry.getValue();
            if (fetches != null) {
              for (FetchImpl fetch : fetches) {
                taskAssign.addFetch(entry.getKey(), fetch);
              }
            }
          }

          WorkerConnectionInfo connectionInfo =
              context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId());

          // TODO send batch request
          BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
          requestProto.addTaskRequest(
              TaskAllocationProto.newBuilder()
                  .setResource(taskRequest.getResponseProto().getResource())
                  .setTaskRequest(taskAssign.getProto())
                  .build());

          requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto());
          context
              .getMasterContext()
              .getEventHandler()
              .handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));

          CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();

          InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId());
          if (addr == null)
            addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());

          AsyncRpcClient tajoWorkerRpc;
          try {
            tajoWorkerRpc =
                RpcClientManager.getInstance()
                    .getClient(addr, TajoWorkerProtocol.class, true, rpcParams);
            TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient =
                tajoWorkerRpc.getStub();
            tajoWorkerRpcClient.allocateTasks(
                callFuture.getController(), requestProto.build(), callFuture);

            BatchAllocationResponse responseProto =
                callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS);

            if (responseProto.getCancellationTaskCount() > 0) {
              for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) {
                cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId())));
                cancellation++;
              }

              if (LOG.isDebugEnabled()) {
                LOG.debug(
                    "Canceled requests: "
                        + responseProto.getCancellationTaskCount()
                        + " from "
                        + addr);
              }
              continue;
            }

            totalAssigned++;
            scheduledObjectNum--;
          } catch (Exception e) {
            LOG.error(e);
          }
        }
      }
    }
    private void doRun() {
      if (myUpdateProgress.isCanceled()) return;

      log(myUpdateProgress, myPass, "Started. ");

      for (ScheduledPass successor : mySuccessorsOnSubmit) {
        int predecessorsToRun = successor.myRunningPredecessorsCount.decrementAndGet();
        if (predecessorsToRun == 0) {
          submit(successor);
        }
      }

      ProgressManager.getInstance()
          .executeProcessUnderProgress(
              () -> {
                boolean success =
                    ApplicationManagerEx.getApplicationEx()
                        .tryRunReadAction(
                            () -> {
                              try {
                                if (DumbService.getInstance(myProject).isDumb()
                                    && !DumbService.isDumbAware(myPass)) {
                                  return;
                                }

                                if (!myUpdateProgress.isCanceled() && !myProject.isDisposed()) {
                                  myPass.collectInformation(myUpdateProgress);
                                }
                              } catch (ProcessCanceledException e) {
                                log(myUpdateProgress, myPass, "Canceled ");

                                if (!myUpdateProgress.isCanceled()) {
                                  myUpdateProgress.cancel(
                                      e); // in case when some smart asses throw PCE just for fun
                                }
                              } catch (RuntimeException | Error e) {
                                myUpdateProgress.cancel(e);
                                LOG.error(e);
                                throw e;
                              }
                            });

                if (!success) {
                  myUpdateProgress.cancel();
                }
              },
              myUpdateProgress);

      log(myUpdateProgress, myPass, "Finished. ");

      if (!myUpdateProgress.isCanceled()) {
        applyInformationToEditorsLater(
            myFileEditor, myPass, myUpdateProgress, myThreadsToStartCountdown);
        for (ScheduledPass successor : mySuccessorsOnCompletion) {
          int predecessorsToRun = successor.myRunningPredecessorsCount.decrementAndGet();
          if (predecessorsToRun == 0) {
            submit(successor);
          }
        }
      }
    }
Example #28
0
    public void assignToLeafTasks(LinkedList<TaskRequestEvent> taskRequests) {
      Collections.shuffle(taskRequests);
      LinkedList<TaskRequestEvent> remoteTaskRequests = new LinkedList<>();
      String queryMasterHostAndPort =
          context
              .getMasterContext()
              .getQueryMasterContext()
              .getWorkerContext()
              .getConnectionInfo()
              .getHostAndQMPort();

      TaskRequestEvent taskRequest;
      while (leafTasks.size() > 0 && (!taskRequests.isEmpty() || !remoteTaskRequests.isEmpty())) {
        int localAssign = 0;
        int rackAssign = 0;

        taskRequest = taskRequests.pollFirst();
        if (taskRequest == null) { // if there are only remote task requests
          taskRequest = remoteTaskRequests.pollFirst();
        }

        // checking if this container is still alive.
        // If not, ignore the task request and stop the task runner
        WorkerConnectionInfo connectionInfo =
            context.getMasterContext().getWorkerMap().get(taskRequest.getWorkerId());
        if (connectionInfo == null) continue;

        // getting the hostname of requested node
        String host = connectionInfo.getHost();

        // if there are no worker matched to the hostname a task request
        if (!leafTaskHostMapping.containsKey(host) && !taskRequests.isEmpty()) {
          String normalizedHost = NetUtils.normalizeHost(host);

          if (!leafTaskHostMapping.containsKey(normalizedHost)) {
            // this case means one of either cases:
            // * there are no blocks which reside in this node.
            // * all blocks which reside in this node are consumed, and this task runner requests a
            // remote task.
            // In this case, we transfer the task request to the remote task request list, and skip
            // the followings.
            remoteTaskRequests.add(taskRequest);
            continue;
          } else {
            host = normalizedHost;
          }
        }

        if (LOG.isDebugEnabled()) {
          LOG.debug(
              "assignToLeafTasks: "
                  + taskRequest.getExecutionBlockId()
                  + ","
                  + "worker="
                  + connectionInfo.getHostAndPeerRpcPort());
        }

        //////////////////////////////////////////////////////////////////////
        // disk or host-local allocation
        //////////////////////////////////////////////////////////////////////
        TaskAttemptId attemptId = allocateLocalTask(host);

        if (attemptId == null) { // if a local task cannot be found
          HostVolumeMapping hostVolumeMapping = leafTaskHostMapping.get(host);

          if (!taskRequests
              .isEmpty()) { // if other requests remains, move to remote list for better locality
            remoteTaskRequests.add(taskRequest);
            candidateWorkers.remove(connectionInfo.getId());
            continue;

          } else {
            if (hostVolumeMapping != null) {
              int nodes = context.getMasterContext().getWorkerMap().size();
              // this part is to control the assignment of tail and remote task balancing per node
              int tailLimit = 1;
              if (remainingScheduledObjectNum() > 0 && nodes > 0) {
                tailLimit = Math.max(remainingScheduledObjectNum() / nodes, 1);
              }

              if (hostVolumeMapping.getRemoteConcurrency()
                  >= tailLimit) { // remote task throttling per node
                continue;
              } else {
                // assign to remote volume
                hostVolumeMapping.increaseConcurrency(HostVolumeMapping.REMOTE);
              }
            }
          }

          //////////////////////////////////////////////////////////////////////
          // rack-local allocation
          //////////////////////////////////////////////////////////////////////
          attemptId = allocateRackTask(host);

          //////////////////////////////////////////////////////////////////////
          // random node allocation
          //////////////////////////////////////////////////////////////////////
          if (attemptId == null && leafTaskNum() > 0) {
            synchronized (leafTasks) {
              attemptId = leafTasks.iterator().next();
              leafTasks.remove(attemptId);
            }
          }

          if (attemptId != null && hostVolumeMapping != null) {
            hostVolumeMapping.lastAssignedVolumeId.put(attemptId, HostVolumeMapping.REMOTE);
          }
          rackAssign++;
        } else {
          localAssign++;
        }

        if (attemptId != null) {
          Task task = stage.getTask(attemptId.getTaskId());
          TaskRequest taskAssign =
              new TaskRequestImpl(
                  attemptId,
                  new ArrayList<>(task.getAllFragments()),
                  "",
                  false,
                  LogicalNodeSerializer.serialize(task.getLogicalPlan()),
                  context.getMasterContext().getQueryContext(),
                  stage.getDataChannel(),
                  stage.getBlock().getEnforcer(),
                  queryMasterHostAndPort);

          if (checkIfInterQuery(stage.getMasterPlan(), stage.getBlock())) {
            taskAssign.setInterQuery();
          }

          // TODO send batch request
          BatchAllocationRequest.Builder requestProto = BatchAllocationRequest.newBuilder();
          requestProto.addTaskRequest(
              TaskAllocationProto.newBuilder()
                  .setResource(taskRequest.getResponseProto().getResource())
                  .setTaskRequest(taskAssign.getProto())
                  .build());

          requestProto.setExecutionBlockId(attemptId.getTaskId().getExecutionBlockId().getProto());
          context
              .getMasterContext()
              .getEventHandler()
              .handle(new TaskAttemptAssignedEvent(attemptId, connectionInfo));

          InetSocketAddress addr = stage.getAssignedWorkerMap().get(connectionInfo.getId());
          if (addr == null)
            addr = new InetSocketAddress(connectionInfo.getHost(), connectionInfo.getPeerRpcPort());

          AsyncRpcClient tajoWorkerRpc = null;
          CallFuture<BatchAllocationResponse> callFuture = new CallFuture<>();
          totalAttempts++;
          try {
            tajoWorkerRpc =
                RpcClientManager.getInstance()
                    .getClient(addr, TajoWorkerProtocol.class, true, rpcParams);

            TajoWorkerProtocol.TajoWorkerProtocolService tajoWorkerRpcClient =
                tajoWorkerRpc.getStub();
            tajoWorkerRpcClient.allocateTasks(
                callFuture.getController(), requestProto.build(), callFuture);

            BatchAllocationResponse responseProto =
                callFuture.get(RpcConstants.FUTURE_TIMEOUT_SECONDS_DEFAULT, TimeUnit.SECONDS);

            if (responseProto.getCancellationTaskCount() > 0) {
              for (TaskAllocationProto proto : responseProto.getCancellationTaskList()) {
                cancel(task.getAttempt(new TaskAttemptId(proto.getTaskRequest().getId())));
                cancellation++;
              }

              if (LOG.isDebugEnabled()) {
                LOG.debug(
                    "Canceled requests: "
                        + responseProto.getCancellationTaskCount()
                        + " from "
                        + addr);
              }
              continue;
            }
          } catch (Exception e) {
            LOG.error(e);
          }
          scheduledObjectNum--;
          totalAssigned++;
          hostLocalAssigned += localAssign;
          rackLocalAssigned += rackAssign;

          if (rackAssign > 0) {
            LOG.info(
                String.format(
                    "Assigned Local/Rack/Total: (%d/%d/%d), "
                        + "Attempted Cancel/Assign/Total: (%d/%d/%d), "
                        + "Locality: %.2f%%, Rack host: %s",
                    hostLocalAssigned,
                    rackLocalAssigned,
                    totalAssigned,
                    cancellation,
                    totalAssigned,
                    totalAttempts,
                    ((double) hostLocalAssigned / (double) totalAssigned) * 100,
                    host));
          }

        } else {
          throw new RuntimeException("Illegal State!!!!!!!!!!!!!!!!!!!!!");
        }
      }
    }
    private ReturnResult checkUserName() {
      Project project = myPanel.getProject();
      GitVcs vcs = GitVcs.getInstance(project);
      assert vcs != null;

      Collection<VirtualFile> notDefined = new ArrayList<VirtualFile>();
      Map<VirtualFile, Pair<String, String>> defined =
          new HashMap<VirtualFile, Pair<String, String>>();
      Collection<VirtualFile> allRoots =
          new ArrayList<VirtualFile>(
              Arrays.asList(ProjectLevelVcsManager.getInstance(project).getRootsUnderVcs(vcs)));

      Collection<VirtualFile> affectedRoots = getSelectedRoots();
      for (VirtualFile root : affectedRoots) {
        try {
          Pair<String, String> nameAndEmail = getUserNameAndEmailFromGitConfig(project, root);
          String name = nameAndEmail.getFirst();
          String email = nameAndEmail.getSecond();
          if (name == null || email == null) {
            notDefined.add(root);
          } else {
            defined.put(root, nameAndEmail);
          }
        } catch (VcsException e) {
          LOG.error("Couldn't get user.name and user.email for root " + root, e);
          // doing nothing - let commit with possibly empty user.name/email
        }
      }

      if (notDefined.isEmpty()) {
        return ReturnResult.COMMIT;
      }

      GitVersion version = vcs.getVersion();
      if (System.getenv("HOME") == null
          && GitVersionSpecialty.DOESNT_DEFINE_HOME_ENV_VAR.existsIn(version)) {
        Messages.showErrorDialog(
            project,
            "You are using Git "
                + version
                + " which doesn't define %HOME% environment variable properly.\n"
                + "Consider updating Git to a newer version "
                + "or define %HOME% to point to the place where the global .gitconfig is stored \n"
                + "(it is usually %USERPROFILE% or %HOMEDRIVE%%HOMEPATH%).",
            "HOME Variable Is Not Defined");
        return ReturnResult.CANCEL;
      }

      if (defined.isEmpty() && allRoots.size() > affectedRoots.size()) {
        allRoots.removeAll(affectedRoots);
        for (VirtualFile root : allRoots) {
          try {
            Pair<String, String> nameAndEmail = getUserNameAndEmailFromGitConfig(project, root);
            String name = nameAndEmail.getFirst();
            String email = nameAndEmail.getSecond();
            if (name != null && email != null) {
              defined.put(root, nameAndEmail);
              break;
            }
          } catch (VcsException e) {
            LOG.error("Couldn't get user.name and user.email for root " + root, e);
            // doing nothing - not critical not to find the values for other roots not affected by
            // commit
          }
        }
      }

      GitUserNameNotDefinedDialog dialog =
          new GitUserNameNotDefinedDialog(project, notDefined, affectedRoots, defined);
      dialog.show();
      if (dialog.isOK()) {
        try {
          if (dialog.isGlobal()) {
            GitConfigUtil.setValue(
                project,
                notDefined.iterator().next(),
                GitConfigUtil.USER_NAME,
                dialog.getUserName(),
                "--global");
            GitConfigUtil.setValue(
                project,
                notDefined.iterator().next(),
                GitConfigUtil.USER_EMAIL,
                dialog.getUserEmail(),
                "--global");
          } else {
            for (VirtualFile root : notDefined) {
              GitConfigUtil.setValue(project, root, GitConfigUtil.USER_NAME, dialog.getUserName());
              GitConfigUtil.setValue(
                  project, root, GitConfigUtil.USER_EMAIL, dialog.getUserEmail());
            }
          }
        } catch (VcsException e) {
          String message = "Couldn't set user.name and user.email";
          LOG.error(message, e);
          Messages.showErrorDialog(myPanel.getComponent(), message);
          return ReturnResult.CANCEL;
        }
        return ReturnResult.COMMIT;
      }
      return ReturnResult.CLOSE_WINDOW;
    }
    @Override
    public ScriptContext call() throws Exception {
      try {
        Scanner scn = new Scanner(znodePath);
        scn.useDelimiter(":");
        String hostName = scn.next(); // host name
        String instance = scn.next(); // instance
        int infoPort = Integer.parseInt(scn.next()); // UI info port
        long serverStartTimestamp = Long.parseLong(scn.next());
        scn.close();

        // Get the --config property from classpath...it's always first
        // in the classpath
        String cp = System.getProperty("java.class.path");
        scn = new Scanner(cp);
        scn.useDelimiter(":");
        String confDir = scn.next();
        scn.close();
        LOG.debug("conf dir [" + confDir + "]");

        // Get -Dwms.home.dir
        String wmsHome = System.getProperty("wms.home.dir");

        // If stop-wms.sh is executed and WMS_MANAGES_ZK then zookeeper
        // is stopped abruptly.
        // Second scenario is when ZooKeeper fails for some reason
        // regardless of whether WMS
        // manages it. When either happens the WmsServer running znodes
        // still exist in ZooKeeper
        // and we see them at next startup. When they eventually timeout
        // we get node deleted events for a server that no longer
        // exists. So, only recognize
        // WmsServer running znodes that have timestamps after last
        // WmsMaster startup.
        if (serverStartTimestamp > startupTimestamp) {
          scriptContext.setHostName(hostName);
          scriptContext.setScriptName("sys_shell.py");
          if (hostName.equalsIgnoreCase(ia.getCanonicalHostName()))
            scriptContext.setCommand(
                "bin/wms-daemon.sh --config " + confDir + " start server " + instance);
          else
            scriptContext.setCommand(
                "pdsh -w "
                    + hostName
                    + " \"cd "
                    + wmsHome
                    + ";bin/wms-daemon.sh --config "
                    + confDir
                    + " start server "
                    + instance
                    + "\"");

          RetryCounter retryCounter = retryCounterFactory.create();
          while (true) {
            if (scriptContext.getStdOut().length() > 0)
              scriptContext.getStdOut().delete(0, scriptContext.getStdOut().length());
            if (scriptContext.getStdErr().length() > 0)
              scriptContext.getStdErr().delete(0, scriptContext.getStdErr().length());
            LOG.info(
                "Restarting WmsServer ["
                    + hostName
                    + ":"
                    + instance
                    + "], script [ "
                    + scriptContext.toString()
                    + " ]");
            ScriptManager.getInstance().runScript(scriptContext);

            if (scriptContext.getExitCode() == 0) {
              LOG.info("WmsServer [" + hostName + ":" + instance + "] restarted");
              break;
            } else {
              StringBuilder sb = new StringBuilder();
              sb.append("exit code [" + scriptContext.getExitCode() + "]");
              if (!scriptContext.getStdOut().toString().isEmpty())
                sb.append(", stdout [" + scriptContext.getStdOut().toString() + "]");
              if (!scriptContext.getStdErr().toString().isEmpty())
                sb.append(", stderr [" + scriptContext.getStdErr().toString() + "]");
              LOG.error(sb.toString());

              if (!retryCounter.shouldRetry()) {
                LOG.error(
                    "WmsServer ["
                        + hostName
                        + ":"
                        + instance
                        + "] restart failed after "
                        + retryCounter.getMaxRetries()
                        + " retries");
                break;
              } else {
                retryCounter.sleepUntilNextRetry();
                retryCounter.useRetry();
              }
            }
          }
        } else {
          LOG.debug(
              "No restart for "
                  + znodePath
                  + "\nbecause WmsServer start time ["
                  + DateFormat.getDateTimeInstance().format(new Date(serverStartTimestamp))
                  + "] was before WmsMaster start time ["
                  + DateFormat.getDateTimeInstance().format(new Date(startupTimestamp))
                  + "]");
        }
      } catch (Exception e) {
        e.printStackTrace();
        LOG.error(e);
      }

      return scriptContext;
    }