Example #1
0
  public static LogEntry process(
      long executiontime, ExecutionPlan plan, OperatorGroup op, QueueManager qm, CacheManager cm)
      throws StreamSpinnerException {
    ANDNode[] node = op.getANDNodes();

    ORNode inputn = node[0].getInputORNodes()[0];
    Queue inputq = qm.getQueue(plan, op.getMasterSet(), inputn);

    Queue[] outputq = new Queue[node.length];
    for (int i = 0; i < node.length; i++) {
      ORNode outputn = node[i].getOutputORNode();
      outputq[i] = qm.getQueue(plan, node[i].getMasterSet(), outputn);
      if (outputq[i] == null)
        throw new StreamSpinnerException("no input queue for " + outputn.toString());
    }

    TupleSet ts = getDeltaTupleSet(executiontime, plan, op, inputn, inputq, cm);

    LogEntry le = null;
    if (op.isCacheConsumer() == true) le = consumeCacheData(executiontime, plan, op, qm, cm);
    else le = new LogEntry(executiontime, op);

    if (ts.first() == false) return le;

    FunctionParameter fp = node[0].getFunctionParameter();
    Schema origschema = ts.getSchema();

    int[] indexes = new int[fp.getArguments().size()];
    for (int i = 0; i < indexes.length; i++)
      indexes[i] = origschema.getIndex(fp.getArguments().getString(i));

    Function f = Function.getInstance(fp, origschema);
    Schema newschema = origschema.append(fp.toString(), f.getReturnType());

    OnMemoryTupleSet result = new OnMemoryTupleSet(newschema);
    do {
      Tuple t = ts.getTuple();
      Object[] values = new Object[indexes.length];
      for (int i = 0; i < values.length; i++) values[i] = t.getObject(indexes[i]);
      Object returnvalue = f.invoke(values);
      Tuple newtuple = t.appendColumn();
      newtuple.setObject(newtuple.size() - 1, returnvalue);
      for (int i = 0; i < node.length; i++) outputq[i].append(newtuple);
      result.append(newtuple);
      le.add(newtuple);
    } while (ts.next());

    ts.close();
    produceCacheData(executiontime, plan, op, cm, result);
    inputq.moveToWindow(op, executiontime);
    return le;
  }
  public List<AbstractMemberMessage> getAllMessages(String recipient) {
    List<AbstractMemberMessage> messages = new LinkedList<>();
    QueueManager queueManager = new QueueManager(getQueueName(recipient));

    AbstractMemberMessage message = queueManager.consume();
    while (message != null) {
      messages.add(message);
      message = queueManager.consume();
    }

    System.out.println(recipient + " read all messages");
    return messages;
  }
  public void sendMessage(AbstractMemberMessage messageObject) {
    QueueManager queueManager = new QueueManager(getQueueName(messageObject.getRecipient()));
    queueManager.produce(messageObject);
    System.out.println(
        messageObject.getSender() + " send a message to " + messageObject.getRecipient());

    if (_listeners.containsKey(messageObject.getRecipient())) {
      try {
        _listeners.get(messageObject.getRecipient()).notifyAboutMessage();
        System.out.println("The recipient was notified about this");
      } catch (RemoteException e) {
        unregisterListener(messageObject.getRecipient());
      }
    }
  }
Example #4
0
  private void refreshPodcast() {
    if (_player == null) return;

    PlayerStatus status = PlayerStatus.getCurrentState(this);
    if (!status.hasActivePodcast()) {
      _player.stop();
      return;
    }

    if (status.getPodcastId() != _currentPodcastId) {
      _currentPodcastId = status.getPodcastId();
      if (!_player.prepare(status.getFilename(), status.getPosition())) {
        _player.stop();
        String toastMessage =
            getResources().getString(R.string.cannot_play_podcast, status.getTitle());
        Toast.makeText(this, toastMessage, Toast.LENGTH_LONG).show();
      } else {
        QueueManager.changeActivePodcast(this, status.getPodcastId());
        _lockscreenManager.setupLockscreenControls(this, status);
        showNotification();
      }
    } else {
      _player.seekTo(status.getPosition());
    }
  }
Example #5
0
  @Before
  public void setup() throws IOException {
    FairScheduler scheduler = new FairScheduler();
    Configuration conf = createConfiguration();
    // All tests assume only one assignment per node update
    conf.set(FairSchedulerConfiguration.ASSIGN_MULTIPLE, "false");
    ResourceManager resourceManager = new ResourceManager();
    resourceManager.init(conf);
    ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());

    String queueName = "root.queue1";
    QueueManager mockMgr = mock(QueueManager.class);
    when(mockMgr.getMaxResources(queueName)).thenReturn(maxResource);
    when(mockMgr.getMinResources(queueName)).thenReturn(Resources.none());

    schedulable = new FSLeafQueue(queueName, mockMgr, scheduler, null);
  }
 @Test
 public void testRemoveEnablesAppOnCousinQueue() {
   FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
   FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
   queueMaxApps.put("root.queue1", 2);
   FSAppAttempt app1 = addApp(leaf1, "user");
   addApp(leaf2, "user");
   addApp(leaf2, "user");
   assertEquals(1, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
   removeApp(app1);
   assertEquals(0, leaf1.getRunnableAppSchedulables().size());
   assertEquals(2, leaf2.getRunnableAppSchedulables().size());
   assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
 }
  @Before
  public void setup() throws Exception {
    Configuration conf = new Configuration();
    clock = new TestFairScheduler.MockClock();
    scheduler = mock(FairScheduler.class);
    when(scheduler.getConf()).thenReturn(new FairSchedulerConfiguration(conf));
    when(scheduler.getClock()).thenReturn(clock);
    AllocationConfiguration allocConf = new AllocationConfiguration(conf);
    when(scheduler.getAllocationConfiguration()).thenReturn(allocConf);

    queueManager = new QueueManager(scheduler);
    queueManager.initialize(conf);
    queueMaxApps = allocConf.queueMaxApps;
    userMaxApps = allocConf.userMaxApps;
    maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
    appNum = 0;
    rmContext = mock(RMContext.class);
    when(rmContext.getEpoch()).thenReturn(0L);
  }
 @Test
 public void testRemoveEnablesOneByQueueOneByUser() {
   FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.leaf1", true);
   FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true);
   queueMaxApps.put("root.queue1.leaf1", 2);
   userMaxApps.put("user1", 1);
   FSAppAttempt app1 = addApp(leaf1, "user1");
   addApp(leaf1, "user2");
   addApp(leaf1, "user3");
   addApp(leaf2, "user1");
   assertEquals(2, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf1.getNonRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getNonRunnableAppSchedulables().size());
   removeApp(app1);
   assertEquals(2, leaf1.getRunnableAppSchedulables().size());
   assertEquals(1, leaf2.getRunnableAppSchedulables().size());
   assertEquals(0, leaf1.getNonRunnableAppSchedulables().size());
   assertEquals(0, leaf2.getNonRunnableAppSchedulables().size());
 }
Example #9
0
 public void cancel() {
   QueueManager.getManager().getQueue().remove(this);
 }
Example #10
0
 public static void play(Context context, long podcastId) {
   QueueManager.changeActivePodcast(context, podcastId);
   PlayerService.sendCommand(context, Constants.PLAYER_COMMAND_REFRESHPODCAST);
 }
Example #11
0
  /**
   * Check the ACLs for a user doing the passed operation.
   *
   * <ul>
   *   <li>If ACLs are disabled, allow all users.
   *   <li>Otherwise, if the operation is not a job operation(for eg. submit-job-to-queue), then
   *       allow only (a) clusterOwner(who started the cluster), (b) cluster administrators and (c)
   *       members of queue-submit-job-acl for the queue.
   *   <li>If the operation is a job operation, then allow only (a) jobOwner, (b) clusterOwner(who
   *       started the cluster), (c) cluster administrators, (d) members of queue admins acl for the
   *       queue and (e) members of job acl for the job operation
   * </ul>
   *
   * @param jobId the job id
   * @param callerUGI the user who is trying to perform the operation
   * @param queue the job queue name
   * @param operation the operation for which authorization is needed
   * @param jobOwner the user who submitted(or is submitting) this job
   * @param jobAcl could be job-view-acl or job-modify-acl depending on the job operation.
   */
  void checkAccess(
      String jobId,
      UserGroupInformation callerUGI,
      String queue,
      Operation operation,
      String jobOwner,
      AccessControlList jobAcl)
      throws AccessControlException {

    String user = callerUGI.getShortUserName();
    String targetResource = jobId + " in queue " + queue;

    if (!aclsEnabled) {
      AuditLogger.logSuccess(user, operation.name(), targetResource);
      return;
    }

    // Allow mapreduce cluster admins to do any queue operation and
    // any job operation
    if (isMRAdmin(callerUGI)) {
      AuditLogger.logSuccess(user, operation.name(), targetResource);
      return;
    }

    if (operation == Operation.SUBMIT_JOB) {
      // This is strictly queue operation(not a job operation)
      if (!queueManager.hasAccess(queue, operation.qACLNeeded, callerUGI)) {
        AuditLogger.logFailure(
            user,
            operation.name(),
            queueManager.getQueueACL(queue, operation.qACLNeeded).toString(),
            targetResource,
            Constants.UNAUTHORIZED_USER);

        throw new AccessControlException(
            "User "
                + callerUGI.getShortUserName()
                + " cannot perform "
                + "operation "
                + operation.name()
                + " on queue "
                + queue
                + ".\n Please run \"hadoop queue -showacls\" "
                + "command to find the queues you have access to .");
      } else {
        AuditLogger.logSuccess(user, operation.name(), targetResource);
        return;
      }
    }

    // Check if callerUGI is queueAdmin(in some cases only), jobOwner or
    // part of job-acl.

    // queueManager and queue are null only when called from
    // TaskTracker(i.e. from TaskLogServlet) for the operation VIEW_TASK_LOGS.
    // Caller of this method takes care of checking if callerUGI is a
    // queue administrator for that operation.
    if (operation == Operation.VIEW_TASK_LOGS) {
      if (jobACLsManager.checkAccess(callerUGI, operation.jobACLNeeded, jobOwner, jobAcl)) {
        AuditLogger.logSuccess(user, operation.name(), targetResource);
        return;
      }
    } else if (queueManager.hasAccess(queue, operation.qACLNeeded, callerUGI)
        || jobACLsManager.checkAccess(callerUGI, operation.jobACLNeeded, jobOwner, jobAcl)) {
      AuditLogger.logSuccess(user, operation.name(), targetResource);
      return;
    }

    AuditLogger.logFailure(
        user, operation.name(), jobAcl.toString(), targetResource, Constants.UNAUTHORIZED_USER);

    throw new AccessControlException(
        "User "
            + callerUGI.getShortUserName()
            + " cannot perform operation "
            + operation.name()
            + " on "
            + jobId
            + " that is in the queue "
            + queue);
  }