/**
  * Monte-Carlo simulation for playout.
  *
  * @param node Node to start at
  * @param timeout Time limit
  * @return Scores for all players
  */
 private int[] playout(Node node, long timeout) {
   if (node instanceof TerminalNode) {
     return ((TerminalNode) node).goal;
   }
   MachineState state = ((NonTerminalNode) node).state;
   while (System.currentTimeMillis() < timeout) {
     if (theMachine.isTerminal(state)) {
       List<Integer> s;
       try {
         s = theMachine.getGoals(state);
       } catch (GoalDefinitionException e) {
         System.err.println("Could not compute goal values.");
         return null;
       }
       int[] scores = new int[s.size()];
       for (int i = 0; i < scores.length; i++) {
         scores[i] = s.get(i);
       }
       return scores;
     }
     try {
       state = theMachine.getNextState(state, theMachine.getRandomJointMove(state));
     } catch (TransitionDefinitionException e) {
       System.err.println("Could not perform state update.");
       return null;
     } catch (MoveDefinitionException e) {
       System.err.println("Could not compute legal moves.");
       return null;
     }
   }
   return null;
 }
 /**
  * Performs Monte-Carlo tree search.
  *
  * @param timeout Time limit
  * @return Best move in the root state
  */
 private Move mcts(long timeout) {
   counter = 0;
   // Move bestMove = null;
   // double bestScore = Double.NEGATIVE_INFINITY;
   while (System.currentTimeMillis() < timeout) {
     mctsStep(timeout);
     /*
     int next = mctsStep(timeout);
     if(next == -1){
     	continue;
     }
     double score = root.q_action[role][next] - uct(root.n, root.n_action[role][next]);
     if(score > bestScore){
     	bestScore = score;
     	bestMove = root.legal.get(role).get(next);
     }
     */
   }
   System.out.println("Did " + counter + " MCTS steps.");
   int best = -1;
   int bestScore = Integer.MIN_VALUE;
   confidence = 0;
   for (int i = 0; i < root.n_action[role].length; i++) {
     if (root.n_action[role][i] > bestScore) {
       bestScore = root.n_action[role][i];
       best = i;
       confidence = (int) (root.q_action[role][i] - uct(root.n, root.n_action[role][i]));
     }
   }
   // System.out.println("MonteCarlo: Computed confidence value of " + confidence + " for move with
   // score " + bestScore + ".");
   if (confidence < MIN_SCORE) {
     confidence = MIN_SCORE;
   }
   return (best == -1) ? null : root.legal.get(role).get(best);
 }
  /**
   * Performs one MCTS step.
   *
   * @param timeout Time limit
   * @return Index of the move performed by the player in the root state
   * @throws TimeoutException Time limit reached
   */
  private int mctsStep(long timeout) {
    List<NonTerminalNode> path = new ArrayList<NonTerminalNode>();
    int[] scores = null;
    Node current = root;
    NonTerminalNode c = (NonTerminalNode) root;
    // int rootMove = -1;

    // Selection
    List<List<Integer>> select = new ArrayList<List<Integer>>();
    List<Move> selectM = new ArrayList<Move>();
    while (current != null) {
      if (System.currentTimeMillis() > timeout) {
        return -1;
      }
      if (current instanceof TerminalNode) {
        scores = ((TerminalNode) current).goal;
        break;
      }
      c = (NonTerminalNode) current;
      path.add(c);
      List<Integer> selectEntry = new ArrayList<Integer>();
      selectM = new ArrayList<Move>();
      for (int p = 0; p < theMachine.getRoles().size(); p++) {
        List<Move> moves = c.legal.get(p);
        double bestval = Double.NEGATIVE_INFINITY;
        Move bestmove;
        int bestindex;
        if (c.n == 0) {
          bestmove = moves.get(0);
          bestindex = 0;
        } else {
          bestmove = null;
          bestindex = -1;
          for (int i = 0; i < moves.size(); i++) {
            Move m = moves.get(i);
            double value = c.q_action[p][i] + uct(c.n, c.n_action[p][i]);
            // System.out.println("Found a move with score " + value);
            if (value > bestval) {
              bestval = value;
              bestmove = m;
              bestindex = i;
            }
          }
        }
        selectEntry.add(bestindex);
        selectM.add(bestmove);
      }
      select.add(selectEntry);
      /*
      if(current == root){
      	rootMove = select.get(select.size() - 1)[role];
      }
      */
      current = c.successors.get(selectEntry);
    }

    // Expansion & Playout
    if (!(current instanceof TerminalNode)) {
      MachineState next;
      try {
        next = theMachine.getNextState(c.state, selectM);
      } catch (TransitionDefinitionException e) {
        System.err.println("Could not perform state update.");
        return -1;
      }
      if (theMachine.isTerminal(next)) {
        current = new TerminalNode(next);
      } else {
        current = new NonTerminalNode(next);
      }
      c.successors.put(select.get(select.size() - 1), current);
      scores = playout(current, timeout);
    }

    if (scores == null) {
      return -1;
    }

    // Backpropagation
    for (int i = 0; i < path.size(); i++) {
      NonTerminalNode node = path.get(i);
      node.n++;
      for (int j = 0; j < theMachine.getRoles().size(); j++) {
        node.n_action[j][select.get(i).get(j)]++;
        node.q_action[j][select.get(i).get(j)] =
            (node.q_action[j][select.get(i).get(j)] * node.n_action[j][select.get(i).get(j)]
                    + scores[j])
                / (node.n_action[j][select.get(i).get(j)] + 1);
      }
    }

    counter++;
    return select.get(0).get(role);
  }